From 6a65a6f4eadf0f209944e735930c9d859af37141 Mon Sep 17 00:00:00 2001 From: Nicole Lo Date: Tue, 30 Jun 2020 19:11:31 +0800 Subject: [PATCH 001/271] Create draft pythonpackage.yml --- .github/workflows/pythonpackage.yml | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000000..ade2056935 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,34 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-18.04, windows-2016] + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Update build tools + run: | + python -m pip install --upgrade pip==18.1 setuptools==30.2.1 wheel + - name: Install dependencies + run: | + python -m pip install -r min-requirements.txt + - name: Install pydra + run: | + python -m pip install .[test] + - name: Pytest tests + run: | + pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra From dcb9023336cc011e830b0e8d3f5fed6e9952f439 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 1 Jul 2020 07:50:56 -0400 Subject: [PATCH 002/271] fix: updates cloudpickle see https://github.com/explosion/srsly/pull/12 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d7dea4e0d6..68ab4ff08b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ classifiers = python_requires = >= 3.7 install_requires = attrs - cloudpickle >= 0.8.0 + cloudpickle >= 1.2.2 filelock >= 3.0.0 etelemetry >= 0.2.0 From 02f86f30532ac655d9b3894525e9d536b9860c25 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 8 Jul 2020 22:50:16 +0800 Subject: [PATCH 003/271] finish all builds even when some builds fail --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ade2056935..77f46d309e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -9,6 +9,7 @@ jobs: build: runs-on: ${{ matrix.os }} + continue-on-error: true strategy: matrix: os: [macos-latest, ubuntu-18.04, windows-2016] From 414aab721dc89121d415afae8fa232d45b0cba28 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 8 Jul 2020 23:13:25 +0800 Subject: [PATCH 004/271] remove azure and travis files --- .azure-pipelines/windows.yml | 43 ------------------ .travis.yml | 85 ------------------------------------ azure-pipelines.yml | 12 ----- 3 files changed, 140 deletions(-) delete mode 100644 .azure-pipelines/windows.yml delete mode 100644 .travis.yml delete mode 100644 azure-pipelines.yml diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml deleted file mode 100644 index 6365a53d74..0000000000 --- a/.azure-pipelines/windows.yml +++ /dev/null @@ -1,43 +0,0 @@ -parameters: - name: '' - vmImage: '' - matrix: [] - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - variables: - DEPENDS: "-r min-requirements.txt" - CHECK_TYPE: test - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - addToPath: true - architecture: '$(PYTHON_ARCH)' - - script: | - echo %PYTHONHASHSEED% - displayName: 'Display hash seed' - - script: | - python -m pip install --upgrade pip==18.1 setuptools==30.2.1 wheel - displayName: 'Update build tools' - - script: | - python -m pip install %DEPENDS% - displayName: 'Install dependencies' - - script: | - python -m pip install .[$(CHECK_TYPE)] - displayName: 'Install pydra' - - script: | - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - displayName: 'Pytest tests' - - script: | - python -m pip install codecov - codecov --file cov.xml - displayName: 'Upload To Codecov' - env: - CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a51146d204..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,85 +0,0 @@ -dist: xenial -sudo: true -language: python - -cache: - directories: - - $HOME/.cache/pip - -python: - - 3.7 - - 3.8 - -services: - - docker - -env: - global: - - CHECK_TYPE="test" - - INSTALL_TYPE="pip" - - INSTALL_DEPENDS="pip setuptools" - - JOBQUEUE="none" - - matrix: - - INSTALL_TYPE="install" - - INSTALL_TYPE="develop" - - INSTALL_TYPE="sdist" - - INSTALL_TYPE="wheel" - - INSTALL_DEPENDS="pip==18.1 setuptools==30.2.1" - - DEPENDS="-r min-requirements.txt" - - CHECK_TYPE="style" - -# This should only fail with pip 10.0.1 with URL requirements (e.g., "package @ URL") -# Useful for testing un-released upstream fixes -matrix: - include: - - python: 3.7 - env: - - INSTALL_TYPE="develop" - - CHECK_TYPE="test_dask" - - os: osx - osx_image: xcode11.2 - language: generic - env: - - JOBQUEUE="osx" - - CONDA_VERSION="3.7" - - os: windows - language: sh - python: "3.7" - env: - - JOBQUEUE="win37" - - python: 3.7 - env: JOBQUEUE="slurm" - - language: go - go: 1.13 - env: - - JOBQUEUE="singularity" - - INSTALL_TYPE="develop" - - python: 3.7 - env: INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0" - - allow_failures: - - python: 3.7 - env: INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0" - - python: 3.7 - env: - - INSTALL_TYPE="develop" - - CHECK_TYPE="test_dask" - - - -before_install: - - source ./ci/${JOBQUEUE}.sh - - travis_before_install - -install: - - travis_install - -before_script: - - travis_before_script - -script: - - travis_script - -after_script: - - travis_after_script diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index fc32bebd67..0000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,12 +0,0 @@ -jobs: -- template: .azure-pipelines/windows.yml - parameters: - name: Windows - vmImage: windows-2019 - matrix: - py37-x86: - PYTHON_VERSION: '3.7' - PYTHON_ARCH: 'x86' - py37-x64: - PYTHON_VERSION: '3.7' - PYTHON_ARCH: 'x64' From 69a53498a2b0a3ca4336e994a7d3cd761779468f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 15 Jul 2020 16:38:22 -0700 Subject: [PATCH 005/271] refactoring the template formatting - separating the argstr case, and the output_template_file in spec; fixing issue with the extension (so the sufix is added properly) --- pydra/engine/core.py | 4 +- pydra/engine/helpers.py | 35 +++++- pydra/engine/helpers_file.py | 110 ++++++++++++------ pydra/engine/specs.py | 31 ++--- pydra/engine/task.py | 24 ++-- pydra/engine/tests/test_shelltask.py | 47 +++++++- .../engine/tests/test_shelltask_inputspec.py | 104 ++++++++++++++++- 7 files changed, 279 insertions(+), 76 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 723b5493f1..7987a40f9a 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -423,9 +423,7 @@ def _collect_outputs(self): self.output_spec = output_from_inputfields(self.output_spec, self.inputs) output_klass = make_klass(self.output_spec) output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) - other_output = output.collect_additional_outputs( - self.input_spec, self.inputs, self.output_dir - ) + other_output = output.collect_additional_outputs(self.inputs, self.output_dir) return attr.evolve(output, **run_output, **other_output) def split(self, splitter, overwrite=False, **kwargs): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index c59ac45d9e..91979f3f18 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -10,7 +10,7 @@ from hashlib import sha256 import subprocess as sp import getpass -import uuid +import re from time import strftime from traceback import format_exception @@ -636,3 +636,36 @@ def position_adjustment(pos_args): cmd_args += el[1] return cmd_args + + +def argstr_formatting(argstr, inputs, value_updates=None): + """ formatting argstr that have form {field_name}, + using values from inputs and updating with value_update if provided + """ + inputs_dict = attr.asdict(inputs) + # if there is a value that has to be updated (e.g. single value from a list) + if value_updates: + inputs_dict.update(value_updates) + # getting all fields that should be formatted, i.e. {field_name}, ... + inp_fields = re.findall("{\w+}", argstr) + val_dict = {} + for fld in inp_fields: + fld_name = fld[1:-1] # extracting the name form {field_name} + fld_value = inputs_dict[fld_name] + if fld_value is attr.NOTHING: + # if value is NOTHING, nothing should be added to the command + val_dict[fld_name] = "" + else: + val_dict[fld_name] = fld_value + + # formatting string based on the val_dict + argstr_formatted = argstr.format(**val_dict) + # removing extra commas and spaces after removing the field that have NOTHING + argstr_formatted = ( + argstr_formatted.replace("[ ", "[") + .replace(" ]", "]") + .replace("[,", "[") + .replace(",]", "]") + .strip() + ) + return argstr_formatted diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 7547073f82..f93c46cea5 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -522,43 +522,85 @@ def template_update(inputs, map_copyfiles=None): f"fields with output_file_template" "has to be a string or Union[str, bool]" ) - inp_val_set = getattr(inputs, fld.name) + dict_[fld.name] = template_update_single(field=fld, inputs_dict=dict_) + # using is and == so it covers list and numpy arrays + return { + k: v + for k, v in dict_.items() + if not (getattr(inputs, k) is v or getattr(inputs, k) == v) + } + + +def template_update_single(field, inputs_dict, spec_type="input"): + """Update a single template from the input_spec or output_spec + based on the value from inputs_dict + (checking the types of the fields, that have "output_file_template)" + """ + from .specs import File + + if spec_type == "input": + if field.type not in [str, ty.Union[str, bool]]: + raise Exception( + f"fields with output_file_template" + "has to be a string or Union[str, bool]" + ) + inp_val_set = inputs_dict[field.name] if inp_val_set is not attr.NOTHING and not isinstance(inp_val_set, (str, bool)): - raise Exception(f"{fld.name} has to be str or bool, but {inp_val_set} set") - if isinstance(inp_val_set, bool) and fld.type is str: raise Exception( - f"type of {fld.name} is str, consider using Union[str, bool]" + f"{field.name} has to be str or bool, but {inp_val_set} set" ) - - if isinstance(inp_val_set, str): - dict_[fld.name] = inp_val_set - elif inp_val_set is False: - # if False, the field should not be used, so setting attr.NOTHING - dict_[fld.name] = attr.NOTHING - else: # True or attr.NOTHING - template = fld.metadata["output_file_template"] - value = template.format(**dict_) - value = removing_nothing(value) - dict_[fld.name] = value - return {k: v for k, v in dict_.items() if getattr(inputs, k) is not v} - - -def removing_nothing(template_str): - """ removing all fields that had NOTHING""" - if "NOTHING" not in template_str: - return template_str - regex = re.compile("[^a-zA-Z_\-]") - fields_str = regex.sub(" ", template_str) - for fld in fields_str.split(): - if "NOTHING" in fld: - template_str = template_str.replace(fld, "") - return ( - template_str.replace("[ ", "[") - .replace(" ]", "]") - .replace(",]", "]") - .replace("[,", "[") - .strip() - ) + if isinstance(inp_val_set, bool) and field.type is str: + raise Exception( + f"type of {field.name} is str, consider using Union[str, bool]" + ) + elif spec_type == "output": + if field.type is not File: + raise Exception( + f"output {field.name} should be a File, but {field.type} set as the type" + ) + else: + raise Exception(f"spec_type can be input or output, but {spec_type} provided") + + if spec_type == "input" and isinstance(inputs_dict[field.name], str): + return inputs_dict[field.name] + elif spec_type == "input" and inputs_dict[field.name] is False: + # if input fld is set to False, the fld shouldn't be used (setting NOTHING) + return attr.NOTHING + else: # inputs_dict[field.name] is True or spec_type is output + template = field.metadata["output_file_template"] + value = _template_formatting(template, inputs_dict) + return value + + +def _template_formatting(template, inputs_dict): + """Formatting a single template based on values from inputs_dict. + Taking into account that field values and template could have file extensions + (assuming that if template has extension, the field value extension is removed, + if field has extension, and no template extension, than it is moved to the end), + """ + inp_fields = re.findall("{\w+}", template) + if len(inp_fields) == 0: + return template + elif len(inp_fields) == 1: + fld_name = inp_fields[0][1:-1] + fld_value = inputs_dict[fld_name] + if fld_value is attr.NOTHING: + return attr.NOTHING + fld_value = str(fld_value) # in case it's a path + if template.endswith(inp_fields[0]): + # if no suffix added in template, the simplest formatting should work + formatted_value = template.format(**{fld_name: fld_value}) + elif "." not in template: # the template doesn't have its own extension + # if the fld_value has extension, it will be moved to the end + filename, *ext = fld_value.split(".", maxsplit=1) + formatted_value = ".".join([template.format(**{fld_name: filename})] + ext) + else: # template has its own extension + # removing fld_value extension if any + filename, *ext = fld_value.split(".", maxsplit=1) + formatted_value = template.format(**{fld_name: filename}) + return formatted_value + else: + raise NotImplementedError("should we allow for more args in the template?") def is_local_file(f): diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 752010b359..ef53fa3479 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -3,6 +3,8 @@ from pathlib import Path import typing as ty +from .helpers_file import template_update_single + def attr_fields(x): return x.__attrs_attrs__ @@ -33,7 +35,7 @@ class SpecInfo: class BaseSpec: """The base dataclass specs for all inputs and outputs.""" - def collect_additional_outputs(self, input_spec, inputs, output_dir): + def collect_additional_outputs(self, inputs, output_dir): """Get additional outputs.""" return {} @@ -322,7 +324,7 @@ class ShellOutSpec(BaseSpec): stderr: ty.Union[File, str] """The process' standard input.""" - def collect_additional_outputs(self, input_spec, inputs, output_dir): + def collect_additional_outputs(self, inputs, output_dir): """Collect additional outputs from shelltask output_spec.""" additional_out = {} for fld in attr_fields(self): @@ -377,26 +379,15 @@ def _field_defaultvalue(self, fld, output_dir): def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" - if "value" in fld.metadata: + if "value" in fld.metadata: # jak output_file_template in input_spec return output_dir / fld.metadata["value"] + # this block is only run if "output_file_template" is provided in output_spec + # if the field is set in input_spec with output_file_template, + # than the field already should have value elif "output_file_template" in fld.metadata: - sfx_tmpl = (output_dir / fld.metadata["output_file_template"]).suffixes - if sfx_tmpl: - # removing suffix from input field if template has it's own suffix - inputs_templ = { - k: v.split(".")[0] - for k, v in inputs.__dict__.items() - if isinstance(v, str) - } - else: - inputs_templ = { - k: v for k, v in inputs.__dict__.items() if isinstance(v, str) - } - out_path = output_dir / fld.metadata["output_file_template"].format( - **inputs_templ - ) - return out_path - + inputs_templ = attr.asdict(inputs) + value = template_update_single(fld, inputs_templ, spec_type="output") + return output_dir / value elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: diff --git a/pydra/engine/task.py b/pydra/engine/task.py index ca0e3e66db..a54fd011e6 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -57,8 +57,8 @@ SingularitySpec, attr_fields, ) -from .helpers import ensure_list, execute, position_adjustment -from .helpers_file import template_update, is_local_file, removing_nothing +from .helpers import ensure_list, execute, position_adjustment, argstr_formatting +from .helpers_file import template_update, is_local_file class FunctionTask(TaskBase): @@ -431,22 +431,24 @@ def _command_pos_args(self, field, state_ind, ind): cmd_add = [] if field.type is bool: + # if value is simply True the original argstr is used, + # if False, nothing is added to the command if value is True: cmd_add.append(argstr) else: sep = field.metadata.get("sep", " ") if argstr.endswith("...") and isinstance(value, list): argstr = argstr.replace("...", "") + # if argstr has a more complex form, with "{input_field}" if "{" in argstr and "}" in argstr: argstr_formatted_l = [] for val in value: - argstr_f = argstr.format(**{field.name: val}).format( - **attr.asdict(self.inputs) + argstr_f = argstr_formatting( + argstr, self.inputs, value_updates={field.name: val} ) - argstr_formatted_l.append(removing_nothing(argstr_f)) - + argstr_formatted_l.append(argstr_f) cmd_el_str = sep.join(argstr_formatted_l) - else: + else: # argstr has a simple form, e.g. "-f", or "--f" cmd_el_str = sep.join([f" {argstr} {val}" for val in value]) else: # in case there are ... when input is not a list @@ -454,15 +456,15 @@ def _command_pos_args(self, field, state_ind, ind): if isinstance(value, list): cmd_el_str = sep.join([str(val) for val in value]) value = cmd_el_str - + # if argstr has a more complex form, with "{input_field}" if "{" in argstr and "}" in argstr: - argstr_f = argstr.format(**attr.asdict(self.inputs)) - cmd_el_str = removing_nothing(argstr_f) - else: + cmd_el_str = argstr_formatting(argstr, self.inputs) + else: # argstr has a simple form, e.g. "-f", or "--f" if value: cmd_el_str = f"{argstr} {value}" else: cmd_el_str = "" + # removing double spacing cmd_el_str = cmd_el_str.strip().replace(" ", " ") if cmd_el_str: cmd_add += cmd_el_str.split(" ") diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index ea421d47ba..24493cb067 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1113,7 +1113,52 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmpdir): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_9(plugin, results_function, tmpdir): +def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): + """ + providing output name using input_spec (name_tamplate in metadata), + the template has a suffix, the extension of the file will be moved to the end + """ + cmd = "cp" + file = tmpdir.join("file.txt") + file.write("content") + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file_orig", + attr.ib( + type=File, + metadata={"position": 2, "help_string": "new file", "argstr": ""}, + ), + ), + ( + "file_copy", + attr.ib( + type=str, + metadata={ + "output_file_template": "{file_orig}_copy", + "help_string": "output file", + "argstr": "", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.file_copy.exists() + assert res.output.file_copy.name == "file_copy.txt" + + +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_inputspec_10(plugin, results_function, tmpdir): """ using input_spec, providing list of files as an input """ file_1 = tmpdir.join("file_1.txt") diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 8b9482b9d2..d177c20d5c 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -479,9 +479,9 @@ def test_shell_cmd_inputs_format_1(): ) shelly = ShellCommandTask( - executable="executable", inpA="inpA", input_spec=my_input_spec + executable="executable", inpA="aaa", input_spec=my_input_spec ) - assert shelly.cmdline == "executable -v inpA" + assert shelly.cmdline == "executable -v aaa" def test_shell_cmd_inputs_format_2(): @@ -1030,6 +1030,98 @@ def test_shell_cmd_inputs_template_6a(): assert shelly.cmdline == "executable inpA" +def test_shell_cmd_inputs_template_7(tmpdir): + """ additional inputs uses output_file_template with a suffix (no extension)""" + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "outA", + "argstr": "", + "output_file_template": "{inpA}_out", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("a_file.txt") + inpA_file.write("content") + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file + ) + + # outA should be formatted in a way that that .txt goes to the end + assert ( + shelly.cmdline + == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out.txt')}" + ) + + +def test_shell_cmd_inputs_template_8(tmpdir): + """additional inputs uses output_file_template with a suffix and an extension""" + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "outA", + "argstr": "", + "output_file_template": "{inpA}_out.txt", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("a_file.t") + inpA_file.write("content") + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file + ) + + # outA should be formatted in a way that inpA extension is removed and the template extension is used + assert ( + shelly.cmdline + == f"executable {tmpdir.join('a_file.t')} {tmpdir.join('a_file_out.txt')}" + ) + + def test_shell_cmd_inputs_di(tmpdir): """ example from #279 """ my_input_spec = SpecInfo( @@ -1222,7 +1314,7 @@ def test_shell_cmd_inputs_di(tmpdir): bases=(ShellSpec,), ) - my_input_file = tmpdir.join("a_file") + my_input_file = tmpdir.join("a_file.ext") my_input_file.write("content") # no input provided @@ -1239,7 +1331,7 @@ def test_shell_cmd_inputs_di(tmpdir): ) assert ( shelly.cmdline - == f"DenoiseImage -i {my_input_file} -s 1 -p 1 -r 2 -o [{my_input_file}_out]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}]" ) # input file name, noiseImage is set to True, so template is used in the utput @@ -1251,7 +1343,7 @@ def test_shell_cmd_inputs_di(tmpdir): ) assert ( shelly.cmdline - == f"DenoiseImage -i {my_input_file} -s 1 -p 1 -r 2 -o [{my_input_file}_out, {my_input_file}_noise]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}, {tmpdir.join('a_file_noise.ext')}]" ) # input file name and help_short @@ -1263,7 +1355,7 @@ def test_shell_cmd_inputs_di(tmpdir): ) assert ( shelly.cmdline - == f"DenoiseImage -i {my_input_file} -s 1 -p 1 -r 2 -h -o [{my_input_file}_out]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -h -o [{tmpdir.join('a_file_out.ext')}]" ) assert shelly.output_names == [ From dfb752034d9dd0d8628aacbd6cb723c9f586d7b5 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 15 Jul 2020 21:52:21 -0700 Subject: [PATCH 006/271] updating the docstrings for messenger and audit --- pydra/engine/audit.py | 29 ++++++++++++++++------------ pydra/utils/messenger.py | 41 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 1b9561c270..5b598fe0c8 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -18,12 +18,13 @@ def __init__(self, audit_flags, messengers, messenger_args, develop=None): ---------- audit_flags : :class:`AuditFlag` Base configuration of auditing. - messengers : - TODO - messenger_args : - TODO - develop : - TODO + messengers : :class:`pydra.util.messenger.Messenger` or list of :class:`pydra.util.messenger.Messenger`, optional + Specify types of messenger used by Audit to send a message. + Could be `PrintMessenger`, `FileMessenger`, or `RemoteRESTMessenger`. + messenger_args : :obj:`dict`, optional + Optional arguments for the `Messenger.send` method. + develop : :obj:`bool`, optional + If True, the local context.jsonld file is used, otherwise the one from github is used. """ self.audit_flags = audit_flags @@ -115,10 +116,10 @@ def audit_message(self, message, flags=None): Parameters ---------- - message : - TODO - flags : - TODO + message : :obj:`dict` + All the information needed to build a message (TODO) + flags : :obj:`bool`, optional + If True and self.audit_flag, the message is sent. """ if self.develop: @@ -149,8 +150,12 @@ def audit_check(self, flag): Parameters ---------- - flag : - TODO + flag : :obj: `bool` + The flag that is checked. + Returns + ------- + bool + Boolean AND for self.oudit_flags and flag """ return self.audit_flags & flag diff --git a/pydra/utils/messenger.py b/pydra/utils/messenger.py index 33f020b378..0058029c95 100644 --- a/pydra/utils/messenger.py +++ b/pydra/utils/messenger.py @@ -56,7 +56,7 @@ class PrintMessenger(Messenger): def send(self, message, **kwargs): """ - Send to standard output. + Send the message to standard output. Parameters ---------- @@ -88,6 +88,10 @@ def send(self, message, append=True, **kwargs): append : :obj:`bool` Do not truncate file when opening (i.e. append to it). + Returns + ------- + str + Returns the unique identifier used in the file's name. """ import json @@ -114,6 +118,11 @@ def send(self, message, **kwargs): message : :obj:`dict` The message to be printed. + Returns + ------- + int + The status code from the `request.post` + """ import requests @@ -134,7 +143,21 @@ def send_message(message, messengers=None, **kwargs): def make_message(obj, context=None): - """Build a message.""" + """ + Build a message using the specific context + + Parameters + ---------- + obj : :obj:`dict` + All the fields of the message (TODO) + context : :obj:`dict`, optional + Dictionary with the link to the context.jsonld file. + + Returns + ------- + dict + The message with the context. + """ if context is None: context = { "@context": "https://raw.githubusercontent.com/nipype/pydra/master/pydra/schema/context.jsonld" @@ -145,7 +168,19 @@ def make_message(obj, context=None): def collect_messages(collected_path, message_path, ld_op="compact"): - """Gather messages.""" + """ + Gather messages. + + Parameters + ---------- + collected_path : :obj:`os.pathlike` + A place to write all of the collected messages. (?TODO) + message_path : :obj:`os.pathlike` + A path with the message file (?TODO) + ld_op : :obj:`str`, optional + Option used by pld.jsonld + """ + import pyld as pld import json from glob import glob From 1b1e0ac6a69a743dd17b449227ed9b57130a410f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 16 Jul 2020 15:32:48 -0700 Subject: [PATCH 007/271] adding keep_extension to the metadata --- pydra/engine/helpers_file.py | 19 ++-- pydra/engine/specs.py | 3 +- pydra/engine/tests/test_shelltask.py | 98 ++++++++++++++++- .../engine/tests/test_shelltask_inputspec.py | 102 +++++++++++++++++- 4 files changed, 212 insertions(+), 10 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index f93c46cea5..89265b0369 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -560,7 +560,6 @@ def template_update_single(field, inputs_dict, spec_type="input"): ) else: raise Exception(f"spec_type can be input or output, but {spec_type} provided") - if spec_type == "input" and isinstance(inputs_dict[field.name], str): return inputs_dict[field.name] elif spec_type == "input" and inputs_dict[field.name] is False: @@ -568,11 +567,15 @@ def template_update_single(field, inputs_dict, spec_type="input"): return attr.NOTHING else: # inputs_dict[field.name] is True or spec_type is output template = field.metadata["output_file_template"] - value = _template_formatting(template, inputs_dict) + # as default, we assume that keep_extension is True + keep_extension = field.metadata.get("keep_extension", True) + value = _template_formatting( + template, inputs_dict, keep_extension=keep_extension + ) return value -def _template_formatting(template, inputs_dict): +def _template_formatting(template, inputs_dict, keep_extension=True): """Formatting a single template based on values from inputs_dict. Taking into account that field values and template could have file extensions (assuming that if template has extension, the field value extension is removed, @@ -587,16 +590,20 @@ def _template_formatting(template, inputs_dict): if fld_value is attr.NOTHING: return attr.NOTHING fld_value = str(fld_value) # in case it's a path + filename, *ext = fld_value.split(".", maxsplit=1) + # if keep_extension is False, the extensions are removed + if keep_extension is False: + ext = [] if template.endswith(inp_fields[0]): # if no suffix added in template, the simplest formatting should work - formatted_value = template.format(**{fld_name: fld_value}) + # recreating fld_value with the updated extension + fld_value_upd = ".".join([filename] + ext) + formatted_value = template.format(**{fld_name: fld_value_upd}) elif "." not in template: # the template doesn't have its own extension # if the fld_value has extension, it will be moved to the end - filename, *ext = fld_value.split(".", maxsplit=1) formatted_value = ".".join([template.format(**{fld_name: filename})] + ext) else: # template has its own extension # removing fld_value extension if any - filename, *ext = fld_value.split(".", maxsplit=1) formatted_value = template.format(**{fld_name: filename}) return formatted_value else: diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index ef53fa3479..797369e300 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -215,6 +215,7 @@ def check_metadata(self): "position", "requires", "separate_ext", + "keep_extension", "xor", "sep", } @@ -379,7 +380,7 @@ def _field_defaultvalue(self, fld, output_dir): def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" - if "value" in fld.metadata: # jak output_file_template in input_spec + if "value" in fld.metadata: return output_dir / fld.metadata["value"] # this block is only run if "output_file_template" is provided in output_spec # if the field is set in input_spec with output_file_template, diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 24493cb067..70e28781e3 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -917,6 +917,7 @@ def test_shell_cmd_inputspec_7(plugin, results_function): res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() + assert res.output.out1.name == "newfile_tmp.txt" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -960,7 +961,7 @@ def test_shell_cmd_inputspec_7a(plugin, results_function): def test_shell_cmd_inputspec_7b(plugin, results_function): """ providing new file and output name using input_spec, - using name_tamplate in metadata + using name_template in metadata """ cmd = "touch" @@ -1115,7 +1116,7 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmpdir): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): """ - providing output name using input_spec (name_tamplate in metadata), + providing output name using input_spec (output_file_template in metadata), the template has a suffix, the extension of the file will be moved to the end """ cmd = "cp" @@ -1157,6 +1158,99 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): assert res.output.file_copy.name == "file_copy.txt" +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): + """ + providing output name using input_spec (output_file_template in metadata) + and the keep_extension is set to False, so the extension is removed completely. + """ + cmd = "cp" + file = tmpdir.join("file.txt") + file.write("content") + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file_orig", + attr.ib( + type=File, + metadata={"position": 2, "help_string": "new file", "argstr": ""}, + ), + ), + ( + "file_copy", + attr.ib( + type=str, + metadata={ + "output_file_template": "{file_orig}_copy", + "keep_extension": False, + "help_string": "output file", + "argstr": "", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.file_copy.exists() + assert res.output.file_copy.name == "file_copy" + + +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): + """ + providing output name using input_spec (output_file_template in metadata) + and the keep_extension is set to False, so the extension is removed completely, + no suffix in the template. + """ + cmd = "cp" + file = tmpdir.join("file.txt") + file.write("content") + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file_orig", + attr.ib( + type=File, + metadata={"position": 2, "help_string": "new file", "argstr": ""}, + ), + ), + ( + "file_copy", + attr.ib( + type=str, + metadata={ + "output_file_template": "{file_orig}", + "keep_extension": False, + "help_string": "output file", + "argstr": "", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.file_copy.exists() + assert res.output.file_copy.name == "file" + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_inputspec_10(plugin, results_function, tmpdir): """ using input_spec, providing list of files as an input """ diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index d177c20d5c..1da24a5c73 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1031,7 +1031,57 @@ def test_shell_cmd_inputs_template_6a(): def test_shell_cmd_inputs_template_7(tmpdir): - """ additional inputs uses output_file_template with a suffix (no extension)""" + """ additional inputs uses output_file_template with a suffix (no extension) + no keep_extension is used + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "outA", + "argstr": "", + "output_file_template": "{inpA}_out", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("a_file.txt") + inpA_file.write("content") + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file + ) + + # outA should be formatted in a way that that .txt goes to the end + assert ( + shelly.cmdline + == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out.txt')}" + ) + + +def test_shell_cmd_inputs_template_7a(tmpdir): + """ additional inputs uses output_file_template with a suffix (no extension) + keep_extension is True (as default) + """ my_input_spec = SpecInfo( name="Input", fields=[ @@ -1055,6 +1105,7 @@ def test_shell_cmd_inputs_template_7(tmpdir): "position": 2, "help_string": "outA", "argstr": "", + "keep_extension": True, "output_file_template": "{inpA}_out", }, ), @@ -1076,6 +1127,55 @@ def test_shell_cmd_inputs_template_7(tmpdir): ) +def test_shell_cmd_inputs_template_7b(tmpdir): + """ additional inputs uses output_file_template with a suffix (no extension) + keep extension is False (so the extension is removed when creating the output) + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "outA", + "argstr": "", + "keep_extension": False, + "output_file_template": "{inpA}_out", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("a_file.txt") + inpA_file.write("content") + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file + ) + + # outA should be formatted in a way that that .txt goes to the end + assert ( + shelly.cmdline + == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out')}" + ) + + def test_shell_cmd_inputs_template_8(tmpdir): """additional inputs uses output_file_template with a suffix and an extension""" my_input_spec = SpecInfo( From 060ff3855cf79043d4916fb21fd63cd34a29e12e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 17 Jul 2020 22:27:13 -0700 Subject: [PATCH 008/271] adding a simple validator to the attrs class (using typing_inspect for dealing with typing special types like ty.Union, etc., fixing some tests) --- pydra/engine/helpers.py | 120 +++++++++++++++++- pydra/engine/tests/test_shelltask.py | 44 +++++++ .../engine/tests/test_shelltask_inputspec.py | 2 +- pydra/engine/tests/test_task.py | 4 +- setup.cfg | 1 + 5 files changed, 163 insertions(+), 8 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index c59ac45d9e..56c00a6dbe 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -13,9 +13,12 @@ import uuid from time import strftime from traceback import format_exception +import typing as ty +import typing_inspect as tyi +import inspect -from .specs import Runtime, File, Directory, attr_fields, Result +from .specs import Runtime, File, Directory, attr_fields, Result, LazyField from .helpers_file import hash_file, hash_dir, copyfile, is_existing_file @@ -234,8 +237,11 @@ def make_klass(spec): if len(item) == 2: if isinstance(item[1], attr._make._CountingAttr): newfields[item[0]] = item[1] + newfields[item[0]].validator(custom_validator) else: - newfields[item[0]] = attr.ib(type=item[1]) + newfields[item[0]] = attr.ib( + type=item[1], validator=custom_validator + ) else: if ( any([isinstance(ii, attr._make._CountingAttr) for ii in item]) @@ -251,17 +257,121 @@ def make_klass(spec): name, tp = item[:2] if isinstance(item[-1], dict) and "help_string" in item[-1]: mdata = item[-1] - newfields[name] = attr.ib(type=tp, metadata=mdata) + newfields[name] = attr.ib( + type=tp, metadata=mdata, validator=custom_validator + ) else: dflt = item[-1] - newfields[name] = attr.ib(type=tp, default=dflt) + newfields[name] = attr.ib( + type=tp, default=dflt, validator=custom_validator + ) elif len(item) == 4: name, tp, dflt, mdata = item - newfields[name] = attr.ib(type=tp, default=dflt, metadata=mdata) + newfields[name] = attr.ib( + type=tp, + default=dflt, + metadata=mdata, + validator=custom_validator, + ) fields = newfields return attr.make_class(spec.name, fields, bases=spec.bases, kw_only=True) +def custom_validator(instance, attribute, value): + """simple custom validation + take into account ty.Union, ty.List, ty.Dict (but only one level depth) + """ + tp_attr = attribute.type + check = True + if ( + value is attr.NOTHING + or value is None + or isinstance(value, LazyField) + or tp_attr in [ty.Any, inspect._empty] + ): + check = False + elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: # what about File + tp = _single_type_update(tp_attr) + cont = None # no container + elif tyi.is_union_type(tp_attr): + tp_attr_list = tyi.get_args(tp_attr) + cont = None + tp, check = _types_updates(tp_attr_list) + elif tp_attr._name == "List": + cont = "List" + tp_attr_list = tyi.get_args(tp_attr) + tp, check = _types_updates(tp_attr_list) + elif tp_attr._name == "Dict": + cont = "Dict" + breakpoint() + else: + breakpoint() + + if check: + if cont is None: + # if tp is not (list,), we are assuming that the value is a list + # due to the splitter, so checking the member types + if isinstance(value, list) and tp != (list,): + return attr.validators.deep_iterable( + member_validator=attr.validators.instance_of( + tp + (attr._make._Nothing,) + ) + )(instance, attribute, value) + else: + return attr.validators.instance_of(tp + (attr._make._Nothing,))( + instance, attribute, value + ) + elif cont == "List": + return attr.validators.deep_iterable( + member_validator=attr.validators.instance_of( + tp + (attr._make._Nothing,) + ) + )(instance, attribute, value) + elif cont == "Dict": + breakpoint() # TODO + else: + raise Exception(f"cont should be None, List or Dict, and not {cont}") + else: + pass + + +def _types_updates(tp_list): + tp_upd_list = [] + check = True + for tp_el in tp_list: + tp_upd = _single_type_update(tp_el, simplify=True) + if tp_upd is None: + check = False + break + else: + tp_upd_list += list(tp_upd) + tp_upd = tuple(set(tp_upd_list)) + return tp_upd, check + + +def _single_type_update(tp, simplify=False): + if isinstance(tp, type) or tp in [File, Directory]: + if tp is str: + return (str, bytes) + elif tp in [File, Directory, os.PathLike]: + return (os.PathLike, str) + elif tp is float: + return (float, int) + else: + return (tp,) + elif simplify: + if tp._name is "List": + return (list,) + elif tp._name is "Dict": + return (dict,) + elif tyi.is_union_type(tp): + return None + else: + raise NotImplementedError(f"not implemented for type {tp}") + else: + return None + + async def read_stream_and_display(stream, display): """ Read from stream line by line until EOF, display, and capture the lines. diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index ea421d47ba..634ff67d79 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1376,6 +1376,50 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function): assert res[1].output.stdout == "hi\n" +def test_shell_cmd_inputspec_typeval_1(): + """ customized input_spec with a type that doesn't match the value + - raise an exception + """ + cmd_exec = "echo" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "text", + attr.ib( + type=int, + metadata={"position": 1, "argstr": "", "help_string": "text"}, + ), + ) + ], + bases=(ShellSpec,), + ) + + with pytest.raises(TypeError): + shelly = ShellCommandTask( + executable=cmd_exec, text="hello", input_spec=my_input_spec + ) + + +def test_shell_cmd_inputspec_typeval_2(): + """ customized input_spec (shorter syntax) with a type that doesn't match the value + - raise an exception + """ + cmd_exec = "echo" + + my_input_spec = SpecInfo( + name="Input", + fields=[("text", int, {"position": 1, "argstr": "", "help_string": "text"})], + bases=(ShellSpec,), + ) + + with pytest.raises(TypeError): + shelly = ShellCommandTask( + executable=cmd_exec, text="hello", input_spec=my_input_spec + ) + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_inputspec_state_1a(plugin, results_function): """ adding state to the input from input_spec diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 8b9482b9d2..41bb785c07 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1192,7 +1192,7 @@ def test_shell_cmd_inputs_di(tmpdir): ( "verbose", attr.ib( - type=bool, + type=int, default=0, metadata={"help_string": "(0)/1. Verbose output. ", "argstr": "-v"}, ), diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 61a3ec3113..80d535be98 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -39,7 +39,7 @@ def test_name_conflict(): def test_numpy(): """ checking if mark.task works for numpy functions""" np = pytest.importorskip("numpy") - fft = mark.annotate({"a": np.ndarray, "return": float})(np.fft.fft) + fft = mark.annotate({"a": np.ndarray, "return": np.ndarray})(np.fft.fft) fft = mark.task(fft)() arr = np.array([[1, 10], [2, 20]]) fft.inputs.a = arr @@ -109,7 +109,7 @@ def testfunc( ) -> ty.NamedTuple("Output", [("fractional", float), ("integer", int)]): import math - return math.modf(a) + return math.modf(a)[0], int(math.modf(a)[1]) funky = testfunc(a=3.5) assert hasattr(funky.inputs, "a") diff --git a/setup.cfg b/setup.cfg index d7dea4e0d6..485d02cc9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ classifiers = python_requires = >= 3.7 install_requires = attrs + typing_inspect cloudpickle >= 0.8.0 filelock >= 3.0.0 etelemetry >= 0.2.0 From 2f47bdbfe0ea1697c46740d5b871a7af38056e93 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 20 Jul 2020 12:59:04 -0700 Subject: [PATCH 009/271] setting attrrs version to >=19.1.0 (for validators.deep_iterable) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 485d02cc9b..b327ac3832 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ classifiers = [options] python_requires = >= 3.7 install_requires = - attrs + attrs >= 19.1.0 typing_inspect cloudpickle >= 0.8.0 filelock >= 3.0.0 From c2cae7f6249b775cb77eb6274162631098bc1a58 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 20 Jul 2020 14:04:28 -0700 Subject: [PATCH 010/271] updating docs requirements --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index b327ac3832..10534f63fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,8 @@ pydra = [options.extras_require] doc = + attrs >= 19.1.0 + typing_inspect cloudpickle filelock packaging From a6ea841c5a158b768bd14f0dba48c9c3a8dfa853 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 20 Jul 2020 14:41:17 -0700 Subject: [PATCH 011/271] updating docs/requirements.txt --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ca468ebf9a..711761e8bd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,5 @@ -attrs +attrs >= 19.1.0 +typing_inspect cloudpickle filelock git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon From 5f37b6b2f9fc3a73038c627be1dd6f24b66b05d2 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 20 Jul 2020 22:49:34 -0700 Subject: [PATCH 012/271] expanding the State's docstrings, small cleaning --- pydra/engine/state.py | 177 ++++++++++++++++++++++++++++++------------ 1 file changed, 128 insertions(+), 49 deletions(-) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index 3c44cc3180..cd5398722f 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -14,12 +14,12 @@ class State: and input values for specific task states (specified by the splitter and the input). * It also contains information about the final groups and the final splitter - if combiner is available.. + if combiner is available. Attributes ---------- name : :obj:`str` - name of the state that is the same as name of the task + name of the state that is the same as a name of the task splitter : :obj:`str`, :obj:`tuple`, :obj:`list` can be a str (name of a single input), tuple for scalar splitter, or list for outer splitter @@ -72,12 +72,12 @@ class State: def __init__(self, name, splitter=None, combiner=None, other_states=None): """ - Initialize state. + Initialize a state. Parameters ---------- name : :obj:`str` - name (should be the same as task name) + name (should be the same as the task's name) splitter : :obj:`str`, or :obj:`tuple`, or :obj:`list` splitter of a task combiner : :obj:`str`, or :obj:`list`) @@ -119,24 +119,9 @@ def splitter(self, splitter): else: self._splitter = None - @property - def splitter_rpn_final(self): - if self.combiner: - _splitter_rpn_final = hlpst.remove_inp_from_splitter_rpn( - deepcopy(self.splitter_rpn), - self.right_combiner_all + self.left_combiner_all, - ) - return _splitter_rpn_final - else: - return self.splitter_rpn - - @property - def splitter_final(self): - """ final splitter, after removing the combined fields""" - return hlpst.rpn2splitter(self.splitter_rpn_final) - @property def splitter_rpn(self): + """splitter in :abbr:`RPN (Reverse Polish Notation)`""" _splitter_rpn = hlpst.splitter2rpn( self.splitter, other_states=self.other_states ) @@ -144,6 +129,10 @@ def splitter_rpn(self): @property def splitter_rpn_compact(self): + """splitter in :abbr:`RPN (Reverse Polish Notation)` + with a compact representation of the Left Part (i.e. without unwrapping + the part that comes from the previous states), e.g., e.g. [_NA, _NB, *] + """ if self.other_states: _splitter_rpn_compact = hlpst.splitter2rpn( self.splitter, other_states=self.other_states, state_fields=False @@ -152,9 +141,28 @@ def splitter_rpn_compact(self): else: return self.splitter_rpn + @property + def splitter_final(self): + """the final splitter, after removing the combined fields""" + return hlpst.rpn2splitter(self.splitter_rpn_final) + + @property + def splitter_rpn_final(self): + if self.combiner: + _splitter_rpn_final = hlpst.remove_inp_from_splitter_rpn( + deepcopy(self.splitter_rpn), + self.right_combiner_all + self.left_combiner_all, + ) + return _splitter_rpn_final + else: + return self.splitter_rpn + @property def right_splitter(self): - """ current state splitter (i.e. the Right part)""" + """the Right Part of the splitter, + i.e. the part that is related to the current task's state only + (doesn't include fields propagated from the previous tasks) + """ lr_flag = self._left_right_check(self.splitter) if lr_flag == "Left": return None @@ -165,6 +173,7 @@ def right_splitter(self): @property def right_splitter_rpn(self): + """the Right Part in the splitter using RPN""" if self.right_splitter: right_splitter_rpn = hlpst.splitter2rpn( self.right_splitter, other_states=self.other_states @@ -175,7 +184,9 @@ def right_splitter_rpn(self): @property def left_splitter(self): - """ splitters from the previous stated (i.e. the Light part)""" + """ the left part of the splitter, + i.e. the part that comes from the previous tasks' states + """ if hasattr(self, "_left_splitter"): return self._left_splitter else: @@ -183,6 +194,7 @@ def left_splitter(self): @property def left_splitter_rpn(self): + """the Left Part of the splitter using RPN""" if self.left_splitter: left_splitter_rpn = hlpst.splitter2rpn( self.left_splitter, other_states=self.other_states @@ -193,7 +205,9 @@ def left_splitter_rpn(self): @property def left_splitter_rpn_compact(self): - # left rpn part, but keeping the names of the nodes, e.g. [_NA, _NB, *] + """the Left Part of the splitter using RPN in a compact form, + (without unwrapping the states from previous nodes), e.g. [_NA, _NB, *] + """ if self.left_splitter: left_splitter_rpn_compact = hlpst.splitter2rpn( self.left_splitter, other_states=self.other_states, state_fields=False @@ -204,7 +218,7 @@ def left_splitter_rpn_compact(self): @property def combiner(self): - """Get the combiner associated to the state.""" + """the combiner associated to the state.""" return self._combiner @combiner.setter @@ -218,35 +232,55 @@ def combiner(self, combiner): @property def right_combiner(self): + """the Right Part of the combiner, + i.e. the part that is related to the current task's state only + (doesn't include fields propagated from the previous tasks) + """ return [comb for comb in self.combiner if self.name in comb] - @property - def left_combiner(self): - if hasattr(self, "_left_combiner"): - return self._left_combiner - else: - return list(set(self.combiner) - set(self.right_combiner)) - @property def right_combiner_all(self): + """the Right Part of the combiner including all the fields + that should be combined (i.e. not only the fields that are explicitly + set, but also the fields that re in the same group/axis and had to be combined + together, e.g., if splitter is (a, b) a and b has to be combined together) + """ if hasattr(self, "_right_combiner_all"): return self._right_combiner_all else: return self.right_combiner + @property + def left_combiner(self): + """ the Left Part of the combiner, + i.e. the part that comes from the previous tasks' states + """ + if hasattr(self, "_left_combiner"): + return self._left_combiner + else: + return list(set(self.combiner) - set(self.right_combiner)) + @property def left_combiner_all(self): + """the Left Part of the combiner including all the fields + that should be combined (i.e. not only the fields that are explicitly + set, but also the fields that re in the same group/axis and had to be combined + together, e.g., if splitter is (a, b) a and b has to be combined together) + """ if hasattr(self, "_left_combiner_all"): - return self._left_combiner_all + return list(set(self._left_combiner_all)) else: return self.left_combiner - @left_combiner_all.setter - def left_combiner_all(self, left_combiner_all): - self._left_combiner_all = list(set(left_combiner_all)) - @property def other_states(self): + """ + specifies the connections with previous states, uses dictionary: + { + name of a previous state: + (previous state, input from current state needed the connection) + } + """ return self._other_states @other_states.setter @@ -266,7 +300,10 @@ def other_states(self, other_states): @property def inner_inputs(self): - """input fields from previous nodes""" + """specifies connections between fields from the current state + with the specific state from the previous states, uses dictionary + ``{input name for current state: the previous state}`` + """ if self.other_states: _inner_inputs = {} for name, (st, inp) in self.other_states.items(): @@ -277,7 +314,15 @@ def inner_inputs(self): return {} def update_connections(self, new_other_states=None, new_combiner=None): - """ updating states connections and input groups""" + """ updating connections, can use a new other_states and combiner + + Parameters + ---------- + new_other_states : :obj:`dict`, optional + dictionary with new other_states, will be set before updating connections + new_combiner : :obj:`str`, or :obj:`list`, optional + new combiner + """ if new_other_states: self.other_states = new_other_states self._connect_splitters() @@ -286,10 +331,10 @@ def update_connections(self, new_other_states=None, new_combiner=None): def _connect_splitters(self): """ - Connect splitters from previous nodes. - Evaluates Left (the part from previous states) and Right (current state) parts. - If left splitter is not provided the splitter has to be completed. - + Connect splitters from the previous nodes. + Evaluates the Left Part of the splitter (i.e. the part from the previous states) + and the Right Part of the splitter(i.e., the current state). + If the left splitter is not provided the splitter has to be completed. """ # TODO: should this be in the left_Splitter property? if self.splitter: @@ -321,7 +366,13 @@ def _connect_splitters(self): self.splitter = deepcopy(self._left_splitter) def _complete_left(self, left=None): - """Add all splitters from previous nodes (completing the Left part).""" + """Add all splitters from the previous nodes (completing the Left part). + + Parameters + ---------- + left : :obj:`str`, or :obj:`list`, or :obj:`tuple`, optional + the left part of the splitter, that has to be completed + """ if left: rpn_left = hlpst.splitter2rpn( left, other_states=self.other_states, state_fields=False @@ -340,7 +391,17 @@ def _left_right_check(self, splitter_part, check_nested=True): Check if splitter_part is purely Left, Right or [Left, Right] if the splitter_part is a list (outer splitter) - String is returned. + Parameters + ---------- + splitter_part : :obj:`str`, or :obj:`list`, or :obj:`tuple` + Part of the splitter that is being check + check_nested : :obj:`bool`, optional + If True, the nested parts are checked. + + Returns + ------- + str + describes the type - "Left" or "Right" If the splitter_part is mixed exception is raised. @@ -369,13 +430,19 @@ def _left_right_check(self, splitter_part, check_nested=True): ) def set_input_groups(self, state_fields=True): - """Evaluate groups, especially the final groups that address the combiner.""" + """Evaluates groups, especially the final groups that address the combiner. + + Parameters + ---------- + state_fields : :obj:`bool` + if False the splitter from the previous states are unwrapped + """ right_splitter_rpn = hlpst.splitter2rpn( self.right_splitter, other_states=self.other_states, state_fields=state_fields, ) - # merging groups from previous nodes if any input come from previous the nodes + # merging groups from previous nodes if any input come from previous states if self.inner_inputs: self._merge_previous_groups() keys_f, group_for_inputs_f, groups_stack_f, combiner_all = hlpst.splits_groups( @@ -403,7 +470,7 @@ def _merge_previous_groups(self): self.group_for_inputs_final = {} self.keys_final = [] if self.left_combiner: - _, _, _, self.left_combiner_all = hlpst.splits_groups( + _, _, _, self._left_combiner_all = hlpst.splits_groups( self.left_splitter_rpn, combiner=self.left_combiner ) for i, left_nm in enumerate(self.left_splitter_rpn_compact): @@ -443,7 +510,7 @@ def _merge_previous_groups(self): raise hlpst.PydraStateError("previous state has to run first") group_for_inputs = group_for_inputs_f_st groups_stack = groups_stack_f_st - self.left_combiner_all += combiner_all_st + self._left_combiner_all += combiner_all_st else: # if no element from st.splitter is in the current combiner, # using st attributes without changes @@ -519,6 +586,12 @@ def prepare_states(self, inputs, cont_dim=None): State Values specific elements from inputs that can be used running interfaces + Parameters + ---------- + inputs : :obj:`dict` + inputs of the task + cont_dim : :obj:`dict` or `None` + container's dimensions for a specific input's fields """ # checking if splitter and combiner have valid forms self.splitter_validation() @@ -591,7 +664,13 @@ def prepare_states_ind(self): return self.states_ind def prepare_states_combined_ind(self, elements_to_remove_comb): - """Prepare the final list of dictionaries with indices after combiner.""" + """Prepare the final list of dictionaries with indices after combiner. + + Parameters + ---------- + elements_to_remove_comb : :obj:`list` + elements of the splitter that should be removed due to the combining + """ partial_rpn_compact = hlpst.remove_inp_from_splitter_rpn( deepcopy(self.splitter_rpn_compact), elements_to_remove_comb ) From f6beba8197f2cc9ff00c226d0cb8c00dc85ab7f7 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 21 Jul 2020 10:13:16 -0700 Subject: [PATCH 013/271] trying to fix the issues with docs build --- pydra/engine/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index cd5398722f..a996ae7612 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -205,7 +205,7 @@ def left_splitter_rpn(self): @property def left_splitter_rpn_compact(self): - """the Left Part of the splitter using RPN in a compact form, + """ the Left Part of the splitter using RPN in a compact form, (without unwrapping the states from previous nodes), e.g. [_NA, _NB, *] """ if self.left_splitter: From 06c8faadca19f78e0538d4c078a3766677614a7f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 21 Jul 2020 11:02:47 -0700 Subject: [PATCH 014/271] Update pydra/engine/state.py Co-authored-by: Satrajit Ghosh --- pydra/engine/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index a996ae7612..f680b225b8 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -205,8 +205,8 @@ def left_splitter_rpn(self): @property def left_splitter_rpn_compact(self): - """ the Left Part of the splitter using RPN in a compact form, - (without unwrapping the states from previous nodes), e.g. [_NA, _NB, *] + r""" the Left Part of the splitter using RPN in a compact form, + (without unwrapping the states from previous nodes), e.g. [\_NA, \_NB, \*] """ if self.left_splitter: left_splitter_rpn_compact = hlpst.splitter2rpn( From b882c54d29e217239010e47bff320bb0e6ea8bf1 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 21 Jul 2020 13:30:35 -0700 Subject: [PATCH 015/271] fixing the docs --- pydra/engine/state.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index f680b225b8..93fa0b4792 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -131,7 +131,7 @@ def splitter_rpn(self): def splitter_rpn_compact(self): """splitter in :abbr:`RPN (Reverse Polish Notation)` with a compact representation of the Left Part (i.e. without unwrapping - the part that comes from the previous states), e.g., e.g. [_NA, _NB, *] + the part that comes from the previous states), e.g., [_NA, _NB, \*] """ if self.other_states: _splitter_rpn_compact = hlpst.splitter2rpn( @@ -205,8 +205,8 @@ def left_splitter_rpn(self): @property def left_splitter_rpn_compact(self): - r""" the Left Part of the splitter using RPN in a compact form, - (without unwrapping the states from previous nodes), e.g. [\_NA, \_NB, \*] + """ the Left Part of the splitter using RPN in a compact form, + (without unwrapping the states from previous nodes), e.g. [_NA, _NB, \*] """ if self.left_splitter: left_splitter_rpn_compact = hlpst.splitter2rpn( @@ -274,12 +274,8 @@ def left_combiner_all(self): @property def other_states(self): - """ - specifies the connections with previous states, uses dictionary: - { - name of a previous state: - (previous state, input from current state needed the connection) - } + """ specifies the connections with previous states, uses dictionary: + {name of a previous state: (previous state, input field from current state)} """ return self._other_states From 7c75b97221bdf7d819c34a21d66c5ca6f2fcedea Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 21 Jul 2020 23:35:35 -0700 Subject: [PATCH 016/271] adding create_dotfile for a simple (one depth) graph representation; [wip]starting the detailed versionthat will unfold all nested workflows --- pydra/engine/core.py | 14 +++++++++ pydra/engine/graph.py | 47 +++++++++++++++++++++++++++++ pydra/engine/tests/test_graph.py | 19 ++++++++++++ pydra/engine/tests/test_workflow.py | 33 ++++++++++++++++++++ 4 files changed, 113 insertions(+) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 844df5ceb3..19dcda91d0 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -1002,6 +1002,20 @@ def _collect_outputs(self): raise ValueError(f"Task {val.name} raised an error") return attr.evolve(output, **output_wf) + def create_dotfile(self, type="simple"): + for task in self.graph.nodes: + # todo: create_connections is also run in _run, can I remove duplication? + self.create_connections(task) + if type == "simple": + dotfile = self.graph.create_dotfile_simple(outdir=self.output_dir) + elif type == "detailed": + pass + else: + raise Exception( + f"type of the graph can be simple or detailed, " f"but {type} provided" + ) + return dotfile + def is_task(obj): """Check whether an object looks like a task.""" diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index 6ba34b543f..0cc6e74800 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -1,5 +1,6 @@ """Data structure to support :class:`~pydra.engine.core.Workflow` tasks.""" from copy import copy +from pathlib import Path from .helpers import ensure_list @@ -318,3 +319,49 @@ def calculate_max_paths(self): for nm in first_nodes: self.max_paths[nm] = {} self._checking_path(node_name=nm, first_name=nm) + + def create_dotfile_simple(self, outdir, name="graph"): + from .core import is_workflow + + dotstr = "digraph G {\n" + for nd in self.nodes: + if is_workflow(nd): + dotstr += f"{nd.name} [shape=box]\n" + else: + dotstr += f"{nd.name}\n" + + for ed in self.edges_names: + dotstr += f"{ed[0]} -> {ed[1]}\n" + + dotstr += "}" + Path(outdir).mkdir(parents=True, exist_ok=True) + dotfile = Path(outdir) / f"{name}.dot" + dotfile.write_text(dotstr) + return dotfile + + def create_dotfile_detailed(self, outdir, name="graph"): + dotstr = "digraph G {\n" + dotstr += self._create_dotfile_single_graph(nodes=self.nodes) + dotstr += "}" + Path(outdir).mkdir(parents=True, exist_ok=True) + dotfile = Path(outdir) / f"{name}.dot" + dotfile.write_text(dotstr) + return dotfile + + def _create_dotfile_single_graph(self, nodes): + from .core import is_workflow + + for nd in nodes: + if is_workflow(nd): + dotstr = ( + f"subgraph cluster_{nd.name} {{\n " + f"compound=true; \n" + f"label = {nd.name};" + ) + dotstr += self._create_dotfile_single_graph(nodes=nd.graph.nodes) + dotstr += "}" + else: + dotstr = f"{nd.name}\n" + + # for ed in self.edges_names: + # dotstr += f"{ed[0]} -> {ed[1]}\n" diff --git a/pydra/engine/tests/test_graph.py b/pydra/engine/tests/test_graph.py index 16c6c00ec5..4e8d18f7b0 100644 --- a/pydra/engine/tests/test_graph.py +++ b/pydra/engine/tests/test_graph.py @@ -452,3 +452,22 @@ def test_copy_1(): assert id(graph.nodes[0]) == id(graph_copy.nodes[0]) assert graph.edges == graph_copy.edges assert id(graph.edges) != (graph_copy.edges) + + +def test_dotfile_1(tmpdir): + graph = DiGraph(nodes=[A, B], edges=[(A, B)]) + dotfile = graph.create_dotfile_simple(outdir=tmpdir) + dotstr_lines = dotfile.read_text().split("\n") + assert "a" in dotstr_lines + assert "b" in dotstr_lines + assert "a -> b" in dotstr_lines + + +def test_dotfile_2(tmpdir): + graph = DiGraph(nodes=[A, B, C, D], edges=[(A, C), (B, C), (C, D)]) + dotfile = graph.create_dotfile_simple(outdir=tmpdir) + dotstr_lines = dotfile.read_text().split("\n") + for nm in ["a", "b", "c", "d"]: + assert nm in dotstr_lines + for ed in ["a -> c", "b -> c", "c -> d"]: + assert ed in dotstr_lines diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 8356ae2330..13f4c2e86a 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3948,3 +3948,36 @@ def test_wf_upstream_error9b(plugin): assert "raised an error" in str(excinfo.value) assert wf.err._errored is True assert wf.follow_err._errored == ["err"] + + +def test_graph_1(tmpdir): + """ workflow with 2 tasks, no splitter""" + wf = Workflow(name="wf_2", input_spec=["x", "y"]) + wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) + wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) + wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) + wf.set_output([("out", wf.add2.lzout.out)]) + wf.inputs.x = 2 + wf.inputs.y = 3 + + dotfile = wf.create_dotfile() + dotstr_lines = dotfile.read_text().split("\n") + assert "mult_1" in dotstr_lines + assert "mult_2" in dotstr_lines + assert "add2" in dotstr_lines + assert "mult_1 -> add2" in dotstr_lines + + +def test_graph_2(tmpdir): + wfnd = Workflow(name="wfnd", input_spec=["x"]) + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.inputs.x = 2 + + wf = Workflow(name="wf", input_spec=["x"]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + dotfile = wf.create_dotfile() + dotstr_lines = dotfile.read_text().split("\n") + assert "wfnd [shape=box]" in dotstr_lines From 6c228e67dd82b413a04e187eade5bb96126a7cb3 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 22 Jul 2020 16:50:06 -0700 Subject: [PATCH 017/271] adding create_dotfile_detailed (that expands inner workflows) and export_graph (that runs dot to create other formats, e.g. png) to the graph class; adding tests --- pydra/engine/core.py | 24 +++- pydra/engine/graph.py | 116 +++++++++++++++-- pydra/engine/tests/test_graph.py | 43 ++++++ pydra/engine/tests/test_workflow.py | 195 +++++++++++++++++++++++++++- pydra/engine/tests/utils.py | 7 + 5 files changed, 366 insertions(+), 19 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 19dcda91d0..aaa2b4eda5 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -1002,19 +1002,33 @@ def _collect_outputs(self): raise ValueError(f"Task {val.name} raised an error") return attr.evolve(output, **output_wf) - def create_dotfile(self, type="simple"): + def create_dotfile(self, type="simple", export=None, name=None): + """creating a graph - dotfile and optionally exporting to other formats""" for task in self.graph.nodes: - # todo: create_connections is also run in _run, can I remove duplication? self.create_connections(task) + if not name: + name = f"graph_{self.name}" if type == "simple": - dotfile = self.graph.create_dotfile_simple(outdir=self.output_dir) + dotfile = self.graph.create_dotfile_simple( + outdir=self.output_dir, name=name + ) elif type == "detailed": - pass + dotfile = self.graph.create_dotfile_detailed( + outdir=self.output_dir, name=name + ) else: raise Exception( f"type of the graph can be simple or detailed, " f"but {type} provided" ) - return dotfile + if not export: + return dotfile + else: + if export is True: + export = ["png"] + formatted_dot = [] + for ext in export: + formatted_dot.append(self.graph.export_graph(dotfile=dotfile, ext=ext)) + return dotfile, formatted_dot def is_task(obj): diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index 0cc6e74800..403fbf27ff 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -1,6 +1,8 @@ """Data structure to support :class:`~pydra.engine.core.Workflow` tasks.""" from copy import copy from pathlib import Path +import subprocess as sp + from .helpers import ensure_list @@ -340,28 +342,118 @@ def create_dotfile_simple(self, outdir, name="graph"): return dotfile def create_dotfile_detailed(self, outdir, name="graph"): - dotstr = "digraph G {\n" - dotstr += self._create_dotfile_single_graph(nodes=self.nodes) + dotstr = "digraph G {\ncompound=true \n" + dotstr += self._create_dotfile_single_graph(nodes=self.nodes, edges=self.edges) dotstr += "}" Path(outdir).mkdir(parents=True, exist_ok=True) dotfile = Path(outdir) / f"{name}.dot" dotfile.write_text(dotstr) return dotfile - def _create_dotfile_single_graph(self, nodes): + def _create_dotfile_single_graph(self, nodes, edges): from .core import is_workflow + wf_asnd = [] + dotstr = "" for nd in nodes: if is_workflow(nd): - dotstr = ( - f"subgraph cluster_{nd.name} {{\n " - f"compound=true; \n" - f"label = {nd.name};" + wf_asnd.append(nd.name) + for task in nd.graph.nodes: + nd.create_connections(task) + dotstr += f"subgraph cluster_{nd.name} {{\n" f"label = {nd.name} \n" + dotstr += self._create_dotfile_single_graph( + nodes=nd.graph.nodes, edges=nd.graph.edges ) - dotstr += self._create_dotfile_single_graph(nodes=nd.graph.nodes) - dotstr += "}" + dotstr += "}\n" else: - dotstr = f"{nd.name}\n" + dotstr += f"{nd.name}\n" - # for ed in self.edges_names: - # dotstr += f"{ed[0]} -> {ed[1]}\n" + dotstr_edg = "" + for ed in edges: + if ed[0].name in wf_asnd and ed[1].name in wf_asnd: + head_nd = list(ed[1].nodes)[0].name + tail_nd = list(ed[0].nodes)[-1].name + dotstr_edg += ( + f"{tail_nd} -> {head_nd} " + f"[ltail=cluster_{ed[0].name}, " + f"lhead=cluster_{ed[1].name}]\n" + ) + elif ed[0].name in wf_asnd: + tail_nd = list(ed[0].nodes)[-1].name + dotstr_edg += ( + f"{tail_nd} -> {ed[1].name} [ltail=cluster_{ed[0].name}]\n" + ) + elif ed[1].name in wf_asnd: + head_nd = list(ed[1].nodes)[0].name + dotstr_edg += ( + f"{ed[0].name} -> {head_nd} [lhead=cluster_{ed[1].name}]\n" + ) + else: + dotstr_edg += f"{ed[0].name} -> {ed[1].name}\n" + dotstr = dotstr + dotstr_edg + return dotstr + + def export_graph(self, dotfile, ext="png"): + """ exporting dotfile to other format, equires the dot command""" + available_ext = [ + "bmp", + "canon", + "cgimage", + "cmap", + "cmapx", + "cmapx_np", + "dot", + "dot_json", + "eps", + "exr", + "fig", + "gif", + "gv", + "icns", + "ico", + "imap", + "imap_np", + "ismap", + "jp2", + "jpe", + "jpeg", + "jpg", + "json", + "json0", + "mp", + "pct", + "pdf", + "pic", + "pict", + "plain", + "plain-ext", + "png", + "pov", + "ps", + "ps2", + "psd", + "sgi", + "svg", + "svgz", + "tga", + "tif", + "tiff", + "tk", + "vml", + "vmlz", + "xdot", + "xdot1.2", + "xdot1.4", + "xdot_json", + ] + if ext not in available_ext: + raise Exception(f"unvalid extension - {ext}, chose from {available_ext}") + + dot_check = sp.run(["which", "dot"], stdout=sp.PIPE, stderr=sp.PIPE) + if not dot_check.stdout: + raise Exception(f"dot command not available, can't create a {ext} file") + + formatted_dot = dotfile.with_suffix(f".{ext}") + cmd = f"dot -T{ext} -o {formatted_dot} {dotfile}" + sp.run(cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE) + return formatted_dot diff --git a/pydra/engine/tests/test_graph.py b/pydra/engine/tests/test_graph.py index 4e8d18f7b0..889d098880 100644 --- a/pydra/engine/tests/test_graph.py +++ b/pydra/engine/tests/test_graph.py @@ -1,4 +1,5 @@ from ..graph import DiGraph +from .utils import DOT_FLAG import pytest @@ -455,6 +456,7 @@ def test_copy_1(): def test_dotfile_1(tmpdir): + """dotfile for graph: a -> b""" graph = DiGraph(nodes=[A, B], edges=[(A, B)]) dotfile = graph.create_dotfile_simple(outdir=tmpdir) dotstr_lines = dotfile.read_text().split("\n") @@ -462,8 +464,45 @@ def test_dotfile_1(tmpdir): assert "b" in dotstr_lines assert "a -> b" in dotstr_lines + if DOT_FLAG: + formatted_dot = graph.export_graph(dotfile) + assert formatted_dot.exists() + def test_dotfile_2(tmpdir): + """dotfile for graph: a -> b -> d, a -> c -> d""" + graph = DiGraph(nodes=[A, B, C, D], edges=[(A, B), (A, C), (B, D), (C, D)]) + dotfile = graph.create_dotfile_simple(outdir=tmpdir) + dotstr_lines = dotfile.read_text().split("\n") + for el in ["a", "b", "c", "d"]: + assert el in dotstr_lines + for el in ["a -> b", "a -> c", "b -> d", "c -> d"]: + assert el in dotstr_lines + + if DOT_FLAG: + formatted_dot = graph.export_graph(dotfile) + assert formatted_dot.exists() + + +def test_dotfile_2det(tmpdir): + """detailed dotfile for graph: a -> b -> d, a -> c -> d + (should be the same as default type, i.e. type=simple) + """ + graph = DiGraph(nodes=[A, B, C, D], edges=[(A, B), (A, C), (B, D), (C, D)]) + dotfile = graph.create_dotfile_detailed(outdir=tmpdir) + dotstr_lines = dotfile.read_text().split("\n") + for el in ["a", "b", "c", "d"]: + assert el in dotstr_lines + for el in ["a -> b", "a -> c", "b -> d", "c -> d"]: + assert el in dotstr_lines + + if DOT_FLAG: + formatted_dot = graph.export_graph(dotfile) + assert formatted_dot.exists() + + +def test_dotfile_3(tmpdir): + """detailed dotfile for graph: a -> c, b -> c, c -> d""" graph = DiGraph(nodes=[A, B, C, D], edges=[(A, C), (B, C), (C, D)]) dotfile = graph.create_dotfile_simple(outdir=tmpdir) dotstr_lines = dotfile.read_text().split("\n") @@ -471,3 +510,7 @@ def test_dotfile_2(tmpdir): assert nm in dotstr_lines for ed in ["a -> c", "b -> c", "c -> d"]: assert ed in dotstr_lines + + if DOT_FLAG: + formatted_dot = graph.export_graph(dotfile) + assert formatted_dot.exists() diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 13f4c2e86a..869758896a 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -22,6 +22,7 @@ fun_write_file, fun_write_file_list, fun_write_file_list2dict, + DOT_FLAG, ) from ..submitter import Submitter from ..core import Workflow @@ -3951,7 +3952,32 @@ def test_wf_upstream_error9b(plugin): def test_graph_1(tmpdir): - """ workflow with 2 tasks, no splitter""" + """creating a graph, wf with two nodes""" + wf = Workflow(name="wf_2", input_spec=["x", "y"]) + wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) + wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) + wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) + wf.set_output([("out", wf.add2.lzout.out)]) + + dotfile = wf.create_dotfile() + dotstr_lines = dotfile.read_text().split("\n") + assert "mult_1" in dotstr_lines + assert "mult_2" in dotstr_lines + assert "add2" in dotstr_lines + assert "mult_1 -> add2" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0] == dotfile_pr.with_suffix(".png") + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_1det(tmpdir): + """creating a detailed graph, wf with two nodes""" wf = Workflow(name="wf_2", input_spec=["x", "y"]) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) @@ -3960,15 +3986,51 @@ def test_graph_1(tmpdir): wf.inputs.x = 2 wf.inputs.y = 3 - dotfile = wf.create_dotfile() + dotfile = wf.create_dotfile(type="detailed") dotstr_lines = dotfile.read_text().split("\n") assert "mult_1" in dotstr_lines assert "mult_2" in dotstr_lines assert "add2" in dotstr_lines assert "mult_1 -> add2" in dotstr_lines + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export=["png", "pdf"], name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 2 + for i, ext in enumerate(["png", "pdf"]): + assert formatted_dot[i] == dotfile_pr.with_suffix(f".{ext}") + assert formatted_dot[i].exists() + print("\n graph in: ", formatted_dot[0]) + def test_graph_2(tmpdir): + """creating a graph, wf with one worfklow as a node""" + wfnd = Workflow(name="wfnd", input_spec=["x"]) + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + + wf = Workflow(name="wf", input_spec=["x"]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + dotfile = wf.create_dotfile() + dotstr_lines = dotfile.read_text().split("\n") + assert "wfnd [shape=box]" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_2det(tmpdir): + """creating a detailed graph, wf with one worfklow as a node""" wfnd = Workflow(name="wfnd", input_spec=["x"]) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) @@ -3978,6 +4040,135 @@ def test_graph_2(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) + dotfile = wf.create_dotfile(type="detailed") + dotstr_lines = dotfile.read_text().split("\n") + assert "subgraph cluster_wfnd {" in dotstr_lines + assert "add2" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export=True, name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_3(tmpdir): + """creating a graph, wf with two nodes (one node is a workflow)""" + wf = Workflow(name="wf", input_spec=["x", "y"]) + wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) + + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + dotfile = wf.create_dotfile() dotstr_lines = dotfile.read_text().split("\n") + assert "mult" in dotstr_lines assert "wfnd [shape=box]" in dotstr_lines + assert "mult -> wfnd" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_3det(tmpdir): + """creating a detailed graph, wf with two nodes (one node is a workflow)""" + wf = Workflow(name="wf", input_spec=["x", "y"]) + wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) + + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + dotfile = wf.create_dotfile(type="detailed") + dotstr_lines = dotfile.read_text().split("\n") + assert "mult" in dotstr_lines + assert "subgraph cluster_wfnd {" in dotstr_lines + assert "add2" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export=True, name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_4det(tmpdir): + """creating a detailed graph, wf with two nodes (one node is a workflow with two nodes + inside). Connection from the node to the inner workflow. + """ + wf = Workflow(name="wf", input_spec=["x", "y"]) + wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) + + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) + wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) + wfnd.add(add2(name="add2_b", x=wfnd.add2_a.lzout.out)) + wfnd.set_output([("out", wfnd.add2_b.lzout.out)]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + dotfile = wf.create_dotfile(type="detailed") + dotstr_lines = dotfile.read_text().split("\n") + for el in ["mult", "add2_a", "add2_b"]: + assert el in dotstr_lines + assert "subgraph cluster_wfnd {" in dotstr_lines + assert "add2_a -> add2_b" in dotstr_lines + assert "mult -> add2_a [lhead=cluster_wfnd]" + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export=True, name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + +def test_graph_5det(tmpdir): + """creating a detailed graph, wf with two nodes (one node is a workflow with two nodes + inside). Connection from the inner workflow to the node. + """ + wf = Workflow(name="wf", input_spec=["x", "y"]) + + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) + wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) + wfnd.add(add2(name="add2_b", x=wfnd.add2_a.lzout.out)) + wfnd.set_output([("out", wfnd.add2_b.lzout.out)]) + wf.add(wfnd) + wf.add(multiply(name="mult", x=wf.wfnd.lzout.out, y=wf.lzin.y)) + wf.set_output([("out", wf.mult.lzout.out)]) + + dotfile = wf.create_dotfile(type="detailed") + dotstr_lines = dotfile.read_text().split("\n") + for el in ["mult", "add2_a", "add2_b"]: + assert el in dotstr_lines + assert "subgraph cluster_wfnd {" in dotstr_lines + assert "add2_a -> add2_b" in dotstr_lines + assert "add2_b -> mult [ltail=cluster_wfnd]" + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export=True, name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index 46c043749d..6ceb6055cd 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -36,6 +36,13 @@ def result_submitter(shell_task, plugin): return shell_task.result() +dot_check = sp.run(["which", "dot"], stdout=sp.PIPE, stderr=sp.PIPE) +if dot_check.stdout: + DOT_FLAG = True +else: + DOT_FLAG = False + + @mark.task def fun_addtwo(a): import time From a17e74fce94bc2fb317cbd4fb2dfdad53a9f1f53 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 13:05:39 -0700 Subject: [PATCH 018/271] Update pydra/engine/audit.py Co-authored-by: Satrajit Ghosh --- pydra/engine/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 5b598fe0c8..3cf32b44d3 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -117,7 +117,7 @@ def audit_message(self, message, flags=None): Parameters ---------- message : :obj:`dict` - All the information needed to build a message (TODO) + A message in Pydra is a JSON-LD message object. flags : :obj:`bool`, optional If True and self.audit_flag, the message is sent. From 96d940f111151d56fee60ef1a6e516a4ee7ce825 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 13:05:58 -0700 Subject: [PATCH 019/271] Update pydra/utils/messenger.py Co-authored-by: Satrajit Ghosh --- pydra/utils/messenger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/utils/messenger.py b/pydra/utils/messenger.py index 0058029c95..deb996cec3 100644 --- a/pydra/utils/messenger.py +++ b/pydra/utils/messenger.py @@ -169,7 +169,7 @@ def make_message(obj, context=None): def collect_messages(collected_path, message_path, ld_op="compact"): """ - Gather messages. + Compile all messages into a single provenance graph. Parameters ---------- From 61f0185f9b008d3d0e635fe6b70820654953d8c8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 13:06:15 -0700 Subject: [PATCH 020/271] Update pydra/utils/messenger.py Co-authored-by: Satrajit Ghosh --- pydra/utils/messenger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/utils/messenger.py b/pydra/utils/messenger.py index deb996cec3..be3e212785 100644 --- a/pydra/utils/messenger.py +++ b/pydra/utils/messenger.py @@ -151,7 +151,7 @@ def make_message(obj, context=None): obj : :obj:`dict` All the fields of the message (TODO) context : :obj:`dict`, optional - Dictionary with the link to the context.jsonld file. + Dictionary with the link to the context file or containing a JSON-LD context. Returns ------- From 64964bab120b2940bb7f9d2c7e74741ef21f66a2 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 13:06:39 -0700 Subject: [PATCH 021/271] Update pydra/utils/messenger.py Co-authored-by: Satrajit Ghosh --- pydra/utils/messenger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/utils/messenger.py b/pydra/utils/messenger.py index be3e212785..65056400b4 100644 --- a/pydra/utils/messenger.py +++ b/pydra/utils/messenger.py @@ -149,7 +149,7 @@ def make_message(obj, context=None): Parameters ---------- obj : :obj:`dict` - All the fields of the message (TODO) + A dictionary containing the non-context information of a message record. context : :obj:`dict`, optional Dictionary with the link to the context file or containing a JSON-LD context. From de31d256a9de29ddd2473ae786e00b1a1879a710 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 13:31:24 -0700 Subject: [PATCH 022/271] small edits --- pydra/engine/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 3cf32b44d3..9e59b3cfc0 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -18,7 +18,7 @@ def __init__(self, audit_flags, messengers, messenger_args, develop=None): ---------- audit_flags : :class:`AuditFlag` Base configuration of auditing. - messengers : :class:`pydra.util.messenger.Messenger` or list of :class:`pydra.util.messenger.Messenger`, optional + messengers : :obj:`pydra.util.messenger.Messenger` or list of :class:`pydra.util.messenger.Messenger`, optional Specify types of messenger used by Audit to send a message. Could be `PrintMessenger`, `FileMessenger`, or `RemoteRESTMessenger`. messenger_args : :obj:`dict`, optional From 0b9c5e70edfb29258b0b7681bbc0eff88c809aef Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 23 Jul 2020 21:46:02 -0700 Subject: [PATCH 023/271] changing the naming for splitter/combiner parts - instead of , parts using and parts --- pydra/engine/state.py | 271 ++++++++++++++------------- pydra/engine/tests/test_node_task.py | 2 +- pydra/engine/tests/test_state.py | 205 ++++++++++---------- 3 files changed, 254 insertions(+), 224 deletions(-) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index 93fa0b4792..60d755b987 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -130,7 +130,7 @@ def splitter_rpn(self): @property def splitter_rpn_compact(self): """splitter in :abbr:`RPN (Reverse Polish Notation)` - with a compact representation of the Left Part (i.e. without unwrapping + with a compact representation of the prev-state part (i.e. without unwrapping the part that comes from the previous states), e.g., [_NA, _NB, \*] """ if self.other_states: @@ -151,68 +151,70 @@ def splitter_rpn_final(self): if self.combiner: _splitter_rpn_final = hlpst.remove_inp_from_splitter_rpn( deepcopy(self.splitter_rpn), - self.right_combiner_all + self.left_combiner_all, + self.current_combiner_all + self.prev_state_combiner_all, ) return _splitter_rpn_final else: return self.splitter_rpn @property - def right_splitter(self): - """the Right Part of the splitter, + def current_splitter(self): + """the current part of the splitter, i.e. the part that is related to the current task's state only (doesn't include fields propagated from the previous tasks) """ - lr_flag = self._left_right_check(self.splitter) - if lr_flag == "Left": + lr_flag = self._prevst_current_check(self.splitter) + if lr_flag == "prev-state": return None - elif lr_flag == "Right": + elif lr_flag == "current": return self.splitter - elif lr_flag == "[Left, Right]": + elif lr_flag == "[prev-state, current]": return self.splitter[1] @property - def right_splitter_rpn(self): - """the Right Part in the splitter using RPN""" - if self.right_splitter: - right_splitter_rpn = hlpst.splitter2rpn( - self.right_splitter, other_states=self.other_states + def current_splitter_rpn(self): + """the current part of the splitter using RPN""" + if self.current_splitter: + current_splitter_rpn = hlpst.splitter2rpn( + self.current_splitter, other_states=self.other_states ) - return right_splitter_rpn + return current_splitter_rpn else: return [] @property - def left_splitter(self): - """ the left part of the splitter, + def prev_state_splitter(self): + """ the prev-state part of the splitter, i.e. the part that comes from the previous tasks' states """ - if hasattr(self, "_left_splitter"): - return self._left_splitter + if hasattr(self, "_prev_state_splitter"): + return self._prev_state_splitter else: return None @property - def left_splitter_rpn(self): - """the Left Part of the splitter using RPN""" - if self.left_splitter: - left_splitter_rpn = hlpst.splitter2rpn( - self.left_splitter, other_states=self.other_states + def prev_state_splitter_rpn(self): + """the prev-state art of the splitter using RPN""" + if self.prev_state_splitter: + prev_state_splitter_rpn = hlpst.splitter2rpn( + self.prev_state_splitter, other_states=self.other_states ) - return left_splitter_rpn + return prev_state_splitter_rpn else: return [] @property - def left_splitter_rpn_compact(self): - """ the Left Part of the splitter using RPN in a compact form, + def prev_state_splitter_rpn_compact(self): + """ the prev-state part of the splitter using RPN in a compact form, (without unwrapping the states from previous nodes), e.g. [_NA, _NB, \*] """ - if self.left_splitter: - left_splitter_rpn_compact = hlpst.splitter2rpn( - self.left_splitter, other_states=self.other_states, state_fields=False + if self.prev_state_splitter: + prev_state_splitter_rpn_compact = hlpst.splitter2rpn( + self.prev_state_splitter, + other_states=self.other_states, + state_fields=False, ) - return left_splitter_rpn_compact + return prev_state_splitter_rpn_compact else: return [] @@ -231,46 +233,46 @@ def combiner(self, combiner): self._combiner = [] @property - def right_combiner(self): - """the Right Part of the combiner, + def current_combiner(self): + """the current part of the combiner, i.e. the part that is related to the current task's state only (doesn't include fields propagated from the previous tasks) """ return [comb for comb in self.combiner if self.name in comb] @property - def right_combiner_all(self): - """the Right Part of the combiner including all the fields + def current_combiner_all(self): + """the current part of the combiner including all the fields that should be combined (i.e. not only the fields that are explicitly set, but also the fields that re in the same group/axis and had to be combined together, e.g., if splitter is (a, b) a and b has to be combined together) """ - if hasattr(self, "_right_combiner_all"): - return self._right_combiner_all + if hasattr(self, "_current_combiner_all"): + return self._current_combiner_all else: - return self.right_combiner + return self.current_combiner @property - def left_combiner(self): - """ the Left Part of the combiner, + def prev_state_combiner(self): + """ the prev-state part of the combiner, i.e. the part that comes from the previous tasks' states """ - if hasattr(self, "_left_combiner"): - return self._left_combiner + if hasattr(self, "_prev_state_combiner"): + return self._prev_state_combiner else: - return list(set(self.combiner) - set(self.right_combiner)) + return list(set(self.combiner) - set(self.current_combiner)) @property - def left_combiner_all(self): - """the Left Part of the combiner including all the fields + def prev_state_combiner_all(self): + """the prev-state part of the combiner including all the fields that should be combined (i.e. not only the fields that are explicitly set, but also the fields that re in the same group/axis and had to be combined together, e.g., if splitter is (a, b) a and b has to be combined together) """ - if hasattr(self, "_left_combiner_all"): - return list(set(self._left_combiner_all)) + if hasattr(self, "_prev_state_combiner_all"): + return list(set(self._prev_state_combiner_all)) else: - return self.left_combiner + return self.prev_state_combiner @property def other_states(self): @@ -328,64 +330,70 @@ def update_connections(self, new_other_states=None, new_combiner=None): def _connect_splitters(self): """ Connect splitters from the previous nodes. - Evaluates the Left Part of the splitter (i.e. the part from the previous states) - and the Right Part of the splitter(i.e., the current state). - If the left splitter is not provided the splitter has to be completed. + Evaluates the prev-state part of the splitter (i.e. the part from the previous states) + and the current part of the splitter(i.e., the current state). + If the prev-state splitter is not provided the splitter has to be completed. """ - # TODO: should this be in the left_Splitter property? + # TODO: should this be in the prev-state_Splitter property? if self.splitter: - # if splitter is string, have to check if this is Left or Right part (Left is required) + # if splitter is string, have to check if this is prev-state or current part (prev-state is required) if isinstance(self.splitter, str): - # so this is the Left part + # so this is the prev-state part if self.splitter.startswith("_"): - self._left_splitter = self._complete_left(left=self.splitter) - else: # this is Right part - self._left_splitter = self._complete_left() + self._prev_state_splitter = self._complete_prev_state( + prev_state=self.splitter + ) + else: # this is the current part + self._prev_state_splitter = self._complete_prev_state() elif isinstance(self.splitter, (tuple, list)): - lr_flag = self._left_right_check(self.splitter) - if lr_flag == "Left": - self._left_splitter = self._complete_left(left=self.splitter) - elif lr_flag == "Right": - self._left_splitter = self._complete_left() - elif lr_flag == "[Left, Right]": - self._left_splitter = self._complete_left(left=self.splitter[0]) + lr_flag = self._prevst_current_check(self.splitter) + if lr_flag == "prev-state": + self._prev_state_splitter = self._complete_prev_state( + prev_state=self.splitter + ) + elif lr_flag == "current": + self._prev_state_splitter = self._complete_prev_state() + elif lr_flag == "[prev-state, current]": + self._prev_state_splitter = self._complete_prev_state( + prev_state=self.splitter[0] + ) else: - # if there is no splitter, I create the Left part - self._left_splitter = self._complete_left() + # if there is no splitter, I create the prev-state part + self._prev_state_splitter = self._complete_prev_state() - if self.right_splitter: + if self.current_splitter: self.splitter = [ - deepcopy(self._left_splitter), - deepcopy(self.right_splitter), + deepcopy(self._prev_state_splitter), + deepcopy(self.current_splitter), ] else: - self.splitter = deepcopy(self._left_splitter) + self.splitter = deepcopy(self._prev_state_splitter) - def _complete_left(self, left=None): - """Add all splitters from the previous nodes (completing the Left part). + def _complete_prev_state(self, prev_state=None): + """Add all splitters from the previous nodes (completing the prev-state part). Parameters ---------- - left : :obj:`str`, or :obj:`list`, or :obj:`tuple`, optional - the left part of the splitter, that has to be completed + prev_state : :obj:`str`, or :obj:`list`, or :obj:`tuple`, optional + the prev-state part of the splitter, that has to be completed """ - if left: - rpn_left = hlpst.splitter2rpn( - left, other_states=self.other_states, state_fields=False + if prev_state: + rpn_prev_state = hlpst.splitter2rpn( + prev_state, other_states=self.other_states, state_fields=False ) for name, (st, inp) in list(self.other_states.items())[::-1]: - if f"_{name}" not in rpn_left and st.splitter_final: - left = [f"_{name}", left] + if f"_{name}" not in rpn_prev_state and st.splitter_final: + prev_state = [f"_{name}", prev_state] else: - left = [f"_{name}" for name in self.other_states] - if len(left) == 1: - left = left[0] - return left + prev_state = [f"_{name}" for name in self.other_states] + if len(prev_state) == 1: + prev_state = prev_state[0] + return prev_state - def _left_right_check(self, splitter_part, check_nested=True): + def _prevst_current_check(self, splitter_part, check_nested=True): """ - Check if splitter_part is purely Left, Right - or [Left, Right] if the splitter_part is a list (outer splitter) + Check if splitter_part is purely prev-state part, the current part, + or mixed ([prev-state, current]) if the splitter_part is a list (outer splitter) Parameters ---------- @@ -397,7 +405,7 @@ def _left_right_check(self, splitter_part, check_nested=True): Returns ------- str - describes the type - "Left" or "Right" + describes the type - "prev-state" or "current" If the splitter_part is mixed exception is raised. @@ -410,19 +418,23 @@ def _left_right_check(self, splitter_part, check_nested=True): True if el.startswith("_") else False for el in inputs_in_splitter ] if all(others_in_splitter): - return "Left" + return "prev-state" elif (not all(others_in_splitter)) and (not any(others_in_splitter)): - return "Right" + return "current" elif ( isinstance(self.splitter, list) and check_nested - and self._left_right_check(self.splitter[0], check_nested=False) == "Left" - and self._left_right_check(self.splitter[1], check_nested=False) == "Right" + and self._prevst_current_check(self.splitter[0], check_nested=False) + == "prev-state" + and self._prevst_current_check(self.splitter[1], check_nested=False) + == "current" ): - return "[Left, Right]" # Left and Right parts separated in outer scalar + return ( + "[prev-state, current]" + ) # the prev-state and the current parts separated in outer scalar else: raise hlpst.PydraStateError( - "Left and Right splitters are mixed - splitter invalid" + "prev-state and current splitters are mixed - splitter invalid" ) def set_input_groups(self, state_fields=True): @@ -433,8 +445,8 @@ def set_input_groups(self, state_fields=True): state_fields : :obj:`bool` if False the splitter from the previous states are unwrapped """ - right_splitter_rpn = hlpst.splitter2rpn( - self.right_splitter, + current_splitter_rpn = hlpst.splitter2rpn( + self.current_splitter, other_states=self.other_states, state_fields=state_fields, ) @@ -442,16 +454,20 @@ def set_input_groups(self, state_fields=True): if self.inner_inputs: self._merge_previous_groups() keys_f, group_for_inputs_f, groups_stack_f, combiner_all = hlpst.splits_groups( - right_splitter_rpn, - combiner=self.right_combiner, + current_splitter_rpn, + combiner=self.current_combiner, inner_inputs=self.inner_inputs, ) - self._right_combiner_all = combiner_all - if self.left_splitter and state_fields: # if splitter has also the left part - self._right_keys_final = keys_f - self._right_group_for_inputs_final = group_for_inputs_f - self._right_groups_stack_final = groups_stack_f - if self.right_splitter: # if Right part, adding groups from current st + self._current_combiner_all = combiner_all + if ( + self.prev_state_splitter and state_fields + ): # if splitter has also the prev-state part + self._current_keys_final = keys_f + self._current_group_for_inputs_final = group_for_inputs_f + self._current_groups_stack_final = groups_stack_f + if ( + self.current_splitter + ): # if the current part, adding groups from current st self._add_current_groups() else: @@ -465,26 +481,28 @@ def _merge_previous_groups(self): self.groups_stack_final = [] self.group_for_inputs_final = {} self.keys_final = [] - if self.left_combiner: - _, _, _, self._left_combiner_all = hlpst.splits_groups( - self.left_splitter_rpn, combiner=self.left_combiner + if self.prev_state_combiner: + _, _, _, self._prev_state_combiner_all = hlpst.splits_groups( + self.prev_state_splitter_rpn, combiner=self.prev_state_combiner ) - for i, left_nm in enumerate(self.left_splitter_rpn_compact): - if left_nm in ["*", "."]: + for i, prev_nm in enumerate(self.prev_state_splitter_rpn_compact): + if prev_nm in ["*", "."]: continue if ( - i + 1 < len(self.left_splitter_rpn_compact) - and self.left_splitter_rpn_compact[i + 1] == "." + i + 1 < len(self.prev_state_splitter_rpn_compact) + and self.prev_state_splitter_rpn_compact[i + 1] == "." ): last_gr = last_gr - 1 - if left_nm[1:] not in self.other_states: + if prev_nm[1:] not in self.other_states: raise hlpst.PydraStateError( - f"can't ask for splitter from {left_nm[1:]}, other nodes that are connected: {self.other_states}" + f"can't ask for splitter from {prev_nm[1:]}, other nodes that are connected: {self.other_states}" ) - st = self.other_states[left_nm[1:]][0] - # checking if left combiner contains any element from the st splitter + st = self.other_states[prev_nm[1:]][0] + # checking if prev-state combiner contains any element from the st splitter st_combiner = [ - comb for comb in self.left_combiner_all if comb in st.splitter_rpn_final + comb + for comb in self.prev_state_combiner_all + if comb in st.splitter_rpn_final ] if not hasattr(st, "keys_final"): st.set_input_groups() @@ -506,7 +524,7 @@ def _merge_previous_groups(self): raise hlpst.PydraStateError("previous state has to run first") group_for_inputs = group_for_inputs_f_st groups_stack = groups_stack_f_st - self._left_combiner_all += combiner_all_st + self._prev_state_combiner_all += combiner_all_st else: # if no element from st.splitter is in the current combiner, # using st attributes without changes @@ -532,15 +550,15 @@ def _merge_previous_groups(self): def _add_current_groups(self): """Add additional groups from the current state.""" - self.keys_final += self._right_keys_final + self.keys_final += self._current_keys_final nr_gr_f = max(self.group_for_inputs_final.values()) + 1 - for inp, grs in self._right_group_for_inputs_final.items(): + for inp, grs in self._current_group_for_inputs_final.items(): if isinstance(grs, int): grs = grs + nr_gr_f else: # a list grs = [gr + nr_gr_f for gr in grs] self.group_for_inputs_final[inp] = grs - for i, stack in enumerate(self._right_groups_stack_final): + for i, stack in enumerate(self._current_groups_stack_final): if i == 0: stack = [gr + nr_gr_f for gr in stack] self.groups_stack_final[-1] += stack @@ -670,12 +688,13 @@ def prepare_states_combined_ind(self, elements_to_remove_comb): partial_rpn_compact = hlpst.remove_inp_from_splitter_rpn( deepcopy(self.splitter_rpn_compact), elements_to_remove_comb ) - # combiner can have parts from the left splitter, so have to have rpn with states + # combiner can have parts from the prev-state splitter, so have to have rpn with states partial_rpn = hlpst.splitter2rpn( hlpst.rpn2splitter(partial_rpn_compact), other_states=self.other_states ) combined_rpn = hlpst.remove_inp_from_splitter_rpn( - deepcopy(partial_rpn), self.right_combiner_all + self.left_combiner_all + deepcopy(partial_rpn), + self.current_combiner_all + self.prev_state_combiner_all, ) if combined_rpn: val_r, key_r = hlpst.splits( @@ -731,10 +750,10 @@ def prepare_inputs(self): if not self.other_states: self.inputs_ind = self.states_ind else: - # elements from the current node (the Right part) - if self.right_splitter_rpn: + # elements from the current node (the current part of the splitter) + if self.current_splitter_rpn: values_inp, keys_inp = hlpst.splits( - self.right_splitter_rpn, + self.current_splitter_rpn, self.inputs, inner_inputs=self.inner_inputs, cont_dim=self.cont_dim, @@ -750,7 +769,7 @@ def prepare_inputs(self): keys_inp_prev = [] inputs_ind_prev = [] connected_to_inner = [] - for ii, el in enumerate(self.left_splitter_rpn_compact): + for ii, el in enumerate(self.prev_state_splitter_rpn_compact): if el in ["*", "."]: continue st, inp = self.other_states[el[1:]] @@ -761,8 +780,8 @@ def prepare_inputs(self): else: # previous states that are not connected to inner splitter st_ind = range(len(st.states_ind_final)) if inputs_ind_prev: - # in case the Left part has scalar parts (not very well tested) - if self.left_splitter_rpn_compact[ii + 1] == ".": + # in case the prev-state part has scalar parts (not very well tested) + if self.prev_state_splitter_rpn_compact[ii + 1] == ".": inputs_ind_prev = hlpst.op["."](inputs_ind_prev, st_ind) else: inputs_ind_prev = hlpst.op["*"](inputs_ind_prev, st_ind) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index 02afde89f7..ef2e1d8018 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -1156,7 +1156,7 @@ def test_task_state_comb_2( assert nn.state.splitter_final == state_splitter_final assert nn.state.splitter_rpn_final == state_rpn_final - assert set(nn.state.right_combiner_all) == set(state_combiner_all) + assert set(nn.state.current_combiner_all) == set(state_combiner_all) # checking the results results = nn.result() diff --git a/pydra/engine/tests/test_state.py b/pydra/engine/tests/test_state.py index b3c46d2484..04152e1fca 100644 --- a/pydra/engine/tests/test_state.py +++ b/pydra/engine/tests/test_state.py @@ -79,10 +79,10 @@ def test_state_1( ): """ single state: testing groups, prepare_states and prepare_inputs""" st = State(name="NA", splitter=splitter) - assert st.splitter == st.right_splitter - assert st.splitter_rpn == st.right_splitter_rpn - assert st.left_splitter is None - assert st.left_combiner_all == [] + assert st.splitter == st.current_splitter + assert st.splitter_rpn == st.current_splitter_rpn + assert st.prev_state_splitter is None + assert st.prev_state_combiner_all == [] st.prepare_states(inputs) assert st.group_for_inputs_final == group_for_inputs @@ -129,10 +129,10 @@ def test_state_connect_1(): st2 = State(name="NB", other_states={"NA": (st1, "b")}) assert st2.splitter == "_NA" assert st2.splitter_rpn == ["NA.a"] - assert st2.left_splitter == st2.splitter - assert st2.left_splitter_rpn == st2.splitter_rpn - assert st2.right_splitter is None - assert st2.right_splitter_rpn == [] + assert st2.prev_state_splitter == st2.splitter + assert st2.prev_state_splitter_rpn == st2.splitter_rpn + assert st2.current_splitter is None + assert st2.current_splitter_rpn == [] st2.prepare_states(inputs={"NA.a": [3, 5]}) assert st2.group_for_inputs_final == {"NA.a": 0} @@ -146,7 +146,7 @@ def test_state_connect_1(): def test_state_connect_1a(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs - the second state has explicit splitter from the first one (Left part) + the second state has explicit splitter from the first one (the prev-state part) """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="_NA", other_states={"NA": (st1, "b")}) @@ -184,17 +184,17 @@ def test_state_connect_1c_exception(splitter2, other_states2): def test_state_connect_2(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs the second state has explicit splitter that contains - splitter from the first node and a new field (Left and Right parts) + splitter from the first node and a new field (the prev-state and current part) """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter=["_NA", "a"], other_states={"NA": (st1, "b")}) assert st2.splitter == ["_NA", "NB.a"] assert st2.splitter_rpn == ["NA.a", "NB.a", "*"] - assert st2.left_splitter == "_NA" - assert st2.left_splitter_rpn == ["NA.a"] - assert st2.right_splitter == "NB.a" - assert st2.right_splitter_rpn == ["NB.a"] + assert st2.prev_state_splitter == "_NA" + assert st2.prev_state_splitter_rpn == ["NA.a"] + assert st2.current_splitter == "NB.a" + assert st2.current_splitter_rpn == ["NB.a"] st2.update_connections() st2.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [1, 2]}) @@ -265,16 +265,16 @@ def test_state_connect_2a(): def test_state_connect_2b(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs - the second state has explicit splitter with a new field (Right part) - splitter from the first node (Left part) has to be added + the second state has explicit splitter with a new field (the current part) + splitter from the first node (the prev-state part) has to be added """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="a", other_states={"NA": (st1, "b")}) assert st2.splitter == ["_NA", "NB.a"] assert st2.splitter_rpn == ["NA.a", "NB.a", "*"] - assert st2.right_splitter == "NB.a" - assert st2.left_splitter == "_NA" + assert st2.current_splitter == "NB.a" + assert st2.prev_state_splitter == "_NA" st2.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [1, 2]}) assert st2.group_for_inputs_final == {"NA.a": 0, "NB.a": 1} @@ -304,7 +304,7 @@ def test_state_connect_2b(): def test_state_connect_3(): """ three 'connected' states: testing groups, prepare_states and prepare_inputs the third state connected to two previous states; - splitter from the previous states (Left part) has to be added + splitter from the previous states (the prev-state part) has to be added """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="a") @@ -312,11 +312,11 @@ def test_state_connect_3(): assert st3.splitter == ["_NA", "_NB"] assert st3.splitter_rpn == ["NA.a", "NB.a", "*"] - assert st3.left_splitter == st3.splitter - assert st3.left_splitter_rpn == st3.splitter_rpn - assert st3.left_splitter_rpn_compact == ["_NA", "_NB", "*"] - assert st3.right_splitter is None - assert st3.right_splitter_rpn == [] + assert st3.prev_state_splitter == st3.splitter + assert st3.prev_state_splitter_rpn == st3.splitter_rpn + assert st3.prev_state_splitter_rpn_compact == ["_NA", "_NB", "*"] + assert st3.current_splitter is None + assert st3.current_splitter_rpn == [] st3.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [30, 50]}) assert st3.group_for_inputs_final == {"NA.a": 0, "NB.a": 1} @@ -388,7 +388,7 @@ def test_state_connect_3b(): """ three 'connected' states: testing groups, prepare_states and prepare_inputs the third state connected to two previous states; the third state has explicit splitter that contains splitter only from the first state. - splitter from the second state has to be added (partial Left part) + splitter from the second state has to be added (partial prev-state part) """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="a") @@ -439,11 +439,11 @@ def test_state_connect_4(): assert st3.splitter == ("_NA", "_NB") assert st3.splitter_rpn == ["NA.a", "NB.a", "."] - assert st3.left_splitter == st3.splitter - assert st3.left_splitter_rpn == st3.splitter_rpn - assert st3.left_splitter_rpn_compact == ["_NA", "_NB", "."] - assert st3.right_splitter is None - assert st3.right_splitter_rpn == [] + assert st3.prev_state_splitter == st3.splitter + assert st3.prev_state_splitter_rpn == st3.splitter_rpn + assert st3.prev_state_splitter_rpn_compact == ["_NA", "_NB", "."] + assert st3.current_splitter is None + assert st3.current_splitter_rpn == [] st3.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [30, 50]}) assert st3.group_for_inputs_final == {"NA.a": 0, "NB.a": 0} @@ -595,11 +595,11 @@ def test_state_connect_innerspl_1(): assert st2.splitter == ["_NA", "NB.b"] assert st2.splitter_rpn == ["NA.a", "NB.b", "*"] - assert st2.left_splitter == "_NA" - assert st2.left_splitter_rpn == ["NA.a"] - assert st2.left_splitter_rpn_compact == ["_NA"] - assert st2.right_splitter == "NB.b" - assert st2.right_splitter_rpn == ["NB.b"] + assert st2.prev_state_splitter == "_NA" + assert st2.prev_state_splitter_rpn == ["NA.a"] + assert st2.prev_state_splitter_rpn_compact == ["_NA"] + assert st2.current_splitter == "NB.b" + assert st2.current_splitter_rpn == ["NB.b"] st2.prepare_states( inputs={"NA.a": [3, 5], "NB.b": [[1, 10, 100], [2, 20, 200]]}, @@ -640,18 +640,18 @@ def test_state_connect_innerspl_1(): def test_state_connect_innerspl_1a(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs, the second state has an inner splitter, - splitter from the first state (Left part) has to be added + splitter from the first state (the prev-state part) has to be added """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="b", other_states={"NA": (st1, "b")}) assert st2.splitter == ["_NA", "NB.b"] assert st2.splitter_rpn == ["NA.a", "NB.b", "*"] - assert st2.left_splitter == "_NA" - assert st2.left_splitter_rpn == ["NA.a"] - assert st2.left_splitter_rpn_compact == ["_NA"] - assert st2.right_splitter == "NB.b" - assert st2.right_splitter_rpn == ["NB.b"] + assert st2.prev_state_splitter == "_NA" + assert st2.prev_state_splitter_rpn == ["NA.a"] + assert st2.prev_state_splitter_rpn_compact == ["_NA"] + assert st2.current_splitter == "NB.b" + assert st2.current_splitter_rpn == ["NB.b"] assert st2.other_states["NA"][1] == "b" @@ -691,7 +691,7 @@ def test_state_connect_innerspl_1a(): def test_state_connect_innerspl_1b(): - """incorrect splitter - Right & Left parts in scalar splitter""" + """incorrect splitter - the current & prev-state parts in scalar splitter""" with pytest.raises(PydraStateError): st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter=("_NA", "b"), other_states={"NA": (st1, "b")}) @@ -700,18 +700,18 @@ def test_state_connect_innerspl_1b(): def test_state_connect_innerspl_2(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs, the second state has one inner splitter and one 'normal' splitter - only Right part of the splitter provided (Left has to be added) + only the current part of the splitter provided (the prev-state has to be added) """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter=["c", "b"], other_states={"NA": (st1, "b")}) assert st2.splitter == ["_NA", ["NB.c", "NB.b"]] assert st2.splitter_rpn == ["NA.a", "NB.c", "NB.b", "*", "*"] - assert st2.left_splitter == "_NA" - assert st2.left_splitter_rpn == ["NA.a"] - assert st2.left_splitter_rpn_compact == ["_NA"] - assert st2.right_splitter == ["NB.c", "NB.b"] - assert st2.right_splitter_rpn == ["NB.c", "NB.b", "*"] + assert st2.prev_state_splitter == "_NA" + assert st2.prev_state_splitter_rpn == ["NA.a"] + assert st2.prev_state_splitter_rpn_compact == ["_NA"] + assert st2.current_splitter == ["NB.c", "NB.b"] + assert st2.current_splitter_rpn == ["NB.c", "NB.b", "*"] st2.prepare_states( inputs={"NA.a": [3, 5], "NB.b": [[1, 10, 100], [2, 20, 200]], "NB.c": [13, 17]}, @@ -770,7 +770,7 @@ def test_state_connect_innerspl_2(): def test_state_connect_innerspl_2a(): """ two 'connected' states: testing groups, prepare_states and prepare_inputs, the second state has one inner splitter and one 'normal' splitter - only Right part of the splitter provided (different order!), + only the current part of the splitter provided (different order!), """ st1 = State(name="NA", splitter="a") @@ -837,7 +837,7 @@ def test_state_connect_innerspl_2a(): def test_state_connect_innerspl_3(): """ three serially 'connected' states: testing groups, prepare_states and prepare_inputs, the second state has one inner splitter and one 'normal' splitter - Left parts have to be added + the prev-state parts of the splitter have to be added """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter=["c", "b"], other_states={"NA": (st1, "b")}) @@ -845,11 +845,11 @@ def test_state_connect_innerspl_3(): assert st3.splitter == ["_NB", "NC.d"] assert st3.splitter_rpn == ["NA.a", "NB.c", "NB.b", "*", "*", "NC.d", "*"] - assert st3.left_splitter == "_NB" - assert st3.left_splitter_rpn == ["NA.a", "NB.c", "NB.b", "*", "*"] - assert st3.left_splitter_rpn_compact == ["_NB"] - assert st3.right_splitter == "NC.d" - assert st3.right_splitter_rpn == ["NC.d"] + assert st3.prev_state_splitter == "_NB" + assert st3.prev_state_splitter_rpn == ["NA.a", "NB.c", "NB.b", "*", "*"] + assert st3.prev_state_splitter_rpn_compact == ["_NB"] + assert st3.current_splitter == "NC.d" + assert st3.current_splitter_rpn == ["NC.d"] st3.prepare_states( inputs={ @@ -976,7 +976,7 @@ def test_state_connect_innerspl_3(): def test_state_connect_innerspl_4(): """ three'connected' states: testing groups, prepare_states and prepare_inputs, - the third one connected to two previous, only Right part of splitter provided + the third one connected to two previous, only the current part of splitter provided """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter=["b", "c"]) @@ -1066,8 +1066,8 @@ def test_state_combine_1(): st = State(name="NA", splitter="a", combiner="a") assert st.splitter == "NA.a" assert st.splitter_rpn == ["NA.a"] - assert st.right_combiner == st.right_combiner_all == st.combiner == ["NA.a"] - assert st.left_combiner == st.left_combiner_all == [] + assert st.current_combiner == st.current_combiner_all == st.combiner == ["NA.a"] + assert st.prev_state_combiner == st.prev_state_combiner_all == [] assert st.splitter_final == None assert st.splitter_rpn_final == [] @@ -1090,8 +1090,8 @@ def test_state_connect_combine_1(): assert st1.splitter_rpn == ["NA.a", "NA.b", "*"] assert st1.splitter_rpn_final == ["NA.b"] assert st1.splitter_final == "NA.b" - assert st1.combiner == st1.right_combiner == st1.right_combiner_all == ["NA.a"] - assert st1.left_combiner_all == st1.left_combiner == [] + assert st1.combiner == st1.current_combiner == st1.current_combiner_all == ["NA.a"] + assert st1.prev_state_combiner_all == st1.prev_state_combiner == [] assert st2.splitter == "_NA" assert st2.splitter_rpn == ["NA.b"] @@ -1202,8 +1202,8 @@ def test_state_connect_combine_3(): assert st2.splitter == ["_NA", "NB.d"] assert st2.splitter_rpn == ["NA.b", "NB.d", "*"] assert st2.splitter_rpn_final == ["NA.b"] - assert st2.left_combiner_all == st2.left_combiner == [] - assert st2.right_combiner_all == st2.right_combiner == st2.combiner == ["NB.d"] + assert st2.prev_state_combiner_all == st2.prev_state_combiner == [] + assert st2.current_combiner_all == st2.current_combiner == st2.combiner == ["NB.d"] st2.prepare_states( inputs={"NA.a": [3, 5], "NA.b": [10, 20], "NB.c": [90, 150], "NB.d": [0, 1]} @@ -1254,7 +1254,7 @@ def test_state_connect_combine_3(): def test_state_connect_innerspl_combine_1(): """one previous node and one inner splitter (and inner splitter combiner); - only Right part provided - Left had to be added""" + only current part provided - the prev-state part had to be added""" st1 = State(name="NA", splitter="a") st2 = State( name="NB", splitter=["c", "b"], combiner=["b"], other_states={"NA": (st1, "b")} @@ -1264,8 +1264,8 @@ def test_state_connect_innerspl_combine_1(): assert st2.splitter_rpn == ["NA.a", "NB.c", "NB.b", "*", "*"] assert st2.splitter_final == ["NA.a", "NB.c"] assert st2.splitter_rpn_final == ["NA.a", "NB.c", "*"] - assert st2.left_combiner_all == st2.left_combiner == [] - assert st2.right_combiner_all == st2.right_combiner == st2.combiner == ["NB.b"] + assert st2.prev_state_combiner_all == st2.prev_state_combiner == [] + assert st2.current_combiner_all == st2.current_combiner == st2.combiner == ["NB.b"] # TODO: i think at the end I should merge [0] and [1], because there are no inner splitters anymore # TODO: didn't include it in my code... # assert st2.groups_stack_final == [[0, 1]] @@ -1333,7 +1333,8 @@ def test_state_connect_innerspl_combine_1(): def test_state_connect_innerspl_combine_2(): """ two 'connected' state, the second has inner and normal splitter, and 'normal' combiner - only Right part splitter provided, Left has to be added + only the current part of the splitter provided, + the prev-state part has to be added """ st1 = State(name="NA", splitter="a") st2 = State( @@ -1405,18 +1406,23 @@ def test_state_connect_innerspl_combine_2(): ] -def test_state_connect_combine_left_1(): +def test_state_connect_combine_prevst_1(): """ two 'connected' states, the first one has the simplest splitter, the second has combiner from the first state - (i.e. from the Left part of the splitter), + (i.e. from the prev-state part of the splitter), """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", other_states={"NA": (st1, "b")}, combiner="NA.a") assert st2.splitter == "_NA" assert st2.splitter_rpn == ["NA.a"] - assert st2.combiner == st2.left_combiner == st2.left_combiner_all == ["NA.a"] - assert st2.right_combiner == st2.right_combiner_all == [] + assert ( + st2.combiner + == st2.prev_state_combiner + == st2.prev_state_combiner_all + == ["NA.a"] + ) + assert st2.current_combiner == st2.current_combiner_all == [] assert st2.splitter_rpn_final == [] st2.prepare_states(inputs={"NA.a": [3, 5]}) @@ -1430,19 +1436,19 @@ def test_state_connect_combine_left_1(): assert st2.inputs_ind == [{"NB.b": 0}, {"NB.b": 1}] -def test_state_connect_combine_left_2(): +def test_state_connect_combine_prevst_2(): """ two 'connected' states, the first one has outer splitter, the second has combiner from the first state - (i.e. from the Left part of the splitter), + (i.e. from the prev-state part of the splitter), """ st1 = State(name="NA", splitter=["a", "b"]) st2 = State(name="NB", other_states={"NA": (st1, "b")}, combiner="NA.a") assert st2.splitter == "_NA" assert st2.splitter_rpn == ["NA.a", "NA.b", "*"] assert st2.combiner == ["NA.a"] - assert st2.left_combiner_all == st2.left_combiner == ["NA.a"] - assert st2.right_combiner_all == st2.right_combiner == [] + assert st2.prev_state_combiner_all == st2.prev_state_combiner == ["NA.a"] + assert st2.current_combiner_all == st2.current_combiner == [] assert st2.splitter_rpn_final == ["NA.b"] st2.prepare_states(inputs={"NA.a": [3, 5], "NA.b": [10, 20]}) @@ -1466,11 +1472,11 @@ def test_state_connect_combine_left_2(): assert st2.inputs_ind == [{"NB.b": 0}, {"NB.b": 1}, {"NB.b": 2}, {"NB.b": 3}] -def test_state_connect_combine_left_3(): +def test_state_connect_combine_prevst_3(): """ three serially 'connected' states, the first one has outer splitter, the third one has combiner from the first state - (i.e. from the Left part of the splitter), + (i.e. from the prev-state part of the splitter), """ st1 = State(name="NA", splitter=["a", "b"]) st2 = State(name="NB", other_states={"NA": (st1, "b")}) @@ -1502,11 +1508,11 @@ def test_state_connect_combine_left_3(): assert st3.inputs_ind == [{"NC.c": 0}, {"NC.c": 1}, {"NC.c": 2}, {"NC.c": 3}] -def test_state_connect_combine_left_4(): +def test_state_connect_combine_prevst_4(): """ three 'connected' states: testing groups, prepare_states and prepare_inputs, the first two states have the simplest splitters, - the third state has only Left splitter, - the third state has also combiner from the Left part + the third state has only the prev-state part of splitter, + the third state has also combiner from the prev-state part """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="a") @@ -1519,8 +1525,13 @@ def test_state_connect_combine_left_4(): assert st3.splitter == ["_NA", "_NB"] assert st3.splitter_rpn == ["NA.a", "NB.a", "*"] assert st3.splitter_rpn_final == ["NB.a"] - assert st3.left_combiner_all == st3.left_combiner == st3.combiner == ["NA.a"] - assert st3.right_combiner_all == st3.right_combiner == [] + assert ( + st3.prev_state_combiner_all + == st3.prev_state_combiner + == st3.combiner + == ["NA.a"] + ) + assert st3.current_combiner_all == st3.current_combiner == [] st3.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [600, 700]}) assert st3.group_for_inputs_final == {"NB.a": 0} @@ -1549,11 +1560,11 @@ def test_state_connect_combine_left_4(): ] -def test_state_connect_combine_left_5(): +def test_state_connect_combine_prevst_5(): """ three 'connected' states: testing groups, prepare_states and prepare_inputs, the first two states have the simplest splitters, - the third state has scalar splitter in the Left part, - the third state has also combiner from the Left part + the third state has scalar splitter in the prev-state part, + the third state has also combiner from the prev-state part """ st1 = State(name="NA", splitter="a") st2 = State(name="NB", splitter="a") @@ -1567,9 +1578,9 @@ def test_state_connect_combine_left_5(): assert st3.splitter_rpn == ["NA.a", "NB.a", "."] st3.set_input_groups() assert st3.splitter_rpn_final == [] - assert set(st3.left_combiner_all) == {"NA.a", "NB.a"} - assert st3.left_combiner == st3.combiner == ["NA.a"] - assert st3.right_combiner_all == st3.right_combiner == [] + assert set(st3.prev_state_combiner_all) == {"NA.a", "NB.a"} + assert st3.prev_state_combiner == st3.combiner == ["NA.a"] + assert st3.current_combiner_all == st3.current_combiner == [] st3.prepare_states(inputs={"NA.a": [3, 5], "NB.a": [600, 700]}) assert st3.group_for_inputs_final == {} @@ -1583,11 +1594,11 @@ def test_state_connect_combine_left_5(): assert st3.inputs_ind == [{"NC.a": 0, "NC.b": 0}, {"NC.a": 1, "NC.b": 1}] -def test_state_connect_combine_left_6(): +def test_state_connect_combine_prevst_6(): """ two 'connected' states, - the first one has outer splitter, the second has an additional Right splitter, + the first one has outer splitter, the second has an additional current splitter, the second also has combiner from the first state - (i.e. from the Left part of the splitter), + (i.e. from the prev-state part of the splitter), """ st1 = State(name="NA", splitter=["a", "b"]) st2 = State( @@ -1596,8 +1607,8 @@ def test_state_connect_combine_left_6(): assert st2.splitter == ["_NA", "NB.c"] assert st2.splitter_rpn == ["NA.a", "NA.b", "*", "NB.c", "*"] assert st2.combiner == ["NA.a"] - assert st2.left_combiner_all == st2.left_combiner == ["NA.a"] - assert st2.right_combiner_all == st2.right_combiner == [] + assert st2.prev_state_combiner_all == st2.prev_state_combiner == ["NA.a"] + assert st2.current_combiner_all == st2.current_combiner == [] assert st2.splitter_rpn_final == ["NA.b", "NB.c", "*"] st2.prepare_states(inputs={"NA.a": [3, 5], "NA.b": [10, 20], "NB.c": [0, 1]}) @@ -1645,7 +1656,7 @@ def test_state_connect_combine_left_6(): @pytest.mark.parametrize( - "splitter, other_states, expected_splitter, expected_left, expected_right", + "splitter, other_states, expected_splitter, expected_prevst, expected_current", [ (None, {"NA": (State(name="NA", splitter="a"), "b")}, "_NA", "_NA", None), ( @@ -1695,13 +1706,13 @@ def test_state_connect_combine_left_6(): ], ) def test_connect_splitters( - splitter, other_states, expected_splitter, expected_left, expected_right + splitter, other_states, expected_splitter, expected_prevst, expected_current ): st = State(name="CN", splitter=splitter, other_states=other_states) st.set_input_groups() assert st.splitter == expected_splitter - assert st.left_splitter == expected_left - assert st.right_splitter == expected_right + assert st.prev_state_splitter == expected_prevst + assert st.current_splitter == expected_current @pytest.mark.parametrize( @@ -1721,7 +1732,7 @@ def test_connect_splitters( def test_connect_splitters_exception_1(splitter, other_states): with pytest.raises(PydraStateError) as excinfo: st = State(name="CN", splitter=splitter, other_states=other_states) - assert "Left and Right splitters are mixed" in str(excinfo.value) + assert "prev-state and current splitters are mixed" in str(excinfo.value) def test_connect_splitters_exception_2(): From 38e8917738b1980f204eb48def9bc350c7ec2130 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 01:40:21 -0400 Subject: [PATCH 024/271] fixing/extending custom_validator --- pydra/engine/helpers.py | 44 +++++++++--- pydra/engine/tests/test_task.py | 117 ++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 9 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index a0d898ea09..131e097839 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -16,6 +16,7 @@ import typing as ty import typing_inspect as tyi import inspect +import warnings from .specs import Runtime, File, Directory, attr_fields, Result, LazyField @@ -286,10 +287,11 @@ def custom_validator(instance, attribute, value): if ( value is attr.NOTHING or value is None + or attribute.name.startswith("_") # e.g. _func or isinstance(value, LazyField) or tp_attr in [ty.Any, inspect._empty] ): - check = False + check = False # no checking of the type elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: # what about File tp = _single_type_update(tp_attr) cont = None # no container @@ -303,9 +305,20 @@ def custom_validator(instance, attribute, value): tp, check = _types_updates(tp_attr_list) elif tp_attr._name == "Dict": cont = "Dict" - breakpoint() + tp_attr_key_val = tyi.get_args(tp_attr) + # assuming that it should have length of 2 for keys and values + if len(tp_attr_key_val) != 2: + check = False + # updating types separately for keys and values + tp_k, check_k = _types_updates([tp_attr_key_val[0]]) + tp_v, check_v = _types_updates([tp_attr_key_val[1]]) + if not (check_k and check_v): + check = False + else: + tp = {"key": tp_k, "val": tp_v} else: - breakpoint() + warnings.warn(f"no checking implemented for value {value} and type {tp_attr}") + check = False if check: if cont is None: @@ -328,7 +341,12 @@ def custom_validator(instance, attribute, value): ) )(instance, attribute, value) elif cont == "Dict": - breakpoint() # TODO + return attr.validators.deep_mapping( + key_validator=attr.validators.instance_of(tp["key"]), + value_validator=attr.validators.instance_of( + tp["val"] + (attr._make._Nothing,) + ), + )(instance, attribute, value) else: raise Exception(f"cont should be None, List or Dict, and not {cont}") else: @@ -336,6 +354,7 @@ def custom_validator(instance, attribute, value): def _types_updates(tp_list): + """updating the tuple with possible types""" tp_upd_list = [] check = True for tp_el in tp_list: @@ -350,6 +369,9 @@ def _types_updates(tp_list): def _single_type_update(tp, simplify=False): + """ updating a single type - e.g. adding bytes if str is required + if simplify is True, than changing typing.List to list etc. + """ if isinstance(tp, type) or tp in [File, Directory]: if tp is str: return (str, bytes) @@ -359,16 +381,20 @@ def _single_type_update(tp, simplify=False): return (float, int) else: return (tp,) - elif simplify: - if tp._name is "List": + elif simplify is True: + if getattr(tp, "_name", None) is "List": return (list,) - elif tp._name is "Dict": + elif getattr(tp, "_name", None) is "Dict": return (dict,) elif tyi.is_union_type(tp): - return None + return tyi.get_args(tp) else: - raise NotImplementedError(f"not implemented for type {tp}") + warnings.warn(f"type check not implemented for type {tp}") + return None else: + warnings.warn( + f"type check not implemented for type {tp}, consider using simplify=True" + ) return None diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 73cdd2226e..cc751b896a 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -137,6 +137,123 @@ def testfunc( ] +def test_annotated_input_func_1(): + """ the function with annotated input (float)""" + + @mark.task + def testfunc(a: float): + return a + + funky = testfunc(a=3.5) + assert getattr(funky.inputs, "a") == 3.5 + + +def test_annotated_input_func_2(): + """ the function with annotated input (int, but float provided)""" + + @mark.task + def testfunc(a: int): + return a + + with pytest.raises(TypeError): + funky = testfunc(a=3.5) + + +def test_annotated_input_func_2a(): + """ the function with annotated input (int, but float provided)""" + + @mark.task + def testfunc(a: int): + return a + + funky = testfunc() + # the error is raised when run (should be improved?) + funky.inputs.a = 3.5 + with pytest.raises(TypeError): + funky() + + +def test_annotated_input_func_3(): + """ the function with annotated input (list)""" + + @mark.task + def testfunc(a: list): + return sum(a) + + funky = testfunc(a=[1, 3.5]) + assert getattr(funky.inputs, "a") == [1, 3.5] + + +def test_annotated_input_func_3a(): + """ the function with annotated input (list of floats)""" + + @mark.task + def testfunc(a: ty.List[float]): + return sum(a) + + funky = testfunc(a=[1.0, 3.5]) + assert getattr(funky.inputs, "a") == [1.0, 3.5] + + +def test_annotated_input_func_3b(): + """ the function with annotated input + (list of floats - int and float provided, should be fine) + """ + + @mark.task + def testfunc(a: ty.List[float]): + return sum(a) + + funky = testfunc(a=[1, 3.5]) + assert getattr(funky.inputs, "a") == [1, 3.5] + + +def test_annotated_input_func_3c_excep(): + """ the function with annotated input + (list of ints - int and float provided, should raise an error) + """ + + @mark.task + def testfunc(a: ty.List[int]): + return sum(a) + + with pytest.raises(TypeError): + funky = testfunc(a=[1, 3.5]) + + +def test_annotated_input_func_4(): + """ the function with annotated input (dictionary)""" + + @mark.task + def testfunc(a: dict): + return sum(a.values()) + + funky = testfunc(a={"el1": 1, "el2": 3.5}) + assert getattr(funky.inputs, "a") == {"el1": 1, "el2": 3.5} + + +def test_annotated_input_func_4a(): + """ the function with annotated input (dictionary of floats)""" + + @mark.task + def testfunc(a: ty.Dict[str, float]): + return sum(a.values()) + + funky = testfunc(a={"el1": 1, "el2": 3.5}) + assert getattr(funky.inputs, "a") == {"el1": 1, "el2": 3.5} + + +def test_annotated_input_func_4b_excep(): + """ the function with annotated input (dictionary of ints, but float provided)""" + + @mark.task + def testfunc(a: ty.Dict[str, int]): + return sum(a.values()) + + with pytest.raises(TypeError): + funky = testfunc(a={"el1": 1, "el2": 3.5}) + + def test_annotated_func_multreturn_exception(): """function has two elements in the return statement, but three element provided in the spec - should raise an error From 78113e2fdabdfb590265e85183d0a2b6f556cfa4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 02:15:58 -0400 Subject: [PATCH 025/271] some cleaning, adding checks for values (for now only allowed_values exists in metadata keys) --- pydra/engine/helpers.py | 75 ++++++++++++------- pydra/engine/specs.py | 15 ---- .../engine/tests/test_shelltask_inputspec.py | 35 +++++---- 3 files changed, 70 insertions(+), 55 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 131e097839..63af9da8ab 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -283,7 +283,7 @@ def custom_validator(instance, attribute, value): take into account ty.Union, ty.List, ty.Dict (but only one level depth) """ tp_attr = attribute.type - check = True + check_type = True if ( value is attr.NOTHING or value is None @@ -291,66 +291,80 @@ def custom_validator(instance, attribute, value): or isinstance(value, LazyField) or tp_attr in [ty.Any, inspect._empty] ): - check = False # no checking of the type + check_type = False # no checking of the type elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: # what about File tp = _single_type_update(tp_attr) cont = None # no container elif tyi.is_union_type(tp_attr): tp_attr_list = tyi.get_args(tp_attr) cont = None - tp, check = _types_updates(tp_attr_list) + tp, check_type = _types_updates(tp_attr_list) elif tp_attr._name == "List": cont = "List" tp_attr_list = tyi.get_args(tp_attr) - tp, check = _types_updates(tp_attr_list) + tp, check_type = _types_updates(tp_attr_list) elif tp_attr._name == "Dict": cont = "Dict" tp_attr_key_val = tyi.get_args(tp_attr) # assuming that it should have length of 2 for keys and values if len(tp_attr_key_val) != 2: - check = False + check_type = False # updating types separately for keys and values tp_k, check_k = _types_updates([tp_attr_key_val[0]]) tp_v, check_v = _types_updates([tp_attr_key_val[1]]) if not (check_k and check_v): - check = False + check_type = False else: tp = {"key": tp_k, "val": tp_v} else: warnings.warn(f"no checking implemented for value {value} and type {tp_attr}") - check = False + check_type = False - if check: + if check_type: if cont is None: # if tp is not (list,), we are assuming that the value is a list # due to the splitter, so checking the member types if isinstance(value, list) and tp != (list,): - return attr.validators.deep_iterable( + validators = [ + attr.validators.deep_iterable( + member_validator=attr.validators.instance_of( + tp + (attr._make._Nothing,) + ) + )(instance, attribute, value) + ] + else: + validators = [ + attr.validators.instance_of(tp + (attr._make._Nothing,))( + instance, attribute, value + ) + ] + elif cont == "List": + validators = [ + attr.validators.deep_iterable( member_validator=attr.validators.instance_of( tp + (attr._make._Nothing,) ) )(instance, attribute, value) - else: - return attr.validators.instance_of(tp + (attr._make._Nothing,))( - instance, attribute, value - ) - elif cont == "List": - return attr.validators.deep_iterable( - member_validator=attr.validators.instance_of( - tp + (attr._make._Nothing,) - ) - )(instance, attribute, value) + ] elif cont == "Dict": - return attr.validators.deep_mapping( - key_validator=attr.validators.instance_of(tp["key"]), - value_validator=attr.validators.instance_of( - tp["val"] + (attr._make._Nothing,) - ), - )(instance, attribute, value) + validators = [ + attr.validators.deep_mapping( + key_validator=attr.validators.instance_of(tp["key"]), + value_validator=attr.validators.instance_of( + tp["val"] + (attr._make._Nothing,) + ), + )(instance, attribute, value) + ] else: raise Exception(f"cont should be None, List or Dict, and not {cont}") else: - pass + validators = [] + + # checking additional requirements for values (e.g. allowed_values) + meta_attr = attribute.metadata + if "allowed_values" in meta_attr: + validators.append(_check_allowed_values(isinstance, attribute, value)) + return validators def _types_updates(tp_list): @@ -398,6 +412,15 @@ def _single_type_update(tp, simplify=False): return None +def _check_allowed_values(instance, attribute, value): + """ checking if the values is in allowed_values""" + allowed = attribute.metadata["allowed_values"] + if value is attr.NOTHING: + pass + elif value not in allowed: + raise ValueError(f"value has to be from {allowed}, but {value} provided") + + async def read_stream_and_display(stream, display): """ Read from stream line by line until EOF, display, and capture the lines. diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 245a4aac34..522716de26 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -293,26 +293,11 @@ def check_fields_input_spec(self): if required_notfound: raise AttributeError(f"{nm} requires {required_notfound}") - # TODO: types might be checked here - self._type_checking() - def _file_check(self, field): file = Path(getattr(self, field.name)) if not file.exists(): raise AttributeError(f"the file from the {field.name} input does not exist") - def _type_checking(self): - """Use fld.type to check the types TODO. - - This may be done through attr validators. - - """ - fields = attr_fields(self) - allowed_keys = ["min_val", "max_val", "range", "enum"] # noqa - for fld in fields: - # TODO - pass - @attr.s(auto_attribs=True, kw_only=True) class ShellOutSpec(BaseSpec): diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 8569666b15..e827902680 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1276,20 +1276,6 @@ def test_shell_cmd_inputs_di(tmpdir): }, ), ), - ( - "noise_model", - attr.ib( - type=int, - metadata={ - "help_string": """ - Rician/(Gaussian) - Employ a Rician or Gaussian noise model. - """, - "allowed_values": ["Rician", "Gaussian"], - "argstr": "-n", - }, - ), - ), ( "shrink_factor", attr.ib( @@ -1463,3 +1449,24 @@ def test_shell_cmd_inputs_di(tmpdir): "correctedImage", "noiseImage", ] + + # adding image_dimensionality that has allowed_values [2, 3, 4] + shelly = ShellCommandTask( + executable="DenoiseImage", + inputImageFilename=my_input_file, + input_spec=my_input_spec, + image_dimensionality=2, + ) + assert ( + shelly.cmdline + == f"DenoiseImage -d 2 -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}]" + ) + + # adding image_dimensionality that has allowed_values [2, 3, 4] and providing 5 - exception should be raised + with pytest.raises(ValueError): + shelly = ShellCommandTask( + executable="DenoiseImage", + inputImageFilename=my_input_file, + input_spec=my_input_spec, + image_dimensionality=5, + ) From 5bf5a7e894ed7a48ca3986c1b3eef8d1b70dd06b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 18:38:56 -0400 Subject: [PATCH 026/271] adding a function to switch on the attrs validation --- pydra/__init__.py | 10 ++++++ pydra/engine/tests/test_shelltask.py | 11 +++---- .../engine/tests/test_shelltask_inputspec.py | 6 ++-- pydra/engine/tests/test_task.py | 32 +++++++++---------- pydra/engine/tests/utils.py | 11 +++++++ 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/pydra/__init__.py b/pydra/__init__.py index b2990793d8..233b2058bd 100644 --- a/pydra/__init__.py +++ b/pydra/__init__.py @@ -35,3 +35,13 @@ def check_latest_version(): if TaskBase._etelemetry_version_data is None: TaskBase._etelemetry_version_data = check_latest_version() + + +# attr run_validators is set to False, but could be changed using use_validator +import attr + +attr.set_run_validators(False) + + +def set_input_validator(flag=False): + attr.set_run_validators(flag) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 29ae8cece3..7db1c5f77a 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -9,8 +9,7 @@ from ..submitter import Submitter from ..core import Workflow from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File -from .utils import result_no_submitter, result_submitter - +from .utils import result_no_submitter, result_submitter, use_validator if sys.platform.startswith("win"): pytest.skip("SLURM not available in windows", allow_module_level=True) @@ -251,7 +250,7 @@ def test_wf_shell_cmd_1(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_1(plugin, results_function): +def test_shell_cmd_inputspec_1(plugin, results_function, use_validator): """ a command with executable, args and one command opt, using a customized input_spec to add the opt to the command in the right place that is specified in metadata["cmd_pos"] @@ -290,7 +289,7 @@ def test_shell_cmd_inputspec_1(plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_2(plugin, results_function): +def test_shell_cmd_inputspec_2(plugin, results_function, use_validator): """ a command with executable, args and two command options, using a customized input_spec to add the opt to the command in the right place that is specified in metadata["cmd_pos"] @@ -1513,7 +1512,7 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function): assert res[1].output.stdout == "hi\n" -def test_shell_cmd_inputspec_typeval_1(): +def test_shell_cmd_inputspec_typeval_1(use_validator): """ customized input_spec with a type that doesn't match the value - raise an exception """ @@ -1539,7 +1538,7 @@ def test_shell_cmd_inputspec_typeval_1(): ) -def test_shell_cmd_inputspec_typeval_2(): +def test_shell_cmd_inputspec_typeval_2(use_validator): """ customized input_spec (shorter syntax) with a type that doesn't match the value - raise an exception """ diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index e827902680..fa5ada3762 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1,12 +1,10 @@ import attr import typing as ty -import os, sys import pytest -from pathlib import Path - from ..task import ShellCommandTask from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File +from .utils import use_validator def test_shell_cmd_execargs_1(): @@ -1220,7 +1218,7 @@ def test_shell_cmd_inputs_template_8(tmpdir): ) -def test_shell_cmd_inputs_di(tmpdir): +def test_shell_cmd_inputs_di(tmpdir, use_validator): """ example from #279 """ my_input_spec = SpecInfo( name="Input", diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index cc751b896a..5199c5c47c 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -5,7 +5,7 @@ from ... import mark from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages -from .utils import gen_basic_wf +from .utils import gen_basic_wf, use_validator no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -34,7 +34,7 @@ def test_name_conflict(): assert "Cannot use names of attributes or methods" in str(excinfo2.value) -def test_numpy(): +def test_numpy(use_validator): """ checking if mark.task works for numpy functions""" np = pytest.importorskip("numpy") fft = mark.annotate({"a": np.ndarray, "return": np.ndarray})(np.fft.fft) @@ -54,7 +54,7 @@ def test_checksum(): ) -def test_annotated_func(): +def test_annotated_func(use_validator): @mark.task def testfunc( a: int, b: float = 0.1 @@ -98,7 +98,7 @@ def testfunc( ] -def test_annotated_func_multreturn(): +def test_annotated_func_multreturn(use_validator): """ the function has two elements in the return statement""" @mark.task @@ -137,7 +137,7 @@ def testfunc( ] -def test_annotated_input_func_1(): +def test_annotated_input_func_1(use_validator): """ the function with annotated input (float)""" @mark.task @@ -148,7 +148,7 @@ def testfunc(a: float): assert getattr(funky.inputs, "a") == 3.5 -def test_annotated_input_func_2(): +def test_annotated_input_func_2(use_validator): """ the function with annotated input (int, but float provided)""" @mark.task @@ -159,7 +159,7 @@ def testfunc(a: int): funky = testfunc(a=3.5) -def test_annotated_input_func_2a(): +def test_annotated_input_func_2a(use_validator): """ the function with annotated input (int, but float provided)""" @mark.task @@ -173,7 +173,7 @@ def testfunc(a: int): funky() -def test_annotated_input_func_3(): +def test_annotated_input_func_3(use_validator): """ the function with annotated input (list)""" @mark.task @@ -195,7 +195,7 @@ def testfunc(a: ty.List[float]): assert getattr(funky.inputs, "a") == [1.0, 3.5] -def test_annotated_input_func_3b(): +def test_annotated_input_func_3b(use_validator): """ the function with annotated input (list of floats - int and float provided, should be fine) """ @@ -208,7 +208,7 @@ def testfunc(a: ty.List[float]): assert getattr(funky.inputs, "a") == [1, 3.5] -def test_annotated_input_func_3c_excep(): +def test_annotated_input_func_3c_excep(use_validator): """ the function with annotated input (list of ints - int and float provided, should raise an error) """ @@ -221,7 +221,7 @@ def testfunc(a: ty.List[int]): funky = testfunc(a=[1, 3.5]) -def test_annotated_input_func_4(): +def test_annotated_input_func_4(use_validator): """ the function with annotated input (dictionary)""" @mark.task @@ -232,7 +232,7 @@ def testfunc(a: dict): assert getattr(funky.inputs, "a") == {"el1": 1, "el2": 3.5} -def test_annotated_input_func_4a(): +def test_annotated_input_func_4a(use_validator): """ the function with annotated input (dictionary of floats)""" @mark.task @@ -243,7 +243,7 @@ def testfunc(a: ty.Dict[str, float]): assert getattr(funky.inputs, "a") == {"el1": 1, "el2": 3.5} -def test_annotated_input_func_4b_excep(): +def test_annotated_input_func_4b_excep(use_validator): """ the function with annotated input (dictionary of ints, but float provided)""" @mark.task @@ -254,7 +254,7 @@ def testfunc(a: ty.Dict[str, int]): funky = testfunc(a={"el1": 1, "el2": 3.5}) -def test_annotated_func_multreturn_exception(): +def test_annotated_func_multreturn_exception(use_validator): """function has two elements in the return statement, but three element provided in the spec - should raise an error """ @@ -453,7 +453,7 @@ def fun_none(x) -> (ty.Any, ty.Any): assert res.output.out2 is None -def test_audit_prov(tmpdir): +def test_audit_prov(tmpdir, use_validator): @mark.task def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): return a + b @@ -474,7 +474,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] assert (tmpdir / funky.checksum / "messages.jsonld").exists() -def test_audit_all(tmpdir): +def test_audit_all(tmpdir, use_validator): @mark.task def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): return a + b diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index 46c043749d..10f0454ff6 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -10,6 +10,7 @@ from ..submitter import Submitter from ... import mark from ..specs import File +from ... import set_input_validator need_docker = pytest.mark.skipif( @@ -214,3 +215,13 @@ def gen_basic_wf(name="basic-wf"): wf.add(fun_addvar(name="task2", a=wf.task1.lzout.out, b=2)) wf.set_output([("out", wf.task2.lzout.out)]) return wf + + +@pytest.fixture(scope="function") +def use_validator(request): + set_input_validator(flag=True) + + def fin(): + set_input_validator(flag=False) + + request.addfinalizer(fin) From 043c6037590fa0169d6841da8f2b91453ef7a52d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 21:02:30 -0400 Subject: [PATCH 027/271] relaxing run time requirements for some of the tests --- pydra/engine/tests/test_workflow.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 8356ae2330..5ef0e11b64 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -2107,7 +2107,7 @@ def test_wf_nostate_cachelocations(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking if the second wf didn't run again assert wf1.output_dir.exists() @@ -2165,7 +2165,7 @@ def test_wf_nostate_cachelocations_a(plugin, tmpdir): # checking execution time (second one should be quick) assert t1 > 2 # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking if both wf.output_dir are created assert wf1.output_dir.exists() @@ -2225,7 +2225,7 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking if the second wf didn't run again assert wf1.output_dir.exists() @@ -2284,7 +2284,7 @@ def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): # checking execution time (the second wf should be fast, nodes do not have to rerun) assert t1 > 2 # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + assert t2 < max(1, t1 - 1) # both wf output_dirs should be created assert wf1.output_dir.exists() @@ -2340,7 +2340,7 @@ def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): # checking execution time (the second wf should be fast, nodes do not have to rerun) assert t1 > 2 # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + assert t2 < max(1, t1 - 1) # both wf output_dirs should be created assert wf1.output_dir.exists() @@ -2522,7 +2522,7 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # tasks should not be recomputed assert len(list(Path(cache_dir1).glob("F*"))) == 2 @@ -2737,7 +2737,7 @@ def test_wf_state_cachelocations(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking all directories assert wf1.output_dir @@ -2873,7 +2873,7 @@ def test_wf_state_cachelocations_updateinp(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking all directories assert wf1.output_dir @@ -3101,7 +3101,7 @@ def test_wf_ndstate_cachelocations(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking all directories assert wf1.output_dir.exists() @@ -3227,7 +3227,7 @@ def test_wf_ndstate_cachelocations_updatespl(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) # checking all directories assert wf1.output_dir.exists() @@ -3343,7 +3343,7 @@ def test_wf_nostate_runtwice_usecache(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) def test_wf_state_runtwice_usecache(plugin, tmpdir): @@ -3392,7 +3392,7 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): if not sys.platform.startswith("win") and plugin == "cf": # checking the execution time assert t1 > 2 - assert t2 < 1 + assert t2 < max(1, t1 - 1) @pytest.fixture From 295c8f7b63b49a6816ab2c6a02f15c9d30cd3c76 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 21:30:42 -0400 Subject: [PATCH 028/271] removing typing_inspect - when py>-3.8 methods from typing are used instead, when py37 __args__ are used; some rearranging/cleaning --- docs/requirements.txt | 1 - pydra/engine/helpers.py | 154 ++++++++++++++++++++++------------------ setup.cfg | 2 - 3 files changed, 84 insertions(+), 73 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 711761e8bd..70aedd8354 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,4 @@ attrs >= 19.1.0 -typing_inspect cloudpickle filelock git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 63af9da8ab..d02c7b24fa 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -14,7 +14,6 @@ from time import strftime from traceback import format_exception import typing as ty -import typing_inspect as tyi import inspect import warnings @@ -282,7 +281,9 @@ def custom_validator(instance, attribute, value): """simple custom validation take into account ty.Union, ty.List, ty.Dict (but only one level depth) """ + validators = [] tp_attr = attribute.type + # a flag that could be changed to False, if the type is not recognized check_type = True if ( value is attr.NOTHING @@ -292,81 +293,74 @@ def custom_validator(instance, attribute, value): or tp_attr in [ty.Any, inspect._empty] ): check_type = False # no checking of the type - elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: # what about File + elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: tp = _single_type_update(tp_attr) - cont = None # no container - elif tyi.is_union_type(tp_attr): - tp_attr_list = tyi.get_args(tp_attr) - cont = None - tp, check_type = _types_updates(tp_attr_list) - elif tp_attr._name == "List": - cont = "List" - tp_attr_list = tyi.get_args(tp_attr) - tp, check_type = _types_updates(tp_attr_list) - elif tp_attr._name == "Dict": - cont = "Dict" - tp_attr_key_val = tyi.get_args(tp_attr) - # assuming that it should have length of 2 for keys and values - if len(tp_attr_key_val) != 2: - check_type = False - # updating types separately for keys and values - tp_k, check_k = _types_updates([tp_attr_key_val[0]]) - tp_v, check_v = _types_updates([tp_attr_key_val[1]]) - if not (check_k and check_v): - check_type = False + cont_type = None + else: # more complex types + cont_type, tp_attr_list = _check_special_type(tp_attr) + if cont_type is ty.Union: + tp, check_type = _types_updates(tp_attr_list) + elif cont_type is list: + tp, check_type = _types_updates(tp_attr_list) + elif cont_type is dict: + # assuming that it should have length of 2 for keys and values + if len(tp_attr_list) != 2: + check_type = False + else: + tp_attr_key, tp_attr_val = tp_attr_list + # updating types separately for keys and values + tp_k, check_k = _types_updates([tp_attr_key]) + tp_v, check_v = _types_updates([tp_attr_val]) + # assuming that I have to be able to check keys and values + if not (check_k and check_v): + check_type = False + else: + tp = {"key": tp_k, "val": tp_v} else: - tp = {"key": tp_k, "val": tp_v} - else: - warnings.warn(f"no checking implemented for value {value} and type {tp_attr}") - check_type = False + warnings.warn( + f"no checking implemented for value {value} and type {tp_attr}" + ) + check_type = False if check_type: - if cont is None: - # if tp is not (list,), we are assuming that the value is a list - # due to the splitter, so checking the member types - if isinstance(value, list) and tp != (list,): - validators = [ - attr.validators.deep_iterable( - member_validator=attr.validators.instance_of( - tp + (attr._make._Nothing,) - ) - )(instance, attribute, value) - ] - else: - validators = [ - attr.validators.instance_of(tp + (attr._make._Nothing,))( - instance, attribute, value - ) - ] - elif cont == "List": - validators = [ - attr.validators.deep_iterable( - member_validator=attr.validators.instance_of( - tp + (attr._make._Nothing,) - ) - )(instance, attribute, value) - ] - elif cont == "Dict": - validators = [ - attr.validators.deep_mapping( - key_validator=attr.validators.instance_of(tp["key"]), - value_validator=attr.validators.instance_of( - tp["val"] + (attr._make._Nothing,) - ), - )(instance, attribute, value) - ] - else: - raise Exception(f"cont should be None, List or Dict, and not {cont}") - else: - validators = [] + validators.append(_type_validator(instance, attribute, value, tp, cont_type)) # checking additional requirements for values (e.g. allowed_values) meta_attr = attribute.metadata if "allowed_values" in meta_attr: - validators.append(_check_allowed_values(isinstance, attribute, value)) + validators.append(_allowed_values_validator(isinstance, attribute, value)) return validators +def _type_validator(instance, attribute, value, tp, cont_type): + if cont_type is None or cont_type is ty.Union: + # if tp is not (list,), we are assuming that the value is a list + # due to the splitter, so checking the member types + if isinstance(value, list) and tp != (list,): + return attr.validators.deep_iterable( + member_validator=attr.validators.instance_of( + tp + (attr._make._Nothing,) + ) + )(instance, attribute, value) + else: + return attr.validators.instance_of(tp + (attr._make._Nothing,))( + instance, attribute, value + ) + elif cont_type is list: + return attr.validators.deep_iterable( + member_validator=attr.validators.instance_of(tp + (attr._make._Nothing,)) + )(instance, attribute, value) + elif cont_type is dict: + return attr.validators.deep_mapping( + key_validator=attr.validators.instance_of(tp["key"]), + value_validator=attr.validators.instance_of( + tp["val"] + (attr._make._Nothing,) + ), + )(instance, attribute, value) + else: + raise Exception(f"cont should be None, List or Dict, and not {cont_type}") + + def _types_updates(tp_list): """updating the tuple with possible types""" tp_upd_list = [] @@ -396,12 +390,13 @@ def _single_type_update(tp, simplify=False): else: return (tp,) elif simplify is True: - if getattr(tp, "_name", None) is "List": + cont_tp, types_list = _check_special_type(tp) + if cont_tp is list: return (list,) - elif getattr(tp, "_name", None) is "Dict": + elif cont_tp is dict: return (dict,) - elif tyi.is_union_type(tp): - return tyi.get_args(tp) + elif cont_tp is ty.Union: + return types_list else: warnings.warn(f"type check not implemented for type {tp}") return None @@ -412,7 +407,26 @@ def _single_type_update(tp, simplify=False): return None -def _check_allowed_values(instance, attribute, value): +def _check_special_type(tp): + """checking if the type is a container: ty.List, ty.Dict or ty.Union """ + if sys.version_info.minor >= 8: + return ty.get_origin(tp), ty.get_args(tp) + else: + if isinstance(tp, type): # simple type + return None, () + else: + if tp._name == "List": + return list, tp.__args__ + elif tp._name == "Dict": + return dict, tp.__args__ + elif tp.__origin__ is ty.Union: + return ty.Union, tp.__args__ + else: + warnings.warn(f"type check not implemented for type {tp}") + return None, () + + +def _allowed_values_validator(instance, attribute, value): """ checking if the values is in allowed_values""" allowed = attribute.metadata["allowed_values"] if value is attr.NOTHING: diff --git a/setup.cfg b/setup.cfg index 10534f63fb..f9948810f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ classifiers = python_requires = >= 3.7 install_requires = attrs >= 19.1.0 - typing_inspect cloudpickle >= 0.8.0 filelock >= 3.0.0 etelemetry >= 0.2.0 @@ -51,7 +50,6 @@ pydra = [options.extras_require] doc = attrs >= 19.1.0 - typing_inspect cloudpickle filelock packaging From 93bb71533d74c442adb06f627983bbccf154b7d8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 31 Jul 2020 22:42:10 -0400 Subject: [PATCH 029/271] improving error/warnings --- pydra/engine/helpers.py | 42 +++++++++++-------- .../engine/tests/test_shelltask_inputspec.py | 3 +- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index d02c7b24fa..ec15ad324c 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -294,14 +294,14 @@ def custom_validator(instance, attribute, value): ): check_type = False # no checking of the type elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: - tp = _single_type_update(tp_attr) + tp = _single_type_update(tp_attr, name=attribute.name) cont_type = None else: # more complex types - cont_type, tp_attr_list = _check_special_type(tp_attr) + cont_type, tp_attr_list = _check_special_type(tp_attr, name=attribute.name) if cont_type is ty.Union: - tp, check_type = _types_updates(tp_attr_list) + tp, check_type = _types_updates(tp_attr_list, name=attribute.name) elif cont_type is list: - tp, check_type = _types_updates(tp_attr_list) + tp, check_type = _types_updates(tp_attr_list, name=attribute.name) elif cont_type is dict: # assuming that it should have length of 2 for keys and values if len(tp_attr_list) != 2: @@ -309,8 +309,8 @@ def custom_validator(instance, attribute, value): else: tp_attr_key, tp_attr_val = tp_attr_list # updating types separately for keys and values - tp_k, check_k = _types_updates([tp_attr_key]) - tp_v, check_v = _types_updates([tp_attr_val]) + tp_k, check_k = _types_updates([tp_attr_key], name=attribute.name) + tp_v, check_v = _types_updates([tp_attr_val], name=attribute.name) # assuming that I have to be able to check keys and values if not (check_k and check_v): check_type = False @@ -318,7 +318,7 @@ def custom_validator(instance, attribute, value): tp = {"key": tp_k, "val": tp_v} else: warnings.warn( - f"no checking implemented for value {value} and type {tp_attr}" + f"no type check for {attribute.name} field, no type check implemented for value {value} and type {tp_attr}" ) check_type = False @@ -358,15 +358,17 @@ def _type_validator(instance, attribute, value, tp, cont_type): ), )(instance, attribute, value) else: - raise Exception(f"cont should be None, List or Dict, and not {cont_type}") + raise Exception( + f"container type of {attribute.name} should be None, list, dict or ty.Union, and not {cont_type}" + ) -def _types_updates(tp_list): +def _types_updates(tp_list, name): """updating the tuple with possible types""" tp_upd_list = [] check = True for tp_el in tp_list: - tp_upd = _single_type_update(tp_el, simplify=True) + tp_upd = _single_type_update(tp_el, name, simplify=True) if tp_upd is None: check = False break @@ -376,7 +378,7 @@ def _types_updates(tp_list): return tp_upd, check -def _single_type_update(tp, simplify=False): +def _single_type_update(tp, name, simplify=False): """ updating a single type - e.g. adding bytes if str is required if simplify is True, than changing typing.List to list etc. """ @@ -390,7 +392,7 @@ def _single_type_update(tp, simplify=False): else: return (tp,) elif simplify is True: - cont_tp, types_list = _check_special_type(tp) + cont_tp, types_list = _check_special_type(tp, name=name) if cont_tp is list: return (list,) elif cont_tp is dict: @@ -398,16 +400,18 @@ def _single_type_update(tp, simplify=False): elif cont_tp is ty.Union: return types_list else: - warnings.warn(f"type check not implemented for type {tp}") + warnings.warn( + f"no type check for {name} field, type check not implemented for type of {tp}" + ) return None else: warnings.warn( - f"type check not implemented for type {tp}, consider using simplify=True" + f"no type check for {name} field, type check not implemented for type - {tp}, consider using simplify=True" ) return None -def _check_special_type(tp): +def _check_special_type(tp, name): """checking if the type is a container: ty.List, ty.Dict or ty.Union """ if sys.version_info.minor >= 8: return ty.get_origin(tp), ty.get_args(tp) @@ -422,7 +426,9 @@ def _check_special_type(tp): elif tp.__origin__ is ty.Union: return ty.Union, tp.__args__ else: - warnings.warn(f"type check not implemented for type {tp}") + warnings.warn( + f"not type check for {name} field, type check not implemented for type {tp}" + ) return None, () @@ -432,7 +438,9 @@ def _allowed_values_validator(instance, attribute, value): if value is attr.NOTHING: pass elif value not in allowed: - raise ValueError(f"value has to be from {allowed}, but {value} provided") + raise ValueError( + f"value of {attribute.name} has to be from {allowed}, but {value} provided" + ) async def read_stream_and_display(stream, display): diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index fa5ada3762..061eec0220 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1461,10 +1461,11 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) # adding image_dimensionality that has allowed_values [2, 3, 4] and providing 5 - exception should be raised - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: shelly = ShellCommandTask( executable="DenoiseImage", inputImageFilename=my_input_file, input_spec=my_input_spec, image_dimensionality=5, ) + assert "value of image_dimensionality" in str(excinfo.value) From 68e838f5d2ff6f05ce6b8349e8a80eb0ab0476df Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 1 Aug 2020 14:57:43 -0400 Subject: [PATCH 030/271] adding tests for validator when input is a list due to the splitter --- pydra/engine/tests/test_task.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 5199c5c47c..e8bb8d507f 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -254,6 +254,33 @@ def testfunc(a: ty.Dict[str, int]): funky = testfunc(a={"el1": 1, "el2": 3.5}) +def test_annotated_input_func_5(use_validator): + """ the function with annotated input (float) + the task has a splitter, so list of float is provided + it should work, the validator tries to guess if this is a field with a splitter + """ + + @mark.task + def testfunc(a: float): + return a + + funky = testfunc(a=[3.5, 2.1]).split("a") + assert getattr(funky.inputs, "a") == [3.5, 2.1] + + +def test_annotated_input_func_6(use_validator): + """ the function with annotated input (int) and splitter + list of float provided - should raise an error (list of int would be fine) + """ + + @mark.task + def testfunc(a: int): + return a + + with pytest.raises(TypeError): + funky = testfunc(a=[3.5, 2.1]).split("a") + + def test_annotated_func_multreturn_exception(use_validator): """function has two elements in the return statement, but three element provided in the spec - should raise an error From a3f9290b6bff157b6d5bc7b6d6359f2a4b25fe43 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 1 Aug 2020 15:05:54 -0400 Subject: [PATCH 031/271] adding better comments --- pydra/engine/helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index ec15ad324c..13c4221f1f 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -280,6 +280,7 @@ def make_klass(spec): def custom_validator(instance, attribute, value): """simple custom validation take into account ty.Union, ty.List, ty.Dict (but only one level depth) + adding an additional validator, if allowe_values provided """ validators = [] tp_attr = attribute.type @@ -333,6 +334,12 @@ def custom_validator(instance, attribute, value): def _type_validator(instance, attribute, value, tp, cont_type): + """ creating a customized type validator, + uses validator.deep_iterable/mapping if the field is a container + (i.e. ty.List or ty.Dict), + it also tries to guess when the value is a list due to the splitter + and validates the elements + """ if cont_type is None or cont_type is ty.Union: # if tp is not (list,), we are assuming that the value is a list # due to the splitter, so checking the member types @@ -364,7 +371,7 @@ def _type_validator(instance, attribute, value, tp, cont_type): def _types_updates(tp_list, name): - """updating the tuple with possible types""" + """updating the type's tuple with possible additional types""" tp_upd_list = [] check = True for tp_el in tp_list: @@ -379,8 +386,9 @@ def _types_updates(tp_list, name): def _single_type_update(tp, name, simplify=False): - """ updating a single type - e.g. adding bytes if str is required + """ updating a single type with other related types - e.g. adding bytes for str if simplify is True, than changing typing.List to list etc. + (assuming that I validate only one depth, so have to simplify at some point) """ if isinstance(tp, type) or tp in [File, Directory]: if tp is str: From 892afc85a35c9c211bdaea73e7293371b1bfd927 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 1 Aug 2020 19:19:59 -0400 Subject: [PATCH 032/271] adding tests for more complex/nested input types --- pydra/engine/helpers.py | 1 + pydra/engine/tests/test_task.py | 55 ++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 13c4221f1f..e2dbb528d1 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -400,6 +400,7 @@ def _single_type_update(tp, name, simplify=False): else: return (tp,) elif simplify is True: + warnings.warn(f"simplify validator for {name} field, checking only one depth") cont_tp, types_list = _check_special_type(tp, name=name) if cont_tp is list: return (list,) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index e8bb8d507f..67c4c4b6e9 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -255,6 +255,59 @@ def testfunc(a: ty.Dict[str, int]): def test_annotated_input_func_5(use_validator): + """ the function with annotated more complex input type (ty.List in ty.Dict) + the validator should simply check if values of dict are lists + so no error for 3.5 + """ + + @mark.task + def testfunc(a: ty.Dict[str, ty.List[int]]): + return sum(a["el1"]) + + funky = testfunc(a={"el1": [1, 3.5]}) + assert getattr(funky.inputs, "a") == {"el1": [1, 3.5]} + + +def test_annotated_input_func_5a_except(use_validator): + """ the function with annotated more complex input type (ty.Dict in ty.Dict) + list is provided as a dict value (instead a dict), so error is raised + """ + + @mark.task + def testfunc(a: ty.Dict[str, ty.Dict[str, float]]): + return sum(a["el1"]) + + with pytest.raises(TypeError): + funky = testfunc(a={"el1": [1, 3.5]}) + + +def test_annotated_input_func_6(use_validator): + """ the function with annotated more complex input type (ty.Union in ty.Dict) + the validator should unpack values from the Union + """ + + @mark.task + def testfunc(a: ty.Dict[str, ty.Union[float, int]]): + return sum(a["el1"]) + + funky = testfunc(a={"el1": 1, "el2": 3.5}) + assert getattr(funky.inputs, "a") == {"el1": 1, "el2": 3.5} + + +def test_annotated_input_func_6a_excep(use_validator): + """ the function with annotated more complex input type (ty.Union in ty.Dict) + the validator should unpack values from the Union and raise an error for 3.5 + """ + + @mark.task + def testfunc(a: ty.Dict[str, ty.Union[str, int]]): + return sum(a["el1"]) + + with pytest.raises(TypeError): + funky = testfunc(a={"el1": 1, "el2": 3.5}) + + +def test_annotated_input_func_7(use_validator): """ the function with annotated input (float) the task has a splitter, so list of float is provided it should work, the validator tries to guess if this is a field with a splitter @@ -268,7 +321,7 @@ def testfunc(a: float): assert getattr(funky.inputs, "a") == [3.5, 2.1] -def test_annotated_input_func_6(use_validator): +def test_annotated_input_func_7a_excep(use_validator): """ the function with annotated input (int) and splitter list of float provided - should raise an error (list of int would be fine) """ From e885b27888afb67adb765b0dc219d4b71b0c4bcd Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 1 Aug 2020 22:11:43 -0400 Subject: [PATCH 033/271] changing to for dotfile tha represents a nested structures of workflows --- pydra/engine/core.py | 6 ++--- pydra/engine/graph.py | 4 ++- pydra/engine/tests/test_graph.py | 8 +++--- pydra/engine/tests/test_workflow.py | 40 ++++++++++++++--------------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index aaa2b4eda5..e42b92b900 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -1012,13 +1012,13 @@ def create_dotfile(self, type="simple", export=None, name=None): dotfile = self.graph.create_dotfile_simple( outdir=self.output_dir, name=name ) - elif type == "detailed": - dotfile = self.graph.create_dotfile_detailed( + elif type == "nested": + dotfile = self.graph.create_dotfile_nested( outdir=self.output_dir, name=name ) else: raise Exception( - f"type of the graph can be simple or detailed, " f"but {type} provided" + f"type of the graph can be simple or nested, " f"but {type} provided" ) if not export: return dotfile diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index 403fbf27ff..cc113b19a3 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -323,6 +323,7 @@ def calculate_max_paths(self): self._checking_path(node_name=nm, first_name=nm) def create_dotfile_simple(self, outdir, name="graph"): + """ creates a simple dotfile (no nested structure)""" from .core import is_workflow dotstr = "digraph G {\n" @@ -341,7 +342,8 @@ def create_dotfile_simple(self, outdir, name="graph"): dotfile.write_text(dotstr) return dotfile - def create_dotfile_detailed(self, outdir, name="graph"): + def create_dotfile_nested(self, outdir, name="graph"): + """dotfile that includes the nested structures for workflows""" dotstr = "digraph G {\ncompound=true \n" dotstr += self._create_dotfile_single_graph(nodes=self.nodes, edges=self.edges) dotstr += "}" diff --git a/pydra/engine/tests/test_graph.py b/pydra/engine/tests/test_graph.py index 889d098880..f5f6f41e48 100644 --- a/pydra/engine/tests/test_graph.py +++ b/pydra/engine/tests/test_graph.py @@ -484,12 +484,12 @@ def test_dotfile_2(tmpdir): assert formatted_dot.exists() -def test_dotfile_2det(tmpdir): - """detailed dotfile for graph: a -> b -> d, a -> c -> d +def test_dotfile_2nest(tmpdir): + """nested dotfile for graph: a -> b -> d, a -> c -> d (should be the same as default type, i.e. type=simple) """ graph = DiGraph(nodes=[A, B, C, D], edges=[(A, B), (A, C), (B, D), (C, D)]) - dotfile = graph.create_dotfile_detailed(outdir=tmpdir) + dotfile = graph.create_dotfile_nested(outdir=tmpdir) dotstr_lines = dotfile.read_text().split("\n") for el in ["a", "b", "c", "d"]: assert el in dotstr_lines @@ -502,7 +502,7 @@ def test_dotfile_2det(tmpdir): def test_dotfile_3(tmpdir): - """detailed dotfile for graph: a -> c, b -> c, c -> d""" + """nested dotfile for graph: a -> c, b -> c, c -> d""" graph = DiGraph(nodes=[A, B, C, D], edges=[(A, C), (B, C), (C, D)]) dotfile = graph.create_dotfile_simple(outdir=tmpdir) dotstr_lines = dotfile.read_text().split("\n") diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 869758896a..17de1c9b0e 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3976,8 +3976,8 @@ def test_graph_1(tmpdir): print("\n graph in: ", formatted_dot[0]) -def test_graph_1det(tmpdir): - """creating a detailed graph, wf with two nodes""" +def test_graph_1nest(tmpdir): + """creating a nested graph, wf with two nodes""" wf = Workflow(name="wf_2", input_spec=["x", "y"]) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) @@ -3986,7 +3986,7 @@ def test_graph_1det(tmpdir): wf.inputs.x = 2 wf.inputs.y = 3 - dotfile = wf.create_dotfile(type="detailed") + dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") assert "mult_1" in dotstr_lines assert "mult_2" in dotstr_lines @@ -3996,7 +3996,7 @@ def test_graph_1det(tmpdir): if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" dotfile_pr, formatted_dot = wf.create_dotfile( - type="detailed", export=["png", "pdf"], name=name + type="nested", export=["png", "pdf"], name=name ) assert dotfile_pr.read_text().split("\n") == dotstr_lines assert len(formatted_dot) == 2 @@ -4029,8 +4029,8 @@ def test_graph_2(tmpdir): print("\n graph in: ", formatted_dot[0]) -def test_graph_2det(tmpdir): - """creating a detailed graph, wf with one worfklow as a node""" +def test_graph_2nest(tmpdir): + """creating a nested graph, wf with one worfklow as a node""" wfnd = Workflow(name="wfnd", input_spec=["x"]) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) @@ -4040,7 +4040,7 @@ def test_graph_2det(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile(type="detailed") + dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") assert "subgraph cluster_wfnd {" in dotstr_lines assert "add2" in dotstr_lines @@ -4048,7 +4048,7 @@ def test_graph_2det(tmpdir): if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" dotfile_pr, formatted_dot = wf.create_dotfile( - type="detailed", export=True, name=name + type="nested", export=True, name=name ) assert dotfile_pr.read_text().split("\n") == dotstr_lines assert len(formatted_dot) == 1 @@ -4082,8 +4082,8 @@ def test_graph_3(tmpdir): print("\n graph in: ", formatted_dot[0]) -def test_graph_3det(tmpdir): - """creating a detailed graph, wf with two nodes (one node is a workflow)""" +def test_graph_3nest(tmpdir): + """creating a nested graph, wf with two nodes (one node is a workflow)""" wf = Workflow(name="wf", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -4093,7 +4093,7 @@ def test_graph_3det(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile(type="detailed") + dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") assert "mult" in dotstr_lines assert "subgraph cluster_wfnd {" in dotstr_lines @@ -4102,7 +4102,7 @@ def test_graph_3det(tmpdir): if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" dotfile_pr, formatted_dot = wf.create_dotfile( - type="detailed", export=True, name=name + type="nested", export=True, name=name ) assert dotfile_pr.read_text().split("\n") == dotstr_lines assert len(formatted_dot) == 1 @@ -4110,8 +4110,8 @@ def test_graph_3det(tmpdir): print("\n graph in: ", formatted_dot[0]) -def test_graph_4det(tmpdir): - """creating a detailed graph, wf with two nodes (one node is a workflow with two nodes +def test_graph_4nest(tmpdir): + """creating a nested graph, wf with two nodes (one node is a workflow with two nodes inside). Connection from the node to the inner workflow. """ wf = Workflow(name="wf", input_spec=["x", "y"]) @@ -4124,7 +4124,7 @@ def test_graph_4det(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile(type="detailed") + dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") for el in ["mult", "add2_a", "add2_b"]: assert el in dotstr_lines @@ -4135,15 +4135,15 @@ def test_graph_4det(tmpdir): if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" dotfile_pr, formatted_dot = wf.create_dotfile( - type="detailed", export=True, name=name + type="nested", export=True, name=name ) assert dotfile_pr.read_text().split("\n") == dotstr_lines assert formatted_dot[0].exists() print("\n graph in: ", formatted_dot[0]) -def test_graph_5det(tmpdir): - """creating a detailed graph, wf with two nodes (one node is a workflow with two nodes +def test_graph_5nest(tmpdir): + """creating a nested graph, wf with two nodes (one node is a workflow with two nodes inside). Connection from the inner workflow to the node. """ wf = Workflow(name="wf", input_spec=["x", "y"]) @@ -4156,7 +4156,7 @@ def test_graph_5det(tmpdir): wf.add(multiply(name="mult", x=wf.wfnd.lzout.out, y=wf.lzin.y)) wf.set_output([("out", wf.mult.lzout.out)]) - dotfile = wf.create_dotfile(type="detailed") + dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") for el in ["mult", "add2_a", "add2_b"]: assert el in dotstr_lines @@ -4167,7 +4167,7 @@ def test_graph_5det(tmpdir): if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" dotfile_pr, formatted_dot = wf.create_dotfile( - type="detailed", export=True, name=name + type="nested", export=True, name=name ) assert dotfile_pr.read_text().split("\n") == dotstr_lines assert formatted_dot[0].exists() From c3954d1a9f75d94db79d91f94bdad1a0ac672804 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 1 Aug 2020 22:55:29 -0400 Subject: [PATCH 034/271] adding blue frames for nodes/workflows with states --- pydra/engine/graph.py | 21 ++++++++-- pydra/engine/tests/test_graph.py | 1 + pydra/engine/tests/test_workflow.py | 62 ++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index cc113b19a3..5df4433f31 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -328,11 +328,19 @@ def create_dotfile_simple(self, outdir, name="graph"): dotstr = "digraph G {\n" for nd in self.nodes: + # breakpoint() if is_workflow(nd): - dotstr += f"{nd.name} [shape=box]\n" + if nd.state: + # adding color for wf with a state + dotstr += f"{nd.name} [shape=box, color=blue]\n" + else: + dotstr += f"{nd.name} [shape=box]\n" else: - dotstr += f"{nd.name}\n" - + if nd.state: + # adding color for nd with a state + dotstr += f"{nd.name} [color=blue]\n" + else: + dotstr += f"{nd.name}\n" for ed in self.edges_names: dotstr += f"{ed[0]} -> {ed[1]}\n" @@ -366,9 +374,14 @@ def _create_dotfile_single_graph(self, nodes, edges): dotstr += self._create_dotfile_single_graph( nodes=nd.graph.nodes, edges=nd.graph.edges ) + if nd.state: + dotstr += "color=blue\n" dotstr += "}\n" else: - dotstr += f"{nd.name}\n" + if nd.state: + dotstr += f"{nd.name} [color=blue]\n" + else: + dotstr += f"{nd.name}\n" dotstr_edg = "" for ed in edges: diff --git a/pydra/engine/tests/test_graph.py b/pydra/engine/tests/test_graph.py index f5f6f41e48..70cefe5c29 100644 --- a/pydra/engine/tests/test_graph.py +++ b/pydra/engine/tests/test_graph.py @@ -6,6 +6,7 @@ class ObjTest: def __init__(self, name): self.name = name + self.state = None A = ObjTest("a") diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 17de1c9b0e..785f9d9e8d 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3983,8 +3983,6 @@ def test_graph_1nest(tmpdir): wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") @@ -4006,6 +4004,36 @@ def test_graph_1nest(tmpdir): print("\n graph in: ", formatted_dot[0]) +def test_graph_1nest_st(tmpdir): + """creating a nested graph, wf with two nodes + some nodes have splitters, should be marked with blue color + """ + wf = Workflow(name="wf_2", input_spec=["x", "y"]) + wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y).split("x")) + wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) + wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) + wf.set_output([("out", wf.add2.lzout.out)]) + + dotfile = wf.create_dotfile(type="nested") + dotstr_lines = dotfile.read_text().split("\n") + assert "mult_1 [color=blue]" in dotstr_lines + assert "mult_2" in dotstr_lines + assert "add2 [color=blue]" in dotstr_lines + assert "mult_1 -> add2" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="nested", export=["png", "pdf"], name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 2 + for i, ext in enumerate(["png", "pdf"]): + assert formatted_dot[i] == dotfile_pr.with_suffix(f".{ext}") + assert formatted_dot[i].exists() + print("\n graph in: ", formatted_dot[0]) + + def test_graph_2(tmpdir): """creating a graph, wf with one worfklow as a node""" wfnd = Workflow(name="wfnd", input_spec=["x"]) @@ -4056,6 +4084,36 @@ def test_graph_2nest(tmpdir): print("\n graph in: ", formatted_dot[0]) +def test_graph_2nest_st(tmpdir): + """creating a nested graph, wf with one worfklow as a node + the inner workflow has a state, so should be blue + """ + wfnd = Workflow(name="wfnd", input_spec=["x"]).split("x") + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.inputs.x = 2 + + wf = Workflow(name="wf", input_spec=["x"]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + dotfile = wf.create_dotfile(type="nested") + dotstr_lines = dotfile.read_text().split("\n") + assert "subgraph cluster_wfnd {" in dotstr_lines + assert "color=blue" in dotstr_lines + assert "add2" in dotstr_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + dotfile_pr, formatted_dot = wf.create_dotfile( + type="nested", export=True, name=name + ) + assert dotfile_pr.read_text().split("\n") == dotstr_lines + assert len(formatted_dot) == 1 + assert formatted_dot[0].exists() + print("\n graph in: ", formatted_dot[0]) + + def test_graph_3(tmpdir): """creating a graph, wf with two nodes (one node is a workflow)""" wf = Workflow(name="wf", input_spec=["x", "y"]) From cb48c2aa873d8d5a60fcafea26012ea5dc1cb7e4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 2 Aug 2020 22:51:46 -0400 Subject: [PATCH 035/271] adding detailed dotfiles with input/output fields of each connection: adding detailed flag wf.create_connections --- pydra/engine/core.py | 45 +++- pydra/engine/graph.py | 77 +++++- pydra/engine/tests/test_workflow.py | 371 +++++++++++++++------------- 3 files changed, 306 insertions(+), 187 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index e42b92b900..4dd1046537 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -739,7 +739,7 @@ def __init__( rerun=rerun, ) - self.graph = DiGraph() + self.graph = DiGraph(name=name) self.name2obj = {} # store output connections @@ -830,7 +830,7 @@ def add(self, task): logger.debug(f"Added {task}") return self - def create_connections(self, task): + def create_connections(self, task, detailed=False): """ Add and connect a particular task to existing nodes in the workflow. @@ -838,7 +838,9 @@ def create_connections(self, task): ---------- task : :class:`TaskBase` The task to be added. - + detailed : :obj:`bool` + If True, `add_edges_description` is run for self.graph to add + a detailed descriptions of the connections (input/output fields names) """ other_states = {} for field in attr_fields(task.inputs): @@ -849,9 +851,12 @@ def create_connections(self, task): # adding an edge to the graph if task id expecting output from a different task if val.name != self.name: # checking if the connection is already in the graph - if (getattr(self, val.name), task) in self.graph.edges: - continue - self.graph.add_edges((getattr(self, val.name), task)) + if (getattr(self, val.name), task) not in self.graph.edges: + self.graph.add_edges((getattr(self, val.name), task)) + if detailed: + self.graph.add_edges_description( + (task.name, field.name, val.name, val.field) + ) logger.debug("Connecting %s to %s", val.name, task.name) if ( @@ -863,6 +868,13 @@ def create_connections(self, task): getattr(self, val.name).state, field.name, ) + else: # LazyField with the wf input + # connections with wf input should be added to the detailed graph description + if detailed: + self.graph.add_edges_description( + (task.name, field.name, val.name, val.field) + ) + # if task has connections state has to be recalculated if other_states: if hasattr(task, "fut_combiner"): @@ -1004,27 +1016,42 @@ def _collect_outputs(self): def create_dotfile(self, type="simple", export=None, name=None): """creating a graph - dotfile and optionally exporting to other formats""" - for task in self.graph.nodes: - self.create_connections(task) if not name: name = f"graph_{self.name}" if type == "simple": + for task in self.graph.nodes: + self.create_connections(task) dotfile = self.graph.create_dotfile_simple( outdir=self.output_dir, name=name ) elif type == "nested": + for task in self.graph.nodes: + self.create_connections(task) dotfile = self.graph.create_dotfile_nested( outdir=self.output_dir, name=name ) + elif type == "detailed": + # create connections with detailed=True + for task in self.graph.nodes: + self.create_connections(task, detailed=True) + # adding wf outputs + for (wf_out, lf) in self._connections: + self.graph.add_edges_description((self.name, wf_out, lf.name, lf.field)) + dotfile = self.graph.create_dotfile_detailed( + outdir=self.output_dir, name=name + ) else: raise Exception( - f"type of the graph can be simple or nested, " f"but {type} provided" + f"type of the graph can be simple, detailed or nested, " + f"but {type} provided" ) if not export: return dotfile else: if export is True: export = ["png"] + elif isinstance(export, str): + export = [export] formatted_dot = [] for ext in export: formatted_dot.append(self.graph.export_graph(dotfile=dotfile, ext=ext)) diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index 5df4433f31..d4151e35ab 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -9,7 +9,7 @@ class DiGraph: """A simple Directed Graph object.""" - def __init__(self, nodes=None, edges=None): + def __init__(self, name=None, nodes=None, edges=None): """ Initialize a directed graph. @@ -22,6 +22,7 @@ def __init__(self, nodes=None, edges=None): the graph. """ + self.name = name self._nodes = [] self.nodes = nodes self._edges = [] @@ -29,6 +30,7 @@ def __init__(self, nodes=None, edges=None): self._create_connections() self._sorted_nodes = None self._node_wip = [] + self._nodes_details = {} def copy(self): """ @@ -95,6 +97,20 @@ def edges_names(self): """Get edges as pairs of the nodes they connect.""" return [(edg[0].name, edg[1].name) for edg in self._edges] + @property + def nodes_details(self): + """ dictionary with details of the nodes + for each task, there are inputs/outputs and connections + (with input/output fields names) + """ + # removing repeated fields from inputs and outputs + for el in self._nodes_details.values(): + el["inputs"] = list(set(el["inputs"])) + el["inputs"].sort() + el["outputs"] = list(set(el["outputs"])) + el["outputs"].sort() + return self._nodes_details + @property def sorted_nodes(self): """Return sorted nodes (runs sorting if needed).""" @@ -139,6 +155,19 @@ def add_edges(self, new_edges): # starting from the previous sorted list, so it's faster self.sorting(presorted=self.sorted_nodes + []) + def add_edges_description(self, new_edge_details): + """ adding detailed description of the connections, filling _nodes_details""" + in_nd, in_fld, out_nd, out_fld = new_edge_details + for key in [in_nd, out_nd]: + self._nodes_details.setdefault( + key, {"inputs": [], "outputs": [], "connections": []} + ) + + if (in_fld, out_nd, out_fld) not in self._nodes_details[in_nd]["connections"]: + self._nodes_details[in_nd]["connections"].append((in_fld, out_nd, out_fld)) + self._nodes_details[in_nd]["inputs"].append(in_fld) + self._nodes_details[out_nd]["outputs"].append(out_fld) + def sorting(self, presorted=None): """ Sort this graph. @@ -350,6 +379,52 @@ def create_dotfile_simple(self, outdir, name="graph"): dotfile.write_text(dotstr) return dotfile + def create_dotfile_detailed(self, outdir, name="graph_det"): + """ creates a detailed dotfile (detailed connections - input/output fields, + but no nested structure) + """ + dotstr = "digraph structs {\n" + dotstr += "node [shape=record];\n" + if not self._nodes_details: + raise Exception("node_details is empty, detailed dotfile can't be created") + for nd_nm, nd_det in self.nodes_details.items(): + if nd_nm == self.name: # the main workflow itself + # wf inputs + wf_inputs_str = f'{{<{nd_det["outputs"][0]}> {nd_det["outputs"][0]}' + for el in nd_det["outputs"][1:]: + wf_inputs_str += f" | <{el}> {el}" + wf_inputs_str += "}" + dotstr += f'struct_{nd_nm} [color=red, label="{{WORKFLOW INPUT: | {wf_inputs_str}}}"];\n' + # wf outputs + wf_outputs_str = f'{{<{nd_det["inputs"][0]}> {nd_det["inputs"][0]}' + for el in nd_det["inputs"][1:]: + wf_outputs_str += f" | <{el}> {el}" + wf_outputs_str += "}" + dotstr += f'struct_{nd_nm}_out [color=red, label="{{WORKFLOW OUTPUT: | {wf_outputs_str}}}"];\n' + # connections to the wf outputs + for con in nd_det["connections"]: + dotstr += ( + f"struct_{con[1]}:{con[2]} -> struct_{nd_nm}_out:{con[0]};\n" + ) + else: # elements of the main workflow + inputs_str = "{INPUT:" + for inp in nd_det["inputs"]: + inputs_str += f" | <{inp}> {inp}" + inputs_str += "}" + outputs_str = "{OUTPUT:" + for out in nd_det["outputs"]: + outputs_str += f" | <{out}> {out}" + outputs_str += "}" + dotstr += f'struct_{nd_nm} [shape=record, label="{inputs_str} | {nd_nm} | {outputs_str}"];\n' + # connections between elements + for con in nd_det["connections"]: + dotstr += f"struct_{con[1]}:{con[2]} -> struct_{nd_nm}:{con[0]};\n" + dotstr += "}" + Path(outdir).mkdir(parents=True, exist_ok=True) + dotfile = Path(outdir) / f"{name}.dot" + dotfile.write_text(dotstr) + return dotfile + def create_dotfile_nested(self, outdir, name="graph"): """dotfile that includes the nested structures for workflows""" dotstr = "digraph G {\ncompound=true \n" diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 785f9d9e8d..792adf487a 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3951,197 +3951,181 @@ def test_wf_upstream_error9b(plugin): assert wf.follow_err._errored == ["err"] -def test_graph_1(tmpdir): - """creating a graph, wf with two nodes""" - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) - wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - - dotfile = wf.create_dotfile() - dotstr_lines = dotfile.read_text().split("\n") - assert "mult_1" in dotstr_lines - assert "mult_2" in dotstr_lines - assert "add2" in dotstr_lines - assert "mult_1 -> add2" in dotstr_lines - - if DOT_FLAG: - name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0] == dotfile_pr.with_suffix(".png") - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) +def exporting_graphs(wf, name): + """ helper function to run dot to create png/pdf files from dotfiles""" + # exporting the simple graph + dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) + assert len(formatted_dot) == 1 + assert formatted_dot[0] == dotfile_pr.with_suffix(".png") + assert formatted_dot[0].exists() + print("\n png of a simple graph in: ", formatted_dot[0]) + # exporting nested graph + dotfile_pr, formatted_dot = wf.create_dotfile( + type="nested", export=["pdf", "png"], name=f"{name}_nest" + ) + assert len(formatted_dot) == 2 + assert formatted_dot[0] == dotfile_pr.with_suffix(".pdf") + assert formatted_dot[0].exists() + print("\n pdf of the nested graph in: ", formatted_dot[0]) + # detailed graph + dotfile_pr, formatted_dot = wf.create_dotfile( + type="detailed", export="pdf", name=f"{name}_det" + ) + assert len(formatted_dot) == 1 + assert formatted_dot[0] == dotfile_pr.with_suffix(".pdf") + assert formatted_dot[0].exists() + print("\n pdf of the detailed graph in: ", formatted_dot[0]) -def test_graph_1nest(tmpdir): - """creating a nested graph, wf with two nodes""" - wf = Workflow(name="wf_2", input_spec=["x", "y"]) +def test_graph_1(tmpdir): + """creating a set of graphs, wf with two nodes""" + wf = Workflow(name="wf", input_spec=["x", "y"]) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") - assert "mult_1" in dotstr_lines - assert "mult_2" in dotstr_lines - assert "add2" in dotstr_lines - assert "mult_1 -> add2" in dotstr_lines + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult_1" in dotstr_s_lines + assert "mult_2" in dotstr_s_lines + assert "add2" in dotstr_s_lines + assert "mult_1 -> add2" in dotstr_s_lines + + # nested graph (should have the same elements) + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") + assert "mult_1" in dotstr_n_lines + assert "mult_2" in dotstr_n_lines + assert "add2" in dotstr_n_lines + assert "mult_1 -> add2" in dotstr_n_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_mult_1:out -> struct_add2:x;" in dotstr_d_lines + # exporting graphs if dot available if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=["png", "pdf"], name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 2 - for i, ext in enumerate(["png", "pdf"]): - assert formatted_dot[i] == dotfile_pr.with_suffix(f".{ext}") - assert formatted_dot[i].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) -def test_graph_1nest_st(tmpdir): - """creating a nested graph, wf with two nodes +def test_graph_1st(tmpdir): + """creating a set of graphs, wf with two nodes some nodes have splitters, should be marked with blue color """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"]) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y).split("x")) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") - assert "mult_1 [color=blue]" in dotstr_lines - assert "mult_2" in dotstr_lines - assert "add2 [color=blue]" in dotstr_lines - assert "mult_1 -> add2" in dotstr_lines + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult_1 [color=blue]" in dotstr_s_lines + assert "mult_2" in dotstr_s_lines + assert "add2 [color=blue]" in dotstr_s_lines + assert "mult_1 -> add2" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") + assert "mult_1 [color=blue]" in dotstr_n_lines + assert "mult_2" in dotstr_n_lines + assert "add2 [color=blue]" in dotstr_n_lines + assert "mult_1 -> add2" in dotstr_n_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_mult_1:out -> struct_add2:x;" in dotstr_d_lines if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=["png", "pdf"], name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 2 - for i, ext in enumerate(["png", "pdf"]): - assert formatted_dot[i] == dotfile_pr.with_suffix(f".{ext}") - assert formatted_dot[i].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) def test_graph_2(tmpdir): """creating a graph, wf with one worfklow as a node""" - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - - dotfile = wf.create_dotfile() - dotstr_lines = dotfile.read_text().split("\n") - assert "wfnd [shape=box]" in dotstr_lines - - if DOT_FLAG: - name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) - - -def test_graph_2nest(tmpdir): - """creating a nested graph, wf with one worfklow as a node""" - wfnd = Workflow(name="wfnd", input_spec=["x"]) + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.inputs.x = 2 - - wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "wfnd [shape=box]" in dotstr_s_lines + + # nested graph dotfile = wf.create_dotfile(type="nested") dotstr_lines = dotfile.read_text().split("\n") assert "subgraph cluster_wfnd {" in dotstr_lines assert "add2" in dotstr_lines + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x}}"];' in dotstr_d_lines + ) + if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=True, name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) -def test_graph_2nest_st(tmpdir): - """creating a nested graph, wf with one worfklow as a node +def test_graph_2st(tmpdir): + """creating a set of graphs, wf with one worfklow as a node the inner workflow has a state, so should be blue """ - wfnd = Workflow(name="wfnd", input_spec=["x"]).split("x") - wfnd.add(add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.inputs.x = 2 - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") - assert "subgraph cluster_wfnd {" in dotstr_lines - assert "color=blue" in dotstr_lines - assert "add2" in dotstr_lines - - if DOT_FLAG: - name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=True, name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) - - -def test_graph_3(tmpdir): - """creating a graph, wf with two nodes (one node is a workflow)""" - wf = Workflow(name="wf", input_spec=["x", "y"]) - wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x).split("x") wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile() - dotstr_lines = dotfile.read_text().split("\n") - assert "mult" in dotstr_lines - assert "wfnd [shape=box]" in dotstr_lines - assert "mult -> wfnd" in dotstr_lines + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "wfnd [shape=box, color=blue]" in dotstr_s_lines + + # nested graph + dotfile_s = wf.create_dotfile(type="nested") + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "subgraph cluster_wfnd {" in dotstr_s_lines + assert "color=blue" in dotstr_s_lines + assert "add2" in dotstr_s_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x}}"];' in dotstr_d_lines + ) + assert "struct_wfnd:out -> struct_wf_out:out;" in dotstr_d_lines if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile(export=True, name=name) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) -def test_graph_3nest(tmpdir): - """creating a nested graph, wf with two nodes (one node is a workflow)""" +def test_graph_3(tmpdir): + """creating a set of graphs, wf with two nodes (one node is a workflow)""" wf = Workflow(name="wf", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -4151,30 +4135,40 @@ def test_graph_3nest(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") - assert "mult" in dotstr_lines - assert "subgraph cluster_wfnd {" in dotstr_lines - assert "add2" in dotstr_lines + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult" in dotstr_s_lines + assert "wfnd [shape=box]" in dotstr_s_lines + assert "mult -> wfnd" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") + assert "mult" in dotstr_n_lines + assert "subgraph cluster_wfnd {" in dotstr_n_lines + assert "add2" in dotstr_n_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_mult:out -> struct_wfnd:x;" in dotstr_d_lines if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=True, name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert len(formatted_dot) == 1 - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) -def test_graph_4nest(tmpdir): - """creating a nested graph, wf with two nodes (one node is a workflow with two nodes +def test_graph_4(tmpdir): + """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the node to the inner workflow. """ wf = Workflow(name="wf", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) wfnd.add(add2(name="add2_b", x=wfnd.add2_a.lzout.out)) @@ -4182,30 +4176,41 @@ def test_graph_4nest(tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult" in dotstr_s_lines + assert "wfnd [shape=box]" in dotstr_s_lines + assert "mult -> wfnd" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") for el in ["mult", "add2_a", "add2_b"]: - assert el in dotstr_lines - assert "subgraph cluster_wfnd {" in dotstr_lines - assert "add2_a -> add2_b" in dotstr_lines + assert el in dotstr_n_lines + assert "subgraph cluster_wfnd {" in dotstr_n_lines + assert "add2_a -> add2_b" in dotstr_n_lines assert "mult -> add2_a [lhead=cluster_wfnd]" + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_wf:y -> struct_mult:y;" in dotstr_d_lines + if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=True, name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) -def test_graph_5nest(tmpdir): - """creating a nested graph, wf with two nodes (one node is a workflow with two nodes +def test_graph_5(tmpdir): + """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the inner workflow to the node. """ wf = Workflow(name="wf", input_spec=["x", "y"]) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) wfnd.add(add2(name="add2_b", x=wfnd.add2_a.lzout.out)) @@ -4214,19 +4219,31 @@ def test_graph_5nest(tmpdir): wf.add(multiply(name="mult", x=wf.wfnd.lzout.out, y=wf.lzin.y)) wf.set_output([("out", wf.mult.lzout.out)]) - dotfile = wf.create_dotfile(type="nested") - dotstr_lines = dotfile.read_text().split("\n") + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult" in dotstr_s_lines + assert "wfnd [shape=box]" in dotstr_s_lines + assert "wfnd -> mult" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") for el in ["mult", "add2_a", "add2_b"]: - assert el in dotstr_lines - assert "subgraph cluster_wfnd {" in dotstr_lines - assert "add2_a -> add2_b" in dotstr_lines + assert el in dotstr_n_lines + assert "subgraph cluster_wfnd {" in dotstr_n_lines + assert "add2_a -> add2_b" in dotstr_n_lines assert "add2_b -> mult [ltail=cluster_wfnd]" + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_wf:x -> struct_wfnd:x;" in dotstr_d_lines + if DOT_FLAG: name = f"graph_{sys._getframe().f_code.co_name}" - dotfile_pr, formatted_dot = wf.create_dotfile( - type="nested", export=True, name=name - ) - assert dotfile_pr.read_text().split("\n") == dotstr_lines - assert formatted_dot[0].exists() - print("\n graph in: ", formatted_dot[0]) + exporting_graphs(wf=wf, name=name) From 0dc48df8018be684301b50d8a5992c9764250285 Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Mon, 3 Aug 2020 13:47:26 +0200 Subject: [PATCH 036/271] add Bas to Zenodo [skip ci] As suggested in https://github.com/nipype/pydra/pull/319#issuecomment-661491395. --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index fa129435d4..806aec46d5 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -49,6 +49,11 @@ "affiliation": "MIT, HMS", "name": "Ghosh, Satrajit", "orcid": "0000-0002-5312-6729" + }, + { + "affiliation": "Microsoft, Station Q", + "name": "Nijholt, Bas", + "orcid": "0000-0003-0383-4986" } ], "keywords": [ From 4e40eec94df2be6bd90962859e0bb756513e5c55 Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Mon, 3 Aug 2020 13:49:01 +0200 Subject: [PATCH 037/271] fix typo "tutoorial" -> "tutorial" --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index df3e383f25..b8eb9b033f 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Learn more about Pydra * `SciPy 2020 Proceedings `_ * `PyCon 2020 Poster `_ -* `Explore Pydra interactively `_ (the tutoorial can be also run using Binder service) +* `Explore Pydra interactively `_ (the tutorial can be also run using Binder service) |Binder| From ab5f780696701cb3d4b4f537364e1e01b8a4d3be Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 3 Aug 2020 23:44:42 -0400 Subject: [PATCH 038/271] Adding a first version of a user guide --- docs/combiner.rst | 66 ++++++++++++ docs/components.rst | 187 +++++++++++++++++++++++++++++++++ docs/images/nd_spl_1.png | Bin 0 -> 30961 bytes docs/images/nd_spl_3.png | Bin 0 -> 26547 bytes docs/images/nd_spl_3_comb1.png | Bin 0 -> 27292 bytes docs/images/nd_spl_3_comb3.png | Bin 0 -> 28176 bytes docs/images/nd_spl_4.png | Bin 0 -> 16849 bytes docs/index.rst | 68 ++++++++++++ docs/input_spec.rst | 155 +++++++++++++++++++++++++++ docs/state.rst | 88 ++++++++++++++++ docs/user_guide.rst | 11 ++ pydra/engine/specs.py | 1 - 12 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 docs/combiner.rst create mode 100644 docs/components.rst create mode 100644 docs/images/nd_spl_1.png create mode 100644 docs/images/nd_spl_3.png create mode 100644 docs/images/nd_spl_3_comb1.png create mode 100644 docs/images/nd_spl_3_comb3.png create mode 100644 docs/images/nd_spl_4.png create mode 100644 docs/input_spec.rst create mode 100644 docs/state.rst create mode 100644 docs/user_guide.rst diff --git a/docs/combiner.rst b/docs/combiner.rst new file mode 100644 index 0000000000..78875e1e55 --- /dev/null +++ b/docs/combiner.rst @@ -0,0 +1,66 @@ +Grouping Task's Output +======================= + +In addition to the splitting the input, *Pydra* supports grouping +or combining the output resulting from the splits. +In order to achieve this for a *Task*, a user can specify a *combiner*. +This can be set by calling ``combine`` method. +Note, the *combiner* only makes sense when a *splitter* is +set first. When *combiner=x*, all values are combined together within one list, +and each element of the list represents an output of the *Task* for the specific +value of the input *x*. Splitting and combining for this example can be written +as follows: + +.. math:: + + S = x &:& ~x=[x_1, x_2, ..., x_n] \mapsto x=x_1, x=x_2, ..., x=x_n, \\ + C = x &:& ~out(x_1), ...,out(x_n) \mapsto out_{comb}=[out(x_1), ...out(x_n)], + +where `S` represents the *splitter*, *C* represents the *combiner*, :math:`x` is the input field, +:math:`out(x_i)` represents the output of the *Task* for :math:`x_i`, and :math:`out_{comb}` +is the final output after applying the *combiner*. + +In the situation where input has multiple fields and an *outer splitter* is used, +there are various ways of combining the output. +Taking as an example the task from the previous section, +user might want to combine all the outputs for one specific value of :math:`x_i` and +all the values of :math:`y`. +In this situation, the combined output would be a two dimensional list, each +inner list for each value of :math:`x`. This can be written as follow: + +.. math:: + + C = y &:& ~out(x_1, y1), out(x_1, y2), ...out(x_n, y_m) \\ + &\longmapsto& ~[[out(x_1, y_1), ..., out(x_1, y_m)], \\ + && ~..., \\ + && ~[out(x_n, y_1), ..., out(x_n, y_m)]]. + + + + +.. figure:: images/nd_spl_3_comb1.png + :figclass: h! + :scale: 75% + + + +However, for the same task the user might want to combine +all values of :math:`x` for specific values of :math:`y`. +One may also need to combine all the values together. +This can be achieved by providing a list of fields, :math:`[x, y]` to the combiner. +When a full combiner is set, i.e. all the fields from +the splitter are also in the combiner, the output is a one dimensional list: + +.. math:: + + C = [x, y] : out(x_1, y1), ...out(x_n, y_m) \longmapsto [out(x_1, y_1), ..., out(x_n, y_m)]. + + +.. figure:: images/nd_spl_3_comb3.png + :figclass: h! + :scale: 75% + +These are the basic examples of the *Pydra*'s *splitter-combiner* concept. It +is important to note, that *Pydra* allows for mixing *splitters* and *combiners* +on various levels of a dataflow. They can be set on a single *Task* or a *Workflow*. +They can be passed from one *Task* to following *Tasks* within the *Workflow*. diff --git a/docs/components.rst b/docs/components.rst new file mode 100644 index 0000000000..6783071b56 --- /dev/null +++ b/docs/components.rst @@ -0,0 +1,187 @@ +Dataflows Components: Task and Workflow +======================================= +A *Task* is the basic runnable component of *Pydra* and is described by the +class ``TaskBase``. A *Task* has named inputs and outputs, thus allowing +construction of dataflows. It can be hashed and executes in a specific working +directory. Any *Pydra*'s *Task* can be used as a function in a script, thus allowing +dual use in *Pydra*'s *Workflows* and in standalone scripts. There are several +classes that inherit from ``TaskBase`` and each has a different application: + + +Function Tasks +-------------- + +* ``FunctionTask`` is a *Task* that executes Python functions. Most Python functions + declared in an existing library, package, or interactively in a terminal can + be converted to a ``FunctionTask`` by using *Pydra*'s decorator - ``mark.task``. + + .. code-block:: python + + import numpy as np + from pydra import mark + fft = mark.annotate({'a': np.ndarray, + 'return': float})(np.fft.fft) + fft_task = mark.task(fft)() + result = fft_task(a=np.random.rand(512)) + + + `fft_task` is now a *Pydra* *Task* and result will contain a *Pydra*'s ``Result`` object. + In addition, the user can use Python's function annotation or another *Pydra* + decorator --- ``mark.annotate`` in order to specify the output. In the + following example, we decorate an arbitrary Python function to create named + outputs: + + .. code-block:: python + + @mark.task + @mark.annotate( + {"return": {"mean": float, "std": float}} + ) + def mean_dev(my_data): + import statistics as st + return st.mean(my_data), st.stdev(my_data) + + result = mean_dev(my_data=[...])() + + When the *Task* is executed `result.output` will contain two attributes: `mean` + and `std`. Named attributes facilitate passing different outputs to + different downstream nodes in a dataflow. + + +.. _shell_command_task: + +Shell Command Tasks +------------------- + +* ``ShellCommandTask`` is a *Task* used to run shell commands and executables. + It can be used with a simple command without any arguments, or with specific + set of arguments and flags, e.g.: + + .. code-block:: python + + ShellCommandTask(executable="pwd") + + ShellCommandTask(executable="ls", args="my_dir") + + The *Task* can accommodate more complex shell commands by allowing the user to + customize inputs and outputs of the commands. + One can generate an input + specification to specify names of inputs, positions in the command, types of + the inputs, and other metadata. + As a specific example, FSL's BET command (Brain + Extraction Tool) can be called on the command line as: + + .. code-block:: python + + bet input_file output_file -m + + Each of the command argument can be treated as a named input to the + ``ShellCommandTask``, and can be included in the input specification. + As shown next, even an output is specified by constructing + the *out_file* field form a template: + + .. code-block:: python + + bet_input_spec = SpecInfo( + name="Input", + fields=[ + ( "in_file", File, + { "help_string": "input file ...", + "position": 1, + "mandatory": True } ), + ( "out_file", str, + { "help_string": "name of output ...", + "position": 2, + "output_file_template": + "{in_file}_br" } ), + ( "mask", bool, + { "help_string": "create binary mask", + "argstr": "-m", } ) ], + bases=(ShellSpec,) ) + + ShellCommandTask(executable="bet", + input_spec=bet_input_spec) + + More details are in the :ref:`Input Specification section`. + +Container Tasks +--------------- +* ``ContainerTask`` class is a child class of ``ShellCommandTask`` and serves as + a parent class for ``DockerTask`` and ``SingularityTask``. Both *Container Tasks* + run shell commands or executables within containers with specific user defined + environments using Docker_ and Singularity_ software respectively. + This might be extremely useful for users and projects that require environment + encapsulation and sharing. + Using container technologies helps improve scientific + workflows reproducibility, one of the key concept behind *Pydra*. + + These *Container Tasks* can be defined by using + ``DockerTask`` and ``SingularityTask`` classes directly, or can be created + automatically from ``ShellCommandTask``, when an optional argument + ``container_info`` is used when creating a *Shell Task*. The following two + types of syntax are equivalent: + + .. code-block:: python + + DockerTask(executable="pwd", image="busybox") + + ShellCommandTask(executable="ls", + container_info=("docker", "busybox")) + +Workflows +--------- +* ``Workflow`` - is a subclass of *Task* that provides support for creating *Pydra* + dataflows. As a subclass, a *Workflow* acts like a *Task* and has inputs, outputs, + is hashable, and is treated as a single unit. Unlike *Tasks*, workflows embed + a directed acyclic graph. Each node of the graph contains a *Task* of any type, + including another *Workflow*, and can be added to the *Workflow* simply by calling + the ``add`` method. The connections between *Tasks* are defined by using so + called *Lazy Inputs* or *Lazy Outputs*. These are special attributes that allow + assignment of values when a *Workflow* is executed rather than at the point of + assignment. The following example creates a *Workflow* from two *Pydra* *Tasks*. + + .. code-block:: python + + # creating workflow with two input fields + wf = Workflow(input_spec=["x", "y"]) + # adding a task and connecting task's input + # to the workflow input + wf.add(mult(name="mlt", + x=wf.lzin.x, y=wf.lzin.y)) + # adding anoter task and connecting + # task's input to the "mult" task's output + wf.add(add2(name="add", x=wf.mlt.lzout.out)) + # setting worflow output + wf.set_output([("out", wf.add.lzout.out)]) + + +Task's State +------------ +All Tasks, including Workflows, can have an optional attribute representing an instance of the State class. +This attribute controls the execution of a Task over different input parameter sets. +This class is at the heart of Pydra's powerful Map-Reduce over arbitrary inputs of nested dataflows feature. +The State class formalizes how users can specify arbitrary combinations. +Its functionality is used to create and track different combinations of input parameters, +and optionally allow limited or complete recombinations. +In order to specify how the inputs should be split into parameter sets, and optionally combined after +the Task execution, the user can set splitter and combiner attributes of the State class. + +.. code-block:: python + + task_with_state = + add2(x=[1, 5]).split("x").combine("x") + +In this example, the ``State`` class is responsible for creating a list of two +separate inputs, *[{x: 1}, {x:5}]*, each run of the *Task* should get one +element from the list. +The results are grouped back when returning the result from the *Task*. +While this example +illustrates mapping and grouping of results over a single parameter, *Pydra* +extends this to arbitrary combinations of input fields and downstream grouping +over nested dataflows. Details of how splitters and combiners power *Pydra*'s +scalable dataflows are described in the next section. + + + +.. _Docker: https://www.docker.com/ +.. _Singularity: https://www.singularity.lbl.gov/ diff --git a/docs/images/nd_spl_1.png b/docs/images/nd_spl_1.png new file mode 100644 index 0000000000000000000000000000000000000000..e4967901dcde2843a01b36245e22123ec53acffb GIT binary patch literal 30961 zcmdqIg;SPa)HX^9NC=34l7fWNO1Ff9bV-+hbVzr%2#9nGNC^l?cL^vU-Q8W%%~|~3 z^Uj$w^Zf-MGs?rW@7;T^b**b%cZj^KI5q|`1_A;C_8SQ?MFa#SdH9Edjs)MRrv)9r zue%>4)a?)uF!68yAtEFuKSV%yjPOQGSlKywYuZIk`TYgj9tkBuKMD$|;sY@mg?1)( z;a3I^D%0E^nh+8%e^Jv=Pozqe3O%ne&yKxI{a%vt+oxw3536yqVsrgSo@9D=p>I4| zIqC3wQgXL6g^g{t_ef`#1FJP*vhOT)m8bWLlqBr6D2ycb$GzD*@H-v}2?qh8;Q#Lb zeaT?1Y7b&0vy|z3Qw22D)Fi}0LP932+uGXRoC5V2;ZY) zV#?i4cC>39O@q`eya`B2yA9fZFYkZ)^r^eM`}EW~K%_bKL1-Hu9-cu*_^W+HQ`L5>3vHq0HdK)kEv>B=C%Y_V+|U;+-`%@+74xD6{Fab&F_3V+KIpK4(wn6y zxw1u`5V8I1i(QED`}ZFI&kl>ecNlpthO>&q0!nJ2>0BSz;(_|mmJ zS!o^d@uR-($F_wmsW?!bms9*1tH(QeR~r{`|JTmq(qtf*?w^` zYXbukD>Dq|=TE!%=g*&0Qc}iNGQOf9?O;;>>0h`MEkT{cWnIytz@mmDZR-1efm@a@ z=n*lo>&^8Q)c&T;%ZC++yBP2{bh=Z{7x!5nw8H6A51 zVmbcv{oY)&yQ(2-8Cx<8nUWGwRG^su;k?hQd2TDHrK6(*NBzZ%{f*&#iCSV}e&-L+ zI-EFZ+JpW5k6~fUwNwbmt>`$6L5g`;n3z$LS!&4{5iluA(LmftF{<%WW1M4Z z#N_c=hllpDj{W`p&z?P#;LH43S}G+gYea^rU!s<$yswSt&5Y};=Q%JqST3c!(wo-n z#G}&O)5FZk8OvmV7usf9oxJ-I!3q?`)#NSd7 z{-z0~z1LAwd$>c2i4Tm0|3k0QyFXR`~djb-r_ zoUrMVb~tQ~$k0a3>yx8nBjKQ+5cx17UkK=rKF1}7dmsVEoxY66BNNb@%_CubGx zp&A$OY&ilyn2YqWDw|AcT1;&0HvafK&L`7O>c%=27K~6F-4acU?(2A;k&kb(x!Gse zwh19xGAk*G1aG?0i=t5p7YUXLmx_ui*rY3#>C~|7;g26b%ts2+=^C{VX5RUuT@L=} zmVWbQzLt=cdi$fy)H;$sf)5)1B6m+~t7y8s-uvuV`ccDURQLQ5loFdtX{2I{Iqkls zyxKoJ#KOjo`D6I+-#?h{$h{~0bNHI0kMD>4(5e48=p~hx!5Fhf@y%W5rQvDw3UMC&qDr=DaU*hD?835N2AONbGyEyy#)lQp+q zWmlwWZEI_5Z10{Q{~X?H=O!Gi-2!Y>mnObXd)^ceXVgis_Pw2;864KSQdy|n2$jh# zW*bt1W0|!(ohbDbCxmAzEGJKY+$n(idS(R-nDtI0cN;x!o6|Gf+9HQP|1>vK^73k= z%Wus#rtNApZGgSWUfJ0R&*vWbWLf0LP^ISz*5wV@WrhbI!6SZ48MbwBh>@Uvks!?+ zcc6`_Mub8hKUCwe8T^!*+J9HXTMm;JDAng)v#Lb3*0-^tEo>i zIwB|*6`A9)U=+N0d3j>&PCHr+n9nl=JfKaV7stlO zUx|z3D;ODN&wodDMg`Mie-17!Mw02`b&+em z3SNa0nveb%J_sWhbgwe{o2ct`bu^GAg-ya~0qu*4{pw2>J>ppjz$5}gX1CIv@a4-F z@d)zj+S;I?AaEmE7ryXA!es>m8J(SNEG{m#S#0NRR8>)lRbveQ^hvASEJlz6dB_*F ziF#+YQBF+1Bz{-qQ@-75kK6Iqvy-Ks6u$bpI$p;uu%9MW#B4^NQg>}vds2RA*D~wW zonD?Dn3eM~Fqn@OzY}A(Uy~1CwC7v-x7Y!`)obZD-Z4#wjRh{J!^V(p5hEibbw*2z zNc}YyahL=()$Lktx{!Jk!;OeXPwM>?96VQxOa6(NV*@N1n4?f47S7*@fG?OLC17p` z3!y9{c}h6|6<}m5$Zw(R1=Zz}nX=NtJ$L(1hE+Lcz*01pLV{hNL7fLSV3%IN{W?{>JBep8FC zWnp2_MkkjhC?f_ZssCUuf42*#TZnJXNU%_}*R#uId`&|tl7dr0$+P;4M8XO$V1Se6q@p`Pp zpwzh6b48e)SSE?na?89M?jJL=Y;R}Sc7LWggGxbhNlD1Zj{~dRfDeh`&|T~{Hoj{O zjMH&79r*U<^)!AE_EUU({IC;31P!>D&$cGYLxT?~++c)xqCxlO=jXS&UWXA+*SXjM zSYT#mhAm}aX8!SqLj-mOo+x>n$??v>VDRUAsp(M81~~0-)QpXdJKEciH-@FsM9a;F zCHXo|r=4c&_d6M23(rRiHN%F;$sazH5CiDA+i>yYc{q+dArj6f*(}9BhA%?IGH!y5 z0H4n~FJf=UJ$m#=j5<;xIM?Ii7-o<=Zv%!s+nA;$z278TI>GjIZ{80Lrva7-m4Bba zx-Rnb=e?_w*-FzvqnSDvRQ{i_etv!u)L$gC+PS%3@_;+aG#HKTJZ9M}kcwlTcvJyn z7P>yogyxseYYRK!Fu^rQi-~c#Jhkx{kj?BT<^J$>!8gv^8-eO$NJxLq?~RhMw~myOs6-jFpVBcfZ0=0g;?qj^2`b6Ue*nV)n=Wu*ygO5`sjRHr%Js9P#Ky)Z zPf7fPagfvQ3>k-6bVNjN<*a8{MfQ*h`*e*1pUb|XC62@1oN$QP!u-6=a%arX-qH$x z*Q1R`-&a2R`uV9A>7+zP`ZzG?S@>RXVa3u>Qu+k*I_(tPq!Y5}Oo4^IymY7HwO_-< zJ3BiwZRE3_sl%s}2$m`Zrt#Ki^~Z>pmqJx2!K5G z3B3OK)}WM>l&<>yW@)v+>EvBAxm~kIhNzMlv=daFij!GZMxz zR!rHq6Q$Z1Et<^~kD&Kxwr;$7EfyXgK5Sj(urpO1%cMCmIcff6e0aFrYPv=Y+}|f# zmh02Sh!B(CpDBk)(jonTAU-Z00adL}F(CoCy$>SRgO) z%eSmqciY>3{Iy@}Ykn-QSz$5GHS5j>cK~d&$IaCV_!RIn5^vs^4&^Gqjymp4nHm@v zeCB#laCmrl7oYYv{(<(vJkiOe+5-r-wFQe}u&G);QDM2cyq@KU zgwQ~lewq7TB?i*s0-sHDOUZVTzP|n`^DnXQ6!XJzsSS zUmpulU>|8{Xi)jTzAGv0Qj5Cl{qON! zQ`MNhKb)3*?cnjTQsan-2)V2CBW&kxm}kR558v(6XBQXv3XYDPvjN@#t)F-v#wq4e zQ{_*v@j~E3N7vEZ+}zfdRUwqGn76R7P+>i5)#MsZE^vN%(0j5wt9v5~@aYE5+4vD?nr!T#Sl{;E(Ks zh9J_sV~_q{qr6jph|SQuT$^)ooe>c92>wt1>)DUtJv2kV_~dun%35C-8i&6}`~O!% zPwe{o+C2_oWqzjK4SvV=hJ#=D!o$aRpe@3*ywtZdZNG%B-5_0QrQnA;^n(nWR6;aJ z@nFTB$oS9CpFdBT5g94Kk}^Me(u&2Yd3IrazJU2Lj3)e|_u%GI^zPlm88>$0dwBvUCyD>*H~)^ms+vSm{cA)-Q7-0gTzb-`9A2A} z>9%kRp&lp7m;~-fm4dIl4#B}gW*YU0{QM>x%Di5wyPb#f@-Kyj4;LfKTsgF>UWx|x zeS34*j3y>F>nU}x(!bDVKaUg;G?-t6L4BYmhh(mg$V$$rfw&0h2!JcZAYp~qypJMe zQ!*0R*89?PddJ54ntaOapE+!NTYOG{F;U`oSjX}u47=Ry)ndo!g1DsQ)2G1`6`RNHaJYo7#Mq1n@|2dA#8iKLZWPjPl!ZJAfLg-S*UITkQ`@%p)5o<7HWLw&yi6R|7knG9=<=hb>lR+v zfWKA~W1GmCb+ozuD|Sdb4i{cKgEO;u*8QN#_F$j0{f8r?e}NiY%rRZx_=>7?FVBQ7t?2hYyeGa`xr2 zv+xAe1D{9$Vxfve*Iv~_0=sz!u$lZ*=NV z--Vi4IIqLmRo=G8rpc|Zi@n_n_c-|x(v=si-2{tZ3MTjg#AkA1e2!alg}zaAQY`Pa z8?wBs@>L6GWtnrZN!FT?wI!MI-!iz~ToKnRW~Q4Cf*pW0{TR;dVY~Eupk8_Z zpi6gVb0C0_aqc8g+(HSdM9qDr`w@8;0z-?_lgDbIYilM{l2THz4#(SRoc1{61=Y4$ zovJGq#vU&}T%`GfB#1c`lI~$!Z9A(l3X_<>N>fKK6 z_X8L{WkxBulg-{N_Bx#O&E=2b%QNEv<~!1oRMyss+>wsE*^Sp37bnlS;$&05RdHkB z&{uc!c%NA+DZ&Zvjc!)RSmqb6F%J(8UNw3l%wPYJ9@nH3q%tLsr9 zZ9`19#!6r-N04qKasFPj`~A9J3Wn5V{A$aX8^f9JT1}2EHb(yXzkej6LtbtzcXQ$e zwQWcX)r7q+g#q3D`NQtcjZG{jbor80+{UI%57Fa<4Ete|b$xs<${=tY(#D<9&%~k= z68>o3A>$j^Z5*6$eSjzIQEs`p7LQAi=lz2A6k^Mq|+|=#idQDM~}c|?wX3rjTpZc6T7wfc(?IBc*|e*FW$AFKtuU{&t&F=0f+)0sIsO`S)r)<|9UJ; zH8P-V(7?T9(=iY=eJ_r@^1Xw3djxK29pvqwcMj>)2&%PqmX`x)q?(9s9|pJzOc5Q` z@r@o0gzcho?I#G?z)0^Q`|+E%20Bqit^xK}OS@aw1T>S6aBBIx)>?=$A8&iJ+)_)d zze~xhT+u|lp~`E%->tD{gb`OCA)4<6hc^sNk)RFfXrU%og0$n={(Ct&xrQ#pObI!; z-Qj#9*IjQXbpE+V7wo?b1wHA|6oxwuSslR-QFsaQ33@Q*sdz=v2$v>rf@+8vn zKTgVPYsZ8t@+lu5NxmxMUC+|TPhlW8TH4y91?rjdPm5jKTKoe7@cQ?cyCNMQqd@HX z*hIDHTUOS8{QbR$FD)##fF&qaxpRxsMAD~2kOy@My=9E<xFwL zZJ~k6neYf7->(dL9_EiC##aJ=wF*9OCM|?S$@$@Wd^buuFt){d^xaeM%!pAil@cE_ z0txsZlNl+&9llsgnoAZVIb3QiM>X;DJ?6kYe^j70fuC*H=%)jM6{uBA(vGl)&qdPZ z#l*#N$ob=UZ4o+KfqBGZUP;IVzUcLO}|7W{iSiLx}ICc z!T=fiBcOjnJp%6&7NhI@M;OskB}lJqZgx5GAjGtC@Z+*>{x#suirmk`EO)8E?CQf#8+UB=c6L}qh08#*c253ECg?esDF9R7VKql7?^ z^#Y3w!%PL!ZREXJCG3Yvd4TfjLh3HCsP$OQj9`}eA0MFHX}rfOUKNP8a?6PG@TfzI zC~E*7o_8gW`0S{c4Ty;Ivx|s#8S+5@xMsG~rsYV;7~PV7SU3NgNde2@4P=-i9reQD z93DVGwQO5mShx*wM`lJwF#3fh@mod3fzOnp!6w7s---rem6Vqs?d=tpmoL{cdrvf@ z<8*)0C~R;&qQ|SQs_IS?@&dmu2G+g^_GDi>@y-Ib6(n&Ow+99W;CMo0JYIyb+NtLV z-Pu7P4JnEtv+~l?(x@nWw->;>HF+aq1Ryay!;;EV4h#r*lcg9bVSl)$0N3K!U07Jy z)3c$zzP`4$HY^Oghy!T|2}lG^AaJa#erOb0O;wd}ZU_GcWL`)D92i)Us&B@)Td3$3 z92Sm_6}M$zIc8=Wa&w<@am6(vA)4K0KAv!KRp#dYhVLPpwA`?~v5}aZO#0|ip8V6g zy1EfaKS~9}`p6jkq@n+x9LMsGwXuQ0;?7Rbi=wYzy^DAe&EQn)k(+Uc#YRW7n+{NO z)&ubt8Xhn);5Y$pAF>9jcC~3hmN2OPh)&W$z0DppVHKB^mX`xtWO_sth6-*hWyapl zt~I-zqr(Pxu!5wdr2k0wzdb!au2&o(RC{O6ad2=D8X792Lg(EBi=FYLv#YCX@ZJ$jIJBONq3y?*i;GK%&om6)Jk?Rm(AXHv2OAZD!y1i3PK^Wv zRPQF_dByx;#%=JUD!}VaTvtKfNm5c0C_xG8L_v?5pdfKY#V>fB>;@$zC9od~3JO$_ zCU8jL&*hPvAu))Xr>CbwZcXTKe%u!*fz)$gMtGg~^im>5z6v9=fAia;Sm3qzUo8Mc^XLj$*ekA(8HI!_ zT$x}9R3I~>SW(&S8eOkNL`+;`HcT(7Jtrp0i%$C5`7!rfj`+<#gM(}k@G~*VFcNmO zwZRg=Fw}=lSRiN3$oNV72c)21By<;dx%7R$hZNL=+?hzRuLGZHsgEjX-P zrzCeVl9~*?3`^9KV`Ck`actDyv-b|P$K{4ON(l=iO7UgNhQ@>mKz0Z2i);!n^r}84 zDH|h6Kti&)von*g$}*DrJEO+2cv5d?em-5a zD<%B{XJ00d?P$nB(YZX{~1J5(?|7Gy3}aU@!lcuqB&b0t z;Hjtg6_=hm0}Y3KX=P>M-uh!)@Qew^u5yFfGLS&{?ZmTOqBUM(5OhpMffv%Y0Gux> z8rsF_X}s+cU0fHiN43_o4Zv`7n*I@c{Tj+J=~g=jbmAbBOOtrutqaXXJtyuC^Ixoj zC4f+ZmYUim$m5e36{MKf`!j=zimWqVhhg3KA?*Oywhbf$taL&`LeM?30a;bES$kkp zeSCacbank;pV3js_X0WMi3kWdtft1=+5&cM&3v*HKvS9EbHv?+6Y`;hTR||Ba`8`F~geZ#=n*)XMBi+bNUW@ zP>ViBGEZ4Cpn2+V5?4qrY|n)8nATIg>Y5siJ>F#DxqItPd@y$JI8?j&D(8PxcrI>k z2in1T>6B!r0flW%Lmxv!3wgE+5(*0~fu{{wJ3rb49>F~hSp^ccv{?sa-OVp@84G| z|HUk8pX|}>Se>dwdE;T^pL%kkp@RUvZ+;=*@~Wimpit?y@IDkuOzO>)nJRrFQ~nMY zso)8lkPyXR7d>T&x(M0hMX_P%(tu=M`=Hw85`d++3(j}1PwyV@OHtm3BuF?|SlBCH z*`VN-So#bFx5xb<4iunbujt{LNT59d2-EzvnG_2Nd;LGc3X4774-wk^1pXb>8`X}L z8>%-`?!DACVTGrmI*{^*zED691&C1K1O@oF9fHuq-X97cL4h#bNut{VQBmGCAWF&u zQSIjL5Dy5;q4Vc*u*V~_fSGX>nG@cI&Yk@O?fcM>aLf>~SDN^sb`P$zyb{#*fo9hL zG;u(GxDTM<9Xz^0S3e-nL$&i@iV(!xic|KCURAXK|zJ!tH{Yn0SsVQ7$v{hKBaN~*){XCC;> zhLTDEpAph2r$18e6`&J$Bcfxkt6=#UGeIL?VeFg!O{Ewx+^1;pJ-Uza8GJAQ462f# zqy~uk@x0{d`)u_V*Etpr20||^`=(D*sTx$srG$s>qNKiuIlV0WN;ysZc#jbmI=Hp% zp+QZhFX5p+MB)J^a-`k?*t3_$CK8nv==`CuH3$MQB45fqMl`5!dy--?e2gifYrU87 zkOVrv6dWW3KT*+|W#PF50oeVAccCRLq~1DM&Zb-#(;Hl8quVjj-gf8H#AX29u_D5l zuuxK~VNBkCU`#I_?{(jYE~K%r*9j4Pj6I=?CQSBkNqi`&3AYOq0WFQ-I zI7K=DB9rP;<^N2&%*8TjZf?$2DX2DP>iI678YXgCTR}cf)I-l5!j9j6{}P-shD&CF zifw`g^fP0~y1KKF;xkPGMCn#tr=Pr%zZQl8KQtKU$=U%++o51W}MV zXg?Yo8*vpsJLG&H-pH0o%2@7sQEUMUx8FTIf?n5`;2M~s{+KzSfGRFsGzijQ^@jse zf`Wn)!lCK)va^sZoGBVC{jUZD!}KxW5g{Zl=k*n%vT<-YIyu>eoO-Fz-)@>a0VK|( z9q)~dQh6QdDJj$S>o`moMz(3ZsLa-fv#W!Qv&5^Ma^xLX9^z@&o$j`6l`c0h5QakV zKtf8owYs{xv0-*9^s*cPHY}kzt9Vz6x>BAp2n8U?H(PG5+U9)l&tmvT77?C<^=yA$ zx2dgqoVC`?$pitFWK>Eg!*7-o?GUknTBlt|y@F#u@Qg)0*GuRam9)rGbOAyH@`i04 z9jc$W8dIl}lgUD!el7BPD_-Te?Ant>O2r%LvTUIaSyYVtZYqL9sz;CQ^qPLKD(`nw`c27UZfWX!N;2V zCC-Oy{gCNCJUZfNN}E?#^K$`23TRSWOG|n$T<-%obeJ7z#F$BJ1yc)MLu&2aN<2=E z$&%Quw&Z7Zf}YIQD8;LO_EPa&|0T3KV3^a>$zs~rCaBaNPG)|t#NL;7OaURv6(&oa zp0AT(YalY(J&Up8bc^3-M_VIjS+d)Fb!)>DU4?o1g(=LX1=XVjILbIbZ|dVcpJr;I zNK${<+uMUABIfo3nWsKEig}P`QaN61tu60m_(JJ9?N;eh;#cp1?AMr?xVXP>``$R5 zhW7F+wos%5`}@uZ26VGwBwT&A#@S1@XAs79dAc92PdV&$)g!y}l^>ay8vA##eP75R zeP!i3lCa%)^&MNNM2!>=&u58^Z=##+cu3aO)z#ho62N15@9EJNr~Qzg4Uz7N5tTum zfr3JRfZcexPpg==9+jx^yB%8?B&YB36(ZJvRu05>Y3}?%r*l7szWCWwFpKYXdidrTaF}|ro-pW3QTReomSO+3s%_gO;@HN1zx%h)Hi7>$ zcpd*j^4z(?Ebq3{fU`RTOMbPBUiGKDua2b+4GkqEBwoJs{$rN=z}g2QESP11dg&NT zJFKn$o~g(AJ@$6JA1iA6S7mma*B)+TY!K~1Mw)eqMtwWLoRVe4_op)bFpwq?9FN6$ z@H(aNQhn1^6*9nDF;jSJqwsnpAz;??Z@W8$IFN#c{67*;MBs)gQNg1RE)`nLxj2B#ep@;cTG)%_-@a zmO0b))hnE1+RI|guJ4nT2f+g& zx>C=NS1va*$hSJnSQgzLsjUR|z0R-=Dq63H$LEHgLH5H+c3LYGd7COM??U;HLIf=R zs)K^icGl@XywB|U`^TkUQwjF4WB2n9S2$>6(X%RBm#IwqVp|D!wvByra&oMUo8ZaX z!jrAsggTt{@2c^-#hXF?f6XiI%as_{G)YNHCbFA;sTR}b`8n7jsJs6>RDeuSrccj` znLB(Ec&Wqn!9UU&KNw{TXY7{d>a0gr36>6=?_+c1*ooFiW+k#2--FOgfo{j_Bf$!) zg0k|1T1&c6ffBVk@C8xYlWec%{w(jqB`eLpG9q$#zb$p`a`bTxmUS%WjhdR8j3);6 z$_kZ$fJb-y^Q~ly!JxE6g`t?Xyy3|&o@dUB^tT}skm;;4rPyn)g8~o7#>POHf~SB_ zw3?Mqm;ledRaP#upCPjN!Up>U)CsMct;JLmgImMaX#Gj@S=Y4g9_!3xf_LI#!*q@7 zK|4!`1Xbj(_I8yUDG|SFb@@DH2+1tW*5E=2c`Pz1{itV0eQVS2209v^BLuuxuU|9k zHjs)6G=D$q-2Ukow8paf2DO1|>27!8@A@C7dK+{W&y7KugDsB7%cehP>cu6KSoJqd zV}6$5w}iE!Ycb}YaN+L_^}?|XSb96mRY)-{Bl=HI^_HV-mt}kbR2^_sg8XDkDY(f z9mFNAlNucz)7+49I1%Fo1O`G-SiSHTU=6T&HsGN2i9}*$QtbT zZ|XYYDcIW|CiB>_4&i^@AQ^~}e6OY!uS{>bu)u#(I)3dV;PPxB7vlhP#iMv{tRG*j z4D=w7rYBj{#-ad!hfT~LY>8E&e{@fZc zpBqAys-&pMTL0`N0m(r5U_6EU`Aob`<|%XE1KT2o8CT2wZOteevo}#R$6{6`#k}c1UaRQ@G2(C2(2Kh)vubxz&|LOlRW=kqxT0IPuUPz29;w zkeYTbef<^3QnaSF_CnLTo`8tRctq;CL*~j*KZRF?*Dj)8TJj^ty)vTh45A`Tazg@R zs^td{rpPJwJGUoYfMf$T6dXbXQ>QZ0N)g}PhR6MzMGOsdG72xQoKd~As z5sG+dfJAJcEmJF)Qo9phIW{%9b?CGy`)=11fBXeua`N3E&TXn}IroF@Q3z-sj{hl; zGwQDOV3>W3fBzhneBcKaN5`w)qz129^8d6bpIz!0k{pGYF9$%NJml~A2=j&9V;~xf zigw}VJjs?zWZdk8Mt;E^FqqC}_Q~0?MxK=CfTmUD(((yi_)GQ+v(y|dopH%VJn(8|GW5xFUGy+@FZ$AY;4W; z;Zz2j0P75IknGwrf?c{Fvb6FyDM|41fvGBkfIF`zKeJZ4{HKS{eHXbKx=KDsW_{5n z(Wq~eJW=c)PqJ>@87)mxrq}*y_@>aan9wIKI{E<-(dzUBomqdPMl2fj&KA-0GtpxO z5QGz2fg662V4*W=R4wzT|;5ttJbnA+m{KJ zbQ|2q2M42ILin}VE#8#En2jG7Y6@4)1~#haKRQhe-y1`-KU{E-;7b{w7-!O| z@d<7ZBgs*wKV$Ad%2WRS+@Kw||3`0w$i#Ygdc#$v9pQZi;Lx6zm|_7)9?X%mURb#N zlM!+HqwSMRr6s|0j|`D?P*>*8M+`mPgVzW^f315mGvY#mf^2JRcE8@4On}D+q4i^M zu>8{pU9`PTS$qOVd+Vpi#r;6}<0qkZ*i4=NTMRK-|NiRucrCWS>vnagNB96yK6{o& zTjX^do&BG9i4u>7;uKS01@*ayEjleHHZuc*6eHn#cEtN z64W&y|6^0N5OnkJcpcgwI!d{X6(C~)@CToP6FXqB>NdmE4<_{0#3O%EkBeb8L8=UZ z1_#=q%kuOw@!jle#EHa|uS>@xo$}r8)0j40_MFUpQ3g15v7a(`hVi>iC}o z>zxvTpOW2FvGBCjY)`9uCM0#=OhcgMNs4nMCdp+!|zX$OH zowA!zUG9fa)6irM5Gf0`wC?p7jy<+smYhP9B7%nl9X|&#xYR&2fQVQB?|9RlNPm9p zu^fGjO1W0)eiOshk@P2ajd$|jMLU=Q_7v*W6AQSMx7kbq;~i|$*Htrh@z9LM{r3l| zTe;mC^RLU_%eHyS8`C2sNn%M&085pRa5KI%0aAchfNIMl0D-qNVFz1mZ&4Y)#_O-J zRLj%i1L7AWU}4c_UM<-ai8+i+T~3FCNaCv^NX2vLD1PZ#0K$jMn@=pFeBXJiz(s>v zMKN9?tpPd21i}>zGY!HLYt-;Eklkvc!2vG#XAO3Of7+qtYx9HsfAc!+ zjB|&;D2JFRuc2%5X&G@wQf8tm5De}vBeMtyXs65j`^)}p(_qB^a!4V|?errIL>>40 zFS%wx*|GV;ZfjF4Ix!IhSw?=^(!~7kk7ona@Z8+oz{10MZmTVh!Qnqn=o0j5HUyVG zHb5}tSrGW2IOMt^1&&nBr>((=;@qlskP&MBSeU^&z@;TPe~m7{!^FHP1hLl?gqIouny@6O;7p}yHwUX3V)NtUr3}O=dQ}wj9+W0>GzYdk5xtJs6)BL7 z+eumfV``&*bGPnRN`$w<=F(p|`?#gR=x;H8W!@7J7y+@&!&5JR^}mUb5J1CwgL(uS zpnRp_f)y_$YY^A^pGl`WyOvX;==?1{QjPCjdiCKXRjkZSKox3@{+YuF!CW%!7;<3dURbv6Kf0%ZCj zCTa*1M$T**PC`^I3F=2<1x2%6)3F)nW^*It{5O;BVGbuuM;Q46ufN=Uw5*q3JK6SDCT}*` zGkW+wp(q3GI}N6B`JNfszaF%6iZIJz4^S3g;k<7iTb1ZgnQj$8l!-s&DI^;i8M|4_HAur3)kJ;k1!!l|C~q9E zS%hefZrs)|MD@B3!{L0#csV925 zUTGINnp7RA4Nwh5y@!1*;PmcdCEh>n{_C+lP!%-`#K8lMJ4i)}TQv%`u{yANCbL(^o5?cf z*FTZ<1XQ+aYuuu*;1c3MM-9!C8 zkx_g$b1$XX%P5AOCh?4d*=*-u3(lL#%^?@wFXl5D>M!+rLk#G2FHaU$F=J<|V)-mb z45w#&k1jx$Jv(76Su|f_6kx@2wyD!VFjl{LQLK8=lS;rmowlcVqLbEr(K%<`oE=^I z^Cy<4Vb_PNa|hR-FUaX>YMtL52flrN5LY)*)X2I0Pn!qm_fwp+%I6m|DbIVm@(Y`1TZU!C)G8IrArVit z|L>Pa+xN1=3#LpY-Sx@kEXhWOV=K0A#w^F&W?g{`;t}?4aXpqg)CHP(6Cjih`Q2ar z9B!1V>jniRRgK@Ts(6*`oDXjpDh2#48s@qg({6DS8z z$uB~?V%v~+zbQVOr9pqdN6^IN^Zkp*_IHg!<~lE-1B;P@at3-2nN1_|i4M9Oo}x(- zj$GX|)|UKOT{-(6<<+A%>AhT@sln&RyNipO*G7w#)&c@?1FN$Cc36Vf{&eG-5)~FV zRvp*K6x!1Hh@0=qP4? z_+j19ZSUsDdR+3v-17MNXzdTp+W|nPpV-h5H>KxlF(_M@OwII1Gm3gvwH%#n94y9i z2Z*#fJ!)RQOhd&PN%(pYp;-*?WaBAhw#VF@1R2x+X8BBeBP$l(EXmaWzgmEvaHW~i zP1@dN#~n{z(~)l>NrD~2yiUj6IDDKbd~|N7#Z{s~%!@Pe-|K5v-AFF76(Osa3%8DEbHq(_l#{FD@sXlR2@2zZ|a(jhp!9dT< zJcECe&wh5o|4PR|yyL5nM~56%xm!tNv%JH7b|46QbsK=bd@3zec#l+X&HPb@#xb=sAnJ6-ouc&{Z5e+8JUhODI&5vEX`bk1QmUbaWi+Fe{tNdIgyK z`^EP+26S8m)t+Rv$2qh5Dam9{=>AH|37CEEbV+~K|J0g3f>b((-kqFzrSIN9C-*p9AJ90VN z(?6}c`iEw{yvlHJ5Tk0x>{3>+j*-0Rm7T($+LJeY9%p83K3Y1;(4a5HP3fzl5Eorn zV(7_-)7wjxU1F@kbrLVI^S825?XW*EMpS3#YOlXlhN8Q?q~uLh%DMBRnQUSBjfa== zDIbSdXngdN)M$~ZM5b3c_Qo*Iwoq|G=sP3b;I+8Jp`<%!VoNi>-nZwc2l6s14>buF zad=n8R2tVE2``Bc!qsj~b*Wa3pnQV5bM|kv%98I!AWeYZ(B$`~L9oeW;O>1dS*nby zkY|WK90FvF^z;)1r58KXZs!m3mY22;H>wmY*7E$;B8YApcI88Amf+noc&Bk_?ho6H zt&QDt3CkJFdX{U=8kBZDpOal7udCQH^RDA>=kGarEEX14=K9uT-+3+#m2@>s42<=M z1iy$NukqlE2nMC)f%~Gy6!E`N-GyVCD~nuT5cK#pm5ONMpRku%7q1oOy-zc|2qYEs z*dMEIl(V|>*?zUZNcoJf*4CzA8k;|Ed(3q)7S_Si!eYX?%w(jC)#qexzBQMLBskY| zx<*qdWW-KL@t6Yp(^A(fhSnLOv>(sDXHk9wngC(T`&8_nOTQ?)tvzju_A`2NmR zCDxiA5fZYV=UeOE#LP8i@5rQG4RnezC6Sa{Fyjd@dOqcgDYqHsx*2skloF`vIkykI z_T%5>Ouf0w?^kf=)cwyE7hR9Z(3+5x)kpGNa{ekgF)NvAx06RrsapeoXWPoxk%zB- zCtwnfu4Gc5mpa(37Ri$s`WqXcOxZac357qes&(VPc=Z*3Ii;o7pfamqO4+KnaO!e$ ztNG^6p^e+VldVMWj1ZZ$jf4HQ*+jQSz0`=4kf8IaeCD(M#<_Ec&GC`EgRRlw?zocK znW`*!?{oV6js3~-xfTBIkqDcg@*RS!$k>Fd-Zq+>Q09|5#dPWn(35@smsu`%OmW!h zlw{z}=j3d+@1NKh;54Zc)GGnL2wM?+`ccZ}V36_u@i8<9{{ zSXdbAiEl}HY0`0XbO@z25fOp7N7e5?<0Z9??<(pGM8(coewuBK8TS8_sUQfq+xc^a zg5)aI=^+(VR(ij>TJ?;%S^unjcW<2}rodJ(>Iw&Me0<{gVwW(wCMyL&yD^67U)ycR zZ`lMfuwQh`q{>}pUc@%Y*S|9e#3C2`Iu%76wvuK?6#QF!s$t5^4Rk(p?(UOk!BUmF z>V!vc6%-@pa0ELh_fx)jjvQoOKK3Yd}dATjCoonnniIAX?Yr&jr!Us zRj&qSwyWOnI*TjU_Jt$WjW@$*^_^B-XY6~jq+A~sN5L;b?v0x2##(Q}y;!^ z@fB5M1xK&^K*YCVGy!D#@yosLJRv@Zb)9@<&+5adZ@--V3I^Qao%Z%mDcsA`a)(0E z&Dyo}tF>bmt&z_b?c9)E28RKz!({ z&QNOeP+smD7f9*3T%uG6&{_VNSod;{jeuTZVvwNx^ ztd^(GSGO9)4sDCb2}VU%p6w6O&}{V}aM&;J$C%FVPCv_a5BfJ@>u57WS{uGSq=mm8 zbmM%^eU$6<-e3fh7JV;MF~j( zX_1gl=`QI;5b2Wc?)ol1_xUTH-|l#anKS3?z4og2YYiQ@N3QlKJ1II?&Om!+-^{pW zdWPKROlbw%Ba%)prHhAxh5N;Lo+Hi`niw8g?hQbGQIN;}IX1q@U)`(cw(Lb=++OMN zVe1ZBKwduQ`qFyK+Ay1by}Xq8B=d`#WlXaf_sYK-D7`DtY@_&Hz14DWc;wVXcyu_& z;Lt{-wCHSmi^av0nxwR3poBK1n4jd#Y2bpG*H1a|&xv_nW38a?H+N+P)`So4m97ux zKD8uzimVSMmtqQK&?|+1D45`R9v0FYQaUF1XX?I2ngu&wek*q&(H{EVa^z8WlH37<@YWQKwrhzZ6OJXDb_NDKB zc#2D&q2uIjD90v1w)azPO7_Y*e@=V7@Ip?@QbD^Y74>Cn!z)YA{%4yn*BHh~;6@+dwk`41JZ~k~FV}^C{1Y z_Q}7C7;iTlxLy`99Kr$nYm<~m%gt)cfKa|IQ1M$Ap7hn_+I+AT|AS-*2W39=fB$ta zwMBbvBvA(Y4L24cIk~)46_TB#)VXPvi(wCQcg}0N?>jgt>mIu!rN5|d8C(^yRQP!< z@v^U~u#3VXI9v=rVaf37PeWE$iqEzUMWLUT@tqDG;Ta8AMUF zjE=_oo0x~JaQv$F%-c6Bf*8NJ=n{T;2O-(ubbS4y7gm98t@>dLe>A(t?yKejc0Qd3 z+%?zDuWXlI>iKK3vCX|7bouoI^j@uc&&ai+5WnnlqBa7 zDBcBJ_CMi@9{)U^MVg8bjE;@HF0g(_R=Ycim)^WJl2(yi!CbHD^1E}lPpd+d~^r_NiPZtX2S3%<10TdpW~KOpuxb;YtjyB(~ZU>xS!8h8!ybqTtxKG(KmSp6Z5L$TiUtL^DK5$G4MvZ~WA zsLG&83B1ye<5@6X=oyFY#&u5Ja9riGwcCSt&zAY?kB^e|r`D!p$w6Ht3nabfTLF`x zTMuU8*OF%UzbrrL%BB~x@@GXN|73DMM~>8d`(KS<7D_x=(Kh|FNf}C)JCjiw4ktcK z-*d+E6OI!si{stj5D>K4i*8I*9Su`sVc8DlXw|!iiV*&ymEK&eLZ6=YO7K_b2_>%gftAZNcT>0Tp*BRlf=> z$6i@ksdYOT+n>!aK3#Pw;W^|8nX}b2zbi#FDF196< zir2;BNmao-l>GN6bYe;VS8aWz$lm=4kLq%$7{PF<;opT{1Pisr5JxkVZ0p6cO3ftD@*(Xgo0AN6?w0qfM! z;0@+e9T%6a&G~N%TKV(3tljlw;vXa zLvUfwS>8#FJ~zI55YTh4{%eQH*yAT(q3Ad{G4T}rCi1fVQQvFU*7iy+yhM?c;~itH z3f@=Pa&mIezyJBt;X687_zU;Schvvs`3@HbTOC~{{aFdn>b{AbV7yAtd~IFCr|7a8 zTY&aMz~eMkmQG?!!v{$b=6S2!eo<}H8J*x@V_fU3d$7ScdMAHTH+x&nkmhxW;NcPh z_z6)wl4;Q54<@lNj5c)SPS2Mw_>j(J;~ms-A(7z_9?g10gNcj#S7eWS`vxlNZL5E8 zTQqaO3R8aw7>ON)0~7MBZiPfFrlK-Q*NyAr<-5+>yfYP(%JTE1r4bcvnpo{uE#xZBle-PM?=RWH%GDPg7-*6Q?@VwGlykNmPtx3YLhQJ`Fqwh^-^`8Yn|huX`FfbNzElW01XG1 zV)=W|zV)4a>$4|$be=f(Aze8U7jZP$yp;cRjK~cIk8f7auf=GVlj`GqjR#7_`(bSV z8J~o@hLQ)&m^R<9zUosHQO9Lu2J*r2n0tcC!+Q3er%VNR)g! z1_Y-5-u=PA35p;zllOtIqGuC{@Z+usgmC?u%!-?ODbm!8XdGzSc<&(0x-HZ=4TwH) z|5MXq=R*23OML4vZe?74MT?0}Mtn{-Tx9&}OPQL6N^E=t3;7P4$G^&)9F}vgak2TE zi7CH&DNKt;E2eAs0dxSsBGaX%WG%H`lcl9dbN(t+MKNFWS#z2gUa1BDz`ENA z5c!cXlMoAWHr>Jt6%ZSsQv*LcpmyVr$u)A9Oqfi1=ZnVRx{?rVOswfG}BS=@+c z%S6;N=VAqek4nbfX#c%|!DE zpr)`3>M;vEkn|@I!ok6Dbv@eMy#t?v>JdP5+1(Mrz=aG92V_mtf&zBy1=m@^ zD&GMg`MvJhy@AG#&UX5hW;)6zwEEjKiA18`=1W=Z3!^)`6oiC@cZ8DuHX0Rrcz7UY z?B0;VPcUiU59!Vb(`=Zv`}XY{JS?tFQ%P39Hyv+g@-&mJdAJhd{Y=vQ9nXm&l~GE`Q%V6Zw9LBb6xHMQ>yqL z&y(GU0#iDQo}QOU;va()gCuD}xsVEN^JCP7g1qO;^YeGf$(gf4n#rgTXc!o%O1on? z-=;dy+``M(<_Xe~D=}YICL!gmvN3i;xTE?|<`G=C*z%B|t*tH7Q0na|FVAH2(czRP zQ|pAzrn#Yw<3=3210~`wku?VFdiNRZX*uVW4kdMKBZyMjTkU!dXa0@K7P&D92WZF5 zpuDfI7e!F)A2bWUEn0|{oI>E=x#K-pUDjk&2&@uJ#5CnZTL~*AWImQx+~zCZ+k;!rkx`ioZLx;Mqm3wSmw0rXH{rmUfodKm7cpOLt8N(A1cCi9?L9rTwEGA3R+5s2% z*~&UcH^iXr)X_UzcD~P5SA@p^X)O9^ZY5v(XJcM{`WM zi}w>=lo>e-srspRF}yNN5AJONdk4rX(sT{AtL##_a<~Q;86TR=>2W2* zelqLX7HcT1eA~nq8M%;GuHv&ZQI-zRW^$5pDF0%{4xp9~5I;QX7 zZ@dR$S@KvZH58~!biQ9rNj@7-`Kqp=zB#f@i)e0X;qmgba0G+a-P415;|6T5GFz9H zmYOJs%Om5*!!$tBYiF}ey@(O@68Stb;ey6RSsueb&iD0yhFS2*6S$!mO91?t&#al( zVS*1g*=D-*Ug<^H0nv#Vrz>u4AT?Ih>oS(hoFXAAZ~AABDQ`9h*-93RZk+^m*hG@> zcKz0bvFNEdV8nrm8kDF*(_x zw6i#9)cB)1e1V1UkxpTq^ieiF+!Qr>91MhRRRlv?S{f7{5%{H7{iMaS&*FceO>}rJ z|9r)R+cl5Kkel!|I! zoSCJIM8NYV0&Z@m%PuII86H0-Kv3}Jwh|hN-ck`n_y-3|mqswa4W_#Yh~IO2#(yZ@ z>-Q-FwTG65=H=`si$d6&At6eDU_|8FonZ1PV|oO;*`TI1cL2fBpxytKDM%wpG@S( zqMWy~ij`I$!>d^6XD}b5ATkB2E;L6zj3ml@8_t-MPrkIMs``QeAOF45t@LdGSylg5 z*`o`yXLoG5!H1}TQ$HsGpH;t!kdBF|_SVhq-_M{Ii~(_+rt-SF6U&8BqE~JRzNOm|IePbKAH3vmqJ=8TjF9qO zssh|0rR0&icY?ipQsSj*jM|Shot(JgMOfsR_&s#I=X+9_xsMVrhP2fMrBQj5{RKzy zR5tG7Jm{I*dGk!CS!npuKN8+Lj*fS!wHI=^I(~@3%ZSvolS`Xz2Ofj?v^YXmW)0kT2mka&ZlUe7K@wza#;W^LW~bv6tH1t$xq$v7e6eA+j!_{y}cgdbbXB^uQRTf>ksVE6EF3iOIDqIn-> z-UTa)7>&`qZOS;d((*`*3vV?m27!r5sm4&1-`~h#&0%19 z;t?QVziw#P?}?~zSioE~QMFq(%po%9Io*lZ$nDL(Ln*d%;#g+uuUrp(*RIeSXht6+ z6Y~!OKv1ruyx%I%PpCDdA6#q?%?^;~TRN!ZuLSD@Jw==oq|6G9s2{{=k;uu9eqLHW zsK#$=s=Nwx;@tQ?^q93ZH_!0WrzMq?{DGYXyriJL-_o`Q8n-4NF|wUb7;DIx`g98t zq+VP%7K1q0-hO93D}wjx*w* zJ1cit=N|0%JiLM*3EZ#bF(xbl0Rcb*1CDm|($jHe2 zfVTnc3R+3P8C&c;7>!79aNLb4%cKNHVq@C(%-7E8u8)9N&4UiX)ExToj19t@|XNdQ)g!CJiE{bjO ztC1R7tkX-<6EOf~k@TwOlwe;$c|Z5-#WYfPa>r$5VR62aUB^IQvvI+JmX>zBV|q#- z9_3wWRv=9R@etA9Nb@3`u;6OGvaY7)Q;%6jYU<2Oe^tq9zGP$RqURo$JDCdc-sdQY z$rwt68}V;=I~*&U$TOpSUB5$nW6XgNZ<*^GEbQ(AnP`BK;0b!{wxptZ5~!-6T%?&l8{u8kkB5<>5JS(n*&QJHp2(l0kz z56kiWx(n+R7;Th0a^2rJEXTtaeBC--rCx~J{|;sN2o56#f`K7pW@hGZ+Nkv1D1c@I zCg4tD^DYnDUcwTyF^58HZFlYWI#`DH+d4i9p@n`)OsqHUA|8l;c6n|!%%k(w^i^nE z4%P*$|GEIU0|20F@%e)L3?KynB9XP_bo`Z?I9WfuoKI2vIqA=AOl}h-%vNV-K?H!- zN$>8hx&(Nrtu1yw^XhZ7(EB6tjF;90+Xr2lqP@RWm*(c?mY4lSx^eNORX{cGWSeqa zT)TcQRR0tIq#zYstSwSgE0zB;J?_*r#g)Lv(ciyA#A>5xRa(~@M992z|Wq9u^^gorB(MtI9) z=H&@{o*yd%uF8|)SmxwrJZ{AJH}<2IA7_?KBu$bIiKY>EAZoC!JGsdhPRk8l5!Z;v zLM=qnxK>xot5diF`kPO8WA{`%JjWtrc2+FgQfL&ISrG4g%&Oslt*ZLxiTa5iK++r> zU%tzd#*_BhoewM;{5?gTpO0ADd2_UtKCM)9&*k(6)2u@!P!hk1yc3w)QR~K{QZzGL z1m*h5XMRLb89Cpqfk99S#kf~e7}dRXUj%~G>v~2&cgywj4#LQi+bL|el>?6xfQysR z7EF5e3cux3N~WOeCcsg``t5#o-;NL|C%58pdUNF|P(&me(AW3&gobHP{LjWG@Y9@t zOI!Wp$6U-~BxF|Mzbxg(;G2O5b2wOeT6VqrfGQ z?k8Mz)mWT)AC;+a&5;nFpLHtj&rxeId*D$M&^}G-ebDoXaT=-z=zbOm!`74bD8T_( zW@Jl0zOM?ur2B9;@v17f#8)diA<=cirgM&@uj3!ZhpvTQu+M-q&2*IAd(i2+F~-yQ zfE*D-WyNpwhXEA_!%ij+RVUlp9sZgKZ30bv9+0lx4dslxx09^wXFd8F_muqz2C~Eio`K5R&-JFb21d zG>^RDkm)2`9(wWsd`ZYuh&as4q9^h)mdxA^OH8~1gG~Z7&q~LPk`iuc4YppQ2>`wb z;#oXz9=zGi%nUSn%_y7$p76OqTh-Fg2=jH@TU0De2*kJMb^q7=84erZWj^NQBd?-*h;^N|3 zTU!B&>+Irkx%?wEh8IUXRmH$y3yzER^%|f|q$9=E6R=3wtVfHa)zln;byZeYc6xdW z4~vM12yi=neSP)s&Rkx<&IW*a5E)nVREYQRCLJ9em@vmD6z=Y4kdu11|668ZVF9e( z9<*y89$xNtu|t@Gjg1|{ODNtvx3J(074t&lHcsR%Z!u$IT13seGhTo_KYH}6#4`QIp^U4WDNt0*KpZ5c( zR!2ppvE)G!9V=^aixEl#*oxRP^e_~#?bBD6N07RSoJOnVsZfUTBO zV@_V)(7?cr_mHB7%s`-}g#`x(2b_J-g5L*vVqXa)wh5Oq#rp%)lKmz2%!N0|HY7^z@2Qfd`lS9b|kkDG_5BhYE>Z_n7+X~PFp z5Jywj=>67<1Gq5URD_|WWk(4sD?Jz29$-=t_yEPp$~qaht-q~7%I{#JsY#5`t#ylR zo6E|{;WB(kg9VhI4h&Q4%y9S$EhLh4Q1gD#CvJ7{f50-NBP}1Mr>BE@2@OFNss-9< zkftn7=PawlfeogizNYEtKFkwS+^9FO1+<2R4?sM^@e~ymAs6MqiDii)E6Kyb0deWG zahiJ|w~~~UblsfjD0$%D4JVVLf&%Kox{|btmX>YU*oS{u`q1l+zy<zO$@XKMxZNk8^!5s(3fh39d+{mMFu z@-sYM&Yx7Go%)-nJ%$w#q2}nAeK&Kq7K{fshi6h$1 z@#j7xz?obA{R6NavAk$u59K#FB5q^W`bLL`s~Q;K$zSgEdn!Em4$&+Wz7U3m+8`_Z zQlR4hXvOeC_UY4B$hd7y*Z08>40?c5>UDKV?REb6ze~`^Sd;nm1X#ru&M#A zJ3T!;z{TMNV$g~!PgmLrfS9+p?chdXVPQe$3Z#v&MJFpAWvmyxy>F0^kO1_WExH=w zH#vR?V+4S^EFs}-I1Bt4CuirgOH@j6D^pWxdFWvQmhCh}Jkd8890Hu_8x0_Uhw&Lr zCH$11K+*!Da7e<$#m0hUZ#j^?ZUkU%z`|+P1ULUXFOuA^sj#dp{YWa{-6Hrz;qmb- z+7;HLoUE*@RmtIIx~i%Y(aWo=^z`(dha2-v--X?u29fTxkQi_vWo76AvO75F2*w<4 zSz%Vz3h+_zD4sOT?KErJI7_B6c1}#_{!|Q~dLktBjg6a|`|G|dQ7mMc0E`3n0RV36 zAa0%pstB;cDl1Q5=ZkC$P)}6Y0n-e6>yLl|4fl@Spl*Ke1R5h=mRN*=4U3Q00VmFI z0r*j4J;fipF}=eD#SMdXZiIA(X(|fL<}!7&Ttl<$1-o_wR8O( z#_1@K=))#}MY3^Fw(%4kDM&|VXJ;>m`}^P8b*aEprmyih2}dh}Dmlnh>)em>v$IzL zmk-7l%r@-kyV=vNzHW1^I8}Ta#8Jo<~}|%EG-S!y1S~{Adv|GQv>Q0A0Hn=yYw{?)Aeox#TcXwN33}f@SIQ@0NVhBeL85A z4Zl+~C&ReF6x{`6wRx`3eQ4Kq99qE3H;L{hC-0tkn42#_Dly8}jkbpl#GOE0hvzF3 zLh+)^nv7}vZ{9d))PsV9Ssy(DUVC^{#60jqxDl45qtRU@LsGSQ{)|S>*T#6^pf7Spw3&tq$a&ZL)2T#_#Elx~KJU)BK z^vc<})W-)M5fdMe6>@A59ud(&2;6Mh4p`>sOOAjsC(L^O{P}$+_6<|Oy_*8FKUGO* zz~=TlCZxEcB3xSTP*guRFYn^g5)3YJ-kM72F$1~C-pP5HsHW-L?HQC&iE5LMJB36w zwY4Z>YHATmtdE^n)f-=N3X2fd^1 z$ie}^8cd>zayr=S6J@wU=j7yU*2HIjb5}a7142h(sfB>;#h!acM=8>Quyg z&^dS%R$YN`*E>@q|Ni;ozbY}F%!QPHDNo4$252GkW#F62x^X86Gbrlu!YHD=Xn^rupr#Y$WE05X(z;=27H1=b2OIzC>CkxT@STzm{ z0jUDuQZP7c=Di&H^JgBCcW?L1PMLT15DkEvklNzm=0Oe9!*4=w!z)-|fvn))QGqc?mW`+g*v>^omFaxw zNlsarp~D6S6+9vpl7-(Z{dGloQHk`>Yd8l?7Px!w?zar(L>ri80sAWaUmP6vEBDi&J_EzsI)4i+q^G3N z(9`>RTpzE);Shs8a-+4(TwPrSzzX4+NfmW0x1}o9Gc7vud-njia(;gPmCv=#+YwAv zb~ZC&d9*m=E9JDbv^3Df=8L=UF((wszrDwd*nU%kjzzq@wgyu3uYF5_d6LBPx96=$ zo?5iFE$ywa9Ye$yZWaRFqRlKg|Du%J3jlx<4F>(Udxm2CO$2Ohxcf%bO6)Ul-@NfK zyqVJgBPFgIRXn5tupXtZsHGKs$qLVF!7^D@GUL&6ICRFancLxx%9R4PJm9Ixm&j01 z#QIOks@(nD8||R}pw@8_VwXHm0<*V=>tqP0Xza<4PMZ_4Y!jtcB!j*zI4~B2_T&Z_ zPc^2Y3zgV8#k}4m*EA-~LLzs%ps(LxH=6Yp1`?wqXiXyEX!`Tud!3 zCok`IamoWH8pUty;(4?H$hfF;3#v#;qQ=%LetQyzMu{S(p`no|@@`o`wUrhu%zkoK zPvfs&|DerVv1nu}H96THW^f3||NEGlo<3kN46XrW=ZwsAyqoadus?X<51bY z5Qb-i{x8HMhRZzUx#}P|X5i)_?$6Fhz+Y-=X`#M1+#`OS*__y6RQT!X=E1?iwRKgA zdz?<5T~{YkdzU#1f;2e65$}KepxyYXGWski@4Zc?YLyzkMJ5bV!LOpr@qKM+$yCx* zl$#sWx5PJn@7hBF+$I#%_=@YCpjHLV%&EC?x5f1%?yn9%yd}`lyl8coSO_)~>>#Sp zjI^|eX;*2IzOZc2%)=1do0Ec^xy0@_8sOUf6`>~e@FA=KUA7~*u_b4b=Rz<)hCra{ zA5=W9X^2gJcn@DZ4EXWGSV<}5*=ZQ~R3(gaX-Rx90N0q?wG7_E{`RlVPE5XYUTQbv z(yyNL;0@7DhC+B?amhP2#8j3%5aL>uECTK>kXK;Mr>}kSeGQH-b=|Y=y*(C2#`|7X zu^dCsVU*1SQ&SYQIt(~5R(J>sxigZ}GcpKm8zH9axQUvaGcUPD&K%lCyGQjKy#}Tt zUBgpjxiVt6QgU+EmX{fGf|tOt3cs@!*fKCs1cO5$Bn2wN*sy^YYZ>m2HlhI-K`mw< z@QG`#af$Kq*AAeP)`Lzku^m(CkOT|7oCW{(pCOU^9r}ws?H4b?@W2^_G_Uur*lC!` zkEgeZZ!}yJAZ;cXJD3aog?*Gnf++Zlz^8@#R`~Phb%;3g>L>SniKur@P7Xz$np=f# zeq7v3W8)uReaY|Ml~SU>zI6+Ka#XRcOwKYB{)+6e5gHS}<6_VZ`R&QjTZ&tqU0v_q zy#oTl%Y6ZCUiDPH=5KWZS9gEL+Sl4ENC@7DJ3PVTcnLYY=ewq{Rcf65PMyZ!x3=4qxjw(=BeoE=@ADyFZVx?E02zhBp{Ih z?x*2f8=Fli7AZb^hA;%ld@9G|$6<8ZNm<$1G+4L2%c3-8dC|YGBs?)u6cWwgy}}@sh}11GUGU%s|NiCW?Dr<_?*J(6+s!BR(D;2+TMA z>hW4WrZ7@dQ&UnVz@QokC*aLz0D%K3YjZ?Wdivt}`lE{4yOsnHE^|OW@QIxvR2t=S zw;^7K5P&$))XM4%wxwLXFej&~q9PLD8g#LPuyQ_ndxPH&;TQG2dtERel0wPU)Kp*p zJ5-%Ab3I{(9%5#7wR7p3<_n&TG2}{^$|LTmXluU#{~Fp8V66;xMZ7!R-#Iu44hRSc z4b@uJI}F0d$4^R5er08)KvT$c{jaWecHl@3!jb3m^gN{`Et!{_`-}A-a5f;l3b^!= z@^XVhBcE=^hn*1QtOD;760aL0g|q;~gY-g2M+Z2~8z@`CH#Ra+h z`SVJrmGH>ONGKYA{@ek9f8>RzwxVJ%6gYtGQ=w}E4*2QehM-Zi|8*AC)buejZeS0r zHSBm8U4%!@Z86jE0oE&EvlbG)E=Rk^A>M`sZ((TI54d^o_91sXaQh>KD!F-i5JSQ; zZC;#5LcPew z+zkHZQct`&d@o?!;RC_C0#i1PpRGYjvFH}|bwU>(ed!A2M`6!sjumwxObkZINh?X^ IOBj0pAGx?UBLDyZ literal 0 HcmV?d00001 diff --git a/docs/images/nd_spl_3.png b/docs/images/nd_spl_3.png new file mode 100644 index 0000000000000000000000000000000000000000..e4e95b4e72d63c3216808c8684c42519705372b5 GIT binary patch literal 26547 zcmYIw1yojDu=Yy{(hbtm4N5mimvl=hoznS&gmg(scS%SKk`mG>-QC>{|Mv6V`@0s) zwGQW;Ju`de+0VowRQatm8Zr?w1Oh>mm61?|Kwuug4-*nBxT4oPcmV!;{!vE92?9aE z`TGL{Nl7DwKqw%x5@PD^Y5NNv6zaAFkmCRq!dR0?O4}-l*>58)bgFq!5Ba&&OZ|%< zUvqL4OY%Q!mc2n9W&B`iD)FPTi}Vla*~2Sntf|?6r@3Hj>uaa2WU92gfuF-d(Wr5~ z@bH$eiqwlUGRQ(Du-zz-D7m-m$iHx_GhWR?vt9Dn(ggvy*fKd35lfC)YPP;2y&5(gzjFGs-Pe# zwesKW5mLDm8FZ?us@hb%{QT&|+ym|H{tC3=r(R;1SfU=}U(^YMB&DUv`5f1KynU=Fl4_BOQOQ$Hz~ybW03LmP`W@u zLITY4-cmI^{*_kdQao+B!N5&0cj`S)Y_&v}RO(=r%Jo{Z>*!{Ne=y0z#+& z3H%Cj!}oAZl&Cc_GP0?1!>g++ask)U`i>R+1UdcskH$VeJ`xh}x+kHSBw+L4-{g*# zmX?~Bm{e6&QPc0hee&}Yb9cWELL=tn;yPMtWjp+c8A!^xBptH)_;9z@71`)^lwo3e z?^&e2`^yim@NN3;#>VGcA&Snx&AX8672X!djXpGDZd_bk8RtKU5IDF1@c#Dp_Qhte zF$+$I-~ft9(KD#Tz(*2t4l`7Up!>0C3T=;=m>7^37a1Yp4l73t{65Wsb2jPKhi4dM zBxGc)in8R8cl!F`;^Kn?1Mfe4_)Uu9e~$pR(Z+%^B{_L!xhN?JFtCrmYcF$ zwzi}=I3LW+g7Doa-v@&yWXiNFDOp%p1O?yC`-OBr++A%ArhZLNXE6=`irMRbDon|L zb+nWz;_IWSsVP6?2{xITo_@K0PQV`tqwg*xWW<6~QE@OLA_Bif4KHe~+3RYt!Z@I$ zq@@0Xm@a+%6_$!BPDs&+hF+`RmJ$)Y<)0he793kxKO zrKKgEQbv>ajiZJJUh3M_;Q|vCRrUMdVGE6J0=&FHLV6~qtDUjnVfG#qMGF>3> z`R&DCkcS8(WAW$$8gTX5azhc}rSyvx2&AdGd8$;0>X#G>HFodR)YQVl1K4PPA{;T^ zHy-3z_^2H}?3WBwRN9`D7Z(@V44NRH*bG}xNq8$(U|j$Eci)Od+}74+^4bGLl|Sj^ z>}*P}Ne+VurQBb0IbI*HkP5iQ^qM>dzki)#n`EfJAIFU%nA`1-+N*l1-{u24BTwGi#UxLk}qM}MQhmj}+|6ax=AlO=8r#_^o zP6`MR*Vay|&lSxhDF(k;-`#C+-cePb-+E!qHC}Gmid3bDMwtM%6<$_K>RH+H`WV=w z;S3@9fyPD5olij1Uq*z_AoM$Dy`7!ZTwEISDinWE0fHIgY~W#Hegr2nfBKLQy`8}waFI9w55iD>LBX_(kiO+Bt(^TTCtG8BhSS5L1nKi|XMJx$P~ z>YGEohpnC6_vq;1i@hlrqs1nVbOAT+*ROY{%9jRHd2XJhFePNbvG6jisHg~lqOY&7 zh=_<-B%jyizGv#|XNZW^voeIl#7p%K0fYQzKu~b@%p!c1l9F;eoX3)x*45Q5(u1QE z{JY@_A3s*q)$t6o`JQbK9?aF%*3|4D9MCDIr~_jGIt2o}bVc6t>hNH7^-rl=@9=QB zzNN03T8Gu=t~2Xa->2`z#eJRCm03!%M#6YGHhP+NzC>%?OkB4Lx7t5)re94>wxMpTH`p zF%$_85RaE5JI=PYwh$Rv*;lV#jgF3fyR*^Moc!suMby$aK0ZD=TB&2t`Mxs*#Ny#X zaL$}axCv$L}&Cnwv!*jU(q@gQhRGEGM4=H`ZehpIgQM34+H*eW`V z7$*APy)+xi62nE7oL7+%WK6KKvtvD^$FvOk8)yf!wRfi*;q|#O?%)YpTH4hp*e_-P z7+^UpMzU&ZI31*4p3vj=Wr+oJk|IzJ{(YO2xcC}4Cs(81tkO_ZJ0H&L{9mAJH#3G- z1ZFZ3J|lcKG2u^gwx#ro-D0Q=4lO?~?|gqoO&RN@jQC&NSzld^O-iCqs#XUPS99!nQ%Q+lw_18Io=)O3L%|bN6UuF?K}j7n8~q0@)iG3rOWLUKCuEu6IF6OIyA^ z5s79tMCY;hPazr>FUZeFBNtBl(8>qDf(Y=40*#21o|7}a{H4zWECIpD$fp=g_hU#^ z)$+d1^(n%&NIKGd7@o2|T1PO-#~zT6$i~q=|0oJ?i3DISM%&B(Ok=gx5rIrNutEPq zIyxws!+3!3u>+@~Q-Yom*6%P9;6+)scrfz+KJ)9E<)qaULBvPYW0+QWBL`CKJ*mzC)_STWUe|xW^KrwB}p&NZ<{6760CaVfN2zBK+_UZ_x;}MmW3B`WN z(I>aSm{2NP`S$8)c$fmI{jZ(coLpTm&(CdK7e$iZyN4VTf`~3ECic)x_coA8OH`xwMC;+YZN=YeZ!)WNV5MyA z4-na^gvoyCjVTQ(;ja}91_mR`o4X|iS_z5!oBCgIfLn6TE0c)@!T^t=p9h@p{vKq* zT?Z%`LIF9Hzz;mjEHecCSoB2Is!z1a3^6``-Vfq>?GL@GQFLh-O=9VjTol!|8Qi$j&5{E z7}N**hFV(K&C%%@p8J?U0BHqCdfIaTTrB1JLfvO1lw^fPaImd(1pg)tp%~ejr+qNX zQT?W(qCzDdh1<1^^oL#@kj)T{4Z|uESq}V5Ol23Yo*pF2eQ^F~2)LoMEJ;W_9|Sxf z@Gd_;KllTom^e5NbH@o^5qc!Ep>?Z)u|nfbg+qaRcmxD+pk^auP`)n$CpbaW8c5Pw zT3Q0FMPgD>xVW-P=hp{eYOSn96B}Qs%03_2p`b&Yoh6C>%|6i6LoY16`0LlFk~S;} zQGmvQQE##%NcdheF`;O1?CSn>%x`G$1JMkd8=8 zckl1pII~JfU;qch{YWx3HYVV9NK$MH2Z@y8;NkJTzkbyqMu3F^;sz^Q^;=L-5J*U4 zn|}n`^F~F#;4u4>C)=OMg5$<4gYp@ia+L3lj4%law|95L*6Qm&U=@Oz1_UB^8$t+< zw-6ac#mKTUTL4npq2;Dwk&zb{7eYSw?k+C1e=|*1^(`?8nC=rG5?~;BcnPmQbfJ<7 z)+2oX5E>a-TUp6KPp_<^(%au}z0}hDx|?hii4xv|1%}eunO}&9r%pT=9T^!J@^NpH zg^DVdRllLWp&?9wa}Feuzo6bMcZ%sr*gp2%pbo5K@Qi$^hng>8ZrUZI>dwMJ^Ex+`xmzFZC zA4>n{AK67L+a=$=ak-#cPabV-(ELUq*Zc!YDCSOUZ-5CHuc&eY!O84?fqZLIq1xKm z*r;x>X{z5785+8xJ@lTP5GWHC7FIML9Revnqww!Ec`Yv|2dsdKipon1z4mR}x;fhd zTN+>G@=*E~b)FguiHVCF8yow;PWbcZPt<4P@GCD_UVzFBteToy+}E#&5Pp9C!IYXR zJ7a`TAeCdNPbKI+0ow?evnqZ3rjGzf#c6108X6kPbZfCy-I4r({fw4*9}nhuz-jR} z@6%+Ip;iS&ojlpNfx$tV2RXJ#DPj(@0RR+L)zu!n!QYCDt*ospd*LFRc7|Xb9v-H0+s1ZEC6vyhKWP=K{3OR>1TNUZS^Fl_oG^&xT|GOHfr3KR ze^#TaJt`sscU+j!!bOBquxKdYIBF0XngI+A#0NAGi%JSO!h1s67~{`~-6!Cpsp*DT zz%x^Ga}=vH=9&g_EHdzJvJA8q|ma^>=ZIiLFgd@$m6Am6a<% zy~1y*KpBYh3^=R%%OZbaX>~@Iu>wVa4M07RXw$wO72)P~?Tw{*{rWZKG(RDzFtq{r zu_W^&IR7b@ugYULPl$&H2o}xdJ~=V*-r?bp$!h@Y5|f5V;9i(FE@XKx0#!z~fnA&A}1?C}d@2b8Hxc`yN6d**C)NEOxqwNP4oe z&x4G?Z-}{VvPLcB6Pc5_tP`_X^SQ&Maz6(LgE|{XGJHYgqNAgOL=Z$pyO;E(G6#8} zFW_VEuyn*S(P-m9MOH`85EQY%ks#!>sQ?mPfFP-DB~YgKXPZ*FROCof;JxK?Wkie{W5B1<4| zot~}-B_T62GeBcWhT#c=MlEUp;|9zQlwjlIaWCm&!@`hJJBWf_$^l0|hTDJ!l4j@~ zB8h8iCI$sPyTc$Ur6waF(0ccdgn)qH#f#+3OvAg<@Nkqpay0U(k&*4C)>fbjc*$Km zR$Q-wk`fPKbqRQ98?VL(5LeLp?quRZlt;=5P(|{+i6S+xSlNYygfgZOKr++Q*9Wqg zQ<8|REJu-cxcI9tU0B&!hFZqP+5Y~pVpYU@)F5A=tMq*a3t~D2qijuL*V+yPidFRO z8v_%QnyMOkrhx(Lq3B~yR#sMdx#8b(xONEZ!b?TKmtVdYYyA!o z&&9>XL3$Jk;4N4^CtnNb0ZK}(`BY|Nt@r_LMWv-6N^#vNB4ZK~G&D77m(gLsHdF+k z`h-hO`}>rE5_C~rpL!?JAyV(5!5Sel}fZF@>XS9i7r z>g6Eah6nVEjm2Ii7y)2n43?0LiTC$m$>CZzj#fVKoCXrZD=*qVr!LAe4F81YF-@i3 zNOlP#{Yz3SQy>qv`bhb|%SQiQZlw|TFierv1k%4EKCO@*a249mcJvMZeA?n1Tkr3q zA86atLH26yq5?!F3r^pisth&|IAlI1Sg8HSo-szkogPK}Reoe3{<$?)(@P95_*S>C zXzGXdD$0vr8L@Ro$|diE1oyF;^fA1O=9e6^n%HQl8?p@LDTz)i=zgmCh;BUMgHW)f z7)5w!pw0nr=(L~eAY;6$gQq`|7nB9s$Hs#<$jbdEuwg@X6Wa`RbC%&(c%ZYA>g<1_ zse+P>EDb5bgKq2u5be3uf3J{dioc?X9mhSlmVGBqc1d0IpAc*bdF#Wal3!AQde~Fo z?ojK0cfSC4M{weSyWju4yYk=Nia-%URn3ud`8U8qbfN!@{LSyQ#U!Q$n7$WSC?oKn zE%M>|?yyA}frY3WfF{_`BUa*~dA1$pMLPa(%s`X#znVDz(@f>9XdSc-Rc0~2BvoN&$=N5FpS$=0B0B4bP12)d9|Ifxsob*TG>fs>9VsC*E zwF7@mA<_nXg`edISZ$mKtk&?)6rI5EAvNnDU=v4Tf}0aZ{!^L`HvUR7n}*DOkqB7w(|Vi|LiUaSSE}(!h=-pufH#-`S*elU>upEtfivF86X$_ z^nY>@fGRn{GraTEKktF_Ouau^FojFFlLQXmCW`P-P_zE`1=IgtfRxoV%%(nKUj7-V zi=lW!{JHO;9TYSSmBli1@fZIpJOA&cfJ1l$Bj1u_gQv6qJsblbrcxi-dS=zrf%VV( zQGhcJ1|i?RdxPdt?>Bq#F|h9e7Qo$EmcQ_ovN#zlc$6gDvDGO1ZZq}&`0CF01Lbe( zlOmwAu`9gzO|1}}lpg}xJI|68HhJHI^CBZ70~0o$yY|3SnH=R(T|)!3FVg`R1Na2s z(IHrkJjj@n2o2KW`;-9=-m7nbqf|Z`8yQjD{vwLS^6CXG&B{Yv>bPFOFIUlm(D>6c zQd8%$#Ss}w4wiW{5)&^uU%%$2ucK!d^i_*l%qz|Nuv4MNCajGOxm5#l7zfv15tY0Pk>CND1s zLvdz3*^QCO$;lNoi%x1|`@wXg8?<6m8RCD1vL20D)Bth;uz9|bAn~gfCMl`T-S~BA z6a>?iuIBPfw)g?asJoR2?JbN(%Jg_nGBA=9%s|2fIsib{@o`sBO$OD7IUYMH!EiFG znQoWs?TlyXTobdkS{uctv_nSgxPOFwa%p`Y-5*y-V-nJLw>hCf)=w~S+Ry1lL<}V* z5%8{lssrx(<;y6jKn8_IBrL71t_nl%(-IT$Ngp^{DKanAhskKW79z%k^YeXVjD>RY zzi~Mil~qrUxPY$ z!3z`HFj$8VWKCXr&5473L0&IDU41` zm@eEf+u$P8aB@0#N0B}}yGRzcwodMwU~M*#CB;ppC03NN_`_4{+KLc--4u? zrhFT>&4V@x zsTAZ}K-Jf7FSNev&#`|vtQQOLB67y>YBbVFaEQNZWW6JJq(9?d-4L`*Q6J*@oZZmC z4;(~HY(?%k0kjB9N^0F)lK3iyvsi-`WE=$_$yZVD<+}&ykGH9en2Uzo>N@tqw824e+E(leR4)6>%|sdb>YGU%Mb z;nL0tU5N50Z7s?>9?I1dew41^Q&P;2gxOzKa8T=&gJ}J>E|9+}%I-Nh}pYJs2 z6}}3G17|MiLAc}eYiel7ZZ6UG&22nwMUzs#Dh4UP_iE?K=kTe9p?9T4^z-d@rZhs^ zr@b6>7EFUz8|a2MP4M%J+bv-^0z{tHlkai-@?+^J29& z=7ofY>Q}itpbgDepNJZUKeqccYK1;0b}Nswrl~ew>#`h6%U_OXj&Srq>T$X~W3Y)&7nO;5EzJqD4n^&lEMNL~v0J-Q@&kkXl}ZJU z_oY?!)8#>i-VYo69{UQwaspuC%u_ti1AU00-QvH^PTJGE4kzWb}VVj6pMKn5YpNjs%9mDtJ=}=aly?sszoCt z*beruJFFcC!B2IrPjP#rly7e#S>fp_zj<67F94cp7U)EXOe#>Ve;ekfpG z9W9Xa_{{qfUk;}62X9&b@b~99Zj82PnXBiepfKu+cmYl_U%{_;4kT_)sVB522PaO! z2vy3`6{|Uh?#u>kMVVO99wM}~W_`HbrcZX{TxQ5mVo1nv-S$E68r<9HR%P9*zX!`m zi;;IbMEgE96M1ipX5*07O6Y%C#~v~w`APKg(Lvi?BJjM7c^wJdqd|tX@?|UP9sYHK zo)2yUF6+W|^!FB5QRPp0TK69GS0Jo>9`0r$Nva$lUh`p#afn(v^j3M4lreiE!o#25 za9FvfBX&C|4U@)^k|BI-M^7>FXxERAK^zI$TgiWPzV-L*P^d;4y}+CFPa zOzG1Fe5HjQM`ltIp}2TQq`;KBje+bmxXX6#n?5REN4$9iA)o(n!@IN1J5Y}!8){ujNz=8B6Zev2P0h*luRh){nSM8^^1fb^R?PR=p$7tR zR%ZxpMWs-P$!6wPlsX?*-5mBO+AggyfKeJIT3Xj*qf|YD!_dcD7K|`CYUO zX@|c9IIYuP@1jWsemG4HebRe`XWp(RAb5Cx^do?w_ghoGkKLItC|SPZN) z5nQmIB=;1Tv)eCbxdQFl#jdVR<9*xEx3|n-GYlkXn!LVFm*NSNoVMUnLlZZ`L9hAn zcdt8{ z?t*#{@!{UbAE#j591uZEQDe^FWZqn49RKfx5G8~#2yI(rdGfOtHt5)h>X$()mR!0S zqEtVMvq9LC2Qri|@hpIrY7e~QVg?wfCtoI%MH z6gHNemiH@GsI=$!<>*Pe4}xcbT69n#rz^|HXuyIWo4r=i1KGO0qaKPowV1VO4n^U+ zj3eTT-_I>P9pJlBc+1Mk@vnHje{$%3WF?yTkv^M|D-eW6-nG%aLA!N#dD&qTEP#Cw z_3rT|^YT_D)@4Jkq8@H#^c8t#Arp6B=LGED5{?oca zzb*36ue+$j3+fyXQsof1ZifZ*2I*mv_^i9?lz!{FIy1fW58b?Vj27N40J0xS_XT(4 zZz1gFi(zzFU;g%j{f_;ma?7-To^FS6t)feW@LO39lX4laHCzCtq|dg7mNqNJ`(lM5 zG3zpg7#U734hbVD>@7YmiO|7(-W|AK*wK<&-wjp@2BoOZpp#Y)V_S#V+5H9V)HE%# zO~%GUS2m$cnr}0AZS1GhKSgpQpqR4{Z1H!u`ml_bZ3eysQ)1&MV_I*6!UQmlb$;xS zOKTy9TF>=1P~%@*eu?=ZLmgGWP4L*wMysYeO+MMan>+I^HZIw?GoPG<;MJoGt$m?b{bgesFAZD|Tec(evcblzrCus=gfp+8E;Gp8_ z_ZSxaAI{xvWvZFyFh(OIe(T(FuhRI*CNlvcGcy{|v6?*Q%&&^IGq+V&QF$pQY2u_d z_TZE#VMR-%8w3u z_9PB#8AJ_5@{bmsDQJ=x;#(iReV_aS(BD?m(Cln&_-q|~2E!~U1mzc3B6QR4fx&OG z$JI6DmH!kz-Rr1pb`2vT5|j(Nt`}vkVsWE(2nyA)Wz?$A`+ZE*in(KWpvnzuo9maJ ziCU-}Q}ZvlC+03SY_SW?qlW4Vv#c@I22Ew6LkL~=-bHFvXP!QBEp@Xi=;-hlMo!Y!5$fxd~}C7JWat_DxLVIHVbD`iUE3>Z)zit;vI zt=_DCWs4AS@wsD&9NrtI=KHCC_LXGOcR4AuJgXfCo1|Mvv#iDG#IGLCxG+eky2E-7 zoO``tCXj7&KBL7}4m>N-Lk92+uOIxYq&qBOW zd#)otFWFbg8tOH4M6agLb;K?}L_+fQsOEwD54B6TZ-0ubdOUvCLbaH*frn zx5)149#$zsch<6VOQN5OUq0&kfGjBZC9HPWS$G>g!&BQ6FQs^FoF3=stJ@tl2gLTa zOWQeff#Ef$7;-$3;^Go`Y^B^-5$Yr1hPBBK;@ar*-5Dkxkzmb%e)9ckP~Szq%5NNY<_Cg*nxy<36{Ev%&o;y6 zy6v(4f%ZIkx7=Cjq*h;k!3-N*JUnnVff-Z#F>U*^B@gnB3NZKm*m#L)e*^=eQ`(nT z^x1+^YD;Nej&-lke5ca#bm#V@j#|6%NSj)*u-fVAN4eH&&31?@5KtnkH|;lTH&|#U z!fkTd^!AauzS*0qvE8#7V412Pfc>OaCq6vi{OUtzk`xAO;fK#@lPf?~=RcDxbkv6z zN0uo~_5xqd`uU!mO1_rT^`2bc#oeUy+m6Ol2}EPm5Wz!G1kJizJm1E~*`F^vmNwPr zg*g=StXi1EH`AP z3Ck3FpV8%UaCzD4salp!1ZN&WylZ4Ox}MUonzJhBNplB-hl@8pSHj~xUse7)d0E5_ z_my>$tUZI2zCejJrz88H~@iDG&@e(2$&Sm7~srKWaW-qM!n}&}$+P}@c zTtYq-Di}6_&a~L-;nC2<(bC>U8??Be@@G7J0@LeWPdeFhc8=v%Z7C+UcVKp+^276n zsl772Iv%zSp)1?PYM-nL3QaK0>}s|9z>LUQi#8u;9!;4kT|ut=R?~IsR>)ReeTE$0 z@rtpmV)@!~$Tw8+g|NxGe{D5V>ebkvKT^@;ZNqr_{8CHRrcZyk+> zW}0cBNJ@_FzUJl1diRQ}mNW6Y!N5t`(UOn03=_XYM!Cy?equra(f~{@OGZ%y+2CeN+6upVd^CS)@7xG2}8n4(FPRRX;4rWow@bE6;lRN+c5>rGI zk<18_YnYnoukKcefAD4W_>#u?tMlED&;gL~rCC5J_Pws4A9*5@JV5&86DD|Kzn1f! zik0+fjD~&1_P$1zRgv_Fy{p$4^~krdwn*p2UDN^kar0!MmPiKT=+r?{T5Ny5kJ~>}h(o3%vP~$&k;porVX==6j4GZc8nwND)KGI5FnS3T@y;BeT z0LI`-?^M2x2Qg*)8cw01p>c6nF55bjbjM*Eq7mKG=`=yl+MBctb)L_Z&;PMn9@MMr zB$ZqWN!NydrPEtpb%3Q1yY)ghL=OqUx(WbrBG{kA7GD5YtwD^Bn~T2L}N=;n(Lt z1D?9|y{mi0cD=L3f(FQgLWcD;r96ubJS*7brI2h3ZdGHvlP zCzn#DC=R2cHVQ|EbvmSdqIpC#z5@HQVIDXh;SAZnc_Jo7?w1oW^{wWvHhP`>8}#c+ zCd;2Kqdt3}3)Y}vI@~z^(DG+ULP41Y!y6Ev)X#AuiIN^2EAEz zNz3ui`RPzjzF{gCCP%IMz$h~GYbU|^@8ooPx0;4uuG$$>-VL>StRCi3N6We zwA8g3V*VcC`x6}4%Rj%^nQ8bm3In)B0=#dMgdOiIX4#gS6*!ME6`|l9Ys4-7jFad||x0wYcQUR;1o)3|VtYSyxz5M>4rKCzB@;%3}8{s7$*h$A|Fm zhX7Nl(HyNg*e@XG=cqb{o0}%4Cf>W2?^R^G$wZpC25TS9GsojQIeZX#(FNSviVYH* ze33RXNJ2S{mSe!g3G5=lEhoR{wSA3`sr__;xf~+sr*n`lakn#v-IKh>dl8S zGBfH}Z_(|1NxMjOzw)J@*!YV*=Vh!qIx{$4N?f{2<;IiyCy(~&cTycuty?gDJgh7! zy?1u$3;b@aAJfajmEM5QX1Wvze{WHQ$%WmfpKCvvO|lybDiQLz5<=!%T-}CNbFMX>-r*R-e7qMQFeU9kdW#E?nMx|IEpJ5(im^ zs>WW#U{S9?fYw}#{S&_HIeJwWy-#$EjL$_h!O<(}K^i zI0W;A0!$92sc7n6zPcD|H;ugi;u!66^rpj_of;D2i)BBaeuY8-JmKj;#O08CW* z^hayUy+4i%Hms_~)vIecUViJ?HD%{X$}_Znl|idf<+vYy{lc!PQ1E3hThEqxc!EO* zU7nBJ3BOB4LQ|7UA?u?w>*2}F2DHoil(6N{$xdBny7t$I`~_JJY3J0#Cu%;>@5G0Z z|41(4JHFRrzi4=RltXnFg^=DV*S+mZ6-<VSdFHq18BR2=1AbXgeFts(R%LqJ;f<0!cX`P} zP+Zl6F7mAE)YQY>pd`qpuW|Ei!;6*kg?CjcKDnp!O4JPZ4UMJI4HMEWG+I0KTGw6K zeD%0U?C6CRwdkq+(PS{5fKHS$i)wHbjzfn0`OmQSdm=WJ1Us+Do~>a3@Km!uEa?t0 zJ4>AkSsff4R-`hBt{5-XgB*)Vy@!w;Pv2Bz7`*EVAFnf&sVt_3xOyUq=bdl z2T;Icznp^a{{6eweg=mYQskwd!fO{eudZ(6IHMrnVKQm|d-S?xmnAalyKotW;k?Rt z_dU%QV(mKmL^I7*bg_G@kL^&+XPi54jKUvwYtPA9U(ZZQwsG!m)9geg0xe#^jeJ6cq3Yg5%J<=hDSQ+Q*^60-zK z8wx`YCY!yQ5mVEzUhuwU>Up?+`Lk4Af~S6SgS$mmUdK|_O749=MGDONrirD+Y5iT` zaZinv_gQ4YNT1wT(N3}l3I8V)IaeH}tT5ku_q!dq5)E;2og0t%@=S+k;z`+|GT`g1 zj!dD=$Lr!ujinaz7I~j=4v9+Dk}Ucs0)vHP2imxs6?xsRI%l!wFA*dw*sVbYQioT9 ztQe0v7aVi~JX{Qse&MTARiFb|lXp@U>F9JSoDa3*LO8SMSekrv87Y~1b_VMUT$A!JN>Et-c)uId6HfMkzB4XO@u+xOn zTNY|QB0+chOe!RCDfF$#UTtBY{`;f$otOOt^WL=QJ`CrXB2fExwNF=>p3zh6$r)R3 zGnz=-jo9z)3=DmoeN8WhXU*lA`nHbdPMCM}WhHMsTXVa`Qytu#)y%O@c4w;TwgPr} z3B1j+yGgb_qqiG82XlDj(jr8MoaatmTb@e!K5vs-tu$TjG-tQQC9OAwm6or5Bt+Q1 zu$?=teO|@_!ipMxh->~jY%8?pWwFCJS=}FfSXaBp4Jx{(bY<{D)yM@m08F~2*Cw^@!%52MwiIE;Z0e5l^nWUuC zjLqr!dE;W4lN+RBN5o`%-LH;yelO4b*V$aY?_E_X-4H_!)f*G<)zKy(Kt8OI$sX;o?VD2x zKUzM)h<|P<3p3ub(-VwlzBiB54akR8pglYNjN>BEf=14@b2<3y^psq%Ro~8T^7{Fj zhtJQLnBMiIEBl`!!n+IBEVZt07Fg1$>9(5RANZooE28bSqQv!{qd!j(oc?i%75wt@ zE7|ADL~3Z?t?QGd!UD61M@704Qk~jk^iq+KBGFQdVg#7~R29kN+@l#~%b1-(UGpX( zPaM2+TiO(-vL&7o;p5y-dHoZOX$MY}c!L|ta;x9u8kJes=)o3mF*OHcG=F{alW%^~ z1F~#7Um_-ci^4_*A>_9?-h7(mX&Cl0^W3Zs*(zloe(hcJVEVMA7*0O#kyol z7~prJ)lyRfGX^IPP4-zNcg*uKMv?gtQE_pq)}oL*Rd?srRAcCEe^1Xk{%8_qYhg(2 zV}4eiw$_6Zva1~2{=#P}NVoFNoivd*-BRjhcg_R#h=NJ9KtM7FU3?|EBy1i9m0a=hiFxN&le#9aZg48=~1jAlFH^Dq@w3)pE%%mKd4%49|;8OUFmXE$ZVaosxcQk{9HFf`MlkE`@5c4+8v1yO_MWyQUWeY@Kbs)!&{e@-bO7M1b zW5+%AHy#lwx6qVb6)g?h-0>#ad5lt6Nc=9fTmEDj7|Zb;c?G%lGnVtUA1o>@f`@E4 ze=pFg>rLZ#&_%Ru?knyin+(d`?@D-y50eGUq_z5uV+~A81f!C+cc6S$R8l&U zQ~*uso-e##b8%J;IFbh{wqA ztt4MT(fL$q$b>@t#Y*m~^yl^^q0OE?If!U^789f6wEv0IEP2DatzOs`Y;* z<55Anmkuu`n`}QY`98u!ezy9uf)q4Cqy&<+yEW`7@>(-o@pd@#S8v_DAetfzEItkd zd>-sM{Joj%sdU4gIyVUGir?&LGk~<_0wM~WDpD@Be(Dw4+2j9KR}}wMS0e2SdZZ?X z<@mr7qSV-sjy_6(+FPI_LP-sH*Rz~iK4>1b~K#I#l#c~^Qp8bF-n~J6CU2)}$kdhVOi8Z0PA3{yaVYtgx%nPz8U8M3UyWCk?45LYHUSTF-F%2W}6} zDu=P7xYpdYcrQ01o0d6IS64Grn&z!1^X-WVkyhy4Y;&W`)icX6DKrS<-a5222ZU%U z-_^GZ=G_*w6K*@xmvOxXT|%uJP6o*_v5ynQUeO|kX5L%}^O915ZmK^tRA4@Z@jJyh zqQZURHhB#x`d0I;Zh^I=eJm)v%Lx?q2!&IYgsMK9&yHI%kTZ~A$loN*i%BcxE*$MG ze#=yY`Bl@H{4wB8Pfvr(bHk7{ld%H%#fuksrUpMeu8(OZ7AISb50WVMZ4Tb&@Y*At z-EL0l7wq6b2oSra2oH-5|=a(M~W`CglzsAn{k?Qx2{|6=EO-WK_l8BJKvNJOx zd+)unvWY_W%1%~7*?UX&9+}5c_R8j1hws(r`~3sHzxcr!ulwBNx~}`ap4a1UoG)%B z`t}ya!!BNHP<`YQlF3quF){ajD4!{eL0^Q|m)!1uPFo4OR# z##dJQ8DRThIF=`htvBu;PCz$+9_!0*MTSByUS2Kj|E?eeFoVTx8xfvbJ7xBbIXPcz za-x&keaQ0iRcRW%OC8_hq?*7=d>cE@N+nR%bS~DZ|K*{>r^?Uyf&Am>1Z`$6_rhB? zqxw5&?~H`0Ye`8sEwttOVtQ<#rA0m_{!UrY*cvheDMyZIdn=Jo~<+)7{gvZf#lC)Nqq3^OvH}@J}i02byY2Z+!$E6+Qg=5E~vQ z6iftVnZ>zw>mB-DM&$YpZaLWY?5^*o zMp=zgriW-DE$;GLFM|Arhh?s(X)HVEYtGr_byYQ2HO=F6?k$RY{j(F>y4MMs+b$Hq zktuCH`-MEcNHSWO*w)soK9%n8Ar}rxoW45H!WOJ6GkDx{qWtuJj0}kCe8_ILHSuM` zDOf({jCch~yaxWbhmeh4_hr%C=0O_{wcmZvS;}y@vOR}dx>!c;x)5u4U^2mv;i_ZjW0!slpQjL@YWe(u-bO42=XHaMDB*kq@0J26`ot#9*f zc3_%wcIia)25?eZaC_#21*V1tQe?a=6}~chsn2TWz}XO3g~EiCz;O*ibnlv+zSw|x z^+TcX%gDWsv9b8B6%$0#I@ho6sTdy=jo~c1Z+DJ>SBX+!s<+3x%y~iMj6onGb(Je} zFk16=s@!IAodU%#o>6lFJpw|#a@1!MT6Q6xP<_w)Gq@>5bk$$U zD#PmJT~T@ZRP#%ap>`y`*kE`($>Hj-GdwmT?6bvsR#<3LNP=7~ax+?D2*ZFA0Y+id}$~LmQulE_5XS=dATJXE!z@#k5QSq6Q>@0km zZ!bQ~VMrW(+FSID(AC8vD%tWE2E8(0Qe&|#>@#LpynE>N@#Qtl$-?Cr7CAR}r$lj2 zk0|qrdj43k>R+kpBxlRR05vr?7s~yO3AP_(F)DFfIAmU!WtY2w&8tPHWHbN1mlK$o0xK40^h*4_zcOC^nn-ptbAx! zdMb+G5!=T4+R$2pR`S$7(qd!vwdM>J$(ulyy!ExUsnv47C5(SdOK+lt+t%R%S||R- zr_@4R4D1!hQ+b_ma2@sUvALOYOxaJC=nQ%|NZt>1#9in*M4)e{O_Y$d8nG2)I^Dwr z@o)0q`n}@1$i5pVjOfeYz3eM>Mk1#lD|`|PuP84!Dc61eMgKt}5GuR}+m_k(Zg^VG zFV{}Z+Pz;0zsvcP3Qq$7Q2xR`+GFt|i)(RT`;=NW_}SPPuqita1y(DAm_!<5QLcv4 z^LOT~Jv=>^Ni8bBGF`F+pF~j$*-N4RCw4dylPpU~;R4cFYFwoyDkyE6WJj#>V{e6( zTY4S;6i=D8`sr`N_b0k5#}f*^$`3!cw0s`=Owox>gCedUm492LF!r7^)eb;xK^iYr ze65b+bK{OCf?TAqmk$%ETd&$WZ#p1>5^=B_h@JE;?+=d`r8r+8`LD~JB1+sZFYH#% zb4>CcN7_G&Dyu9{{0!Qu(J%ad1Lwz&pNfHA<)!*Sb;%=l9tRXXhyVg!FgVOsaj!O* zdOI$H0@wE3eWp8(l7Z%M_`*H2^nX?65~Pb1BJAb!|_;Va3m$|1;-~(c0$fg zTStMe#NKpGdJUQ15#I*+HHsYhy18S~SpMRflT*aa&X*C&yq|A|!6s`=7RR7bF8W(7 zJ;8e;srT=R(KHKRB=Q!uUEWl;A+

$9iq7WUQllFaCS7tmtrVZWZdbFV+>&Iq`rl z1Ll_g6L8_JdDvi4~AXm(}LI z<4 zkHo?XAQ3P!=J(q=KdP*k3zG+tk+KjzL2<#?Pr?y}*Hsr>smXx10TspEHeMk7AnT{f zh>c}&Qd{uU{PpR^AjVdJSLQ2|&N$yda;cq-+w za~sP=|0bJy<%)6FUHc=G7~0yoeX`oRJo>!X$;gzn?f)UwL`7Tn+6M@A=Pg)4%R{gd z-s}_@;w8#^C;0_YREXnr71@GoJ)yCHGnzqGO$#5CUACyQ+J=I@jWIF!!HAnr{}?H4 z=@QkjfR62-Ojn((u@OyRW^=>I=y&*4R{4R9`WBzpCW&aQf3I@trfWROxIdf^8=V>6 zS~k6P#q*DZx*VLGGYxh#eV0{o{bd6|V~YMuMHDmD6j zT(HF15nhEpK(c~F(zerW@M}G~=l&gGUFbD10!fPD; ztdpIGu1TZO;o*d#tQ))_Abjz{YfHGSt)W^5-FOqXcotQ$-428p`|;4^!iK1YD zqtd*JNa2b0YJ@UH>Sd+-vXcUp4GoR>hv!3&Nt8=Sneab^B(@iPZq0p5K=(WsTm5P~ z0lr$HU1QdrJKB{sub7(1l!v9=Prd+U~j zS?FQ;ltavpqOR>`()IuQB;mfSChJMpqnwfUQ>NQX@;E+m+sINH<_%UKV+RKVMYesx z#kiE197`SCs|ixOf{Uu!+5+m&XGK zsd3El@nBw^K)VFR+5_*B zEgZzi=r+haj?QzN$ppWVYZty=@|BW1ZeL(x>NYV)576>oY)Ajp@yiYRG>Wl}h41k1 zsu&}X4JxgUW~PXdsrcn(Ga9xm;@c=-R#ueNzb5;rdDP-^T#2jnQufR%p5`-OLy0C! zSS=ZakZ#qP9o5p=>CDVzCsF#?h071j8BJC6Okit=M_sww-xMH|trl7gT#^YD7z=y* z!kLl3G;!S8!E3c!K8Tj)sNnOBMJLX||Fm9}W!bIW`Q(48u+#djhveVO2(qw>QrD z*|@n4NOu$kXPqGsltj|m&qh-@P_7aEj2?+?nDBUoqA+zqwq8$1gI`y!wvYA}W@i35 z6B(dt^7J5mf5uFH=;n*+)g3n%?o-jCf$61!-(^u8w1t`2s@~-Nz;n%IPY{Rb%mdff zGb#P{SNgov{{#WL0NY6YIr~G@?=yOIKJ;9uCVgaNjgzULjG~*O@^AL@5~q+ln3~3= ziEIX0Don!~gI7+xxHu_9P$`+ieYOWjn!Bq;90_vQ@$o6!3TpyeCnt|@$MM^rj(G_3 zs6IALZ)A_%xpJ;&p=3{Z!o>V&Wv)}=bJPc`_N`C$`yZ1HqQX?hQoSXOUs_9z^gWki zPNOp5ZY-;KHBS}MCa!FFqtgGLD?fuNS7S$B(HGxN1`$Q(>dYpi8gDL_S!sX-va+%L zhkvj?Q@S;2ba_+#(%jsnD+VD#lNff#*DF5mS%wL`pA ze?L?{3Oybg81$k>{ka`y#qaRRIO6U@!%|J45V9ufphU}^!N|(olRcCsog1*CUYp8> znIW1$^CT1KVTnU>>c!eCw%eYU_8&EKx@0*s21VRZ5f3Jc$lQv3M6EFCrad>3pLqu3EX>_6faC1qr=f%3i zUgS9pu_|ZmX4>~XOG*3$lEfQ%Wmwf0Fm_@AV!#Ai8Y?*L<$WxPIq~)d(_%KRXp=t9 znVQtrq|Gl>M|{ESYsAwyU2n0%_Nd9Ay65rf;J)qA%%1|9%;YI@%!T#Kw7=uP)S)^BWf$Gr55(M*%{wygDWw$?JTwceAMf53M; zE}*n@cWFmnm(~64r{$jhV~5MNPw9!=4Q=pt$=sJsn7lI>f0hdyIV^r2Rx*L-xO|kX z_k}Y{K9_`Ruhn^vV(PWLFJu~UOB*917@PjY57z41D;q`&9oR9R@m+(K002es_Pqj5 zE%&uDZa0FDeO%>3W0WF61;^(D)7Ao&{_!cK*?Gm)GlLGjA07VCE=k-BmE3i55~1Y# zEa;>%#7kXxXWGU`;JovN2K&0htkUJaNH_Hsg;FpEH{pM8hQhinH{Caq$Fc9;5Kla* zQYR4RWxuCw7txpa?%~y`oNr@Z-Ko4DSA1xWe`y%BI*kNF*7L`}JIu|CPICuzLSbU> z!Y?9k;pQ5DGN=1xfwMO$DK|G3m?!kq_C|b2aRXyAGA_|`R-vK0%gba^27PH3W)|$2 zu)7T9(b%Je!qNoUy*U3Ja`$Pl$;p3ZGZ<4>8^+)q73r0Gwqgr1qG^^JFImHb5;JAW$FJuP0vi;Hd9NsBBsu< zIJ498M5=y;Xu7R%m>$wVfEj?}LLxme#`j0Dfol z)czEH2UNrLTMR8vvxP)E5{PL`A3kLGsG;s1Xz-`PaCpU2HBVufd#JtpVt6|>@<=h@ z^XIaLfT?(?f$WK2o;41-+f{!y)Czy!DHr;^`-X+AFe~dV=41NDj|?#~b4?y`ajBz? zeXq~=nCj}t!>8xl!xSu%2a-9j1-FTPPrL^22hho~`3qz5tmKn#xdwtScsbJ0#PSOL zPH`>_y-n4%YuttJe)V}Pu;q8vW)$Y-tpIhUqQ@#|#ppr7S@iJrsLt$R%ka9>`Zaf* zRD+dhY8=Et-U}`3YURf*V3@al>Y%a)hoTRu>#2<#)>gj%Q_GHHo73eJb&a z3^#BW*vbX#1&|KWHD)b@OsVE(iHx*{H2r$4%)@o^Z*>Ak9*6q{J=Ta8$}oO1GBQ?H zR#lfTlT%DA{#dRwBfNyT+3s|d;^MSrVhk3Iqj^dh!k+5tpSL7vZ-iJswLRmze*^DI z7OCiLuqf?~U)bxySFi)Vk}0F%-;$O7b;JRpv;|kkeDJPdzkKrlJ_X;49U%IO;Op7H zk81yaeN;*uZDV7zG%IOgv69N`aN6sGId#?0NNliECL`U5yu+psYMz*{a}sS(5zWL~ z9}n7v?uFgOSvkMP?83q@sqD(i%I+D2nUN`Scy6EV1|B{EfsVHJY2T6~A6VZ&krTbG zIwjg0>c#p`CK%-8H9FDqq_pZmm>TpAY5KyjwJI&3*(=$kD zCWhKt-P{!NyJ!F))Al#u9|902Ky(K^=Le7>;G=xj63^UO$?%H|390eBY|71Df|o?d z4-?N$*Z`&itlQTG=K!aMYDM^CFD4d{muC@$gdu8vvuk{QybA_==gr+KSL@n3J6AS} zn~!#Pu@EcnYpRyD`jPNn{gKYU0sz zfhusOfK(M27?_`*54ajJF|7GWKxf;{*12vWYZ3lWbgF=Bz5AvC^q5#F@jb~vveIpM z^AhzPfjG5!Cqs{ofMR$UU3}7|NH8}DL|D7H>W{WIJ6l^%B#{1UjM{KLQu9cE{`~pk zlaLl&Ojl)suvaIDfae;$g7tw1ia=y#Wi48FGg_D;0b0yqHIhRe3ZPlQdWx|FJ;^1i(Af$5jAG&3`^ z=HjWZuMafN$Vj4eMVf%2r1haZK|nUu=-Ov2-QEHe5+Ued_4M?(t_HUOwrLZvqYtd! zi$kLg%cq`^0GjtqXA$T$nWJkkj)2_(GRo|u!w3G7bdbw1LI*q{@nB?Op_JuZ41QfJ z!9r$XW_I`9y(}g`vjX%9V5|rPkOxvy*0;B#mVwzLmpcNWmZ@Jv-_EcD3}GBHGc(`4 zdk3;0iG?blQ1UvU2=VZ0ywI%LZ(taqh7jQoXUexie};tM!OtfOmY^p&6S5)S%RE>i z!VUn;%+l)WBoY~(2OLZOh46L?G{p$}MU$O$_DVcp(-RUt+$ z7vWDGNN6T`cXoJaXb3{4`fXbCz7%ePs7~nCEUiG7am20uHM3|M z8XLo@qKUP2N>UP(+&||?hSWoY{j2~&R6}FJzz?&uzK)M5fn{G=2`d3W=zl&v^Exfe zX!}FRC`wLf4x8q(-miF=T8YOAZZ0lYu3VuM_C9V6yaxS-g8&Tky$BxH^;hipNZ_Lm zFHEJg8)1*k2dn3@=r($C3kak@NfeC;H9{FM$3TFGGIN;A3CtS6HS_fJ1P=9l7+Vtp zLC?V8yVv~)fdI;rpPwI3Dit8+Nl8r|9Oeu#0BOP~Y%2pyKl=yD<%aXxNN!mn zl_)y!?c0OH!~-|5uyBKS&U^pqAtTw6>$Nie zJomY^@&#aa0ORz0ua|9hLEMK<9{9}rfQlfwb!(V3MrLe!y2rd6j0zy#0llTCMx5EHa+W z{QiB`gzFzFmR^s#pD7KPvwdm&&aH2+TFQZGx3I`lW<1~Rq`^V}kr!%2UF-rxV-i!MSc6^c=7oE8B5^+rhF z+?+xCb4^WB@6!D4n+o>i7hBZ!>ZD*Cumfm?h4u5@9UoT;!t4`en_Y3_YZM#+o<9bI z@jLz#-1>@H9u1SK8Ad`$1VgG28W90LLn=^r_L7Pulpe@tfJQ?gOyh`?UsRQqi8mzz zl%t}U>VPihs|hV`cuGEA*(~I`zP@_A9tlt=qTq7VZjwyp5XNuDvjLq8^jD89qKYvs_<3VvF4GlOUGpDaJargnc$j$RgD}R zBu3DZ;$I8r>;AA0%hYA=3lk~+^(#M$n=d}J{fUl;W+|ld-1kdNNT{r?21`Ei>lc`K zTOh9i)qObYxeRN7DBtTvnQaCzPqDGFrl}#VhIm(gBvF$4-YA;|WB~mQyrJMT0Oi8s z1s4eyHW&c<{AcK`+Z#jxPh%m|F)(ld_;wD221pd@qI^*J?6YS)Py=ufqUhT~*a5&1 zqoJe>*98vQU0`Eh0zH-nnu7Ds-~FQ|0{{?(isWMkv!JKfIQY(Gv0A9?$x~qC!$l1@ zgn#^lYy+wvz^RKWv^S6q9-P4bz*A$QG;e8F~~71u$I#JUmf?q?8o8nU;qnvT*GL zZXkf~)B!~Z%ycXSIGg>I!D|S(7XT2mS%VDTK0fuG!|!o9u>U$xe(+X{43G`9b$^k{ zqaZI&-~@pd!wozLYH<-H2@1lAl`}B0yiBnaQEV#3q-;#9jdC?%8URaSs6$NFD>I;3jr9i z#tFt20jYX?4ON40fzFj{iCD%4T1(ns|y;|Z2t2ZN|O zRC3m<6|NeA+Gx-khq4GtOG^v#^Jm`I#SU&la&krpnIcd-4XskxCGQW zq8mVv z@54nhoeK&X!g@e~L?X|BydwYxK5*m5M@Qpi=pl{*cMnbiJnlwmutQDsu;2>?g$;0^ z@qVWV`FpraQq6)y)Iwc`>~M_%KV@Wk`fQ?s??0dAa=#8DY7P##yk+VIL)_p>2A`9- zs_HF-GAti^dlc-NfeEV1dM7aEb{mc8zx~J+jt>9rx&Q4`=9`JGd4`R?yJroz@E~NL LD@qoL8@~NNdpqk< literal 0 HcmV?d00001 diff --git a/docs/images/nd_spl_3_comb1.png b/docs/images/nd_spl_3_comb1.png new file mode 100644 index 0000000000000000000000000000000000000000..dd19db24f92d4e640a1c38d88dcc2ee0cd942a64 GIT binary patch literal 27292 zcmYhj1yq;K^FI6mk(TZT=|)7lk?!siq`SKXq`ON*P^7y-N*biQySw4Pe4qDs&U=n= zyzgc2+1Z)dxvrTtTv1-)4I%*|1Oj;@B`Ky1fj~`y-+J)S;K>}9AvX96!Cq3!2?9aF zeEAIpNzcTCK*%6cV#2EKnFos=e5#9tkQ4SIR@G$kI8|k%L}_}d#48c8#HEW`7W$Gl z9I5*s{D<0TALrDPmBo`y@MWmd4kZa3ks;3HOay>l>eYW0yW3X-_s|h$HghBsa;;|j$d3{OiZY5 zsU*-Ss4FR12vlyWsQi!}o1EnM@Zo5=t*x=K@ynMliCBCs&6A0B+bOkOr6kY(B?ZdIEV}npUUS^SyNM! zmNr{%)M?24g%y6wKr*#R)zZ*#C0`=uuSsQ4P-nHpcw}TGJPfn}Y@s488ylNKHXi~! zyuGutZnb$_S{gp@Poh8`xWTqqv{Ccwq9R%rmNFo|UXv>~7uN~Hc5SFEb!c$#!D7?& z*A7nj!2cF^PZbMsbK<9p*+}+con1~;)SEcTgF!b5G)iXXqleq`GJ`h# zs?S4PLm8{ico-B2MsN@pY%Ij?zCMeEdWW8#o~Wp(qobp&Om8TNva&KYHMQIE($MfQ zEaY!rpNFUCKl5sNlvjbNhEE|d(PSd&)Y;SB(UKAU-^DQkdwlp?s>*#om z<_fbSiHbm-vrr1b2U^dR8-XuaO_$0^Nr5$^dv4hv#PsrVb8l{MTUl8R^!44|-y`u- z6j=q_;$kFjkK|aHnQhwhGEAm54daC;bIgR;T;_v#>xMw z9_)g;x;g;?K{*lfDRJhr2p@Tzq^qmz&!0by6`h1S7~vB7OirK7M6!^<+Kyb}Hjp5! zY-|W9DAX9?_BhC==!0!&DUI@Hrh#;LYrLs`Pe=}t4Rd(|IHEa% zen*>|muuZf^%T%+=-@kCN!%M%xeg|IsBDB|L#1O`GU z294whmi_#>ySHa)YuoCyEf4XJj6{x^44SXA^YZdakfhXy4QS-jY!96INDh#|4r*#hn-hPI!~qV^u|uRrxhA6B~9hqGQiS}2hKJJTAG`ytE=6KLd7Ubbm~wbXxP{_j+^}p3%YW0Lt8HbgAg@x7H_-Q zXft0MmG71}z8fDO&qV5RcR?3OMkAl;ez6mml!RTkTIJ~I=y|pg-qt3FWBD%jFZjeB zUbvXFg2MK|6<=v-X_cFU~@ys>TW?(VsK``g>S>+nRU=jS@*hEQ$dL5(Q&Pf&a91u`i_ zL`3dod?y-4MtOC0ZW#(7q+K2@53qS=kR!rI$~e8phG*)`XLp5kL;4b9AY*U7ku zY**R?_V)Igkct6)2fg6fKoSv1{V$v6X z)hH(}F5c>OjS||~(A30E>T$9ntfG?e{kw^%=qsmtlVpjB$w`T5qTAbZGm?9owNrN> zDt!`S-IpLmpqPu46c&TInVAy-d0kzi%8H7kcjzE$X=!PZbbi*Z zF#aea0&VamunQYlg&zVkGM~p8v^;{y0t`oc%fP~7_YL8PYG0}SmRP0H z-HR=`Rh|X!)*%hHHk(KIdp2iDef<~ri247fig|eOc5-rB>g^jCV1M@x8X_hp#@c4C zuA#BLxjE_iw7<6YZ|R|~hV4@i7B)68DQsL^LPA1zQtOs&HY8Ee&aGNxaIjtMjDbMH z7*mc9;RT`TPiN;X7a655u<0jST3Wmh)_BM3c=-4&o|p6>Jt`_PzrQ+U%85@-{_GKq zK;;5XuB9fovfSJikaGZCO^uJwH@O`H*ZB487YHsWqymF{*0YtSVBpl$)DTN#SQtDa z;vGoT4(ID|S@eN_THD$AJwH8yn5U(w$y!!k;~N}klb>ktSLF)=EsKx9pc4Ecfo zK54FCwt$Z`B}NEww;dPJQG39vgRt{qZ=LT@>+D3d`L3M7-{dQn!c@=tJt z16TA$3-1E?gpmZ45$<5yDCz3zYWX20DcGgX!&q`GaI(ZgPWP2bD3igDk-=l2+!$eE zm~wFVcR&`Kf@~LWShaolh0w^TUUN0z69m@7Znr6#<&Agvy&1ge8AN8(fpy=s-|`;xKn1ZvLR(?i-Lt6@S8D}18HOP)aFFMr+u-> z@XFd&-)j$RXBxG-8GmjP`^4^)1=HYY`%4%0i?BF2IL&sCCn6GI@ei7+@g zhSSXDCgHL9#`H|D)6xvr#Eg2OT=c;5168#Vp`nG0*K#EI_yf|@PoOPVhX`{E;S!Ue z7{`ETp0ix^-kh%AF1&jfCT2B;VSNX!u1@v==kV_rSVe>7BR}gOe`yr_GYC(vV5iH1 zYOJ6FbBT%Y5fYO+#`0GdxXZmg;NYC7n3+*{A0x?-1NTRA1;LIvIXd2OulYlfy~m=Y z3J8MhxH&ak;#)aJZFDo^KNzPuX?xC#`t<1)4$IL>i|rB&r>%b&{AdYby*(747c8@` z5U;>HjG5WT)~63{%=J!i_4Up-sB4m*o}M`ZJ~zD7l=3fjfG6nt@ceY$=CVI^YV_M5 zlA8xbEel|is3;m1Omj;|I>$;MrlHk<=gDXu8D(%`9RnQ{A!lH3j2{aOGd(m8j(P9) zc2Jo?M>11eTS*xl^b7+n1qJkxA99{QuuD|vfcBib&JrEqZ_9mkX%R@Vu@LAz7IXqc z6BGZRyT^8&e;{|_(#Oj|d7@BNf+i}MlJc;PjSo>_;`#V^cqeYMl7lZn7LFuFK~8S7 zQ^?4Oj1zZa`q|~JmNX(BH^yN4WY4KVt3NUC=+|afSHo42YPW|Agh(QU)FB*t24Q0g zR!+FSKEtlC+L-67H<9?~_aNUB_TgW-7VlQ2S5r`kY5OE%3=5(i3&3DM6lt4n)7WhY zPR?IVO|1;#=jOk{Lq54;s`nub4yoaPo*aWhB{{i>ZfsmR-(EH#6-0rD0ZN00_T657 zL)dW;8_in=W@e)k zF!EW@U%nXrMawOD^K=WR+cunoHeYQ|9v1Z)mj#-R&d>_!-+0Grr+>p+@lS8vPoWbN zACmmG2ceFapzQ2oQ{ZZCUym1_^M3*@f& zLnDcf>~9pTe!jg_VWtBRlI2tLLWT2voXAjA(mxS)ilK&&xW*sz$Es^;0HguA31RA@ z7ZoMMp!U|n5*it{ebpaE;T(fLiN~}*F)(jh65+G&G^56|HjAP#z|&9_INM9#eZn& zt02qb^Y}L~fJAK&Cijx`6{{4s3;Oz3nF(v+Ny@_Tdj)QoJiYJoxI%XKgytdkzG)8~ zQ|#zt>7skd@CphFx(!)*J|RBrVb4?}Fls`UT3@51=YEHI|Ng{mL#xU^E>4J(+t&4L zqYq@yof}9Mr~rJur^K+h+?#x`KwQWkRkYv7srS0Rez#;bEu@I%ei~@EzQ6xUL4nnd zWEs{PDM}n<%f+RoJylNm#4u2hmz>qi*!Wvn+41o)OA#k6BTTVI(E0ZBf@oYaEVIGZ z{0$Wq+1ppLnY24{(J4$iTie@k2`v9hNqK|{l*7f0U0rL1;s<)7aw{r=qTXdpPUrK_ zJC;R#hJrHOS1Zog!8XO-;T*x7q$H}? z`PGTZj-M*+ORei~gMtFeSmvO-uKeTPYLV(zK%1Bb;IatCkhBjVegE#SQ&%5T%LE@7 zNx;!RJ&kM2f$D(x)bNF;)^hTfQoi`9kpTq`6;*CIT6u$1dWj7oA&ndDJB&=OQ!h)# zRJrKm#U@URf0U`-1y*07*#|g8uXeNLW9BQwAIzr#A`l)b%2}l zBX=c-fA^^e85b9qhK8o~4Ufk=5FtR|zdZ;0HOC>8=N|-jJFXlr{hFG(&1{y}4VoTX zx1~SS=y`o-hs%0~)y>UKAR57i4xDR5hNU54QBj*ZNWS z+_JwLud&K)CX$qdARv@`u#>`FU0+|Ho+1Ehz>0*d6oBlChz`TVloT3jYG$p<4G>iI zZbdoMWNWLdk&%!HSq=RGFH+taAO=oqG@Tu1PbsPJ$Vgd0FMwb3Nrk?u^poxFD*(o( zr>A#!cLTcx`1;SO6)FmdA^=Sh1D@sYz`*`uQyoCsD52m$e8<6&#r6p{kn9giNfldy zouXoFa&mHTa4?txlqw)YL5c_Kkvp zVScqEJ3AYw3UhtHVTi*Zxl+^$88#oJskOE0YHDiA%9T!kg2Zq^!1)c>CnO{!j`#0p zq`=f*kAb+r*p+^&3inRkN>XCPNt)1r*l5p{85k%6<^&`J!Of%y35EfPMUP`f9k{Q< z#Q<#m#I6wj^?Z8-;1z#aYD{>TXnpvB4m4hnDfn77{7Gq={EdZZ3NX3V)Nk}42FJl- z8;C!M=8kTKQYtE#y2QwNL6>mw{^|_g4TNGcG7N%(&*Oy(fSEzt)rLuv1&1C`-C98% z3K;4FILNT|Ko&%x3qLdhfvsi7R;_hDWAJ5{y zX;{R>`t17AoAJg~Qd-(_vC(BPW+}D*4~P$AV`KgOmRAR}KsIn3VO{Y4ab)d+o+53; z(O17YLlKVkx;_Fh7m%ql^Ycii1+0N&0G#o-?5P5<1u$y}_A8(^s8qkEM0whq%KZBJ z8=zbu6=gyn*buE2PmV%2PJ6(@0-o#eV73adDo8ktcmm$;G_ue0ufR_F-`4G~6cq%m zyyxK9+}gT2UM3Cg95b($h#?{7c7%Zd9L?!*wt@au{N(x_agwy0q@?@pnW>>6S)Am> z$qDO+56SWIB52=>iozP^gf$JT;NiEHne$pmTCIrC)67k+wiI?u~JH7zZ=16tpp;-Dt7RKM;h ze{G(VBS0iGrYr`_i-?Gbi~lGsm7fWCfXevj3}W+FY}XGqn24}JN=ixrS$*zR`YoP7 zpYXh2+uBPu8c~oRzl!kfWhSF6{46N}FS;gjV5osmpN+)8B(Fb2>`<^?-$NYWf?h8t z_YVw!02D&}aUKf`%lGL4NTS){Xs-b|FZlEAzhr@e7CtU_pOS zV4|9vy*)e{%Ja=}8Q~%rQvE(+_aLS;!nr}4 zhNocZ@MtB0cI?O>yoEwLT+wz6xiu#`9wJ=6sh!w-nf-#g!#aZkO$?UsKJM1%C|#^{ zWgfZe4e6*+6)_AC5KxceYJaRCOt+(sijfn{G4$UYVqlJs6xv{pQ!q#Q!hdrR%{)Un ziG3@0#dGj7_Vi_J6+AlhcH0a^Bd|z{F(6yd{QqP_2XOLBm~%&6n&f*h_VK@YRofpT z5WeZ3*hr}o2;^dh|L-Z@J#ZEQbiLJiYIMsRe|goH)fES^$#3{6h-KkndFx>00V%|a z{wH$JyHvjrDL$YIvC`F7Jm{8i0Vv5qY)D%Mt7wHl6>(NzijDsigdBLQ0pIm=WnNZ{ z5PYuTzt359ZKN0khiL)X!hqgrz8;c|b~v*B2%EG+x(Wx6sm5%;99aJ?1QdPh62eSh z76KMc)A-+Doc1TwEMa7Ao?Oh!7ugztY%`1^^}v``x@y1&kzdxg``@bT!TK24m~TE{ z2EB}J`A_xWfPVe48L5>iT%r@s*?9DSI!1au*al6W%-}sw`2lQGxCAe}v%QP`Hs-kj zVIJB&FcIVN6R512>L4MwfsQQXEdzmlM4)=(ssGjCp!1{>&p8pzyL_MgnOseTR9uYn_U)dSr- z*!)lTuVJqIR^n&soP}tXUSxIvGM8(?EQ1;jXX9s}xS1El_57!}<&KAe3TB-X8|tf< zv2HJ8i3SZ=pB-}BW0+EpGfv8D18x0eAi}<%!AI0GH*a zcH$$@@;=3XGanm19pW+GeArh8Q@F(aPxYx1tk2&9UK`-({aSt1$UON!9Xq|*T6XOS zWQkw~zNDA(p8++z8ke3o?5L~x-$MW4VTKTevJVx^E6RUT9b8~ZYvbvWjh*uz}gm;Vt$AHuZuz*H# zKMFvK2XIxZ&%NvMl3tk6`JqHr)V$GZ&QkKsf7&%Pp3ef$UjBc(iaD50P)Uw}~i`e_!XLIxhPILPuv zMMVKWBIfo@B7ZDCA)(Wt85t2peqgXX>zz%xBq(^&8}wcOc0GY@U(;rP9xdwKd~+ zY2cK>bA~_wRMt>cg@&w9I&cxS|4AD$7+fg){Tt*O(`m;8eCGzOp7=yWs@l>V1iaFs zGJ2vai|w&9Tc;BkmP$qrp3>qZRL?%n3*V(afB5iWYs)X|7MiO4;eL69f{}}h>n*=c za~~yl!xA+nQ`$)Xz%>D@A`J&s-tEox*lMDk0U#SW*{f2`Kl9l9DyXP%FdMEFYbkgY zB_82<-m^b*T7B`JAwde~Z4|Q07y9eHpdl!3K$(>SCYWfNH zwn|T-GZH(}zQh^3VxqPzd&av?;}cdJe0+TQUOIYO zc{z%TA)eZ&3U=R6liCrnjRTZFpPa;dBV*K9hN zi-9cRVslMaZVtX3r)q@aZ}6$9QmZo6H3$i09NSj;`xe&zo_-SfnP`L)NCs&CHqT9M zDW&j)sPhGjDQU$1X4Ly#tDf^PGn$r|QoWX8hgE1{g)SHgq0N<@ot>m&b&87~Ca({B z_&d_l2pfZCB_%$D=sriO*8ml#G%{}9r;UU@M36}e8RllpZz<*X@UmzLnxC+?icMh< z3;!FBFVz!J($jOejprn-D&N8Jro-xA(5gze+>a@nHThkLFC{DOyjMlja!{l5YPOtE)J=XFG~ zd5WKhvJ1F{e6sUvI{wxoH9dTk*jZ(v7$H7)wYTT;+@K{L@zy^-29*K;+y)B=9dzrJ zOUl|=hOw}>^jht?I63!A+u8!4$^<>pnQkcH*J_Q{TiPKx#P>Wj=TDbM&Uc+|Wf~-c zD_+~toq?|@uEPTi9v&VRKG?X{>Qq{uYDa%XUXYTEZgM#mpSm7-a(fK#8kXdF=uUBD zjS`=4{p6*2Q`{oq_U7NJLR^&3AfxwFeO=1*&yPsBr>3$;=PV&24i2{(+CQehFx}rO zTN)ZD87;ZiQFVr5`goT*J=RHzA+>(rG1no?kaI=S`#g-TSQ9{8-8d(Q+bn>KEhgSV zjt2h+QIwVzt3Pwr3HF+_(r#b4HIZQE8f@;JUhx}ov@*RmYR4QwcSbltud6}R-FQEB z#!82BjF*|zu#*Z=b4K#9nxi*5(^z&_S37pC&|tXTo&wLmbq}Z?+PiA8yuA^8V%*sD znw@z(FK+z2mxwSI1V@O6XU`)7&ImvYfHMM~16=~mA33c+4+7KdUr*$&i?s9Cq9S4QBSZ!Ef`cS|Q&si$VP_%pWa`iP#A8=b8KFF6THi zJG*-*Q}_#gP$$r#ouCBBYBw62lw>OS`YrrgW5;FSb%>Q4OXPbJx&?LvZSl|lRM`W6 z{lluV0Chl~=B!(y5x<|RzgivxkN7>07X9RvXL9%%7zVD7bjJ-;hL)F8hEV^2B(FgE zXSC}%+@<nHt}snUQ5 z*wk-1KIHxSti&(okL8#mGdjruWnVCtL3gXgs z1J$LbX%NUkF7Cy5weuOlAZYcwPz_Sa@Av=nYGt#X4K`(ET| zNE~ojn@OfB2SZ}-U)M3iMW~;dehv40=34R4)Z~(2Z;n1ME6Z7KaMCEx@lMlJE)Od` zdHp=;U18vA&a@+U({i(eyK409QYoK9Nvc&ANJ>H?zTu(gy=~RNQx9O6g2ApWe%CvZNKx&bJO!1*X=g=zzel$E}x@PvO8|)z?3{x{ZP2j7Ee0I zoWPMRenRi_;h!^@hrA?3oyQm|9iX0_6HOeU{UZcZq&`&TGRhThY$)tO;fD@ zp`hptA%^eeab(PeB1X?D zvKRUKWQO5Dh^xJME7`sqmu1M|TKRV$6J2(zP1aw%hzx5s)1`^q59|j(kZYXZm08E; zO-n8<<*!zSEG_po=n_ajF7?jw7%-do_086MTNw!(g(kGsp0k{%&ubkEbp%gK?Z6vz zIH{VL)E8Ya2%~k48rEMoGCf$XFgR=E1Y=2(nXzQzx{@6LA`0mWmD_Kp5=<^7XA{+@ z`xadrE0Rpo;IdWYf#sw)pllTQPgR+58%h&zb(|mQc<_gMWd~>I_LJqlvg#eGXX6h_ zns71fKO@teUE5blLrp;HWin?Thj1U=HMlrQn8E&6yAIHwQ|>BRfTNC_692UL2`WfT zpnZ>=s@ZA2MtCOR`FStmGV_r+ipwi6nY@x@_^196wnN! z$G>>b@>oB^20xI>5D3~BgB+3f@8A$aFmP}zKGkXZJGhiSH{>|$SHs_H1S=46dwT5K zvdI9$*DDDSotVfG^r8lz5R#`#P#N!{1~kl-4gXGU-t65dm?wVH+-^HR#Ffpe7&Q!z^Z&H3$|6cic3F^p&lgNA+4R!c^Yp0Vb7WM~f?}|d zjKxcO-F1VNTBN{NepxgI{|R`Of__TOHR-SDO(X|E_nAk=x$C-Gl@#T=Ps7GsRJ)Em zv~9IvAyRRKF#hk`-Hu1TBR-nh@R?kW6HdF96)(K7+@t^yU#bWbYLy~RQVw2p5V)G* zqdr9vnRp)}}cj_2;Pjk?A=vzq~lv^2Ws`JtgdGy59N zcfZyu8gQbtZkarY0%jlXtG?izeRxN=f4+OEQ%?xBuCmgHqoVul&}sPes4vvuxE=o; zrnKn0`-xE2q|8*``X)C)$NrgvDCU}otp0QEn7Q_=YF6Ic#}fgzvrR((R}=rv!f`M^ z&&npmu5b-c#B5ZBvc&gU*a+O4sX>yYsS1DC+>+uX9Ctf~dR`sKQTrABEZwq&q1jg3 zq7DKuE~Fy_4OLXcmC?Dw`s}jV?1X~}Qy*N=m*l5+bmjG|D-dYCL2qvut_>4Zz zD?Or3oudyiLlFP?wSRuZ>b}D_&88#N^1W<$Qr2PwKUFO=I76O)3f?TgN&OTuCR1QF zb*y_aLJ(L(wWC<%oKykA`#ShOmZhLz;vrl;z>GM|0y zj|B`w*2;7^?Mk!)yAdsf!VGMEnM=OFLSm1YTYS?9x!E@-GjNICP%ymZH5pZWm&TZi z=izwUze&eF_`=OW+b*;8vl#o9%t>^6J}r81Y!i8({N_@LKH-mzUF7E66M_enK_?B{WX;x@)QM&{zdi&iywf43yO@qkT?rX zpCj0+`_)em@m~nkmG7NxwwzCr(YPTZ8u@GDII?t>YU4;vM|H_YUT~{picJL(Qu1ZP z6&*_FTyO6PvJtqv@8n98J(`sX-zQ{V`qWFdan^6LUZwAQwv0~GN`pXJ z$cfxc-%&c88cNK;9rZ(VF6H9xia5co-kzxUMLryH^=s`~N2g~f=!JiiJx4j5cHce^{TVgc?%!nS@IrBTJd;5U z`lgKWxPU6PVsCb|%x{1HD}VE1DzruJ)_uEMJm+Xrz@a0ewsI;7%X{r2xyQ0^2nG;GGFEY}B9tikIm z{v3yVxxpTXhX6fky?fmBsY1dYi)7NPutXP{A=Y@3|UYR!W28uyrz+H zdo6m?6^q;Qj~3%653ZG5O)g;(L-`an@421H^>ric5`F&Xa>{YxO^d* zX>&UERS<4NzYh!w5-H{1UJJq3+J#d%jPOfi3`l@LR!mJo_X+y@Y8|#>INxzlMvIe{ zbRxL7i?)%r`NHU;Y*DB}{Ab-G$~9JJ`r1ITLB7z*XQVsGKQ!^+ICSK|%$Fe|g_uvZ zY3Qv4KH=V*%uA&byK+~GE(zMrma!gDe@(7^6g@YXV=qqX=a~9qwF7=O6Nw+r6 z-WM?It|Q(ggwh+{&jb$_zU`YUOZ*KFD*PfNC#l!MF+Q4Fe)y!Mf@777C5~fkc#*}p z-Uqm|AN(@#a!hOl3x|RLhZN&B6~Z60GgUG_{5bDEDeu(f#HCZlv{S zd|b9IM{jo1^gF!BN>a6o&)Na=1p~oBQKI1^Ii){4w!2q{L50lQ8Sy z{}u~$NFF|14Q);i(>Jps^_yFtH*N%X#BG&g3;1@C zD!sUX$M=G`Z7%uEqZ({eY>2;bkYCR2asK)R@|VllqtiRJ#9LMNFW7!N+9cROqL zT+@eEaLE5GA61vGn>mf_>JFB=GH+X>BB$_wEjB6-{xIqyr~66ANM%R|X}_PDLHK-L zkB1)Gi5?pBtE}u5WMXRc;aC^&tVd({aGN^G;k#JLLA{Y2gzU+eg`IIyia`haHcBXeshR(myMNO(%0K+Dl+Nkt>47yiO8fKM1KTW&sM1i7cmLS8=1SLW zN1qP`R=xJuVtygQLcFfB1U%g(6x-kE`G}V0TaW8456l_}w#0=0E-ty8smc~o)Z1^& zXv(Xu3GHt7Svg>Qm=8dMq03YDXgEBaxMSKTfl+;U=zy&J`9ZA1?5^j7@vbs|-#W7q z&ynT)zAg;af#cGpSb^jqzo>cIlxjM^Pg+JrM*0Sigyr44e@R&N&1D<)O)1wU=h$l4 z$PoEM!RNMDIzG^)_!zeyTlWhsSF{%$`J-@5qzg5DK}}XKAQLr;+0MZ-}RQ zOmgJhxtn_{7VDVrEiErq6qUIgr^b8lqjMD64?Ng!@HULD*Gbm|e8cW%9weWk(wh&U z6cnkNk;4r=(0UD7?YVRhdMQeKv6()Gc({4X!@wc-|4ywlA$R4pLSr%&oS}zGQB`LYtX+0K{MWRa7l`r`bHVQv8Og1~ z@q)Acab`FTiOG#WE&7}sZVC*5eo{9kt6 ze)+SuJ*a*a3UF%;P7sA>KYS|Td)jubC`jtB$>A+CFkpk?o~RBlHZch`Y{y8?G$Q;{ zQNPFhL~z&Qw)EHqM6_s)OM_Pg$Z$yr+)wc{Zk@Y!R*@ke9-6dTa&m{g=UB8g&ASS# zM|k<_YQ39VOP^Zp%Bv%t^=z`fCnk0p&ukZ5zdP?WxMwV&rqOM1caA%^Y&XXPs;0oDdxqMMysJ~Z$!6aWnJy}vbdOn9?$`?841g$4xp2_QqT-_oRp$_ zNj5|Uw}m-$!Rz5k&s$hvKdsH@k(tScsVU?fhsg=ty}1v6ES&W~{HP>+%EwEX3`Yfl z(9;Kht^A;^QBB897fqY`Q2hOOrN?~-_f+)1f4_&6S@ihcIYslIZ&~4MHSc!Oah7JU z9qG0D4!!^I*+$_l7P$W+Bm$i`KFiHT%`|Bk8!hXG@}bq#z>C+q=X-&PvSWJFEth=S z(fyv5z7*S?%8upU9%%HqArQD89HdUtsC0QgP01Wl6D&sNofGos?RnelVLfYSF(s~b zJagN5zRegLp9l(PrY9=YfG!Ogz+t`p=n>=sQ{RAmf8zd7DC^3rnP2kwPiIFi? zU!r?a_EtFQ5;p8RD&4FWH}&WB)aBvIv{SAfEhVED35uvxYXp3>ND?8C!8G>meT`aX z7@vV#Wi_=@yLJ^CCi$^neg%}>*SqK2jd|ZH>FIbfn@qpodL#zCs=2`-vKypM#YwJk zp%IbQ{#sF*JN)OW`Kw=}Q9ctI_$b~v)TDB2jRtHAwlAMc`Y6dM~ShIoMI9OTZ(L`6y2R~02G<vg_qwltE*ai6ND%K!PP#2lvQg0Tw!FdxU-n^OR8;Kf zT7!f5Ul|@II@hLm^!k#$A$h;i_?*Phy2OiW^QiTxGFkxzsdN;0>kPAO%%J1pJ#yB6 z_6=hSb>P7LIUqRBlm@UldM&!rpndq5dzqo1!SCte@?>dUC3LtI!D}7?{CKPHaB|Y4 z)ik(t^H3Pc0`w}Q(1&Nsnz<h`6n+uO^5_$Plv;L|4o3CI%y#7ymWZZ7?6|GR_C z5S1SaO+E*Srp2x;jFo5S(hLmtTnlFn=MYG8hC1q@pkZ{M!F^<+s`^2zCsgj1DZ{u_ zRj*SyZ(o^Z`qyv90}>mir-BxfKu|9Xi8&v9hI>K$D7LvqkBs^L5@Hhkj2qwjrLd6N zhypK3Qocini-)`*-sp0obxeCE0d5r2SQ6z$g#+GBM(XnQ?XsMeV>$HAjSVPB8`sDa zV4vyfVHg>CjnX-srt(uzk;N;IB>L7qi8!L}lJvuT{FK1PQJt>!W6BfO7iO}}*KgqG zrdf2hL{6M+wk=ys29$mRm#1sn!QLBbD0z@4|wEDJ5*)rwM8WKW(>K@DM{m z%C)#{==Pl&vlhMO4XC)@ibs}HwQx5(^4}S;S@gZSx3qT99ehy#D7-4C3@!7drl!bO zWkmSuGU2Bzszsuij+HU1rmcp=4*Yp<^cmJykg(U=D$n#m!$9XXSN{G{3+v-on|W*? zGawy(z(c+$3XUyB^%kdR_5IGwZ5D@eM-hAp5#_)~z(E^M;cGu-?n=IeK!mvj&J)HH ztZmpP4r~Tf*)rTaz5FR?Rb;J;m1LLuck6CBaiY(hb3ho zFgrVU#wkfsqT4|K$UbWx%+}9#+IJcRhrFBrt>@uN=87gnAF%{AAwDraesw!-S)~;B z=J=d+ye!=kBBU@p@=%m2nZl7e;}je&M5d~$3c5GH*+r+Fu?@N06KqWvbQe_brDwNj zVuD0nJH6^+jvTaprMKx^OpcnL@o$dih@j)1cl3O5Iele*)W&XF>EXYOeL>#3>AgE& zm-EL@qc0(T36;I&zycw z8PV!_K2J|A4Jz77+?QTbV>?D!6~Tt(jay~uZONjKzkZTT)REq)(CQab%BMw_WVrp1 zuWEKVhP`9c;?9vEc+k2oNGc0O7vSYoEsEBEhOj_#D$7F@ggF|Jir4ZJPRNZzrWB5#VRI?oSbKs+E|UneJg;R-CV< z?2@KXRTawCWOOy^>C321w!vDNziabF!~Q8K%o-5u?n+<6_y#00G@Q>}&-vn^3eO0OLMKAvAc%%bLE;t>)iV0;^sK@;>M6ASKyl5sHhpm+4q*Q_omxQ&0 zK?MpL&=7!PW!C!W9co&f9L@QOKOy7s?zTA888r#`W1A-qo8M|PFrDou(*o1bD3#d-cK)u>Q=20SpT1w^5EN-1uaf>$o>PzKs2G9Ey(1*A zXP7kSmS{Vj-yALQjXc~f>^>$8nx&>G*!iR;oYT{oF}o#v`ej#S_288wSNO3QK-L!&gD zt_MlxE;K5n9}AjDerp7C{qqbRa=7c9OoUhJ3;+C99123x>Sbf`KAFp$^-kq5oV@U0 zv7y)nK^FW64d2zv^dAM{-g^X^di5etvp*yzAwbN<2wX{L9qhN+IMk*>G9ms+!&VwR zP7N9KklfZ*5{RP4^K@F9<7xUp9SheyQwO-6BiUKZ$dGW;>G#lc%jS15E|Um z0a;wMav)g|n&;(_mYP;UZV?*xX=$eRk;m^Srietf#1>}KO{(~wyPhw1Uy@_+2;R^X zVPGuzu@Z0+K+vAnF5RmfcsgP81t+uX(8sLQ>wJt?()Xu8cmBl0!_=aEV!+XzerZX8 zl%!<;-!)>Q6o)gr15FKk zR9-+p+$q;^KYUFM2jKjH-Yc%|Q6VSimvUa{4U<0?&vb>gcG;eF8Xw`dlw^{JDyNjK z)>CqDncQYF*`~g4qGU{vs+k?#UaBc4rzAY9r?z6^eg6@wEB|2B6`9<$X zK%jE2D(w;M3DOrQ|JEP8<~d%>xUQ^nTxyFbDK6oxVI?o3En9r>I?~iq^0@y>Dx5?{ z!;+Gdaa?IxG3Yn&;&J1~8GiZ8wJkTO(`SvV-uJIn%oyL}k;l%@2k)Eiy!zfc;gl0v z{>lRMU=dEV`TZK<*S`|!sb7dU-d@k%h;%lB`}g6MPtc=IOs+u)rV^++5vHTzGGje` zSuO!R|Dbjz5qC{h&~-KV|4KXSsHnfM-va^;ptK@>rBOPh8yrDVK?E#9x)cEk$&o=i zq@@LvMo~&iVgMDyAq13=k{ViKsDXibHs5>iTEDy2z3chonLiBp)O^l4`|Q2Xe!ou7 zVS>BEV7ZLji2b4YNy8W}Ww)tsF*hUL8ul{Hj21r|sd?JaX($Mu^5QQ?&L-;z*Tinp z@0wveW-DO%VDRgWaRCVq7Ee~0b@zKkncZ$g=5}l;9*bi(sV-U?Iw8wDb>p8iX1u653UMuXFv;@IZvMT0d{%NM7iIY>pGx2&ZW#VCh`=U@v<&6Vc zqI%@1+uqAhF_XIQ-ktK~MkQbBAQ%|Wl##6@Ym25zwgf96)%j(#VHvcm`XHm>X6Bc} zG&FI3{jy(7zaplXVcI}F0Wau~Xx{905FK#^zH>+81>4*bs?(<+KT=;Up?mFS>+Vcc zN6+BS%0U>%c#J3#7O<`Q-_|n(hEr#bwMNo`JkdN>+xJy~zjB54?xw8u=W*E?8CdtV z#12>5_caxGdMj5T-tc*dL=kA#X^f4H1;eV!Ey+1!G1E)+V_jOe-aI|66!_T7#BFfG zoF7&an=M-cQ%gEeT#tdp_FwpHm1n%m z8beifUBAPg>qXL*)oRBm^E{6QFv5>S2?zH>tc~+UpO3vZFGntx^(^iFwNyG#QQ3yJ zq>nQ+6cZEuVl8G226WSDsX~Z#8BOUN%Kc3S`w(9+Jkr^GtLE&@_ak+EsPtjqd-{9a zN$KfI6D{3yho>&yk4io`Z$G{9K7=ZSqnr49tZZo^vAqU$RCKkjTz!+7g+t$X(`o1L z16Y!l);t#Uz;C-mv8zAx@%?}w3;oq4C+TUJj!F&U5)x=@+5Rk4-SZek1_x6g6uoDD z4?mX#5P`=NCHKt+cneT)w-Rb=()zh|m@qW|ztQg-Sn1l@5~iq5 zFDyJ9dbov+7fS40Y1h$kp?&U(ja`wK%qU|^lyC45R7$r|?zziK*8;)0NVRwuAz92A zy`9QPS^n{rlvj(+rYG&0xxLa6WSG{65C04)o)IHakU^y?>A)*LcJ)S#(l+fqTid$I zAT4ce$-ShjqPe{eK_5S&s{K3;vo51tM@e^)hc!nPA_hJ#E2G2iJFC=CxieE_UHT;H zUb1evd{k8IO4)L_(UGXtd}s3)OOOXOU|pO||5q)|Q~r9oYsy2CsMcVbn3(O10SVTm zQ!Q3;pi%Nx$$Aw(>GqQ^C7MI5{DL^&%0jzB{LLn^fUOTp0(#Qs9|zvpPNPMw0ljj( zB_{(;p`_qZ6Sw?aA@=w$eVEWCe};-aw^p-|a#8~Q26VH})~hcTPV>eGrgiM#bh+SK zI&g*pzs=r26}Yj%9CXL0r20%_e4zr3X<-KYanQQ7DDMw$M%gkjqRe_S3F?5q1;uF% zkRtEW!dgJn)2|ZCr|kPDXAH2r`tCHGlgc45!7f)2t>YlAuH9oF{^;1s;S>u zjWF79YBObWo!vdDvsJW7Q|tM+O~~!65emKBi+}7ypY?li@@5C}($#tS(oxF+RGmnz zYciS~R(*esqAIdlQcq^61UCuhR9D;8XV975UF=9~&#C%*fpb^U+aiyNPiPIy^QI<$Q<8*5&Ebz18r=K>=-c`Unab|BJ?{dH={wuS zUMBqeHZhNis2497S7DfOGa{}P3w<=J*!HWjE_Ch*u&kJ#9#pfc*4vTk8%c|&U!BM)F7{k+RdhX=`IQC5PGS!#zVzEB zRTbAU3baD$3ngJe3W zj6Wa}Pm39UT;0ujSUECQT*~ruV1RSnT{PQZ6SwuxIEn3v3dsrTpOUh!nAQFm8226B z5;4oHl*uWQ?6Dq9KXqfSc6HdFWTtUXnTniD+P>?1!7Vd1$*TNeQkl{gWh0Heak=(H zs|=>B?A;OfsW%|?UU(%Rjny9UU3hUAJ~`JQ;n~94 z9C#&H6}9`rso&1N{yb@@9;ycErXDaSOgoR&$8Q}}?&ROFs{Zy|8LIYUZpNe^`+3jV z^h3coyS*C6o5Di7S3_vu+7?9iKWeBF&kY`u6c1u!Y8J~eLK+;>&US2klKhLJiS@j& z2$s)G?UT>+3pIT-W=`#*iCLSnfyd_CQzGn7x7r6EGmkDJf^VV^E?SZ@m-hHVxfER+ z=(>IM0ZVb_jQ7l1{Dt6hyNP-DEBOWzrnTO^BPRNnd~ygnxfct9!scCH zMuF9-P%9Mpf}x}Nl>heaC#$a;Xwn1W?R!Jsmm*`v>;3eM&koz}nWh-s{od${cbqwK zy*o3d=YtBpSLs{(#X~fyd>0}Rz6?B7sqdR%|`B2f~@cjgSJd&1pR&k3@QAK7W%VmvoNA<>o zWZL=8IrP8JK0VEnrOI%U_LH8NSlUJRtWX?xw7)SKnGeAX7CPYs zPqb?;EH3m^mS?JsxdEDIZN{1_`ikEfX1AFimRI8$8(8I-$U_#zOj#?Z*+fyw*$#V&b8LwAKd{TLaU>a5?w ziF2(g%a^aK;G_86rUqAy^z-EReD|uWJ(EENFP+UNb9QAfdI@CvIsFSPVghHPIb^+4 zt6!XGxA(uJWlK#keC$D7x~K2!y9be4EQj*l*Y-_hcnLb%5Mbr3MI#^PNRyGRue*Xc zROxc414AD*nYQ*+=s#ybKKz;e^eU9dB2lRMe*2xEAVlM-(9I>6yxe!ct~!MmoW_$G zJ$wk2vL+dY|7D(^6EZCKv9bcy_YSS#@k%(EgQ zx%BK(FMV*+{I1m0n%+1(Rnqn7*jV&7(Fl~q2?;p^++D7A)*xh|c3`mJg;Bj3MY`Bno6k%|gT^W6K^;=n z@CtPPWMl~m3fCfrrR{`^7(Jdo1+nj++ic9oI;FEUd6SZp+8SfE<`Ksvq3YLm_WJkh zWMtQ9glNt=4=fKKJAMMB&}`q+81Sp45oMBh(Z)e6T9R5U!dU)gwV3{-@!lI|;l)Qg z#&1VH#r`#uFG`RWPOZ9R6g*PgTh1)t~-s z5W$tSi^90i?VYl2dnJwcKb&+`%XaUc!>3Iz$l9yTor50m`b-{%#0D!7sVCzw@K+@H z2K-OsqQ)U$8U?Apf!QPbN=5)>{p&N_FuRel9-EOMBMYz!AX1T$-8%8#o-~t+5k2kb z?R_8D6{(p6_$O7rZTQBqMzTsudn+{)O3KRE2E(b(b9jI;&GbqE=gD>YiiSpW>K=py zAXb1N9sb$K=it?xv8V8dkR5m=*3e?lJA_u%Re59%gCtrKk#Lj!&*oJDb2;& z+8yMnECM&RKU~+IJ2OEMI|J-&YX!WSzTFq z>g6RNDk^jF;w6NSpC2rFqc%DO5Ik}j94su>j*c1e@smqSA|vaJ*#M#gHUNECZb89? z&vpX+CftnY6y0X95|}UMSD*bIRrK@Q=knjRoc*38wVe5lv7HF2tGzv6RCn+1-;00# zfG{>GHC5q`JTKwa-Ma;ah4M;D!m(}jpFS}n0Yfo;55k04jYI}ys5I6Modrc2{K|P$2kpUWS*ZAxCfI6zUU!wR= zfE6?u2+7W$KcDc95eelzh99?+QLG3`8XB5Mj~>BsqNhbvR7L?W#?APpy?tkU+j*#L zaynMM%>-hCjc@UMc6N54OLqcpfS#UtrJb&W!#oJP&<9KAH}Y@3kx9kH9+B) zbYL1~A}ul@VdC-z)$QGlzrN0?#RPy}LdTeOs{?Hk{2O#i#-{X5FCA zJSnfNOzg6QZv4I8JS~?JXK!yG#oyHZ9;g9;3G3@~iySEUZcZOIy|BQ}%DR~`L8RvK z0EssmO#=8QsI#E~K`3n6gkooyEzI`cTYOh~Gg~d1Q3@N;~@ z3g^0IOt=*cQuW^4Y9el6D=I1g;8jpi5P8h?#D=7KMJX`M#7t|~0Ldi%=+dNM7y%k~ z>Kdu3Ne!f7xK8$l78{);OnVFmV8~i&d4mT^&Cvk(ijQY{0HrC`(+v$O%F4W|w*vqA${@ZBX4?%cf#c850@urO6s zKp9I83JStt&~PI4uo=NQsd0G)r}fr2 z(S;y`f6UJp)rR)WMWKbs4KjKxB&(grl#F(H6*EYs(93}T1^j@i^*D-r8FhaCFa7=a z(K>fSd4-3Ehr83UKz$Pz6l5Hfk(Yl5REQ9yy1Ke}%8kN)07R_61=S&77S@wps%`<) z0zQ!U=(3Y|@W~@m?9Or2tnmhmqq}=0jIsK9W#oKo2%WRDGaZ|_CZejkT25X*YV|TR zf)Xqi^wh+v5->4B?6K`2a4>bYp~0esV&N=5Yo&~Ng#srbt!LIP zmQyu7G4b-Jj^ne^>Z+=l*YgYV@*1`{Q?Hd`AlidAQc_fmaD?jimF4BN4iX>$PnS##4x18{e{usckkaH;rw7NqGtSjA??fLrp zbRn$(7X`$m<;k^^NSjL?o%T{`C#a8wtwOa1Fh>gP=7jI8hI>|~vI zOx#>BTp#_5Asx7!j;aFgAm~!f4>wY%si}1?NUNxv+X)f7!4%W2LFSz z5_FV-larRfBzw9=YGMcG10YK=v)Y1ers9JD8Zd?VWURtADkI|`q=Su3-iHr>oO|m1 zYq~v_gM)(!2~tP^M%YT80UFcL+m9b*xVW+}ip|Z%Ow)HDr*0Dg{ z7+}4jq13MNU3Q>MRZ>vsoIlUPQry+<6IpoS?|9?>$i$&1G5gc2PMBR(P`hOc5zNfY zynKB2zr%BGUBDl)_@t%RlT3mcGjG;ZRb^WF`}7eHewF$Is83!>3Yl*}Ks5m@sMxU3 zbLDr)AdH8_g@wr@x@rcX*Vx&?Yy+Nd1=Nv1V*#ti$Hxah0%Fy<32$TLbQ{+K7hM%j zJ)%2N_{}Xq)e0Z(^U%T2hZp3X)?zU;GxKA?qXfEi@4!IY@dn5!%gf6P3oimk(%T78 zjZI73^74Z)f1|L_YvEJrLz-LnwvMSTEiJ9AbO#?O>rq~>gQcaVwJ|4#O(^a+5x_Dq z3QnGk29qA4nL`tcU+&^6-5E4TkBy9&m>+$5rq_RujI`&SXwg55hiMQ1Q_oN+kTq!Y zFRrX~0b&A+&B%Es+wsqZHjTpmbvo`87cdWD zj-;Y^eoOEK4Nc1`L4_0CkdnqmRb}OUHq7oHZ+I4mft>$saa;fpD}1&88xiczegODG@{fVIzPQ$0CSk3Kw+Le#0Hmj54}>8Xu8_ zL1@0@u+e914j@znYT{VLkB*?z&g15$?#x!5k;)DyL}`2(z9ddXeyy1}tf7RbNk2O0 zbe|C71$k!*y;Qp1C{c@hs*@)-u-Utrn1(dK(`3RSuW0!1Fs3p}qW9tkQ8{xG6ikK3 zeNItcIM$g>pC^}1d34>HOOL3Y3z6XqLzi)hd~j<8MVot!sb?k8QZJiPeKT;90Fs84 za-23*opV+)Tj%H`l9nPXWx=sjiHN>by^5oIlX7KfDf%fp+XWK|GgxyZ zQYXp)Ct={kl%#XX;z|JFyN#L=o~`(8`1e9?NyBE8R;-p?oB}$2F{0=tErkIYPUQGZ zw|U-mUg39v0UiN_>tqXXcxd?b5^io;3$j%@f!ynY|Q1$4f;P(b7^>9`6(ox}Jp5uG75DL`9xPhD&6!#hT}d^1h7;3@9PPUkhW# zN$_IAh4Z<&FE^vy-n=42eG|=jD}?xVS~6Qx(CLuZEwU((mO=`yMECs_Cj2q)+k(IV z6*%>I!kh$eY`E|{Zf=2Q)Ko4JPUX%K7D0Stkj(y5qo)_dzK zQ7iH^6DuVvRjQugZ5ed@lqMw=`Bw@Y%kf`b=6MLFOP2zmQ&C>Z$Fp_4hf8XlYewtAUmDpGHFS=sUB+;8db?(R|BU#{#? z16p313bl|34B5bTp#K!jmy^*#*S*1w7SK{+IzK^DJQt(QuLR{UZ|}UDk?$V1hCl*H zNAzJZx0Jxqk;gl;r3Gmho?cla5Md@v61ms?Y|x6Skl^!(7$yPs*3i&!of_tK@XTRw zga2#2J^Mrf4OZWOcQOnd5gm8$#1$zSnU`^KYu(DL4L{ zb&q+*4{6}w!EeWX<}x2{({-MFYvF%nNl*=+KJDqLAAAJf4R92}eIM835>Y<94ATzy zyugn!H=lUuNhwI1FOLD+B`3EHUKIHHDx9m}*^f6pQ%>yo+1Hnz1Jn8HN;t#PAsHF1 z@X;Ot|DV9?&A&0z9z==3w;Ubc1qEFOYiVUQwHXtv!o-vX=GUyjTff};%-OS6<@Zr) zXjsS_9OP2=eB7D5^LH%wmLQ89^SN`&i;GIR&!Z>u0pkxgT#$8&Hw6o_VlasbBlnGr zw60!_eYnVg-n|S=nS1>+9>;=tBa8MjwhHedfK+ zi!3TG7Lt+ifS?$>1+TxO1PGQCz@@g&<>;~DWfn#;A!TJ{A(%*zg>>41(veKg^fFX7WBlDFCYtoD2_+jyT11QvUxi zI^ivR4z#5|gk7O?5VFGT{}oH|JfMAg0bKTX@1ml>)rk!ckLyR+(}sOEYVa<%=>p6! z4Bni!+^M)h;?cJU>^pBff=xhXqP=~tac4ZEOOoGD@8(UI*oW)f3t3lAz<^+2xM6nf z^Wb{fzp;y6Fwia`M+WtX{dGSCTyVT-aN!_|6c!O#uV)ECmRDDIfB)|G_=~J`~<>2@|GYliIYku@q~owIfc9bQ;CSLZ&L$5AYr7DWgHp9f&=fXFg7?H?U|!RY4YB`+se z(CYy1*?+_p5N}O)QBqt2UQAR}6tS&s4*zvbH#9kUK$%$-V!4BziB*VK&zvbMD@#>J zH-W!TyrN)uTJ-*pi3tg#{)q_;L>yYtSCe?^3A^JZLY?h%t(9}YOgq{oqNhh*QZnuX z6?ARkB;ki@W}MdkzP?I8H@kL?DVts-x$fIUU!MW^D6ZuYdgtfc;&ENIscC6-Kvi!r zH8q9q~BUSJQGC1 z?p|Kz4c_Z;0t7D54mLJ3^{Y+w5N|a4?SQrK2T&uBY;-R4W3e{`S$_`?*VWbrhlSBm zQh-Mp9v%)`CyeqXcEHlGDaQX9Y$_5$$9CaW0std@fX2lt@dH@-N=n9Z#~lZ}eS8iM zRsr`H78<&@zpqYQyG%uY%{Kl$0pc}t)Vh|1g~ijSTNSogFtRg`{9zXbNY(v}ZRSw% zegcM~QV~|`FGDr~D30xuSi4Ff{DV)P?*M-Ijg40?{!e`K2ej(ekgNehvl>3Xe*r83 zErwE_jj%L12@ynS3Wx=PY?gVmJ=+w*cKCHx7AF!)9RBaHA_y?ANWp&ti?cOJa_JcK8Z41na~ z>ij%(6(Q$H>?|asAQXppFCb6|fm;WS!bQNhNvXh$#Guz3lU)V1mMWYsI<^`bWFg2F zhFdyAzf&Io2oFL+I$GMbv_bg^R3$`z7YAPoEF>^L0NWlGT);{r-mGFsO~u7D*VV0WWj_FVJ{ue&*9++#^3c`63F?5{=zB2(xRfW>gu*2%gW)6L*7<^4UUeQUPy#8 zu@zXWcXkGKgDa}t3VJz=UNbY!6zoyAbFs2whOW2s2&Qd7OF_7UzxBhW-oe3P28V+r z4Wy2B%*`>dqX#5gM8-qfry;{zRde(q%}Su*X5cGr?(C#2y}N5`GB2Isz1dVmmv(%xwS(dtgA0pPzD-nUV1plvCcm&F}oB zuP=cy#ff#}XV_2F!PX#Ve{7ZC5ujHoqDH*pFwS}xJmR$hcWL9~6fMG~;$3Fn!`T4C z43LOJV`E~>5x}TAqQ@_RSM~~6xg)(dcUj|DM5xhn`$vBW^1P7ji&(wFcnpW7 z^*(g^l$w#@UT4}M%7}nyEJgf*Fe~l)gA4FQSDW@fgoPbP76EPo^=2A&U&xSiYQ;`G z+Zh9dG?aQpGfK(HwH1*qExzxQbRR#V!5>oY&K*aHy9>6SGDj3B%U` zCtCnW?!b@RiCP2gMsYt;%m^sYIl}7A(-o z%U&O)sVyxhAz@%*Yz#Z2XvCQqv`Nay1iN65-v(rVx2BdBylbdNQc#$mn=6EVgiGWM zdG+)iVry#)dN^bgxEZy_5ScdtVVp8q)E1O&ySLe|XL(2-M`Lj)!!t}gR`=uAFDFMw z{PuQM7sh;TZH>|VV>}C($&nF13yXm3`QNMfG_)YRi%8d=A|PR{!lF9>|DS6XhsS6l WYb_%`WLUy4$*yVL)ck<72>dVNDZfwv literal 0 HcmV?d00001 diff --git a/docs/images/nd_spl_3_comb3.png b/docs/images/nd_spl_3_comb3.png new file mode 100644 index 0000000000000000000000000000000000000000..b50fad23f5650880533920136234254029a61509 GIT binary patch literal 28176 zcmXtA1yq&I)4zn2lt?2X9n#$`U6Rs`bV-*8($d}CrF6F--JR0iUElKl&v(z^2>a~a zot>Th%{&Gx%1fXi;UhsH5ELm%u}=^P)HwLjL4XD$TpJ!%;2%Ug$5$z$3%?{Wa!rkCe{q9~cNFCxTr=5ub!2ku;K}jOoxPr=lvz%M%wDrvM*va!C8^ zFC$eI6@0=$Z|B8&dlg!zy~!eekE?^_mZpFJxF6r;ZT*8Jzto1JL&f~{e0p%Rv#ZR= z$S5u@4i676Dl$J_ZgKkhl`D1R_g1Hpl9FDlr<9ZwA{v^ys%lMD6#=hvX>c&2bO(Zc zW_4%~#?OyUF2}k;LP9z^X)P^}U~hO7eDcCKYQ<}0(HlSpww2&p*}^7-lM?nJ@JWRdbpn-3!cHx7hSAw4xN z4%|_0R@NIlJeoZ_^xz++O+h5gxM6cuX39mAp*V~o$4K-%X*NbiMg|6JM@QlLEkW&B za}1Gq{$ig1;VKh7c3HgtQrPSfDf&{!(Y+FR)+gqeA>VH4pboHb%v7Lvsd zCRtI@9Iq|yTJUx6o0E>-UWrLx5<|ox=Qj+M(;*FwjUjy7aImo2E$(sK3HNt*XEAIp zhx0x9Ojk!s1Lp{m9s%PFoj-2Rw^LG5Ds)>kCw<%Rua5}{386@6Xmrh7IhG><=*mb=p0Chr!(%mN^SnCW9&tHdw!XkX`tjR- z{ZBkxY;<&SQBjA_L=2UJ^TF&JNOW}c-Q|9-<-yOt0|SC?XT~=tYo9-V{vw6f$^EsQ z1YDFA76k9?4+TYvm&& z6IE7@c{~7r+OD)N?NAW%e4YJm@>>1)ld>`y1;x}!)brClT32O`M1+9rN&BBa0W&8l zzuYhP;L(U~Zf>}^xe@I&#J*<%$|&i2g-4AaXO7(JA7{A=+wu@$6;|J z{QN{eeFB#9?P_9Bu82#~dwRnq*f53($DkT4MQ!bOclPgZMy+tn$ACsN=(UDLBLr4g zSAUEkIy^pR+!I327Om3Bbt`RH0Kyx;whLxj8S7BKd@r>*WbDz5eRk zGz+Y51E7y;B=Twadh>*i9tJjl<_k55hxIda^TFgDI@ zYkU41LnRKrB?c^GTh`bYU|NBAWuuBU8{kf7i3Vl}`edV_NGccheS6qiZt)=CutImj zG>j0*D%EMKHQ*(JH2^2vz+sbkOLEZwWsaB()&YtJo?>rm`|9hcirzPp$pJ}ta5!G( zr=z2L>MbfM(Q0x&Tny9dki*Txlg$G&j+s0| z(gq6)`*6FR@em{7jo9`6*+P5)vfxKD}bml$9OK6!yD$CnhWm1!;6X*>j72~CYYHZ}%m^*H?}zS8PN|KY=Igu0qq zI*(KF?vF*c^DQS}x976@HvYu;0XJ(h5D)dOvC-Yw*jR>{ZC?Hm$jgD-ujuIU{h110 zm!rjvezPw=@M+ZirWpq^07tvOR) zmY=cXHd>}tS6E05=OTMmbSRC5SdtDvA4G%JDZe`&vPK2zBJ+lxNf*_KyrRdAfTYMIBv^}h(IUFa)3j` z#B?zoOcW>pbx;GbRCTr09RkGI%nV!!P|bwr448qgKr|}3Wb(o>bi{8q2|^GE6Y|99 zD2d8MDR=^WK>GQ=E)XLDQ3LdkY~TOs?gsV_p?Dd%=G4^GQMtF0qF*RrpkI6pyXhdV zlEf(iMsT(5ioo#9p@7d*>a%uIdb;=Nzn*|sdICyClYkl|3pO#9%sB>V2s$1f9zK4* z?n($kWLB1eiVCLY;I~Q;bU;H+F89@;)L5eezJC3RiyTmxBZL5(rvnSgEG#TE8%YPY zhsE5|vdVT&2wlh=#~?T?Y~x%PmKF#Y1QfPsp#r@JaAp? zKxx%Ixf>s9*4qgyX1$k`0W|p2@DrL2L0eiPpao`=x?Ug<2?nHaiW6c(AXc-GG~TYh zsO)eUbC;K?=IbAKR*3Je;USUH&@wW!zhJN#Fzr=7+}?t}b3O+1pyn?*1?H^hNSr)#2i_D~86^H(wtgN=(cd=#1mT!rOKxybIHr4k(2R~=0*l|F)D?+{r{zeS*ZJ(#_zL?d3UXgm3n zBtWQX-Qj&ntKEv6l{K0Llgj_v*Vpdq!}6(tfxfMtkCgR1TfIF@YMRy~{3!_qg}CD5 z`kTMs#(@xw0?!8&T{wJa|200oz9}3;-x${A=vx6*sWu`S@%J>Hbw7Aie~;@M_oik( zjW#0Smc5~DZ5T8({}E=yz0zxK&zHMgQBxImV9|rB{=m)sS&_hxG#UyE4;}LDXsM|n zHy6RG-g0JjFzGIJVCw+cm_^b5JVAgO!Pul@vH58s620A)_+bwV`5l5z6EtLUTEvrY zX9t0T>6^bl?BwMea|93rf;0{V1;n3PbvB5m^^xd(ecy_US8T4`pNZhm0%(+zxt}Je zW@}L@zFFF$Ic}}CtOueoxiVGq$YA1#YQGD^gE>7N9nU}A-DR@Uc$`^VQ&fEwL$&@C z<0@7#(fgBtDlF>F>5mtN2xTd%sH!p}=O7q!SeU$8Ue>pKO{F06{G_N3pexsQ%9 z{$N$-<5TaEP~kE;<{Lr0-zMMyTc|QR3ej7+t8TI(luboI;bbG=LXVDyK#EJ&+M&3- z@KscAkuqQ9<>m2+q)3n%A_2RYo|k8$rWOxAkUzu4MtuYK^oN>ST>}ZLyNF3im2RxEsnogk zU)bi+IfbbS^+O4*4Y#4Xs>OjDP*dNdb6;pwDqebYgfB?4HjuB@%uX+0B_7=sWs5Puw9RvoK za47g2+uKxo)c#$)@TMa+_jM*rxFD1Hf(J8>0DK6)o0}U*7MT~Um^pDwhy4T5w79MQ zVyu#1r*XZX<)t(|-<=PTifaDT#@RDTG(RvS3@Kb=?Gp?u4ht*<4?$AKMvRYi#wMe-7sWS5l@ z5)lm>vw--DJSZ$ImEVJ-zd=&@C8ANJjs>9{u%6nJzLL;BKF_KpYQa26{y~aSP`4;Q zKQ2@nbOEytA|QATUU_LL->@*q$1bMAVrUG+jSX1lmlz0M#Xu+-$XeXzgMLcMjR~#%5SJ(D<`sQXPCUjZZ%r)z#^*+r7uV0Z@6}c0;r34iI z%v_{U!0_S6BqX?e{d#eIz3q}YC~M~c&B(~a3WM~Dg+)jcKVR;xSlIWI1H3RI`Y_+; zyM4xp02EU64HrQ4&@A8zNN#SEPo}4QbI1;sQFIz71bc^Q3_QHoP(XJ(uENV)KCb_~}GuqjWwumZa^i6*Qvu2x5Jc%l)XS5-q-)ZH49% zYfL0bshX#7V!_V8F_~p3adF=afbPxD(-XZTLS1U=+8iKZL}O;I^l8g1ZUj-8i;If{ z&IJ0qoeiO65TQU35D+X+933t&Szt;?Oi#8qyD6$p{eybE>kxddW#^85c;p~4-9CrRez-!%s@*{hCmt`MoVy3IZ;tx`{hXNU%$gxe2SBa z{~)ZaldDH!sp@tHbNH3Z*O=%NB^rB3CvA*ERZ3_(2sFF&TS(b6jfhX^h1eZT;S(+{u6H0Dl`n8?nH$L#ue8LmT0!M?`Cj^=*%NVl zSh#ASI5)R_A*F&E9{I+}=VZMXofz%UAK)VC)S)5T^$-e*O1JZhb&t#M;e^0Cx}vA1 z-mJ)g_|g)@XV426h*4h(sbI#D@PBfEaIC(b>uAef(C2#!fp!}zE*v(tzAf6-6?FK) zh+xMQR-+3%gv${Q2`SUXR#7crR53CJhA{1}#Nst}dzD#a^4&tK01l&w!TZR+LX<#0 z+lVsQ5Wjz~NAY)Z6tl3PotK1OgMG-}?F_mEb$+C}VO`2V!90QFnHBGE+nCP^X-u zrj@8;LAbTIzd!MX4S->LcJ-L=9u{;6ps_&Qg^rGngF}^j`X<=lv%)o0^&e z8Nt!fQGb8G7|j`ybMN2A#yf&oZaRrd>BbVZGJFoJM3CCXj)vd{Z2-U}*U7}h1bhV_ zA0K=Rq&SOBE)|7j0Q+RY%gu$Y8DNZ*B%$b09cJg`tgERx9M6+6H8pi|s?qNhnulUr z5I>omn!>`yR#R6mDJqKkYYb^;vijE@3;-&O0j8}^Yilup9KR901N@hLPA2y zMnQh_8dwQ>09^o-A|@^ltou4x%usMxGanxvT_0RU=ZTmJNL~O%$Vf;Q8slYUWsg^f zS|HJji;cy9Dn-wwdC|CEAq^lFcNntXowZ@Lt>R9|D=#kx$zyglbe#Je&lgif4KhY* z>e12B_J)rusH}|g!zOf$j3YpG!7WQ%J@z7;@q(pR*yy+Z=`-Q3|M>}!8?yd+xGV!5 z(VS03GZlKaP7S!Um!vnN(r_CI$;ru!iy&*G0T7LvBLhUSXcCAebeT0e>Q&<_Vq#+X z_?=JGm~wJM#DdF#-r(RMgH9tgpx9d}NF)pP)vFfITZf06)8?;R z6ns2s%Q+Hgj*gDyrWeL6pLKL}q@`&A-}Cc7x~{$d2l6A>%#Qv=Q^w4-97Sr7WOHzE z0A0@;`_hS^paNj105oFn?O{TqVrp^>DJ7*hWqabuA~j1_;U$I)0ziYjA6x-MIrYwF zTT~x6U?_z%qCzYcUse^TL`G53p| zEN}%kr$$d&&zYYX#}SebYxQpE9B6XXi#-6b1R~>+ew@-JCI+tRPBx8)oq<8Vk{d^{2*(^UVdt91U(#P=+!Vd3Do?2P(~k@CBqtOCT09K(EfJP!awoy*xjKjr|GV5F<-1sfe2 z8tU%Ot*oqc+#UwtmWbDiTGl69UjOlO|F=@ibreocPQaD4xL+EY`M~+DZEw2(jH|m_ z4B+TMm;op(oTm?*CKlG^$=b_$05%>;(@0bP}ibd-G-k8w(Bf#1M3Wcn#tv`QG0+?1I zjCrYYf&7mjKeEIBaTcmqbc>p+I|O-}FoXl(Za9ti?H(i4>ylB~>bR;esujlIDaMPj z!^8yo`hsq9;f&(bGB5y-SXG$wg(yz?@pL^9{1yNVcR;{Ld3kxTQK6AAEMlI{FRIz} zhEB!LpK0jmWCmv;LYkV5rKP2{wQ7^TY%l5pha9U_+s5qf`C^%+kW;I=Wklvul{pmWq z{gK2M#3b4HxVW;?QcNcjS1`d4Oz?i00F;4_mX`F)IJFFjAlNWFJR7TUk`52yQ6`j) z0=q}MHY6kjpr;rQwlJY#VaO#!qVTZG-ivGsgxu-Xut}F z<>Z#Iaqc~Y(oO$_!$KOYO2bGP#-P8YrfMrGMb&${O62LD{%~grg6gdO+%7RliF|`F zn+-*5u}Ts?YL&3lU6MHParXa5X)i;_Vx1A?4!}bG)}7xX66x_zvOZPJ3Y#ZncK=FT zK#xjcS|zgzh4Tbte^i{OAZ6A}TKMpt)1X~q0~_ULv{={veS8~kw47{c1In9bL>PhZ zbzj;8rCSW4@^lQ}tQaz+Xt06FCt$K-Q3AjvD3I(Kw|)nUzlHNO90QcmUSi-$H~Ets z^=NZ2TXcR?_@K$QDw?O;iLt(77yKUT|A)2zH*AP;uZLz&1qI}=y98`!Z$9=7EzTVg znC+;jM6lf~yFojZ?tkZC%*sqh&VHY7jeQbq^s?W~f1D6P|58f_9j!8G@^%7nqPtui zMnGkc0wLCUA=w`&M0fSex>oMSeOi|G0@keTWogjN>>* zo(UF4{PN$z3%JrvouqT%%)#bh0qFnEyj)BEYgee6ycMj#3|5f(_}>Z(rsNNG;^bg> zA(H&%(0KnH+IB(6Ts?ALia9~9Gnr{MCig$itIsMb!1+e?alF1EuXx$d670urhfc({ zfE9M7JZKWr2UtN~`X4K>fDXLM)B&s@{|)B;eVH5Q{sZK9t<`lvb)x6}p)$m3|GQ`C zz;TA-`M4q zHT9uG0sjjtTETX~w*PH+3l_$icVtc|O9KK#b^5;>&6qboS{3F65>yid2%Z=HZ-s#f z=f{Dr24~$t>%vDcdxq$L!jcX=6OsxT)B*gN@&4Z+5fBR3d4rT!o`+?KFs;s0{|QAq z7TbdL8`>?7aa17kmg@i8PF-={ng|R|{Woa)zrk(AsH|U&jg6h1oqKy0z)IM+=|&i6 zsHv$rIGiP*5$9xQXXaDxuJ?AwZUpr1nD6hP;zBS(YHDiKGM0gRni(IraAxC4D=#Xd zITC!^QCwR3fsRgDO%2_l-{ncFSt_o$IC zu3Jxm`cC^kQ9%=$H5W5mP^B8P?D?^8SPvB_$oLD*EM z@QL-5W0cz=-)3lnce&>>&K;oB;dtlgv=-(Qlaqcabc0ayc%`kRxVZMPEno-4Uaj7b zA?GtePFoWbJ7dhuuU>`C&o?$cXWSaI1dCDr#1#+;E;c6FW7Vq^`?P10cDuG_04fDC zveNh#hPj-QbtwVv6|F50p%#yI);743zavJmtOF*N(O>_+7og2kdZ@oIzeO%A+|&A4 z-- zl#-QBWu9tPj}T~K^Dw`jh)S^B8hnn=NQq0qJNqXkWj=NuWEP)8k3;|2*urAol$G&4 z2)gJ%t)j3HIPtjXXpB9Nu`g`T3*Prn%YycMH;Br;yE|wH<8cX1)RvX_?(;J)zfRw* zDG5qT1FVwSW7^8#fgaOt9$YLX<<{1amaeWDm3*>5r`2NR8g@vC^kR5SN=(v;z?EwE zM6d9*wvG!g*DNCMGw@(Q=)QdU6122O4eI=!UTBAcnI9@DJLhe3TeF(ON;Bci)YNYL z`C3oe5U4+GGo*5|v%a0|!^i=F0w_d^ijwj0Y+V#hJk>p7kLZ{(dk&K5kWq-2hoU8M z<0HWP^ZI77@|zK4-Fj0z*~Mdi!}zfMoFMq`LQ z6?ar7?rgU%Z%<4JHZ~Eb*jT<~AmCvibe)l-w1pLE)TqA>p0_ZEKeXwxc#_qM(uD`d{+B$yyirIt+ z70M*tB1?I`{OG~JKww2!z|GAI@-D0NF_liO`JO8E{?>$ya~`U~4Z3gD1Jd>BMCq0s z2Sn`d*`=k50$J)RvG#4Y6gO>0MYbd2~0U+mB;?2g}vclMr=n zy)q<2>{#XHX}e2+TRoJ5ft@BV=sdZ!vf`64KvXgov+3`+g@cWwectA9-;%#0o|NK(wXHdOasO`4C_T`2xtZH+_?`(OUnF* zHICRj8tcR8pH=V72nD_-X0T>lk4*adMu50qS849FBZsR*igli38~R&m>Eb^VhP9|7 zG?*?FcsvU7QCA{qS@o@S0XK)6(JX>8v-FjEQF74j@wM7LknkFRqRNMuzN{|@A*!21 zWO-Sj#p6mqk8E-nV3 z5;S_>)7Fvkibp#_Why)`KL;#D9gb(I+m;~+414ODHpx;t7GGAMIUH1^NlZ0euHLrX z#@=z8j0{h$_iCkc*{7D6S0rLPCduXx#TM!0^0=0+e0+13XMl?3$(Z!Z@qCb5p4YiC z#C$V~4jIlUe*Iwhx6YN-@<`Vvi}39Uh1*cLASjtEt~H7tcpvb^x&t87uS*sSWz$JZ z{q;#Uo*=kN`lQG0y|@w`9XIat26-Mt#A@9Z_Xs>Qm;MO_f548ri-}eT*80Ye({_<> z{5rlozJid{@;jO%B6?LOEYGZ%g4U=xpP0`B6-4XKz0f0B3i+N}KESZ?{IR>2x#&dkX|+BrOZ zbsAO0SJ8!48#%kW51tkg?-3{F*xM~G1z~n*<$WcW^t+j6kvrk_4qJCYr z>6ex%EDY*Mp|AgNYXli>{g_s}1a;lBW3T(5RL9R4%j3>gvqP#|o79JfxV7Go0kfQl z-GdDudU}2|)m5ajjg1dkyW|vfS0}!P6ey(excr-30ys#zwwx}1vS@ODYYQ6_!}VeD%|A?RJG&G&Zs{hY z54~3eOvY>3HU2{lAJ}ng4Te%bCaHU>Z~eA-{cGE{cJ8u{TXJ{BlDnyb2#2 z09cbt%{g*+d%qkW6omP5ipdgAQc}aL^Uf9t=MbggR}jCByZeaav8nZ)L0gkrnGJUv z8Cu$Yv-z-k7e@WjOyPeTgRdZPepwY|au48{Vf5QN?JL)W6Ws&vGbUytDVsc921D>h4KI2AyUfB$yieCW(qsJlN| z^W(Dl7GRK=*Rw;na1~?@=y^-nQsgJ6Jza+5M8fBec_VoMjzLXBD|qQ+!`US-D`)`< zd}G7QM;kpkhRKdl`nf#_Kqmn%v_?Gr&nlrq9|S)-*C^{*KDcjkz2!f*nL5im$Is?1 z)s(0W_%N;xxy?p)}ZO6GBSix|U zTWK7Fl(XJ(;(lwU^sD_y1B668ipObp2f1mNt4@h6!+FHrwYcya-<(73%deVO_n$*s zO*i1+1TdUPQahQfqN>h=*;9fXiSY%2G2w17Jx>-?2UIW|+9XsQy_Lp0NX*k?RrMkv_1rbtA!=Rgm zynL;O=jwcwqq>QEe0>5Ae$rZ_l(sa=SDq!~*^}AcMi_SZl_3a|8YPdPQht^>`?-+pBn>c0Z#HS?(g(2kljwd|fizTL0ns zzZ%0*)rPs1c6%--?ye3PnSmD7md!QTu6PsS*HklG?iZyvmnT#Lk=_9L~~vd5dNqx0UkmK>)i4sLj# z&FCxz^;WnY5gSFwNJ$+w6j`CS7Axj1&N-WN`*IhXZ6DpQpB>lsN$d>Srcwbp|J zQtA+_-hS|1Xm&c!PlMH2dE8Rf);M7$fg)?Oi_M#h-BmMYaZz@-JhBiCKtV!9O+UM@ z`MF}+@z;pRveLWi)+uVPO_HP_GAxo>KV zo38iV#DB>DB94(6NV>jiI_E&J?7aV382b0(@`915LB6{B{^#jo;&zs3d2eptr?{!O z(CHz1TAHWpyPR;sTyA22z)4aCryRXMqZ6#ALm7w=x0N#-UIt=*o5}n%f_g$K&ijRM zCmX3?DHP%d)di4+3HnTmLdNS?B24`$wnttjC&vhunAiuj z;+~zc;d4j=z)c}U#EsxS{juzK2nJ-s)N9>yDD~@XV(tO#k-3DUWA!AIvq9xR#@sh5m&9Grz2;N8$cQFSC-ZpqHZh zr*Rckg@Iv*WTxOmxq$%W(e!4w%hQPh1SbQOu1Z!SGY&@Q&BINFbogGXR#j!CiGyX< zL(AfvvY6)5_Th)kb2a*;Dv`bC%kJ0_8b~IuP{e|LDb?)Jat$xrK~X03+n~h6#P?lc zuFGM+%gcw4|D^SYNV~al6T%!uNM)D~)?(?qO7NqK_)Wp-3FLvU&|hY@x&jz@Cs#x~s2e!Boo~|Cr zJ!HjySedO-7bG%V70d9RZ)k7>aI9jP{_)+4u7P#$`9RCjirbUGu*VD(OYqmTt1_<{ z#z7*%U%Lj3%WW;EH-ZmY9K7@gE22Tf)ij?zEo^ZKBBD}Gmc1PtV-8J=KXdl!eq~z` zl!&7)zf2y}XI7EQ^VL1WZ|ERDP`uUK{`?39EKoi>u3nPFs#n~>p1S{}{t=nI2Djr9 zo(#B);FPyXvXog2rEd4G=c&~D7W=ifvw0;IEq1qj5RJC46A{FDL*ZAHcZYrp&;KH7 zjpH0gmiYC<7KuNQGN!{(3DiR zdc7qu;2Uh7Baa7dk})nB+KHZ1+e*uK+cthiwUkM5^x28AUh zO?FrPH#A0bD}P$N&G@o=SoLbzZ)|bSsH;T-Q5T+|0o_8B>1@cpDQzvSDHz$4iG%K!xYmq4-I~hJBxXpZ@UYEyC;RnHr}WC0pHOG-CR`)u)Qe z`qk0hN1Vlq-wrlPQt#3Hc2Gy#ytZpIMb~u(_i?M3$=p1wolWJXln{AGyq`|;noAe$ zDit~8CqX<8&mcW*UWjP*@inpOH{wca**K%9$K;Xt6Xe{m%Fr%WhtYs z7<9RXs_cKrtd`2jf^rmSHu!MB`A_U%U!!36+||_p&}5+Gd-Xi>aMN7}^3tHS(E`Sd z#H6t3UFwOGRh+SrVV&!Cyq{vEhx7GyKF3gU8O5c=58DSFKF*C*4mREf%j=aA=VrCD z^TiYnza6AQ`nww{?H}))7P$GQlS7a~AejnncdXBSKWy|+&Y+aK9fei~J;B^_ltDZ8jrWrZ2&Ai;WkQ53 zEE+#Xm@e0-g2L&FousNN1w_xEX*I6eFWYgeWHkx{NX#|>V2C~Iym9t1%G*{2fzA-+hF4Jb;qOA zSC31gA~WiG+>Z5hpZ=MPQBLMf)wq~|lwv2<<-_LG?J<_KRzjrab(t`<=5SU*9c=6M z!;it$6?%LXX=5`Sr7)EldnYrkLm%(_YVP-ui3iusw(}WoQ&%n(*7Gh-4QM9In{$Uf zgk2gh6cbqkdrRpx$T4VmHeZb}`rr}a;XJA-mjq@<8?HdLRJbmW?oBxEg-PkeH6qb= zmRV|N2tLwxrnj&?64m_1el4+AP+b@~%UUo3p*!5$+a>1Rq-!ORvZLUnn%JIa_JOYM zd3ljVj{cS>VkW-_v+B3u@BNRO4xjb|kus8t=l4mZcFG5WXKxZ+%zug1%OrN_&kk4G zx7le-`c@Y$9@tV{rMY20z0*jTjE=!rkOCn*-sgq~PZ8~L@CphGtCfj(+D24Dl$@*- zwiD*%+1dI0%N%yB%UENPhYf%MiI`AUNUS#(8{0 z+2_bBgWNfUX<`o+(A1e7gJ#ZK<8Lk!H0&AGZUI|~q4ds4^{7ADE)wdS?ABvGC+r$_ zk|beYWUT~gO_PlD&+2QRHUy{LPLC=HF_PTe(OX97Puey=$(UO`A37VALQ6rL_~rz> zQtt%O?#@oP7szihN!zIz#z#lNn+Oodq4jiLaW?H9`g{!<4e@8aNT}5=c=?&~l{OE@ z&EI&hw>1#y$~lB72LGy`wD`w#orQ$PKb_az7K_)V!y)qKa-4kF!g|A#X8Gj{CMK4h z>q)s!twK7h_ky>s^9gy-U}~7smgiC$BQ=>sxUBog7}CER&SRZ;Tc=dMYs-0c1Bt2l zdft0$vqAa~eLe&@6uDm@B*$9eZO3^wOYDcHPt$2sM{+8p^zyoDt(ixRis|f z#>UdVailT2Qk`JpHxJk2;ceUGP{c{yKH0{!lw5t!G_N@Z%Gw6FZi0 z_W&b3eHpI;S6YjAec(Z7}DlI;rqJ5IKv%NbklpXteMw%ZW7GK%Smnfa0U zsQBeQx9FGxJ4{c;Uu;y;mWup3{8#cOWp@DP3|eC{@PBiDHI;7E6fH;z_v4k?1CfH2 z=U`M7yxIqiLq5yD#p3(ZM;f>6p~DMJp3?VUcs}VrlHLgCeOl>mUO!o#j3}CShoL6) zKRfN~dR*jx_R(wcOiP=(8C+>J4OG*kzvpJ%Vz7^nKXSb|ew)hogG?}j*+Ru@g^mSM zSW*PaN?gcnPdZ_(cH`yIzx)4`-IrOL4fL_K+K&#bj~ZLT(;&Tq03={Y;%AfR8kg;H zbV>@hr$5QU#ffN~w27L-WCtrPb4R9v&s@0n!Q@aw{1sfdv}cBhg3&c=3ct<2Au8V_ zx0REO&ICQ}+!BRY-dNCK^E#Bfm=&Bx<1DAUu;x(7)riH_e<8oESBp)#>3(~#BWb!y z!-Z*Jpg)4Z#OWVjgj^-d?^ar++xe%@iW{4ouW?i3U6%j+*(b1X@p>1i*7zZ#$(2L3 z#@^E3#df*CTb>of>vzum7gEp5h=W770neBMABjXIBoZ#!@gTCY0Y0~#V>M|P)e008 zKiuIwYtT2at4c2(n`SI5+88>&s;d1AbJbKI7)Yy;3bsW-@gIMa1fL`E8PofD_-~X_ z@Uh%if2XsWjlcWqpo7|3*=sI*+^3I@&-)SSna1<)1siB@S?d|Z+Ao^6U2faj=u@{> z;(7d@<~-xFJN-|)M2ZaqB}Vm!?5->g!Q0c+tji8we3U7 zwTOj5IN90f*y6j22`$Z!nrmo3`c9}H3Qjh+JPA&o6-taDxNLo^s<0AGA+4bvQ_3aB z=amvrbrpoOP0ajn@9W0EOO4ptUp<-G zIqwsNw>SCaY6&nlt`9{fo<#}ZtUY`Fos7aBlw|dIP#oSw0 z5x;tMi}p7INDx3VE^M_lEG#VrQ5&~wi7TK42Ab;C+1Y1Tv&fp9W<0U*wi=xpt`AQ> z_AZ{bJO~ik!U^xrjyWYXVfN1a9_aUm2zB-QHG5J)^y42PtL3A*+S=6O@86g`PmaaM zdraHDTgjz!Bg?2_IS&m5joahqtPuT&073Yx0~6~ztsiU^^?jF}pWEenIks{N!Z)(Z zH8uaJ?A~d?OvpQ&2_~A{D6Dy)-wt|>vJV@f7YqBr@gx6QXxh{-k@(o#!ZJz7)@uWr z)CZs5+60cgn{7K(%rPpz(e)%G_G1lV%Ud7gr{X?#yUMwMe`u{J`7S~Qq23hO1y!w` z-8~EgD8$ey*b-&kUWupY9CsNY{xSoSbjnT1iKujD@e0g!Q|u@C#ZS{V^^r=hXD4_M z!F?dN%;|M>+)9Y2p?Owwi&T$fq@(ZhL0VaV7GXP@dY;RHKz@ke@z1kxQdLdKZyzs< zSLoJNX}QQjc8YVOlkc)BNite|C@`Sqds(HW2}3)Ow6)*2<&TNiT9Wt8xrK{sP>do%~%(8 zJ>JmJ(fgfSDc3kT%^q`}8M5Q<_lBQihJQb%P{C6D_b>-q!qt_su(5HdY5rmHPqcdf zW$fdvUM~!*MusTHsoE~uY%%sWC%q=LPPO~NW%K^cwLM1Bds&$y3VwE{Kr4ZukfB@CX@PK{u-|qu4^}g=^~` zg(ep5^ji>e&W@}o*p>!d8uhv%N!VP~jpfkK`dsZj_c}LPZ-k-)9N11bDJD&+y^rltDMTNm zvK5VAxotgXjp98$n@8kTHzPxyR-XpZyzqW+NA3`$R)PLl@MdEyCYHmwIStb$EG&yAQ)*6oxr$F7evJ;4RTIQWKJ#?Z=x#PydRDO7P&oO#4D0&}VSFI>HadyUZZSjex&iuAKII`~d7IO^|w zR^uKU_vq}N5Xa(S@!z)B6f41MZ|7E~pY8KM6%q1Cf_}_@R;)^Q>rJL30;Lr`Po=16 zUe!hk9ur-jHm)t<-r;Y3!%7#QM7OyrSV2=R9zF`)hdv|0Pa2W7R)))2FcR1Jl@t`M z_l}zu|6=PjOo!6TD#}_W-Fb$YDNOPnpH`BtkItD|biK*~*cn2+=x7)1_7DP^8jXf; zq-wW}xFW3gB+ubdY^Vzh%kPwQhc(#(&Q)X*HT*ZG0KbH!~sIv$m2lKt+4`xV67aH>48_HgAXqS2x*Kaf77 z&f>ONl`l|9RrM7lPPHxVc3``L&*kxe@BRll#x9lG97VsQ#F57$BzTp$O5z$_Q-zV2pY6Xl6Fi6{{M%G;&%)zc5mqF{L_%Ld?v< zkz4%9NUw#Nh#3=-E`UmG&Ya!mEcj5MdB#TVd3(0v>_zcQW=xh+S*GOLPr{z-d=#Y0 zswzx>jcGn}yYS(bqKMK6(xD(FB`Gk3v>@F%a~AJ=uJ^jmw=*An zu9=6q=Z?MCUTf|3|E-hokfNoXm5h#dGw~Kf-FnniP}rXp@~#6*@BXRUwCCQukM)=2 zOqiXzQ?TJCB#WON<4tQNPYk$KrO7;=!gk`)a^pGYQi+Zm1xFtH6B9$hwDb&bn9@w* z{>;$e@hW``+n}g=wZ%VgVQbzvi5IsM_c5+gOiJ2LF&Mi!@qv=0K6tOYqm#`=HVz%o z+tp$GB}hp)`O?U1f|-x|-3(&Wu|^`UOAlrGJhBu+CoSF|@6SAc9NxnaNWMR8fF100 zF1FM(GQ5j2dvn|1g^^o9mN-2Fot5&so0ro0N6!%FQ_|kk!MTEJyuUA^!~5;?#MVqgq4#$NKFF+vm z7X(t#C+Lq}Bw4-WqWiOJQHtA$b>B?Ole%Khku22}?tuYq@qXBYN8W;v(CIBhY;akm zY|TP=%jtm_XV#%vrK{p;q7(VPU~-FEli8by2%Gom=i(h}IbvugXJ(oX%YocWr8-DQU+i~YILwN*ht6sT=TP$K>-lYTr0HiSfabw6~qT_xu! z3C=e4foJ0F=~?AWE@<1*3nvK z0g{sfvw-;RRQ8Go#g{jsdKweF?;RNU36z2DF!mv(I5E*?=cJ6=&U%lZAK(1j zFWQujm5Kb`JJ`j8ebU&+koU1{Am88Jr}N`rsa^^lK=0R}lO2bj#C(PlN0gxtld!e* zeMfnO^zoJ?w|#zR6H}9GL3i4-XIpr|7DeXO+{-GmQb9jyf3lNaEbRmSfwjNo&Pb_G zt}zs%B&DV{de3j=HNKK@MK}|na$~P%T`6=As?etg$K`w^`pal0sl8q@dr+*MEFNsm z3V&sH4r8Fl!#f=xT}XSmI(?E7kNA>xj9Pbq8bK za?FbF=CSpnBl;?tzQtT=j1rZ#)Cl~fRkgQ?IcBUBAHz@;db;+E^jcLO%kGJR6iwRv zh^ckp7?x+iM5h*8nEBhYQ=SeH=lltd*?u=+ z)0;)rSTWMRNwfA{6t^Y7p!sub8~GE)>zU#vgAtxuy1M`13e1cu`{NT2be=tMuHBLzG1y|QClkq_Y+G1@6=T>r0- z=V2|*EQ^rrxN|%QR6Izpny}NeU21S@NUZD)ZG2*DI#>AJMOtV|0o_NUyoif&P33{& zkt5$)<}e%C?vAICUv*^o`ES*cn(2w5x|B>!jVfxsp3=L7T$%9^mKHqS95}yuI`0(o z^l5pT4X=n`kln1^D>us*uhnVxC9=*cHI`PC8``Wdm%bVLy8OIti2cx^p&_i0Ek*|I zASNME3neI-?|C*C>po}QS&%cGtY_v}b7DgP{4wx6C4;iIg`1{4QMK%BpZ_5W^*R3^ zg`FVYOmC2hoXmn>Iez-#T>&Y6v7ol-i5aQI`Z=#%AKS6+v_fBDC+Ey zBfq#nVT*5Y8asZzKOA}lugRsxOD*zvXBgQoRFmiA=BAOcUlQ@LH0&V{YFa||I*$=N zgw1*%pY82btQ4btmsXS-6^rkFSBxU{O|yH?UIyJLnjBo#PtE2@25!*HAdjF)wh<$( z!BR>8mi?Kf&Nk;5>X$wyKeESeMh=~v{*KUDxY6Sy;(6QNT0%miv$t2-%k?;Cd+kSc zN;YBhxPJjzI1@)QlqY<;DuS08 z0^L+>cEWt>*0Zf4bn26qs1F}=be2(BFJ9N#MWR{PYqJw_f69ne(`7HsT`N(~3=+C4u} za6x>&_{?xl6HQb6AkTYtrs%8M zURj#q3clLZRN&^S&Kmm*)OyFLJyr+#IF#*Vt@bGt;@H?)?VlHp*jGG3GTts5mTbJU$CoT~VW6)ifp zPaR7(KCIxtjLf$_(tP>pJTUwQ`oTk zn7{T&>X^%AA^y#YzN=_5vfacJAMNLcLY@T8m!VXodV3)Qz2b6TDVlTWBXjm;LBTzX z+Jiqsu@7qlVy1d-1>Pf(1aMG`z7DgIE7hL6iZS`#M#pJ=n}0l;#6s8T0JZsz<|k+M z!s~!28FoGN8CUO=o+SO0o!vBW8ZuM+!c3`>>>$l$$P!==qLN60D z9hE>_z8f9gJd}#$>alHcx68}QZd5dS)WR|jw1I`L3^S<3$Lyv=lr?8K-9Uegz$^PQ z^R!n=ulCx5R|_U`a;tJM^BW?-q4NLJj>*ufhz}?yjwhzlocaQzU$B88o|j4UhjMI8 z{O0z)TO~{(KH8tK{-mTlr?FGMURF+t5&hs+0f?keRdZfLHZv>(Ht#3f8W$xNak0*+ zErWxS3PO^yvL$@jvZWF>-h?%sr-}2*Si`$7cz?7NoeuPU_g8n5;^L7j%>XVMqy;?= z5BqN4_RhIR*h9g>>eQoOLYCWLzt1jOeG;65*h3ZwyxqW(iGF2CraW9sPoFG~T84goRp31=^@~b-{vEJo zHr#(T%O}EP0+tf5sUGbLO7x%{eLh1!@o8Mh#vdDR@7vzDXS4_UNJ)V_WzavwJibx# z;YTHp#^VcC%Han|_BJ-3-{(s>Y<;k4jze!R@=yM{SH8`b=~SnnMJbkfPoe+rR7rSZ zrg@IFql1R+7cwQLO(NggX6S%5U(ft~`mESWMrqpZvy|9Wi zvi)fVJ>l<1qyi#2;s&Zj8lS9b?Zwo`acp~psAX|%{$VEH_7jSDP5JQYlbBiFkH>+B zN8yK`bC26-TFI|{hwW(5ynk%tp}z9PQ7Rk-X0>az>lm$T>`~rd1wqPp$H|Eoc?-Je z!a_YL3GtNG)zxKSU_c&%4(6w{2H^gjd3eIjyK4gF?7IH;HDraYNk&U%+EXGWm!;CK zV;Jdctxgw}Ljv!bO*Ab!jMpi(wk{Owrgmc=MC7J29LdrWq;y2lKurb?qRovw!82Y! zkBt1?7%`&*wXE(v9+Od%3#sA-ZUu_D!67@Pr&{*G$U|1=g6`MKcaE-&W?=#1rZIHv zZ|T10m-<;GEG+C?gXy1~lP;sr{!}iie6&Cql9ZMfx9X2DCPS+H6&Vm~JJ|kQB0&(s zsKmC!%F0Tu`X|H|xQ<~421B*40;ppck)*VgbI5DSpfB%d)e(g4#=(!(lw>ZgJlXAJ ze=1^pwVb~P%@)M?xbK`fRyofTN9m}JX>gn;e)*_8R)yod0D&_@gbK%9QedG(sC=UR zV1bVtag$%W_Us$Clm2^z|DP}NxImGrZ$N;cLV-Fbzo;lcxvm8TRd?lOW@Z9$>G<%F zzwUOo79Z{K&=5bDcxtC2FfD%VOXJX=X zYrgaQ_wUc#5*O}RS^_Tcx=3nl^E54EMdJ?CFaUe-3P37DE;a!m2LNSkN+AFh`1EZ| z>Oyx%2>EVe=84C@calw?s(qFt7)bfq-t^!qOdoow>g9k6D>Z+?RAUAK{?hiK5 z)QpCTYH@)fPh32_{q?Eh4}X|t+|%U)igum@BecM{6gREAw-*jn*JTJo`qI);9hc{= z)C>$V;g}E}g7u6^08EUI9)6D`@~o(>MYp%}0qX1;`7aeo-tEXok1AY89~B-gYT~?xn;&evaq3g&%pu6jdzEU-RxVa8{E{OcFi%u`TYCuwQU^XXX@FYie{nE z;yBMprU|5yPq*HybKX)@Q@ejZKQ;AtwnnUV2b8|*rAa4u%z-Kj_z2*>)>c(TJ$~Gs zsdy|Je%M(OU}zNxkU+ROg@wW=$+(oFfHegmqMn|ft!?(ZcNdXAo3m|D%?<2L$Ug&u z0N7`N3Yh(=@EbR7B*w&C5VMW?f;PEzDy#84k4t}(j0v$waf85o5Pmh-)HhvrGMS3rP~e?6%dk+`zFycy?iLix|1 zI#6BXciZOVt7oD4aiOM-$hxUcik zKGz0Oo`Q4#2Y@14)cWhC%O-2S2f;iZqMt5n4NY*VcX-8%sm_SSx{LN#cy3 z2W&W^qN6tU@D&N=qYt%bFS8?u%7ujm1QAJ#JOtZs z$$qwnEY&Rv81mc|JU$E(PRa zZmtus2UR%0#V;!>0~kV9=-vf$17qXAp!;@nBX?p~d|+>Fefb-|%*Bgm=21pw`!EE^ z3SjF|l8>FF>$5={#?3tkj7=~c2!e+f={)e!qW*#3(auu7Q87T$G(tn`)6$rQ%#Dq! z0CMYUCh+nKP=_#_$H&LP!NHwEuJ`VhR8&M?AVV1Y&o#H$iXm@vbBWA2X-P{PuRI(_ z-N?D^)b+YA%LfM`At8a$gggW%dDp~R7G(tzI#7wHrat+qC3sVB;T--WK=kbG`2(2- zwkZqZo{NWXFLVP<4)}cmSzbp&?m(LWnLo(VN_l**uqL*}w#+0Yoo7n~@jU3%gsbvO zN<4=POpH4yFJ-|%h1ciLpS8IkU@?jcMy~;Nib9c>ymZG~SJlM|L3RpIo7fS3?Na>MHu!T zq-b+1SYnt>BfCy^c6_g1J)@AL1YAAzWlGg%f)pzAeD%SorKP{PI>8IQ^GFJGE^ zxIw26My*8$m`=e-kxnG|kKmKYNIaJK?~Y|ALiDMfs~&Y}6kx+SL&^RvFTcLNkZKi1 z;soiYFRr8FvKGe1vH&*&i!2M-(TmC2N~b#m7Y)F40L2Q1TUKxC`>zmXz0y|-oJp5- ze$MhvlDt~NN{D{oUOA3|u8Hj($no9!R2fLnK5)&kUkc6 zP(YNZt{!D+f!zwM0}iKM8E{E}R16OkK_XN(fFsqbjAbc;>cOquT>`|sAtVTp6kx=E z|7KvzfdvxSRzKBy04$xDxL$U<)VOPL8?cKs)YRtY=7dvY9u0>dD=O&I<~?O$S%W9V z9)cgQvDW2OYRs8Tyr&HF7+k4_6n%@o;}A1o#Mx)2r>U5k9 z;HrW~SzTRy{SV9q_Ho4#>^H|U)nW`$V8rmE15;KnLu?DINh7~C&5Wx&ZLVI$ONX5! z0DpA;{P~F22@lfgc6Nc^OoR;G-Q9uR3DnIaybAgtq@k-17?^w8+xOkwqbHsMhXU@>3pv%jNgcr?d6>f6Fc(Ph)xf~0b1to# z%gfh(x5pqms5APdyPI1fobBh&)7yux(!mG6ee>)OP+359bRU2F9au5BsACe@6t9FHm4l^9Nf|&Mup(V`^~XUl zMg#-}`DsZL+svvwFUF{kReG?S??QeNPUwofe5^WWsc~JQmaqwO$lVuR!C3Rr zf^2ST`eB!#OnbHWTi%M6UnZ-z9N3>k;>9c+ixecGWx~;G_ZjdG13qzVQ;^J!t7cci zA90=i5e1w5r0S#eW7Q-#v7KABjO~q)IH>(d_!{!X&11*(>=V4mE_tNAI`omNJfdBT zrJS3>s4T3~%$Z+EawEd)cc0?G?h&Gl2>0G z*G5EV?@KvWUC!4`Td`g)Hk60&Z>x?Csh3YQnz~sE`@!+)Bg*sp?iGiB9jr55gx`0= zcQ%s$rHOUP2>RL%=VwaZU-aso1!jnI+vPEwzaQQQ3pWk3lYF{`Ay2rzWk&6Q-|VJ9 zqRc$&5m0NIxNvOL+?`+Aqfs$rkmT=2q_#cL!@Y!XsULMN0q|ZavOM4F-lOm#cS|X_czT>9|6TgsBcu2Q zgIG8hs(v`0{PI?@X;;22oD1g>zB9unZ{B|Kv+jKHvkwRfof$613Y_MPZ|8l3t9hyl z@!VgAp#vj=1%te^LtKT(Zqw$)s#VS#qib-^8tvr$BDTbQy^T8J#Z7S1v5ZO(<^|o1 z4ySXsBse>ZcPbF@HLI0k4BiIO&A>g25?2}1;3L#HG`hIJ`tj_jI&jo?*3s)zHgAmF z&raknUcZQ%30Z4nOG^uIZ150n9v)z!UGosk3YzyUDjKT*h6LDp_#(X{r#nZ>Ux=oihAI$gS${rSU6Q5TnQ8}>X;}u&SY&^YQr#xi(o&% z1*UW)I!1p01rio>3N{~5=u*U5b z=j2qIAU{OvPtpDTw(jmC)G=>Wv$H|hU!bF_D}UvRb zNXh65l)PQy<%I=aDXZ~bUpBZ6;4>}hGe8?mpD_b6nzuxSRJpvFU?e{KU4j*RO$~_$>=o`m`2XzkmSU5AJf`?psnkX({TtfPi0z&K#)gt)%GD(fPT#r!BTEYfQ=7 z!0;5JKi30nFH;i}F-Ee(Z1|r^9rmv5T5*iPG2etB?b{MOEO6AD4B#I?QVOblLIykD zs@gj^7#J7;cpO&zknT59z{diovC4BAW+41Jct$10;AF+ecNZuCDrJ*8tdAzZ=`8H>iYG_5t-)RT*Qzwj#RvJe+X#kI|D z2G*4)gA(Dvg9p}?;y-{Z242s*cdBD8%>bK7Yn!#1D2Awsg~BN*qop3eBT4QO$hW4Z z8JLR5L-19A+-*x|U}%Vo%oP#8!%gy+dR$U+Ia%9xuAK@-Irs?B|@*^7`6;QJv-tjXSiwN_imk9r14A zRJ0b6lM4cvCAz{bMU*zR6N*y`cBmm1E$Hs2%Ydl>Q5$4swhW?f;%L`0sRA4iEEj+x zr~Od1wa1VFgK5}neHlC!zlE*~a6Ny=!4(E148UQk-f43S2n)x@Lh`B^+#2wRPESq< z`DWMG*H>2+Jz1Q)$)78=DgW6#NJyYi%9FAGO-;ujnKhl))TGA1kSb=^`fc>f7jXqPEaZds5)Pqj1SA)mGL_ollE7xl z_$}Y%3jl^C+)hWwVVHrxe*IZliEEy=gr+mx)H&hqkb?yaoSK??10|l*n>H)UqEC@e z!RCye@q~!&_sGc8=IJ;HSK{M^g@vC;P7e3?QzHlw)zv?e=0#+I3X`vd%>ZI1oz9K{ za&o^LAyQsh*;#%F!vYqZ&dyF|DVG=HbQkX;H8~%{Eb9U{7rN8HUfJpbDvyke3>qaM z2pk#~D8Rc%=98W~m!k^-DEjwrqlKknKny|mld{njh|?j9(8K{qM&KrXDlWFFa77sl zi2$sRpMP_6Q!X5%@K_B}n^Rye!7qi7W*hC%$g22&HrmC*qZmvUAK%}#H8=xE0#$xV zY(>85Tk7g|!Wt8Lve`Cb@#P|cm#=S>%y0>hxybLq!MhJuN^q(E7m6xKHQ`WtUxs!c z7goKi$FHSavMGb91MYAgG+6O1`tLQ487M2TQ6gu5=6X#>M@L;9zb>la)2GWaGHubL z40Lp>Q;m&aFk!E1YAQ=7r-=o!Dm#$lv8q<;z7zXUGSpYk6gv;5* z#S|d(w6xp<7`V~U^aIEL{C|SMcIX|WN2iUL|2C3E%2OoeP`VNZzCKx_kd)T+c9NBZpKV1-s&^p;JBh^Ppf0E|Ey znc#!P?isN1@81#NaId34NM~kje6lle<9`vQga53Z%5uJ_)m(>eX60`w)8FJcwmzOCaTp}7qFm1RbI2WxIj^b#3nf@=^N?A2r%fOxAQO94$QK7Neu zT`U`QT^T#+LP=mT`;Gjx&thV(W%XG*i6m=B={okLJ9~IQ-zUoOf95s5OZmkOkLu}d zOJ%vae{^>KgAOtq524Jc$nS=PyM~6qR%J`v z-*3cbsB?Z7V-8UYhq(;vXSXA?R>*9g4h#+k|9}bkH8C;q>sL}r3fKDy9BGsCdsK;S zkV|ZXZb!yKhR`Bor56WLfZ+ozeqO$O30)%O{Z}@&woEHrFR-+LIfA?rB(eG5Phg~B z>OylU$QHnWGoFUDm2BVOAn@SffhV&1Ys1xovlXuieK=qWmqPqn;f8Lv7mrYB@9v%e z1s)7TAt50M7@?gAB6BwPAhD^do03o#j>Xu7GZE*!TU(SmPkM=ya|YTpdDeXX9HaJb zZf;J#3Z-uG+%eb9z;!G#M8lPA=Y zl8{`ajyWq3;Rhchj#-AnauWvuH=-os9&Bc8FhKHkHSU50Bsumb0;X^fpv+OQ$g1QH z)YjBsc8kM5-_KRj1_=?V%d~ejX~>Y7h6cefD)N`$FXb@xPbqK@;H&<9*&59xw{P9r z1e$Nlp6Gr@aU-V2zJpqWycc#l-jZU1Z-Iw_LR~m&OJpKoQi%p;LxlC~+5SwW&^+8l zxYef6eC1~~JTmb9?`VI*0mJz!s2%qm$=iI3s4LZ8vzj2sL^yQSnhB5D2=2xUf6~0_O();UYf+R~Da&;=q3>HsWgb5C|I9 zzkhI$q!e5TgcKqnETHI;a=7TKIdn^cbaL&GftFcr?)W7)B70ov2Qp3LjN>txSePUI zrzyHPsUst$_jNr~T1i269KnkC^CG?$!>mQetZn?g47&+=pV}0Tw+%?z#LiQXuUU>C zh6cCqZ@t>mG5!12;RFTA!Rsh0DsE^Y%k$jCBTn`hGS6B#+1c4SIb99LGmL}J-@ot& z)|Qr*W@fu9D`SNU)JRB3=;-J~L_~>7A=t#6HK9oK7N0J-CYRZa=8g17R`M1EJG0Djv2W7+zGbol@ z-$bD!`mSzon}CnG`EUISeSLjGA|fLXM+p&;p5b9SDyqEtdR)lC!GVXTC*F$yTIlC9 zf7|iSKf4b6{QU0j_owT9+uPen@bLYf5D_skldb^t+uPftqa!CLCoeBAT&`d6y;31y z3p_nN#lnesy&i6wn|Z*!@{6?DSy^k_+wtk?{9X?(?(Xh7Iy&shbjMi{4&ePi!wU!t z3-9dhadB})F`AmM2TE&er-4c3IXSs`@YSE;WDb+jv=*zy#&6#s zuG2mgFV^$?b(s^vTSII%f4^zKL<}jUe55a6Cu}l9F=b7Pjh*Zc!ZOg;S8u;o5EI*; z`lTTh@Pdh*J#T#Xj2=ni#g!HI($;XYkdTn|a;v405#_Nu#ZS>}S6A27Af*t2zi2q( z;>jG=(qF!O`TA9%L{9JBCoZnM+*~yc4ey5=Td;Ism>@KFj^JR@A4~?lvhQOw>Z~HK zxax$%{@ol+Jk2w*%X`C~iIQFlsk4yR%OU<`EA#Z=VKKP__va_>8zD*ZlOMiUSDE>hP zh7uA2#;;*Hby8SR(B0dsuAzZR$SG@^K53e2b0EsZi2FiFsB3L)4ae0x6!`>G_|@u6 z=rdAe{k85O4y%RsGG+AV&wWo$Y~H?o+bo;=x!gG$f&&ghrmPq2(b@H_AQ#w_7WeC3 z0YZ`Qwn9!1D0pXQXX>{RQohJOW7E@ZHmg3_+1brd0jaL~cWms*mKGKsuqV$Ht0E=` zF3*VqIbf5xeH(!W967a4hjR#MI1zNvFv3q~^VV%av*LPst7UCZzjdk=2$hwUM@B|~ zv0zmZQ9Pe%ddGn1t7mO3(>Y$SIT&v}U;E2)z82E1UjgpUDlO&ne)50|Z1%jrdbmA* z^X3hMPAe-Ln^u)EdU)SQI=aOwlhHUjjqVbp`)|R|+&L&jUUvJTU`eNN3e=XsK`1FH zy{2q9Q_iC*wD9P7V7#SOj3Jc@n;pOJ#xw*M@FmZBzK3`$b2advNFF!L_J8~aV@V-u9?SiU%^(*qE zlAW%Pw7@V01qD}!^V7@Q8w1hk(&L{T_Y}d2J%twBj`<>C-re0{fDM0O-$zDA*DBYA z>qQ3`6d`A5xbfo^IR)Em+x@9uPmlM-g@thK`_pCjSBDzj>1tKR5*tzK<2i%oS| zSw7xeR0054d5FVBvsXoI|8~~DBl4Q2qM{O&N3T?+<>lpt^bZY9Oih6;0#L`s#8lWa z3UD)>-&-4|%D}*&p)tSZMFyLFhUzcxz7D27IXS6Ps_6>m!=8M&w)S_yI2juspUeB{ zp}LwisO~4lv-JQ6d3kvUM@LG|(c2gr1?3XeS9o~0Pmk`No{!)R(Cf4|kMQw7-DD%v zox3zY-d``bdTOYtsYytf?f%Vu_>stAoz~QJe=u7Gz2fn_yI5$j2j-)pr8T^21FW?B zHWYt-XGc6!w%k`pn@5-#SeW7mr6ncgO8p+x4sJzdBYsxjsur8 z>pKi!;wC27U{74%`s{Tsp$FyU~1YqAh=TK5oqDWa36{wiCkdl&m*q8bM4yr$Joj(aby{;1VVP$14`Sr{8 zCP0zVO8mQYJ|-Ts@RCwBEAZQRcz9=}&piEYg-~Z_XC00GKCck~_meM^zM+M1hq@ks zt=;MYf&$Xx59yzbguvPvtQI$vgj7{kjg7a!;km&o3JI|AE=P9fI0JqTgdB9nftV!S zWz>9pULeBIYH~vOR#a33;2xaCvMY?X$E*2)Xlf8ZFlg43))!`FWs!K@WL}L&?e6Y^ z1N6@nRLZo6^($@&fE)Gk^$jQdRB690{e)v}eLR}ZACidu@X?d=8r^jzRz<$Ny!>Sq z0s%qt(P9&Lyc*eMHFP?RAAKkHYkV*WLV|}&P$9tU0>DaQHVOta z5w=1DL6A|^>`er(gzmBE6Dlg|-VhnRPj`1WK&;2hX&pE?xYsEtGxPKACo3H_b((En zi~apSVD?}Q;Y%8!=H`2chg$AwbcntkhRnB6L4a+ieYOS$2GEf?bM`D86tJk*z~3&l zxQEP&3kr6gZ4NFjF2a$rvR02I%{m`1wRqgxJ!V^2Smep1FO^{GGbgfz{53N(d-L{f zh-(QaI%;-FiDV@@StPJ+pBs+FG1KZ_8g!W{f z+S=OM)&>JBBa_P2qhHb7+zcK7_l@)iKwxSLiiE_(Ny6m#cxQkJU}wq2!hYhjng5QV zf!I9UoC5cjn3VK)_X7CaXWsx~goppLhnni@4<9}xx!X}E6c!f3YY6Fg-dk-jkdZxu zB+U;-q_IJ#EEXG;)YKZ(#QvC8kEC*k@FP2Cf_Ne%Bot>m zP644C_;^59mPOz55q>0nbFdik%s%nc~SQrs}t7~SaCM)~jLI3&l1#pM}%vyoVK|n$ZOrm}s zkptj(ZfZ*T{~!MjV4R9|OUT7Vs7r0}9#zB5oB2!w0|UJ7S25xW;$&bx9i5$Ie0 zmWz!DX=!7{Z+G@O{ZL3-{Gt4wK6osjhqzCbOa1sfGe#`=%CtL<7WLWCAP`$y+ZMOW z*KI@;H2>a8SNC)N_^d^(>aTKDM#m53sx~W-6mBHw)MpQ#oBOW?z1F8LCtaB3c9cIp zd9NHTo-8`+n%)feu67A1y^&6Rov-&q#Bn3cSqSjezpTt!Uq4G#N-_yPER4|z^dNTBxv}wMig8IsQV>8q8c5KH!>R9B_*&dLlGq|5;qvmzp>I8& zX#ecBzQIFrazeKl{q5tAwpzEWyRR%R{_UemlGTS#cqJvI?~!Y*1J$ZlCX36gclUiO znR#5v1iwF^2r)tX5z~>WBSBD@!tc#TO8U8bz0UGECZ=k`Yvo@){7>JAxHBrua+lfy z&BnhCzL`*ur?d#6;Qg)eamD%~+c^R_KKN?b#+5)Zm@`%Dcb)!^4MDzmnnc8*P0}blab= zr1CHC@M+SAlfpmtZL+(S)F*yWPdd zGcoZ%mhxqEoKG${5fIq^i1Pv`x!CCVk)Gb_=)3sbiD9ttix)2(930HcKGA@@LU)Sx z1{NFxfmm6+BtZN9+h$8LTa1S0du3$?48A08&`>b-Jf`i*#~bEPqtM!%^XHo&oclf1 z2N`jQrDcKI-{T%M({aR!iPocx?B^u>?BIzsG&I1$yP%}_0S|7p)a;5*k33nBLU?lW zt-*d@`-GkO84JtJ5%)mcwaERI>^4s3gkpK+9hY!w1mx;!tW*o@kjEX7on1$#w^wkv zbu4dqnvM`-XOF$EvA@5ch}((v`aQ*SPA4IH7L$>Qi3!anr(fk?r}FXyMG#~Q9h+V^ zG|0-qL8sY0u%l6;shRqGMf=ut8_x8C7E;Bq2&tC8HSmwznKLo_&wCeEZTi=`RIk?Am%N@+BzT5*3|1YAM z&gn-vZhA-bUmB7gA5U$8v^`8PQ;ni;u=?oE$d35&V*(43=+6lKAPI?bZx{z}WYGuJ zN_e-6Z@<6ba$#3iy<>aR=;S|~EFgj~W3e-h+2~lDY^zCMqK)3zcydaPOVE|_X}S4+ z*?TM>v)Ps06nj?#USMZ;cd^ZTxxM{!^YTjxgkE&KNKeO{J@cJK0%BetSlJ<646nOx zPq6WU_YLF5R_91+C&{Z=c_WAww^Khi^--kJK z8f;ly7==ZB#vZXj40?Hayx0_|Mew)tqk^HKkr7B%?a#NQT;Kk!{{|74^z(+jae^IK zjEzG@?>jnHju||rAfNvytzd-qzW^y!&+!VY0le5%aL8)B4k<;>5ich?mj~|FFqRWG z6+MR4qWM$|hXwom%nYuWr8vfVKr`6P;o)Hpiy6{mL3|Z$jNII@U&(&S!y|}T(Z)m- zh6AsO!<|>!;ZRh3C_qHme6t_7v9-0pcJKG-D7CfBOE^f_VniY+0lB!`Xu{PE zyn{dz81*_q%m(lTb3I_<;o$)UCSrR`2CgALe(cQE6feAG6YFdT!Lk7(Za_dlPfyRy zke=sFV76+xvyc-aBI4@mDm^{D--=j@I*AYI!Qr9xLj8BO?DrrY#s%*TBIVVgiDBfU>p`KENff-JWlUlkmqmzbn84_cGqBV=I1`tVpzH zuI6FIA08i%md;-{i_!0>m!kkb4-6E>hX4G#>i~uBgRJP|$B&>Il7ti({3a+nn;hgK zwY4~&9f$}Zs0S4ssa$y`DE6z~P#;?d2hBRGR8YyP|M>8M^hsXxExl&@xCDZ~P*1Q4 z2rj?}QMf?mnP_w0{Am6gH*ZtAnf*in%HOT`Sod)u_sS$k081Uu_3Lt>$^rm@((Hp8? zkj~}-5>D`nX=(p{$x2V=(@S_@0Z||~1ySNO9zk!noGf_VjEa)gs>;gW0|OFYzTk5^ zS%frv9x-GFg$58mpP!!-5T63fGBg_Xc0Bjpn<>ZE(tQPA3)Ie28 zNYWP;G$ke|T@^n#ZuCcj<77p^!QleHy=WLA9xg7Y+l46}EwaBSq20ewP|aA% zlFioU>A@A%pBdCVR8&;Z(9l33;*a3>VPlrWp`Sm0?(FRJ_lpyU_l2{}rnl;?MZD|{ zeHi!%-2=r$CMKroG93{9KZ3bI+WY$Y78Vw|yWt@q@UE_|-XoNkl|9)SCV+@bNYK#J z^En?8Lh9@5Rn^tMl=V^YND@|d)JL*J8A^GAY=798nwt9gXHdolC16O#)pZ<m{%SqhX27F){efw3-13%)0y3NzGO_<>QC$JX8*N&1uEmEKe}aRO(Qk1aXs zP|_55QxK3Rp;zTOc6Pra>6JPM;~OV%>UY+gAr9gD1(9R?B+%2FBIT1L1lb=lLKWg< zKoB|p&*VJ4sf;^DTj6iEdav6F56FR??-364=BIL1~W=BOfGW*r+1CB*VIlAb|$e7Kq|`QE_p@ z?Cd8hs$NJ&RtYKu#5L9B$w@K;^u*D`CI?ri~3g87yY!f%2>(!&m!3fGwB% z(=u_(Y>B<-z8&Jy(q1YLPug~Ng8I*@Ziu|^Z&1=EC!5XbWki%{?kE3sM{ef>F`a9x ztIMnESXdb&I?~aM(W8)eDKS9zLR_5laeB9)|9#a`OF)<%z9Glw{9@^A;zd8}jq$$M z82LVN5gr_2Nl6bjHz#_;#Kf(wUjNGXziRmk`_Q8Y{?$n*i>I@*Mv@cNu4*c(4h%p! zy|grQ_L18Y4#|#lGyMzSMyr=n0&MHk<0^k0y@t9vD4)BzNwafs#JdE(tU{EC=*9{Z z&Gr}b3nWG*;vMN3w5}{{2m&y8Gi--KzkZqQ$L-9J-ZWKHe3Uc^rIt#fXz`xTeFH)O z?sxBO0m^{lygl)zl?i_q4C2nl<`_w^@fx0-V}Bsq-5D#}UZ+j7T=#qawjpqYL5d-2 zz7*1AK|(LP8_u^uX>>}UsmrIm;SuRNTKqTb`|Zj7V!3JZE_ye2nZ*i&e9-Rc<`+D( z?b2nuI+!JReb%SAJWDPnRedn7qxP3KGAatQ;*Uz17K)0Jl6IAGEI=d9RH7k#3d%JK zu)?h#n`f0q-ye}KQ%PXP#tZZ;#_vbnXN!y>bH(Eb$1^F>H??3 z4X*7Sv*7OmH`}d=4zchkU*GGyAJ|;L1dD{frktFdv>SajB4n+xMLA0KUc)%xV*NY2 z*&P%e#nloexxkn>1pH)APagxu0;k;se2}vV{KpSNXuS1@1M3@}_dbu0kN*5%)dxOT zy${ z6UvGSb<8%R^X$JtD@`ham=qWjp5E&~(jTDE5*HN~#gBFG#T`qlUpW+`?lCphT#{*L2Hb3+pQ`dUH#b9x zdB1$-1#^xASpl-odm5TQ6B8DUwh5A3enD8;&0qN4*mLS~6AwOOe=cs>nh7dBoR|+I zIOH=O%g}Cd3tEWTfwNs{7ckk5giakVHuEYXcCiZD5pjRo+iOu#bA7y<1uKPk-BBV? zNxedY^dZ)KwogC9pzS%ePpFM2LHV&Rr&$#uM6*x2^}%^BoW&Rs!Ln%fhnnw3 zu`~%a80kUNWhNF2HIfaIQIgf{Ix0HOCmQETOTg{^GQ+in06%nVmFc!FweQzGy7>|H z079TbtNH9uhwt6d{9Dj+VT8WK1D>>8wFHmFxSTKkh`UllO+;LiypuMBNrrlkATm0|FgTsIe&fb?*M63_`TdK2r3)TK0UI-JBS21+qYo22ld7)n=W zNaFzFxth>P>w9*!tBRLTC0`1b&{L)BYHKn0NlYxv>9zgsx{lXz@x7dn4%9&8s$K(P z6b&X0>wT zljU6X$wz7p`6)>I0RqdMSpn#{)eP_^?<{h1Hzg#XC-7P)ZkR_`fs=2bbq8XtFYC8% ze)s@ToOxR{s$BiM7~Nf`$y^CBOoEGHn_uHoQ&Qj$s0y6zfk8TbVltT0mzbuKnVU=X z#q>k3t--n-5%&{jv`m8i_#>oY2o@(PX%1M9VsiRNuNi9wZ^2R+hz%u3~$KJ4yHdX(nH|K$Pz!!(<|17osZ)tKu( zLuF6qHC<`v4-+;sbBu8Nel}iMH8OZ%0`t7ng+f0btILM;=rZ!~@PwuFmlPF!I94~z zR4_TL^S9RF@&4p-b8>gg&githzFxdM{(I0Qjd{`&Y?+`|WTva)cLw6Uk+Ux$aV%4~p%z+?L%H{V2vvec;_44{9mQr`#q(*$H+C;-w6XNi? zkEX}N&_8`-7?;c(Z+>-P1xjD;l(g9z$R36+cT35R<#jq+`0vcwp!o2kH}%yE_*w)8 zV{y!M2>dNKQ;_WlY@S6k?r(aTDV)}zYi>2?t*i6&6(f*EuU-Z7@mN}mlSsLTLFD1@l!sXUuvw!v|DDD<}(d3k)3&Q5}YSvmN^$PhscqO*&{_s1Xn zE;m-IltQL9VH)k;g?V{X`DG%;y&s+MG^!D~o&K`3?j}Y>fmYpgy@Z4W6DupU@cI&k z-EN{A)Po0`P$%p_^XpI=gh=uU4GV7gJLg5N3+mT{)t1w7ed~aS5wwsuimA{^-s(zt zw?8o9;~ygFJ|AQc)nSWw-zpQzP5;8>ylXx;m{suJy8N)*YSI12LIz_uf8sL;y?~?x zK}6)K=8%!Y%QZAr?2P%U=o{beM7*-6R)?x8X%kAy+*R_xv2>%K$*iJeZ`jR(i$NnQ zR+L&N|9wz>uj?b+IztNqdoq#RPWaPvVQ;UW3M1OJF8FdvHScYFmoL(Z`7*5Lo11U6 zGW*egM^y#~2Tq3r&OtOt%>9vX$qfraGEB1lBeBIa-qzdpj*go=uzIHSC4BNiduqU; zi#$t;xk^a8wtgx>;15m#aGsaP?wDkvzc#HG%s@^mlg>wE!L68rCZG9^`^#7U;Ut27 z@4|eOvD(z62BT#}=hBiA)#mnlmO%1)DM$3s=YeEBIv*5qyTpF-;xsy*u8A<;&`wk` z;r`Tak?LtCReGqsnSPV(HD6a~A3p*aYxueiIyLCH3vCsT&ej;xGVbr{r6`SZYm&X0 zm{@R{3FJf$Ud(YjWffnuGQ9dwHauc>GOWR|xX?=5da;>Ih-GiG0B;>h;2E9l$aFT^ z{zk)GdDQjlLICzuTU2C;ztAR?TpJj!^Xhgzr=xw0_1(KLQsmhj0`ran*1d8$p>7Kb ziklyejY*FY9S`xxOE_=a=!i}EGb$cctRiL*HVu$u5)$w%XKIyGO3~B(m5vVyt{!Yp zFrPs0Tmh6Ks*gV2d3at&tv6^M*=$#^X%O@0%)M~H&X>hgP?p29;J)Yyp?dQCTp&R$ zoq}sPZAyNL>PK6us^X2)`iSEi06>R>oZctXJ<2IKo3+vyr5h&5+D2+a#$uVJwtX^~5#`QM6^Fe@{Jl-14f zHp03a+hQw&?LIYWYGr>H2o^0d_&3`MA0X|9%zZyEU*`7!4v>fS$>puv+~Pdghaoo&ei?T~xl`Bn zjK;+j@NEELn7N&@6RN+U+-uJ_=UaRH?N4X^vV>uPbrP+DSPVwmOR(mx^e?F6$=Q&d<6ZqZ#>a zN|7(8jc2FHSU!%UKap^NHp(5ZRJWOJzwC63JIX`)t{LX~n$^xeK~X^gg@OS=lUQOF zMNljL=QnanB8n1Olsgk5*MbwSkY+xAVHIU%W0v~Wj})EzS@XGdxwG?~AlIh+k&Psy ztn@0N7mkhT%6MugItoJ^X#PR9ogLfdNGAaXjo;l?)jk-j=a zB%=HEjr=Is3~gNCwJ_SpbHzE|1C_G;(bG3=r*{|#B(8~Rw`cR?Ons{uYjq{1nQ0Lb z?Xbko_0v4b%hQuP3ETLR=!|wq`i~O@&PVjui8)p$P>F+~z%w!F7zim7Q|6PY)qYNH zuGW8Tu83#uDi6|REy-~?bq#pF9T2Yt#*pRRJ*Rkkk&Uqmg>4S&rT!+=oc5yPhBNx_ z4w6HyFVMXkj>vOg;rJWO_K7Z0lJ;DkiD0Q@YrOL**np8X4GsloH5Lw@Bcnt_re)^+ z>y<@(y@ZvG87xdF4foUw=*wleQX5i{lRt;V*kn@rMOeIF<)w0*;4uo0Fy)kzavVcd zQ&okeNlxaM{IvBI&d)AB-(B}UEA`Qgd9P!KxV}Yj>-xeq{u}muMCRH^(d1b{frsns zwZTM1IlZ(989$It`%WPl8uOKvmWDM;#wvfWvH3mQ+f*h3%B4~HEX*nu13taRj($J1I`7!wA zE7(C51?ygxg4x8`2NFd{JM;?+)l}Ad8p}B+)63vlW?Tb40w)UUdjJXB^M6z<4g)4# zo%i#M5CihgaY~+j1L*NBy#<4$t%Q8(F*rAu7NS5 zm5~0`)4bAZO%JDZO71Sgt<4UG`N7_;te$acxAsqrk3WAJh-q*=t~S9;QgRc$ z!y;g!V#oQbm_EDjdo`?6U7GnUZK=U?61w-|Y4B)ycXkdP?NhzfkW91pG5ttbST`V) zs3ho6JUk(|BILqi=UyC9w?y42TdCF>4>GLtEbO(#3V6G2E2qRS$E%q(dQT5e*~*Ha z#1E)ub<&&FOm6^~eSx>Cs{FgrR&7Qtn%|Qa4)R1vLkVHrH<~M93h{a1=s6DVP88D; ze|F?3{j+UyyuJD=TU%7rctFCrs3B`vaAY;+IfULSuv^bmH{r^U{^2QHKF!>;=$iR0 z6cJZ%Vn|@JPvDe8fp^@!Fp1a z&|_YnQsWR5(!zoy^jM2nu^i4mM?*{Te;(A+-oCOKB1!+wg5lnF4-1PWd%Ukxn&?xN zmzwM_S6pScM_&ocMR8^(24qNoEAW~Gl&I!C?@Jp4@B_~ihjGlm4?lRL4Y?(rdV?hZ zJy^6sJ`sOcL7eLyIXi)lWC7p68!4FUl8=ruUHawGuafC#{i@${MU%JWV=LBg8;0E{ zxJ$EblU`Glu963^Xrr-}eO^lD*=o9;O)Sg5a|bTtj`SX`_}LW7EHr}D39rer$@-oF z-yG6zTrpc>2Qj_2Tn;1=wTP3owZC0p6YFCbNAkuaJiWM(biewX09^(AHrG*)w{8!1 zGS3YlW8)kfr!E5%s+eVs=lR-gc31ing;WG7Yb@?5?o~2+MXo$qd&!@K8Z_yDT;ISz znMvmd7x~#L@~Nwf3ywajA{y1o^4)igUi@}&US{Rwr`WZvCk**0gLF_w>ahdGdtx& zx=jllh^c8Xr;I8 zE-vsA)E7{^Qa@xH9BkVyim4ie`z)j&RNR6HDWBa$Z<)XNkbrj)udS^!-Ft7RV%z5R zXeFkzifa2&YU^ub?B%y4gx8f-%#ijW&8hKtGv~jB_Iv&GCUBp{4(YSl#G44#H#cd% z|CTK_v9j_?kNYqbt-vs@b6ItJw>U5|B&V#*&K`Tm%jxK_w@{e!C{<&@js&UZcAPlM zk;;_iT<%3r)+z-oA)0Rf!|TGr^@ZNa=7+ycf5yge`8q#6-eT>0NzqzeZ%H{FekzE1 zoehedwdEEorQi2eMzk9Urwj$6TfDGW4Sz4`2`{AX*VL7(8ClL3s$H2mC@FM2@!Z~b zA<<+Jg~(b0s8h(3c8$85&`Zcj7O97?BP}bxDw2^nt?s~huB1p%5Bs;DDp{p$w|}BN zse){njkZ}i>A5nmaM(i@Yqd1KLhUEnEon=J2lueJ5gEr==9O~ zA^swN6zvSv`SDc4m%GWmG4$NV^5mEGJofJV#+t(sh-_07?A5QixSb;&KR*>Q2x+C5 zo(|Mpj}DJqURb&rHC|Nz1P5@MIOFE-M_y{Gmig~RL@_<{sv45h+MPj5g!$SM8z;PN zzKTC^_>@MQR4{0zX;#r|$li+=*A>LN+nj5E{o8IB5_?1js%QK|z}l(eqMX)&R;%+Z zy`TBa+}xHi%MHf_dq9>j-T~*2wgl|to+r!2ks1vzCPsEs3vWuw|3)n@D*6H`xO392 zm(ls!^sY93L~FCnu+@NSx#ey71NGBa29ogWAxu0oovA8m>E`^Bl7n52!humP;Y>_^!=bNoBgE;HCuDvPvFuWDG|XR z1}IYlLkqvV>p5WWXdndzHxJvVkB9TlC8eb!1FpM=Ectc*HOT3ml*oM@ceFTYeL<1% z2nYxj4KNg9MtUy|kBw`rJ6?AW7np7Ljvl+O2qEgr-c?UA`%1ij3FG}mdYTs|xx({4 zJq!b?pIdDjrwwZpt8LX>)gUk#8E`w~T`aE4RU57Vc}yTt_>S4lBZ&N#-nPNab_oyj zjA=J)ILR$Ei;IfPM++{c78r4Vf;_!!$z!6(_)B=JBMmv-?m|P=89jW>`xl4?D=Xip z*XUV4_5LAC<#Z@Yb_yR(W}jXXFRYTijw$3*ehfgh7il(HdEG5s2Zu?Vv(kf> z=HkAz<2s_c0uoXayZD?aXE^REdybN$O+CDG)_Dw4A?BX?_fS_P%iYqf{c5>$Q z)o#tyhQMfq#G5b%B)dIXM}^%ptzRXQ*x(8V9IC(A$@N;-f}O)VT~E< zWof6Nn80DZb+fQA?IM6G5gQ)v?5;KZKh+7f_a9vLne5CBrdZC(3$tbu2LH} zxpoExPZ@D!rj7v@v+u-Z>utTS$9TOmfh}}KP8%H zQuDryfEK5-yi8egTU%Q*ngCroXy&V`&Rn54ZD{?Nprl?hh>vr3=is__?V?uj5D-X~ zmrqA{bQ0&P8IY2m3dx1z+ua?tRoACrAx{x|;c!4bWUz+1H{KV;6ozGQZ;znq@~V1d z;_h-E5PKQ!rYHyEsrU({Z=?MA;_0=zK_wa6mGA80PkD+;*{0SHMH>{W_L5CKJv|?W zGt@m$D2pDbu0A-QA6b{;qbYq^5C5vZ-}?Y}J0m0XtQ4V!3v~1iD#a8XTvE+yHNLGZ zd*v(AaJaA))l#h?LVzeF;+>FH58w3E5pBcShCx@-t+0}rg9B^n&#(=$_wRRGAqN4KLx#MFzB4l09Y{6Cz#!9^bxYkd$&I-4HL3wS5kGdU_(@C{Y>X8wtwpP)iJ7ThnKPMlAf35iEoa zxINtgV>v4oq5V{*U8dE@L_;HSWAZJDxT!Gozw)uwd@dt-pQev0{z`skX7D4v@CP7@ zz~}u^C$>#l{0!TcFmSmHH0*)+=k?QjJ$DWdX=!Odgiqms3No|RCT?n8&*;^JcCY$z1X z^4~R~wq4~7*`Ta}@}6Vyc5lz3TC+=J^PoV19B@}in8X*?iF}rS532jJ7_1oxKxO#! zzzEc%^Z7?#9@49Af_->}p&&00xVON)X4g&6gb{HiUe?OXzou-Upr_s!Ss~-ks-&gd zYz4H2f8(0f*47&I{e1l6jT6|lJD*OglgnkN-IB(3X~IteTki{lnwbSniJu1q?*j$` zpeZz95!7idt@i%To;hlxrGg%$`W$Hq9Fk1RYEQo$6Z&z&UP?>-M9inOoBq;ul)bpDxHD%H^w z;4?;-7Y4cpz=7;=zoJWRvg-<#>ZI+YdH%<{Ptx{s`f!H#+Oaf2?fF_CdU|ZElAgfR zlV{WQvDU}Iu|kw2P#vYk`S{+HJPeiGpB)ewI(-E1ze_7cVl@aD-Rvwl3g z=m9u|LCo_$GBLB8H?b)EOYYOdxewq74=2=g_c&&;rd6-K7QQ>D?h8MVkZNeMH5mIH z`_U$jcA^tNF{nnS@{@dW{zKcAxK!y;#B#{wIU4u);dhZ32Fa^7F@@xf^?>jW-4rgz zJ6TiHK#bJ%TfmZUYG6u3Ib9s{bRJJG|N@CR{(VNMY6^g^nMJgJr+n=^wJYd-10< znlLbonTbhddJLP&lQZQ^hLXfir_C+$l$PnJw^#*FA$p<1g{G`@_;b%a>5*%FgcX^urS^LTANL(C_)0LBSSXey| zn)zmeq%qJ7O!yc4y!iQNU6U}7TlKo`^*c8{Pukn~NZ86#b`#UYP<+`IL&EX#wl%2p zAKD|#_DFexJ~vEAO${$q8tEILh^VWoo^b5#?C8El%tSneorbj%d7iHIfKfpu$^!Aa z=EypCQUDDCxJ&3#FK*rAss^icHc!0sC@JC}`x?ch;2>p%+R;>!b}@%-YiBo>?f;+6(B2!b(Kd@jy>YL4te#d_uwOU_zkDWV`%B7*I3RrLh&~B`C#CH_rhRs^| z?uW=T~J(<(dwljvyZLs%cxiX9`aN7|M=Cvc54AW z`j0p3;nRToxT@v%5(2~{CPN8Kmv!Etfj(Pl1nLQZ8NLiyGOD!7ht7b36i2Vc1)ZWb zG1*osR-SadCg9l(`S}yjzhbljL9PQZBLSxl;T!OI#w{T)ZvhMR6DQ|Fo43xF!|jcY z2f!RY-x_8CA0VDNrw{C^C@%-JfUUM>r^6&)-{%4^AL0R%95DUIw@1@mc>t$09T6Sh z(A->IU43o^X@JZM@XWS{63JVxK(OM%L+rh^e)pa1M$4}s5boU}5rFcOdj(Rf_`%In z7r^-{AmM0oI<#|eC~4_gSa<*ux`%WyM>%fIsOV@oKp}jC?iDd1lUJ{4WoLH=827o? zuY9Prd;AL`vYOnlOCuxYKmwgJ(F(j$lzg@v(B{Po_##d&>FepqvC#ix9O`uiD5-d> z$@DBOX=`cm($kl&Vn)e!gu2~bn5n5HDy}1D#?b-ap}m{|b{#Y~xu995W@bITy=UZk z00V&4H>)e8)j_9^x&krLYpEvM9st|upH8#3)(L2RlG!b%O4PE8icledq(=gzoq$^f zG&i@qs#;n^FJG41ZGHg?x93P0Jg#ROf3n5q=H>`EtQ6(t)q(gXG7>u;PWVR(XpRFe zcSw&p*luOzsZM{i`^$YK2;gr0Q+-NFU0z>1+uN@nr$j_Vq@;iiQAhNxa6R7wi4qv> z1qq+W)=(m#Pyp&*A?-)+`zyNm!CxdCv^d3pN(9tCiXUpOH2nSj!TVXT5+?JMal2g< z08&=;z($*cy}d?@8(Vl^NLW~_!|rQ=mxuFpZSL3futm_jdJggJDEPFnxCr!1yHZ}k zH}K&jD>LX(@mX19)YSQ4>{Fe*ygYL#^zEBBr!LKaUz?ec?UUIZfm>h1r)Sr?SNmmS6A-1uMGMsK>Wf!i|2VprC-Dl9iL=a4^FJiao}YqcDAtn@=u@ zzl4X_G%Rp&1F7usOvU=iYS(kb7pCQG0=5dk=Act0fNYbPkT78ZL=@}b)chVCR4Gvv z$dnyP=jS&aBL$?Lr>4`>Q*zOt1+%xnc))(*vRfp4lL4|{(EWN<{#-D4|6+F>_`(`k z03e7|sFwKnJR>L12JAF1u$b)ZS3nl^;0$i;*RM!PO@&_N|1kv?`}Tq6-Me@41w@bx zuvHLQps{i%hZzBt9n^ZmuA$0!cn+vJ0Jac7rN(4WfxHE1!Vb5$RovHsq-7XL>L&k* zoQE*YNJvPY0Q(mRVIs-JrVb=?<=roK^eilB4;TVv2kj2$*bEHTfgEJoB6ECKlYY9l zH%}p78uIeZn@K>g11oU4EF~_^2=rIG2M|c?mp2cXwTintJ3z{MXFCqS41&y)HO zK*5>a+S+=E2OAFflau4)O~AxK{8FafV#+!POdRs9ZyoeC52kgzf5B*gq!yTf)YIIv zBec&ys(~C4>g~%(DE%)XIh=>ele4hc1F`HwI>1~I+ykJ6TM6fXQUyZOP|}w*T zMcoDDI}C1aZUO>=^T0efQYxygnF<4tr4)TKVPR%ovugm#R>%o>5pWk^WIbyKR^xuT zhX6TVZrjErNayo(JKGot^^B-|`5rrXg+Jz>S`21+)PyWC0OU^QtpWgHqp4-CuC9O< z&~_L@l$x=;1N`Lx=)b=UASz0T5ePd6yic%bw6wH;$H#9Cy4aR80Nw*1bPXQTA(JWl zzSAQpsiYPF_>;3CU>5_RV-xxVwop+{t}rMWK%kD-?IzG?f};r7yTgBTzXHwq{tG(q zUkWE`lG4&6!2AHbDX6R80sI2q^E5Eb{9vwTA4GRRfFrM@^iOj?$`7-ELb-L@AwY}_ z;j~>72GmjDl>lQLR6dg*EJe=2mMzxXl7j=y!BGcb3v3?1P2jCV7>~&)lz51bffOCQ z9>8gu4{QoHUA|oRgG$e8|FjxaxFL>p37=gpYe4M@1%IOfA|WCzTq>yR^Zx)hHm(H# literal 0 HcmV?d00001 diff --git a/docs/index.rst b/docs/index.rst index b288971345..9da78d005d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,10 +6,78 @@ Welcome to Pydra: A simple dataflow engine with scalable semantics's documentation! =================================================================================== +Pydra is a new lightweight dataflow engine written in Python. +Pydra is developed as an open-source project in the neuroimaging community, +but it is designed as a general-purpose dataflow engine to support any scientific domain. + +Scientific workflows often require sophisticated analyses that encompass a large collection +of algorithms. +The algorithms, that were originally not necessarily designed to work together, +and were written by different authors. +Some may be written in Python, while others might require calling external programs. +It is a common practice to create semi-manual workflows that require the scientists +to handle the files and interact with partial results from algorithms and external tools. +This approach is conceptually simple and easy to implement, but the resulting workflow +is often time consuming, error-prone and difficult to share with others. +Consistency, reproducibility and scalability demand scientific workflows +to be organized into fully automated pipelines. +This was the motivation behind Pydra - a new dataflow engine written in Python. + +The Pydra package is a part of the second generation of the Nipype_ ecosystem +--- an open-source framework that provides a uniform interface to existing neuroimaging +software and facilitates interaction between different software components. +The Nipype project was born in the neuroimaging community, and has been helping scientists +build workflows for a decade, providing a uniform interface to such neuroimaging packages +as FSL_, ANTs_, AFNI_, FreeSurfer_ and SPM_. +This flexibility has made it an ideal basis for popular preprocessing tools, +such as fMRIPrep_ and C-PAC_. +The second generation of Nipype ecosystem is meant to provide additional flexibility +and is being developed with reproducibility, ease of use, and scalability in mind. +Pydra itself is a standalone project and is designed as a general-purpose dataflow engine +to support any scientific domain. + +The goal of Pydra is to provide a lightweight dataflow engine for computational graph construction, +manipulation, and distributed execution, as well as ensuring reproducibility of scientific pipelines. +In Pydra, a dataflow is represented as a directed acyclic graph, where each node represents a Python +function, execution of an external tool, or another reusable dataflow. +The combination of several key features makes Pydra a customizable and powerful dataflow engine: + +- Composable dataflows: Any node of a dataflow graph can be another dataflow, allowing for nested + dataflows of arbitrary depths and encouraging creating reusable dataflows. + +- Flexible semantics for creating nested loops over input sets: Any Task or dataflow can be run + over input parameter sets and the outputs can be recombined (similar concept to Map-Reduce_ model, + but Pydra extends this to graphs with nested dataflows). + +- A content-addressable global cache: Hash values are computed for each graph and each Task. + This supports reusing of previously computed and stored dataflows and Tasks. + +- Support for Python functions and external (shell) commands: Pydra can decorate and use existing + functions in Python libraries alongside external command line tools, allowing easy integration + of existing code and software. + +- Native container execution support: Any dataflow or Task can be executed in an associated container + (via Docker or Singularity) enabling greater consistency for reproducibility. + +- Auditing and provenance tracking: Pydra provides a simple JSON-LD-based message passing mechanism + to capture the dataflow execution activties as a provenance graph. These messages track inputs + and outputs of each task in a dataflow, and the resources consumed by the task. + +.. _Nipype: https://nipype.readthedocs.io/en/latest/ +.. _FSL: https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FSL +.. _ANTs: http://stnava.github.io/ANTs/ +.. _AFNI: https://afni.nimh.nih.gov/ +.. _FreeSurfer: https://surfer.nmr.mgh.harvard.edu/ +.. _SPM: https://www.fil.ion.ucl.ac.uk/spm/ +.. _fMRIPrep: https://fmriprep.org/en/stable/ +.. _C-PAC: https://fcp-indi.github.io/docs/latest/index +.. _Map-Reduce: https://en.wikipedia.org/wiki/MapReduce + .. toctree:: :maxdepth: 2 :caption: Contents: + user_guide changes api diff --git a/docs/input_spec.rst b/docs/input_spec.rst new file mode 100644 index 0000000000..3fc8ac8216 --- /dev/null +++ b/docs/input_spec.rst @@ -0,0 +1,155 @@ +.. _Input Specification section: + +Input Specification +=================== + +As it was mentioned in :ref:`shell_command_task`, the user can customize the input and output +for the `ShellCommandTask`. +In this section, more examples of the input specification will be provided. + + +Let's start from the previous example: + +.. code-block:: python + + bet_input_spec = SpecInfo( + name="Input", + fields=[ + ( "in_file", File, + { "help_string": "input file ...", + "position": 1, + "mandatory": True } ), + ( "out_file", str, + { "help_string": "name of output ...", + "position": 2, + "output_file_template": + "{in_file}_br" } ), + ( "mask", bool, + { "help_string": "create binary mask", + "argstr": "-m", } ) ], + bases=(ShellSpec,) ) + + ShellCommandTask(executable="bet", + input_spec=bet_input_spec) + + + +In order to create an input specification, a new `SpecInfo` object has to be created. +The field `name` specifiest the typo of the spec and it should be always "Input" for +the input specification. +The field `bases` specifies the "base specification" you want to use (can think about it as a +`parent class`) and it will usually contains `ShellSpec` only, unless you want to build on top of +your other specification (this will not be cover in this section). +The part that should be always customised is the `fields` part. +Each element of the `fields` is a separate input field that is added to the specification. +In this example, a three-elements tuples - with name, type and dictionary with additional +information - are used. +But this is only one of the supported syntax, more options will be described below. + +Adding a New Field to the Spec +------------------------------ + +Pydra uses `attr` classes to represent the input specification, and the full syntax for each field +is: + +.. code-block:: python + + field1 = ("field1_name", attr.ib(type=<'field1_type'>, metadata=<'dictionary with metadata'>) + +However, we allow for shorter syntax, that does not include `attr.ib`: + +- providing only name and the type + +.. code-block:: python + + field1 = ("field1_name", <'field1_type'>) + + +- providing name, type and metadata (as in the example above) + +.. code-block:: python + + field1 = ("field1_name", <'field1_type'>, <'dictionary with metadata'>)) + +- providing name, type and default value + +.. code-block:: python + + field1 = ("field1_name", <'field1_type'>, <'default value'>) + +- providing name, type, default value and metadata + +.. code-block:: python + + field1 = ("field1_name", <'field1_type'>, <'default value', <'dictionary with metadata'>)) + + +Each of the shorter versions will be converted to the `(name, attr.ib(...)`. + +Type can be provided as a simple python type (e.g. `str`, `int`, `float`, etc.) +or can be more complex by using `typing.List`, `typing.Dict` and `typing.Union`. + + +Metadata +-------- + +In the example we used multiple keys in the metadata dictionary including `help_string`, +`position`, etc. In this section all allowed key will be described: + +`help_string` (`str`, mandatory): + A short description of the input field. + +`mandatory` (`bool`, default: `False`): + If `True` user has to provide a value for the field. + +`sep` (`str`): + A separator if a list is provided as a value. + +`argstr` (`str`): + A flag or string that is used in the command before the value, e.g. `-v` or `-v {inp_field}`, + but it could be and empty string, `""`. + If `...` are used, e.g. `-v...`, the flag is used before every element if a list is provided + as a value. + If no `argstr` is used the field is not part of the command. + +`position` (`int`): + Position of the field in the command, could be positive or negative integer. + If nothing is provided the field will be inserted between all fields with positive positions + and fields with negative positions. + +`allowed_values` (`list`): + List of allowed values for the field. + +`requires` (`list`): + List of field names that are required together with the field. + +`xor` (`list`): + List of field names that are mutually exclusive with the field. + +`keep_extension` (`bool`, default: `True`): + A flag that specifies if the file extension should be removed from the field value. + +`copyfile` (`bool`, default: `False`): + If `True`, a hard link is created for the input file in the output directory. + If hard link not possible, the file is copied to the output directory. + +`container_path` (`bool`, default: `False`, only for `ContainerTask`): + If `True` a path will be consider as a path inside the container (and not as a local path). + +`output_file_template` (`str`): + If provided, the field is treated also as an output field and it is added to the output spec. + The template can use other fields, e.g. `{file1}`. + +`output_field_name` (`str`, used together with `output_file_template`) + If provided the field is added to the output spec with changed name. + +`readonly` (`bool`, default: `False`): + If `True` the input field can't be provided by the user but it aggregates other input fields + (for example the fields with `argstr: -o {fldA} {fldB}`). + + +Validators +---------- +Pydra allows for using simple validator for types and `allowev_values`. +The validators are disabled by default, but can be enabled by calling +`pydra.set_input_validator(flag=True)`. diff --git a/docs/state.rst b/docs/state.rst new file mode 100644 index 0000000000..c99dadd00a --- /dev/null +++ b/docs/state.rst @@ -0,0 +1,88 @@ +State and Nested Loops over Input +================================= + +One of the main goals of creating Pydra was to support flexible evaluation of a Task or a Workflow +over combinations of input parameters. +This is the key feature that distinguishes it from most other dataflow engines. +This is similar to the concept of the Map-Reduce_, but extends it to work over arbitrary nested graphs. +In complex dataflows, this would typically involve significant overhead for data management +and use of multiple nested loops. +In Pydra, this is controlled by setting specific State related attributes through Task methods. +In order to set input splitting (or mapping), Pydra requires setting up a splitter. +This is done using Task's split method. +The simplest example would be a Task that has one field x in the input, and therefore there +is only one way of splitting its input. +Assuming that the user provides a list as a value of x, Pydra splits the list, so each copy +of the Task will get one element of the list. +This can be represented as follow: + +.. math:: + + S = x: x=[x_1, x_2, ..., x_n] \longmapsto x=x_1, x=x_2, ..., x=x_n~, + +where S represents the splitter, and x is the input field. +This is also represented in the diagram, where :math:`x=[1, 2, 3]` as an example, and the coloured +nodes represent stateless copies of the original Task after splitting the input, +(these are the runnables that are executed). + +.. image:: images/nd_spl_1.png + :scale: 50 % + +Types of Splitter +----------------- +Whenever a *Task* has more complicated inputs, +i.e. multiple fields, there are two ways of creating the mapping, +each one is used for different application. +These *splitters* are called *scalar splitter* and *outer splitter*. +They use a special, but Python-based syntax as described next. + +Scalar Splitter +--------------- +A *scalar splitter* performs element-wise mapping and requires that the lists of +values for two or more fields to have the same length. The *scalar splitter* uses +Python tuples and its operation is therefore represented by a parenthesis, ``()``: + +.. math:: + + S = (x, y) : x=[x_1, x_2, .., x_n],~y=[y_1, y_2, .., y_n] \mapsto (x, y)=(x_1, y_1),..., (x, y)=(x_n, y_n), + + +where `S` represents the *splitter*, `x` and `y` are the input fields. +This is also represented as a diagram: + +.. figure:: images/nd_spl_4.png + :figclass: h! + :scale: 80% + + +Outer Splitter +-------------- + +The second option of mapping the input, when there are multiple fields, is +provided by the *outer splitter*. The *outer splitter* creates all combination +of the input values and does not require the lists to have the same lengths. +The *outer splitter* uses Python's list syntax and is represented by square +brackets, ``[]``: + +.. math:: + + S = [x, y] &:& x=[x_1, x_2, ..., x_n],~~ y=[y_1, y_2, ..., y_m], \\ + &\mapsto& (x, y)=(x_1, y_1), (x, y)=(x_1, y_2)..., (x, y)=(x_n, y_m). + + +The *outer splitter* for a node with two input fields is schematically represented in the diagram: + +.. figure:: images/nd_spl_3.png + :figclass: h! + :scale: 80% + + +Different types of splitters can be combined over inputs such as +`[inp1, (inp2, inp3)]`. In this example an *outer splitter* provides all +combinations of values of `inp1` with pairwise combinations of values of `inp2` +and `inp3`. This can be extended to arbitrary complexity. +In additional, the output can be merge at the end if needed. +This will be explained in the next section. + + +.. _Map-Reduce: https://en.wikipedia.org/wiki/MapReduce diff --git a/docs/user_guide.rst b/docs/user_guide.rst new file mode 100644 index 0000000000..642b85252a --- /dev/null +++ b/docs/user_guide.rst @@ -0,0 +1,11 @@ +User Guide +========== + + + +.. toctree:: + + components + state + combiner + input_spec diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 245a4aac34..d698f60d33 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -214,7 +214,6 @@ def check_metadata(self): "output_file_template", "position", "requires", - "separate_ext", "keep_extension", "xor", "sep", From 98a0989972afac99af6d72876a7aeb45fc121768 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 3 Aug 2020 23:47:44 -0400 Subject: [PATCH 039/271] fixing zenodo [skip ci] --- .zenodo.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 806aec46d5..749fddafb7 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -45,15 +45,15 @@ "name": "Mentch, Jeff", "orcid": "0000-0002-7762-8678" }, - { - "affiliation": "MIT, HMS", - "name": "Ghosh, Satrajit", - "orcid": "0000-0002-5312-6729" - }, { "affiliation": "Microsoft, Station Q", "name": "Nijholt, Bas", "orcid": "0000-0003-0383-4986" + }, + { + "affiliation": "MIT, HMS", + "name": "Ghosh, Satrajit", + "orcid": "0000-0002-5312-6729" } ], "keywords": [ From 833a7544dbd13f855c680b789d5b1e9a673088a9 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 4 Aug 2020 00:13:27 -0400 Subject: [PATCH 040/271] updating release notes [skip ci] --- docs/changes.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 0e195a1088..6c6858891b 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,26 @@ Release Notes ============= +0.8.0 +----- + +* refactoring template formatting for ``input_spec`` +* fixing issues with input fields with extension (and using them in templates) +* adding simple validators to input spec (using ``attr.validator``) +* adding ``create_dotfile`` for workflows, that creates graphs as dotfiles (can convert to other formats if dot available) +* adding a simple user guide with ``input_spec`` description +* expanding docstrings for ``State``, ``audit`` and ``messanger`` +* updating syntax to newer python + +0.7.0 +----- + +* refactoring the error handling by padra: improving raised errors, removing nodes from the workflow graph that can't be run +* refactoring of the ``input_spec``: adapting better to the nipype interfaces +* switching from ``pkg_resources.declare_namespace`` to the stdlib ``pkgutil.extend_path`` +* moving ``readme`` to rst format + + 0.6.2 ----- From 3c38a1a171daeb95fa563811408933787bb569ee Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 17:16:58 +0800 Subject: [PATCH 041/271] compare relative rather than absolute runtime workflows in no cache workflow tests --- pydra/engine/tests/test_workflow.py | 64 +++++++++++------------------ 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index a77f787fdb..5a366004d0 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -2162,10 +2162,8 @@ def test_wf_nostate_cachelocations_a(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking execution time (second one should be quick) - assert t1 > 2 - # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + # execution time should be a bit shorter + assert t1 - 1 > t2 # checking if both wf.output_dir are created assert wf1.output_dir.exists() @@ -2223,9 +2221,8 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking execution time - assert t1 > 2 - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 - 1 > t2 # checking if the second wf didn't run again assert wf1.output_dir.exists() @@ -2281,10 +2278,8 @@ def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking execution time (the second wf should be fast, nodes do not have to rerun) - assert t1 > 2 - # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 / 2 > t2 # both wf output_dirs should be created assert wf1.output_dir.exists() @@ -2337,10 +2332,8 @@ def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking execution time (the second wf should be fast, nodes do not have to rerun) - assert t1 > 2 - # testing relative values (windows or slurm takes much longer to create wf itself) - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 / 2 > t2 # both wf output_dirs should be created assert wf1.output_dir.exists() @@ -2461,8 +2454,8 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateTrue(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - assert t1 > 2 - assert t2 > 2 + # runtime for recomputed workflows should be about the same + assert abs(t1 - t2) < t1 / 2 def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): @@ -2520,9 +2513,8 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the time - assert t1 > 2 - assert t2 < 1 + # execution time should be a bit shorter + assert t1 - 1 > t2 # tasks should not be recomputed assert len(list(Path(cache_dir1).glob("F*"))) == 2 @@ -2531,7 +2523,7 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): def test_wf_nostate_cachelocations_taskrerun_wfrerun_propagateFalse(plugin, tmpdir): """ - Two identical wfs with provided cache_dir, and cache_locations for teh second wf; + Two identical wfs with provided cache_dir, and cache_locations for the second wf; submitter doesn't have rerun, but wf has rerun=True, since propagate_rerun=False, only tasks that have rerun=True will be rerun """ @@ -2735,9 +2727,8 @@ def test_wf_state_cachelocations(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 / 2 > t2 # checking all directories assert wf1.output_dir @@ -2871,9 +2862,8 @@ def test_wf_state_cachelocations_updateinp(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 / 2 > t2 # checking all directories assert wf1.output_dir @@ -3099,9 +3089,8 @@ def test_wf_ndstate_cachelocations(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for second run should be much shorter + assert t1 / 2 > t2 # checking all directories assert wf1.output_dir.exists() @@ -3225,9 +3214,8 @@ def test_wf_ndstate_cachelocations_updatespl(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for wf2 should be much shorter + assert t1 / 2 > t2 # checking all directories assert wf1.output_dir.exists() @@ -3341,9 +3329,8 @@ def test_wf_nostate_runtwice_usecache(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for wf2 should be much shorter + assert t1 / 2 > t2 def test_wf_state_runtwice_usecache(plugin, tmpdir): @@ -3390,9 +3377,8 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): assert cache_dir_content == os.listdir(wf1.cache_dir) # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # checking the execution time - assert t1 > 2 - assert t2 < 1 + # execution time for wf2 should be much shorter + assert t1 / 2 > t2 @pytest.fixture From e4d527987675028635b8b3646b613d2a47b8b44b Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 18:06:50 +0800 Subject: [PATCH 042/271] update os in github actions workflow --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 77f46d309e..192d7483af 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -12,7 +12,7 @@ jobs: continue-on-error: true strategy: matrix: - os: [macos-latest, ubuntu-18.04, windows-2016] + os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] steps: From 63a8cf33cf1bccc28d61d1ed52c73404ea395a35 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 18:11:31 +0800 Subject: [PATCH 043/271] update attrs requirement --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 68ab4ff08b..a30b22dbe8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ classifiers = [options] python_requires = >= 3.7 install_requires = - attrs + attrs >= 19.1.0 cloudpickle >= 1.2.2 filelock >= 3.0.0 etelemetry >= 0.2.0 From 0db97ee55a5918efc51fc5e870c72039e54e4d39 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 18:22:27 +0800 Subject: [PATCH 044/271] update pip version to 20.2.1 --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 192d7483af..73c74a2da5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -23,7 +23,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update build tools run: | - python -m pip install --upgrade pip==18.1 setuptools==30.2.1 wheel + python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - name: Install dependencies run: | python -m pip install -r min-requirements.txt From fe3bb8f01534329311b3a2655b46721d0c656422 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 19:03:07 +0800 Subject: [PATCH 045/271] ignore test_dockertask in github actions --- .github/workflows/pythonpackage.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73c74a2da5..5c9d7ec94e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -1,6 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + name: Python package on: [push, pull_request] @@ -31,5 +32,7 @@ jobs: run: | python -m pip install .[test] - name: Pytest tests + env: + IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # not running containers in this action run: | - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra From 7722394a5d6e0f4e985b5577744308cb23cb5e01 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 6 Aug 2020 19:40:53 +0800 Subject: [PATCH 046/271] mark flaky tests in test_workflow.py --- pydra/engine/tests/test_workflow.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 5a366004d0..e9e38a39ea 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -2229,6 +2229,7 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): """ the same as previous test, but wf output names differ, @@ -2278,14 +2279,15 @@ def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # execution time for second run should be much shorter - assert t1 / 2 > t2 + # execution time should be a bit shorter + assert t1 - 1 > t2 # both wf output_dirs should be created assert wf1.output_dir.exists() assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): """ the same as previous test, but wf names and output names differ, @@ -2332,14 +2334,15 @@ def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": - # execution time for second run should be much shorter - assert t1 / 2 > t2 + # execution time should be a bit shorter + assert t1 - 1 > t2 # both wf output_dirs should be created assert wf1.output_dir.exists() assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_forcererun(plugin, tmpdir): """ Two identical wfs with provided cache_dir; @@ -2397,6 +2400,7 @@ def test_wf_nostate_cachelocations_forcererun(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_wftaskrerun_propagateTrue(plugin, tmpdir): """ Two identical wfs with provided cache_dir and cache_locations for the second one; @@ -2458,6 +2462,7 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateTrue(plugin, tmpdir): assert abs(t1 - t2) < t1 / 2 +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): """ Two identical wfs with provided cache_dir and cache_locations for the second one; @@ -2676,6 +2681,7 @@ def test_wf_nostate_nodecachelocations_upd(plugin, tmpdir): assert len(list(Path(cache_dir2).glob("F*"))) == 1 +@pytest.mark.flaky(reruns=3) def test_wf_state_cachelocations(plugin, tmpdir): """ Two identical wfs (with states) with provided cache_dir; @@ -2808,6 +2814,7 @@ def test_wf_state_cachelocations_forcererun(plugin, tmpdir): assert odir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_state_cachelocations_updateinp(plugin, tmpdir): """ Two identical wfs (with states) with provided cache_dir; From 9a3df05ce6799d8b82c7c810e13cb32917ebc1ba Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 20:50:42 -0400 Subject: [PATCH 047/271] fixing issues with numpy arrays provided as an input --- pydra/engine/helpers.py | 5 +++- pydra/engine/tests/test_node_task.py | 43 +++++++++++++++++++++------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index e2dbb528d1..4809d933c8 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -44,10 +44,13 @@ def ensure_list(obj, tuple2list=False): """ if obj is None: return [] - if isinstance(obj, list): + # list or numpy.array (this might need some extra flag in case an array has to be converted) + elif isinstance(obj, list) or hasattr(obj, "__array__"): return obj elif tuple2list and isinstance(obj, tuple): return list(obj) + elif isinstance(obj, list): + return obj return [obj] diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index ef2e1d8018..248665c0a0 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -59,9 +59,16 @@ def test_task_init_2(): "splitter, state_splitter, state_rpn, states_ind, states_val", [("a", "NA.a", ["NA.a"], [{"NA.a": 0}, {"NA.a": 1}], [{"NA.a": 3}, {"NA.a": 5}])], ) -def test_task_init_3(splitter, state_splitter, state_rpn, states_ind, states_val): +@pytest.mark.parametrize("input_type", ["list", "array"]) +def test_task_init_3( + splitter, state_splitter, state_rpn, states_ind, states_val, input_type +): """ task with inputs and splitter""" - nn = fun_addtwo(name="NA", a=[3, 5]).split(splitter=splitter) + a_in = [3, 5] + if input_type == "array": + a_in = np.array(a_in) + + nn = fun_addtwo(name="NA", a=a_in).split(splitter=splitter) assert np.allclose(nn.inputs.a, [3, 5]) assert nn.state.splitter == state_splitter @@ -101,9 +108,15 @@ def test_task_init_3(splitter, state_splitter, state_rpn, states_ind, states_val ), ], ) -def test_task_init_3a(splitter, state_splitter, state_rpn, states_ind, states_val): +@pytest.mark.parametrize("input_type", ["list", "array"]) +def test_task_init_3a( + splitter, state_splitter, state_rpn, states_ind, states_val, input_type +): """ task with inputs and splitter""" - nn = fun_addvar(name="NA", a=[3, 5], b=[10, 20]).split(splitter=splitter) + a_in, b_in = [3, 5], [10, 20] + if input_type == "array": + a_in, b_in = np.array(a_in), np.array(b_in) + nn = fun_addvar(name="NA", a=a_in, b=b_in).split(splitter=splitter) assert np.allclose(nn.inputs.a, [3, 5]) assert np.allclose(nn.inputs.b, [10, 20]) @@ -704,9 +717,14 @@ def test_task_nostate_cachelocations_updated(plugin, tmpdir): @pytest.mark.flaky(reruns=2) # when dask -def test_task_state_1(plugin_dask_opt): +@pytest.mark.parametrize("input_type", ["list", "array"]) +def test_task_state_1(plugin_dask_opt, input_type): """ task with the simplest splitter""" - nn = fun_addtwo(name="NA").split(splitter="a", a=[3, 5]) + a_in = [3, 5] + if input_type == "array": + a_in = np.array(a_in) + + nn = fun_addtwo(name="NA").split(splitter="a", a=a_in) assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] @@ -818,14 +836,17 @@ def test_task_state_singl_1(plugin): ), ], ) +@pytest.mark.parametrize("input_type", ["list", "array"]) def test_task_state_2( - plugin, splitter, state_splitter, state_rpn, expected, expected_ind + plugin, splitter, state_splitter, state_rpn, expected, expected_ind, input_type ): """ Tasks with two inputs and a splitter (no combiner)""" - nn = fun_addvar(name="NA").split(splitter=splitter, a=[3, 5], b=[10, 20]) - - assert nn.inputs.a == [3, 5] - assert nn.inputs.b == [10, 20] + a_in, b_in = [3, 5], [10, 20] + if input_type == "array": + a_in, b_in = np.array(a_in), np.array(b_in) + nn = fun_addvar(name="NA").split(splitter=splitter, a=a_in, b=b_in) + assert (nn.inputs.a == np.array([3, 5])).all() + assert (nn.inputs.b == np.array([10, 20])).all() assert nn.state.splitter == state_splitter assert nn.state.splitter_rpn == state_rpn assert nn.state.splitter_final == state_splitter From 96c6e8811d34867c7d618730cbfd888c259ed3d3 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 7 Aug 2020 22:36:44 -0400 Subject: [PATCH 048/271] adding a mixed option for input_type (for mixing python lists and np arrays) --- pydra/engine/tests/test_node_task.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index 248665c0a0..ccb16ba4fc 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -108,7 +108,7 @@ def test_task_init_3( ), ], ) -@pytest.mark.parametrize("input_type", ["list", "array"]) +@pytest.mark.parametrize("input_type", ["list", "array", "mixed"]) def test_task_init_3a( splitter, state_splitter, state_rpn, states_ind, states_val, input_type ): @@ -116,6 +116,8 @@ def test_task_init_3a( a_in, b_in = [3, 5], [10, 20] if input_type == "array": a_in, b_in = np.array(a_in), np.array(b_in) + elif input_type == "mixed": + a_in = np.array(a_in) nn = fun_addvar(name="NA", a=a_in, b=b_in).split(splitter=splitter) assert np.allclose(nn.inputs.a, [3, 5]) @@ -836,7 +838,7 @@ def test_task_state_singl_1(plugin): ), ], ) -@pytest.mark.parametrize("input_type", ["list", "array"]) +@pytest.mark.parametrize("input_type", ["list", "array", "mixed"]) def test_task_state_2( plugin, splitter, state_splitter, state_rpn, expected, expected_ind, input_type ): @@ -844,6 +846,8 @@ def test_task_state_2( a_in, b_in = [3, 5], [10, 20] if input_type == "array": a_in, b_in = np.array(a_in), np.array(b_in) + elif input_type == "mixed": + a_in = np.array(a_in) nn = fun_addvar(name="NA").split(splitter=splitter, a=a_in, b=b_in) assert (nn.inputs.a == np.array([3, 5])).all() assert (nn.inputs.b == np.array([10, 20])).all() From 160955645684ac6e56cf90ea778ecb52a85c0385 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 10 Aug 2020 15:14:13 -0400 Subject: [PATCH 049/271] adding tests for graphs for workflows with a combiner; adding blue colors to the arrows that are from nodes that have the final splitter (i.e. the state should be passed to the following nd) --- pydra/engine/graph.py | 14 ++++- pydra/engine/tests/test_workflow.py | 89 ++++++++++++++++++++++++++++- pydra/engine/tests/utils.py | 5 ++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/pydra/engine/graph.py b/pydra/engine/graph.py index d4151e35ab..b04b0ac95f 100644 --- a/pydra/engine/graph.py +++ b/pydra/engine/graph.py @@ -357,7 +357,6 @@ def create_dotfile_simple(self, outdir, name="graph"): dotstr = "digraph G {\n" for nd in self.nodes: - # breakpoint() if is_workflow(nd): if nd.state: # adding color for wf with a state @@ -370,8 +369,13 @@ def create_dotfile_simple(self, outdir, name="graph"): dotstr += f"{nd.name} [color=blue]\n" else: dotstr += f"{nd.name}\n" - for ed in self.edges_names: - dotstr += f"{ed[0]} -> {ed[1]}\n" + for ed in self.edges: + # if the tails nd has a final state, than the state is passed + # to the next node and the arrow is blue + if ed[0].state and ed[0].state.splitter_rpn_final: + dotstr += f"{ed[0].name} -> {ed[1].name} [color=blue]\n" + else: + dotstr += f"{ed[0].name} -> {ed[1].name}\n" dotstr += "}" Path(outdir).mkdir(parents=True, exist_ok=True) @@ -480,6 +484,10 @@ def _create_dotfile_single_graph(self, nodes, edges): ) else: dotstr_edg += f"{ed[0].name} -> {ed[1].name}\n" + # if tail has a state and a final splitter (not combined) than the state + # is passed to the next state and the arrow is blue + if ed[0].state and ed[0].state.splitter_rpn_final: + dotstr_edg = dotstr_edg[:-1] + " [color=blue]\n" dotstr = dotstr + dotstr_edg return dotstr diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index c343be4fff..edbbb2f71d 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -22,6 +22,7 @@ fun_write_file, fun_write_file_list, fun_write_file_list2dict, + list_sum, DOT_FLAG, ) from ..submitter import Submitter @@ -4032,7 +4033,7 @@ def test_graph_1st(tmpdir): assert "mult_1 [color=blue]" in dotstr_s_lines assert "mult_2" in dotstr_s_lines assert "add2 [color=blue]" in dotstr_s_lines - assert "mult_1 -> add2" in dotstr_s_lines + assert "mult_1 -> add2 [color=blue]" in dotstr_s_lines # nested graph dotfile_n = wf.create_dotfile(type="nested") @@ -4040,7 +4041,7 @@ def test_graph_1st(tmpdir): assert "mult_1 [color=blue]" in dotstr_n_lines assert "mult_2" in dotstr_n_lines assert "add2 [color=blue]" in dotstr_n_lines - assert "mult_1 -> add2" in dotstr_n_lines + assert "mult_1 -> add2 [color=blue]" in dotstr_n_lines # detailed graph dotfile_d = wf.create_dotfile(type="detailed") @@ -4056,6 +4057,48 @@ def test_graph_1st(tmpdir): exporting_graphs(wf=wf, name=name) +def test_graph_1st_cmb(tmpdir): + """creating a set of graphs, wf with three nodes + the first one has a splitter, the second has a combiner, so the third one is stateless + first two nodes should be blue and the arrow between them should be blue + """ + wf = Workflow(name="wf", input_spec=["x", "y"]) + wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split("x")) + wf.add(add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) + wf.add(list_sum(name="sum", x=wf.add2.lzout.out)) + wf.set_output([("out", wf.sum.lzout.out)]) + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult [color=blue]" in dotstr_s_lines + assert "add2 [color=blue]" in dotstr_s_lines + assert "sum" in dotstr_s_lines + assert "mult -> add2 [color=blue]" in dotstr_s_lines + assert "add2 -> sum" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") + assert "mult [color=blue]" in dotstr_n_lines + assert "add2 [color=blue]" in dotstr_n_lines + assert "sum" in dotstr_n_lines + assert "mult -> add2 [color=blue]" in dotstr_n_lines + assert "add2 -> sum" in dotstr_n_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_mult:out -> struct_add2:x;" in dotstr_d_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + exporting_graphs(wf=wf, name=name) + + def test_graph_2(tmpdir): """creating a graph, wf with one worfklow as a node""" wf = Workflow(name="wf", input_spec=["x"]) @@ -4163,6 +4206,48 @@ def test_graph_3(tmpdir): exporting_graphs(wf=wf, name=name) +def test_graph_3st(tmpdir): + """creating a set of graphs, wf with two nodes (one node is a workflow) + the first node has a state and it should be passed to the second node + (blue node and a wfasnd, and blue arrow from the node to the wfasnd) + """ + wf = Workflow(name="wf", input_spec=["x", "y"]) + wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split("x")) + + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) + wfnd.add(add2(name="add2", x=wfnd.lzin.x)) + wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + + # simple graph + dotfile_s = wf.create_dotfile() + dotstr_s_lines = dotfile_s.read_text().split("\n") + assert "mult [color=blue]" in dotstr_s_lines + assert "wfnd [shape=box, color=blue]" in dotstr_s_lines + assert "mult -> wfnd [color=blue]" in dotstr_s_lines + + # nested graph + dotfile_n = wf.create_dotfile(type="nested") + dotstr_n_lines = dotfile_n.read_text().split("\n") + assert "mult [color=blue]" in dotstr_n_lines + assert "subgraph cluster_wfnd {" in dotstr_n_lines + assert "add2" in dotstr_n_lines + + # detailed graph + dotfile_d = wf.create_dotfile(type="detailed") + dotstr_d_lines = dotfile_d.read_text().split("\n") + assert ( + 'struct_wf [color=red, label="{WORKFLOW INPUT: | { x | y}}"];' + in dotstr_d_lines + ) + assert "struct_mult:out -> struct_wfnd:x;" in dotstr_d_lines + + if DOT_FLAG: + name = f"graph_{sys._getframe().f_code.co_name}" + exporting_graphs(wf=wf, name=name) + + def test_graph_4(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the node to the inner workflow. diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index 9eae2ba41a..be51a29fc5 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -152,6 +152,11 @@ def list_output(x): return [x, 2 * x, 3 * x] +@mark.task +def list_sum(x): + return sum(x) + + @mark.task def fun_dict(d): kv_list = [f"{k}:{v}" for (k, v) in d.items()] From 2854e608b83a54cad8698bbe9be325bf33c56b29 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 14 Aug 2020 01:41:38 -0400 Subject: [PATCH 050/271] adding a new type MultiInputObj (converts everything to a list); adding converters to make_klass for types that have a class method converter (e.g. MultiInputPath); adding __setattr__ to BaseSpec to be able validate and convert inputs that are set after initialization (using attr.validate to validate and manually calling converters); refactoring the way how input is created from input_spec in TaskBase --- pydra/engine/core.py | 44 ++++++++++++++------------- pydra/engine/helpers.py | 12 ++++---- pydra/engine/specs.py | 26 ++++++++++++++++ pydra/engine/tests/test_task.py | 53 +++++++++++++++++++++++++++++++-- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 61a29be65e..2a280f9793 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -130,16 +130,26 @@ def __init__( if name in dir(self): raise ValueError("Cannot use names of attributes or methods as task name") self.name = name + if not inputs: + inputs = {} if not self.input_spec: raise Exception("No input_spec in class: %s" % self.__class__.__name__) klass = make_klass(self.input_spec) - # todo should be used to input_check in spec?? - self.inputs = klass( - **{ - (f.name[1:] if f.name.startswith("_") else f.name): f.default - for f in attr.fields(klass) - } - ) + inp_dict = {} + for f in attr.fields(klass): + if f.name.startswith("_"): + # in attrs names that starts with "_" could be set when name provided w/o "_" + name = f.name[1:] + else: + name = f.name + if name in inputs: + inp_dict[name] = inputs[name] + else: + # if the field not in inputs, the default value should be used + inp_dict[name] = f.default + self.inputs = klass(**inp_dict) + # checking if metadata is set properly + self.inputs.check_metadata() self.input_names = [ field.name for field in attr.fields(klass) @@ -154,18 +164,7 @@ def __init__( self._done = False if self._input_sets is None: self._input_sets = {} - if inputs: - if isinstance(inputs, dict): - inputs = {k: v for k, v in inputs.items() if k in self.input_names} - elif Path(inputs).is_file(): - inputs = json.loads(Path(inputs).read_text()) - elif isinstance(inputs, str): - if self._input_sets is None or inputs not in self._input_sets: - raise ValueError(f"Unknown input set {inputs!r}") - inputs = self._input_sets[inputs] - self.inputs = attr.evolve(self.inputs, **inputs) - self.inputs.check_metadata() - self.state_inputs = inputs + self.state_inputs = {k: v for k, v in inputs.items() if k in self.input_names} self.audit = Audit( audit_flags=audit_flags, @@ -412,8 +411,11 @@ def _run(self, rerun=False, **kwargs): self.hooks.post_run_task(self, result) self.audit.finalize_audit(result) save(odir, result=result, task=self) - for k, v in orig_inputs.items(): - setattr(self.inputs, k, v) + # # function etc. shouldn't change anyway, so removing + orig_inputs = dict( + (k, v) for (k, v) in orig_inputs.items() if not k.startswith("_") + ) + self.inputs = attr.evolve(self.inputs, **orig_inputs) os.chdir(cwd) self.hooks.post_run(self, result) return result diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index e2dbb528d1..e7ca69f90c 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -235,13 +235,12 @@ def make_klass(spec): newfields = dict() for item in fields: if len(item) == 2: + name = item[0] if isinstance(item[1], attr._make._CountingAttr): - newfields[item[0]] = item[1] - newfields[item[0]].validator(custom_validator) + newfields[name] = item[1] + newfields[name].validator(custom_validator) else: - newfields[item[0]] = attr.ib( - type=item[1], validator=custom_validator - ) + newfields[name] = attr.ib(type=item[1], validator=custom_validator) else: if ( any([isinstance(ii, attr._make._CountingAttr) for ii in item]) @@ -273,6 +272,9 @@ def make_klass(spec): metadata=mdata, validator=custom_validator, ) + # if type has converter, e.g. MultiInputObj + if hasattr(newfields[name].type, "converter"): + newfields[name].converter = newfields[name].type.converter fields = newfields return attr.make_class(spec.name, fields, bases=spec.bases, kw_only=True) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 08061d8c60..9cf90637b8 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -2,6 +2,7 @@ import attr from pathlib import Path import typing as ty +import inspect from .helpers_file import template_update_single @@ -18,6 +19,16 @@ class Directory: """An :obj:`os.pathlike` object, designating a folder.""" +class MultiInputObj: + """An :obj: ty.List[`os.pathlike`] object""" + + @classmethod + def converter(cls, value): + from .helpers import ensure_list + + return ensure_list(value) + + @attr.s(auto_attribs=True, kw_only=True) class SpecInfo: """Base data structure for metadata of specifications.""" @@ -35,6 +46,21 @@ class SpecInfo: class BaseSpec: """The base dataclass specs for all inputs and outputs.""" + def __setattr__(self, name, value): + """changing settatr, so the converter and validator is run + if input is set after __init__ + """ + if inspect.stack()[1][3] == "__init__": # or name.startswith("_"): + super().__setattr__(name, value) + else: + tp = attr.fields_dict(self.__class__)[name].type + # if the type has a converter, e.g., MultiInputObj + if hasattr(tp, "converter"): + value = tp.converter(value) + super().__setattr__(name, value) + # validate all fields that have set a validator + attr.validate(self) + def collect_additional_outputs(self, inputs, output_dir): """Get additional outputs.""" return {} diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 67c4c4b6e9..07263d47b5 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -6,6 +6,7 @@ from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator +from ..specs import MultiInputObj, File no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -167,10 +168,8 @@ def testfunc(a: int): return a funky = testfunc() - # the error is raised when run (should be improved?) - funky.inputs.a = 3.5 with pytest.raises(TypeError): - funky() + funky.inputs.a = 3.5 def test_annotated_input_func_3(use_validator): @@ -334,6 +333,54 @@ def testfunc(a: int): funky = testfunc(a=[3.5, 2.1]).split("a") +def test_annotated_input_func_8(): + """ the function with annotated input as MultiInputObj + a single value is provided and should be converted to a list + """ + + @mark.task + def testfunc(a: MultiInputObj): + return len(a) + + funky = testfunc(a=3.5) + assert getattr(funky.inputs, "a") == [3.5] + res = funky() + assert res.output.out == 1 + + +def test_annotated_input_func_8a(): + """ the function with annotated input as MultiInputObj + a 1-el list is provided so shouldn't be changed + """ + + @mark.task + def testfunc(a: MultiInputObj): + return len(a) + + funky = testfunc(a=[3.5]) + assert getattr(funky.inputs, "a") == [3.5] + res = funky() + assert res.output.out == 1 + + +def test_annotated_input_func_8b(): + """ the function with annotated input as MultiInputObj + a single value is provided after initial. the task + (input should still be converted to a list) + """ + + @mark.task + def testfunc(a: MultiInputObj): + return len(a) + + funky = testfunc() + # setting a after init + funky.inputs.a = 3.5 + assert getattr(funky.inputs, "a") == [3.5] + res = funky() + assert res.output.out == 1 + + def test_annotated_func_multreturn_exception(use_validator): """function has two elements in the return statement, but three element provided in the spec - should raise an error From d0c38d042d600560765971a256182166d0a4a1d4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 15 Aug 2020 00:05:39 -0400 Subject: [PATCH 051/271] refactoring the spec classes (adding FunctionInput, moving some methods to BaseSpec), so FunctionTask can have input_spec (that overwrites the function annotation if provided) --- pydra/engine/specs.py | 152 +++++++++++------ pydra/engine/task.py | 27 ++-- pydra/engine/tests/test_node_task.py | 5 +- pydra/engine/tests/test_task.py | 233 ++++++++++++++++++++++++++- pydra/engine/tests/utils.py | 10 +- 5 files changed, 350 insertions(+), 77 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 9cf90637b8..9328d24cb2 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -99,12 +99,57 @@ def retrieve_values(self, wf, state_index=None): for field, value in temp_values.items(): setattr(self, field, value) + def check_fields_input_spec(self): + """ + Check fields from input spec based on the medatada. + + e.g., if xor, requires are fulfilled, if value provided when mandatory. + + """ + fields = attr_fields(self) + names = [] + require_to_check = {} + for fld in fields: + mdata = fld.metadata + # checking if the mandatory field is provided + if getattr(self, fld.name) is attr.NOTHING: + if mdata.get("mandatory"): + raise AttributeError( + f"{fld.name} is mandatory, but no value provided" + ) + else: + continue + names.append(fld.name) + + # checking if fields meet the xor and requires are + if "xor" in mdata: + if [el for el in mdata["xor"] if el in names]: + raise AttributeError( + f"{fld.name} is mutually exclusive with {mdata['xor']}" + ) + + if "requires" in mdata: + if [el for el in mdata["requires"] if el not in names]: + # will check after adding all fields to names + require_to_check[fld.name] = mdata["requires"] + + if fld.type is File: + self._file_check_n_bindings(fld) + + for nm, required in require_to_check.items(): + required_notfound = [el for el in required if el not in names] + if required_notfound: + raise AttributeError(f"{nm} requires {required_notfound}") + + def _file_check_n_bindings(self, field): + """for tasks without container, this is simple check if the file exists""" + file = Path(getattr(self, field.name)) + if not file.exists(): + raise AttributeError(f"the file from the {field.name} input does not exist") + def check_metadata(self): """Check contained metadata.""" - def check_fields_input_spec(self): - """Check input fields.""" - def template_update(self): """Update template.""" @@ -189,6 +234,56 @@ class RuntimeSpec: network: bool = False +@attr.s(auto_attribs=True, kw_only=True) +class FunctionSpec(BaseSpec): + """Specification for a process invoked from a shell.""" + + def check_metadata(self): + """ + Check the metadata for fields in input_spec and fields. + + Also sets the default values when available and needed. + + """ + supported_keys = { + "allowed_values", + "copyfile", + "help_string", + "mandatory", + # "readonly", #likely not needed + # "output_field_name", #likely not needed + # "output_file_template", #likely not needed + "requires", + "keep_extension", + "xor", + "sep", + } + # special inputs, don't have to follow rules for standard inputs + special_input = ["_func", "_graph_checksums"] + + fields = [fld for fld in attr_fields(self) if fld.name not in special_input] + for fld in fields: + mdata = fld.metadata + # checking keys from metadata + if set(mdata.keys()) - supported_keys: + raise AttributeError( + f"only these keys are supported {supported_keys}, but " + f"{set(mdata.keys()) - supported_keys} provided" + ) + # checking if the help string is provided (required field) + if "help_string" not in mdata: + raise AttributeError(f"{fld.name} doesn't have help_string field") + # not allowing for default if the field is mandatory + if not fld.default == attr.NOTHING and mdata.get("mandatory"): + raise AttributeError( + "default value should not be set when the field is mandatory" + ) + # setting default if value not provided and default is available + if getattr(self, fld.name) is None: + if not fld.default == attr.NOTHING: + setattr(self, fld.name, fld.default) + + @attr.s(auto_attribs=True, kw_only=True) class ShellSpec(BaseSpec): """Specification for a process invoked from a shell.""" @@ -276,56 +371,9 @@ def check_metadata(self): if not fld.default == attr.NOTHING: setattr(self, fld.name, fld.default) - def check_fields_input_spec(self): - """ - Check fields from input spec based on the medatada. - - e.g., if xor, requires are fulfilled, if value provided when mandatory. - - """ - fields = attr_fields(self) - names = [] - require_to_check = {} - for fld in fields: - mdata = fld.metadata - # checking if the mandatory field is provided - if getattr(self, fld.name) is attr.NOTHING: - if mdata.get("mandatory"): - raise AttributeError( - f"{fld.name} is mandatory, but no value provided" - ) - else: - continue - names.append(fld.name) - - # checking if fields meet the xor and requires are - if "xor" in mdata: - if [el for el in mdata["xor"] if el in names]: - raise AttributeError( - f"{fld.name} is mutually exclusive with {mdata['xor']}" - ) - - if "requires" in mdata: - if [el for el in mdata["requires"] if el not in names]: - # will check after adding all fields to names - require_to_check[fld.name] = mdata["requires"] - - if fld.type is File: - self._file_check(fld) - - for nm, required in require_to_check.items(): - required_notfound = [el for el in required if el not in names] - if required_notfound: - raise AttributeError(f"{nm} requires {required_notfound}") - - def _file_check(self, field): - file = Path(getattr(self, field.name)) - if not file.exists(): - raise AttributeError(f"the file from the {field.name} input does not exist") - @attr.s(auto_attribs=True, kw_only=True) -class ShellOutSpec(BaseSpec): +class ShellOutSpec: """Output specification of a generic shell process.""" return_code: int @@ -432,7 +480,7 @@ class ContainerSpec(ShellSpec): ] = attr.ib(default=None, metadata={"help_string": "bindings"}) """Mount points to be bound into the container.""" - def _file_check(self, field): + def _file_check_n_bindings(self, field): file = Path(getattr(self, field.name)) if field.metadata.get("container_path"): # if the path is in a container the input should be treated as a str (hash as a str) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index bce758eee1..60eaa54c9d 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -104,31 +104,26 @@ def __init__( """ if input_spec is None: - input_spec = SpecInfo( - name="Inputs", - fields=[ + fields = [] + for val in inspect.signature(func).parameters.values(): + if val.default is not inspect.Signature.empty: + val_dflt = val.default + else: + val_dflt = attr.NOTHING + fields.append( ( val.name, attr.ib( - default=val.default, + default=val_dflt, type=val.annotation, metadata={ "help_string": f"{val.name} parameter from {func.__name__}" }, ), ) - if val.default is not inspect.Signature.empty - else ( - val.name, - attr.ib( - type=val.annotation, metadata={"help_string": val.name} - ), - ) - for val in inspect.signature(func).parameters.values() - ] - + [("_func", attr.ib(default=cp.dumps(func), type=str))], - bases=(BaseSpec,), - ) + ) + fields.append(("_func", attr.ib(default=cp.dumps(func), type=str))) + input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseSpec,)) else: input_spec.fields.append( ("_func", attr.ib(default=cp.dumps(func), type=str)) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index ef2e1d8018..79339dc9ee 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -485,7 +485,7 @@ def test_task_nostate_4(plugin, tmpdir): assert nn.output_dir.exists() -def test_task_nostate_5(plugin, tmpdir): +def test_task_nostate_5(tmpdir): """ task with a dictionary of files as an input""" file1 = tmpdir.join("file1.txt") with open(file1, "w") as f: @@ -497,8 +497,7 @@ def test_task_nostate_5(plugin, tmpdir): nn = fun_file_list(name="NA", filename_list=[file1, file2]) - with Submitter(plugin=plugin) as sub: - sub(nn) + nn() # checking the results results = nn.result() diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 07263d47b5..a0e20f875e 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1,12 +1,13 @@ import typing as ty import os, sys +import attr import pytest from ... import mark from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator -from ..specs import MultiInputObj, File +from ..specs import MultiInputObj, SpecInfo, FunctionSpec no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -546,6 +547,236 @@ def no_annots(c, d): assert result.output.out == (20.2, 13.8) +def test_input_spec_func_1(use_validator): + """ the function w/o annotated, but input_spec is used """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[("a", attr.ib(type=float, metadata={"help_string": "input a"}))], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=3.5, input_spec=my_input_spec) + assert getattr(funky.inputs, "a") == 3.5 + + +def test_input_spec_func_1a_except(use_validator): + """ the function w/o annotated, but input_spec is used + a TypeError is raised (float is provided instead of int) + """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[("a", attr.ib(type=int, metadata={"help_string": "input a"}))], + bases=(FunctionSpec,), + ) + with pytest.raises(TypeError): + funky = testfunc(a=3.5, input_spec=my_input_spec) + + +def test_input_spec_func_1b_except(use_validator): + """ the function w/o annotated, but input_spec is used + metadata checks raise an error + """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "a", + attr.ib(type=float, metadata={"position": 1, "help_string": "input a"}), + ) + ], + bases=(FunctionSpec,), + ) + with pytest.raises(AttributeError, match="only these keys are supported"): + funky = testfunc(a=3.5, input_spec=my_input_spec) + + +def test_input_spec_func_1d_except(use_validator): + """ the function w/o annotated, but input_spec is used + input_spec doesn't contain 'a' input, an error is raised + """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo(name="Input", fields=[], bases=(FunctionSpec,)) + funky = testfunc(a=3.5, input_spec=my_input_spec) + with pytest.raises(TypeError, match="missing 1 required positional argument"): + funky() + + +def test_input_spec_func_2(use_validator): + """ the function with annotation, and the task has input_spec, + input_spec changes the type of the input (so error is not raised) + """ + + @mark.task + def testfunc(a: int): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[("a", attr.ib(type=float, metadata={"help_string": "input a"}))], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=3.5, input_spec=my_input_spec) + assert getattr(funky.inputs, "a") == 3.5 + + +def test_input_spec_func_3(use_validator): + """ the function w/o annotated, but input_spec is used + additional keys (allowed_values) are used in metadata + """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "a", + attr.ib( + type=int, + metadata={"help_string": "input a", "allowed_values": [0, 1, 2]}, + ), + ) + ], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=2, input_spec=my_input_spec) + assert getattr(funky.inputs, "a") == 2 + + +def test_input_spec_func_3a_except(use_validator): + """ the function w/o annotated, but input_spec is used + allowed_values is used in metadata and the ValueError is raised + """ + + @mark.task + def testfunc(a): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "a", + attr.ib( + type=int, + metadata={"help_string": "input a", "allowed_values": [0, 1, 2]}, + ), + ) + ], + bases=(FunctionSpec,), + ) + + with pytest.raises(ValueError, match="value of a has to be"): + funky = testfunc(a=3, input_spec=my_input_spec) + + +def test_input_spec_func_4(use_validator): + """ the function with a default value for b + but b is set as mandatory in the input_spec, so error is raised if not provided + """ + + @mark.task + def testfunc(a, b=1): + return a + b + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "a", + attr.ib( + type=int, metadata={"help_string": "input a", "mandatory": True} + ), + ), + ( + "b", + attr.ib( + type=int, metadata={"help_string": "input b", "mandatory": True} + ), + ), + ], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=2, input_spec=my_input_spec) + with pytest.raises(Exception, match="b is mandatory"): + funky() + + +def test_input_spec_func_4a(use_validator): + """ the function with a default value for b and metadata in the input_spec + has a different default value, so value from the function is overwritten + """ + + @mark.task + def testfunc(a, b=1): + return a + b + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "a", + attr.ib( + type=int, metadata={"help_string": "input a", "mandatory": True} + ), + ), + ("b", attr.ib(type=int, default=10, metadata={"help_string": "input b"})), + ], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=2, input_spec=my_input_spec) + res = funky() + assert res.output.out == 12 + + +def test_input_spec_func_5(): + """ the FunctionTask with input_spec, a input has MultiInputObj type + a single value is provided and should be converted to a list + """ + + @mark.task + def testfunc(a): + return len(a) + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ("a", attr.ib(type=MultiInputObj, metadata={"help_string": "input a"})) + ], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=3.5, input_spec=my_input_spec) + assert getattr(funky.inputs, "a") == [3.5] + res = funky() + assert res.output.out == 1 + + def test_exception_func(): @mark.task def raise_exception(c, d): diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index 9eae2ba41a..9de388c9f0 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -1,7 +1,7 @@ # Tasks for testing import time import sys, shutil -import typing as tp +import typing as ty from pathlib import Path import subprocess as sp import pytest @@ -159,14 +159,14 @@ def fun_dict(d): @mark.task -def fun_write_file(filename: tp.Union[str, File, Path], text="hello"): +def fun_write_file(filename: ty.Union[str, File, Path], text="hello"): with open(filename, "w") as f: f.write(text) return Path(filename).absolute() @mark.task -def fun_write_file_list(filename_list: tp.List[tp.Union[str, File, Path]], text="hi"): +def fun_write_file_list(filename_list: ty.List[ty.Union[str, File, Path]], text="hi"): for ii, filename in enumerate(filename_list): with open(filename, "w") as f: f.write(f"from file {ii}: {text}") @@ -176,7 +176,7 @@ def fun_write_file_list(filename_list: tp.List[tp.Union[str, File, Path]], text= @mark.task def fun_write_file_list2dict( - filename_list: tp.List[tp.Union[str, File, Path]], text="hi" + filename_list: ty.List[ty.Union[str, File, Path]], text="hi" ): filename_dict = {} for ii, filename in enumerate(filename_list): @@ -196,7 +196,7 @@ def fun_file(filename: File): @mark.task -def fun_file_list(filename_list: File): +def fun_file_list(filename_list: ty.List[File]): txt_list = [] for filename in filename_list: with open(filename) as f: From cafd1a37ad49af60982bc6285c5b3369ad6ecabd Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 13 Aug 2020 18:59:50 +0800 Subject: [PATCH 052/271] test slurm docker on github actions --- .github/workflows/slurm-ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/slurm-ci.yml diff --git a/.github/workflows/slurm-ci.yml b/.github/workflows/slurm-ci.yml new file mode 100644 index 0000000000..7a7995f8a7 --- /dev/null +++ b/.github/workflows/slurm-ci.yml @@ -0,0 +1,33 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +# use containers in github actions +# https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container +# https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions + + +# using go and docker in github actions +# https://steele.blue/tiny-github-actions/ + +name: Run SLURM tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + continue-on-error: true + env: + DOCKER_IMAGE: mgxd/slurm:19.05.1 + + steps: + - name: Pull docker image + run: docker pull $DOCKER_IMAGE + - name: Have image running in background + run: | + docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE + echo "Allowing ports/daemons time to start" && sleep 10 + # uses: actions/setup-python@v2 + + +# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 From 1b6056c9bdcb9755cb86b43909e62839fc720081 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 13 Aug 2020 19:14:15 +0800 Subject: [PATCH 053/271] run sacct after pulling docker in github actions slurm test --- .github/workflows/slurm-ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/slurm-ci.yml b/.github/workflows/slurm-ci.yml index 7a7995f8a7..1d5b95abd9 100644 --- a/.github/workflows/slurm-ci.yml +++ b/.github/workflows/slurm-ci.yml @@ -24,10 +24,19 @@ jobs: - name: Pull docker image run: docker pull $DOCKER_IMAGE - name: Have image running in background + run: docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE + - name: Display previous jobs with sacct run: | - docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE echo "Allowing ports/daemons time to start" && sleep 10 - # uses: actions/setup-python@v2 + docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ + && supervisorctl restart slurmdbd \ + && supervisorctl restart slurmctld \ + && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" + docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null + if [ $? -ne 0 ]; then + echo "Slurm docker image error" + exit 1 + fi # https://github.community/t/confused-with-runs-on-and-container-options/16258/2 From 71fa13abf402568b5978016705c4a56421945ae0 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 13 Aug 2020 23:55:34 +0800 Subject: [PATCH 054/271] setup python, run pytest and upload to codecov --- .github/workflows/slurm-ci.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/slurm-ci.yml b/.github/workflows/slurm-ci.yml index 1d5b95abd9..5af4070f63 100644 --- a/.github/workflows/slurm-ci.yml +++ b/.github/workflows/slurm-ci.yml @@ -1,10 +1,7 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - # use containers in github actions # https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container # https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions - +# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 # using go and docker in github actions # https://steele.blue/tiny-github-actions/ @@ -21,10 +18,12 @@ jobs: DOCKER_IMAGE: mgxd/slurm:19.05.1 steps: + - uses: actions/checkout@v2 - name: Pull docker image - run: docker pull $DOCKER_IMAGE - - name: Have image running in background - run: docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE + run: | + docker pull $DOCKER_IMAGE + # Have image running in background + docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE - name: Display previous jobs with sacct run: | echo "Allowing ports/daemons time to start" && sleep 10 @@ -37,6 +36,14 @@ jobs: echo "Slurm docker image error" exit 1 fi - - -# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 + - name: Setup Python + run: | + docker exec slurm bash -c "ls -la && echo list top level dir" + docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" + docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" + - name: Run pytest + run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" + - name: Upload to codecov + run: | + docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" + docker rm -f slurm From 9eea0c2e6dbcc1298c2dbd53c958aa1066174d29 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 17:01:31 +0800 Subject: [PATCH 055/271] rename file and add codecov upload --- .../workflows/{pythonpackage.yml => testpydra.yml} | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) rename .github/workflows/{pythonpackage.yml => testpydra.yml} (71%) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/testpydra.yml similarity index 71% rename from .github/workflows/pythonpackage.yml rename to .github/workflows/testpydra.yml index 5c9d7ec94e..6b3e438428 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/testpydra.yml @@ -1,8 +1,6 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package +name: Test Pydra on: [push, pull_request] @@ -31,8 +29,9 @@ jobs: - name: Install pydra run: | python -m pip install .[test] - - name: Pytest tests + - name: Pytest env: - IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # not running containers in this action - run: | - pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # docker tests run in a seperate action for now + run: pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + - name: Upload to codecov + run: codecov --root /pydra -f /pydra/cov.xml -F unittests From 87777dd778d36e12067c3d553796d1bd2095593d Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 17:19:44 +0800 Subject: [PATCH 056/271] run dockertask tests on github action --- .github/workflows/testpydradocker.yml | 37 +++++++++++++++++++++++++ pydra/engine/tests/test_dockertask.py | 39 +-------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/testpydradocker.yml diff --git a/.github/workflows/testpydradocker.yml b/.github/workflows/testpydradocker.yml new file mode 100644 index 0000000000..f9bb6d92b8 --- /dev/null +++ b/.github/workflows/testpydradocker.yml @@ -0,0 +1,37 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions + +name: Test Dockertask + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Update build tools + run: | + python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel + - name: Install dependencies + run: | + python -m pip install -r min-requirements.txt + - name: Install pydra + run: | + python -m pip install .[test] + - name: Pytest + run: | + pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + - name: Upload to codecov + run: | + codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 3472e4d061..1b2060361d 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -6,10 +6,9 @@ from ..submitter import Submitter from ..core import Workflow from ..specs import ShellOutSpec, SpecInfo, File, DockerSpec -from .utils import no_win, need_docker +from .utils import need_docker -@no_win @need_docker def test_docker_1_nosubm(): """ simple command in a container, a default bindings and working directory is added @@ -31,7 +30,6 @@ def test_docker_1_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1(plugin): """ simple command in a container, a default bindings and working directory is added @@ -50,7 +48,6 @@ def test_docker_1(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1_dockerflag(plugin): """ simple command in a container, a default bindings and working directory is added @@ -71,7 +68,6 @@ def test_docker_1_dockerflag(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1_dockerflag_exception(plugin): """using ShellComandTask with container_info=("docker"), no image provided""" @@ -83,7 +79,6 @@ def test_docker_1_dockerflag_exception(plugin): assert "container_info has to have 2 or 3 elements" in str(excinfo.value) -@no_win @need_docker def test_docker_2_nosubm(): """ a command with arguments, cmd and args given as executable @@ -103,7 +98,6 @@ def test_docker_2_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2(plugin): """ a command with arguments, cmd and args given as executable @@ -125,7 +119,6 @@ def test_docker_2(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2_dockerflag(plugin): """ a command with arguments, cmd and args given as executable @@ -149,7 +142,6 @@ def test_docker_2_dockerflag(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2a_nosubm(): """ a command with arguments, using executable and args @@ -174,7 +166,6 @@ def test_docker_2a_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2a(plugin): """ a command with arguments, using executable and args @@ -201,7 +192,6 @@ def test_docker_2a(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3(plugin, tmpdir): """ a simple command in container with bindings, @@ -224,7 +214,6 @@ def test_docker_3(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3_dockerflag(plugin, tmpdir): """ a simple command in container with bindings, @@ -250,7 +239,6 @@ def test_docker_3_dockerflag(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3_dockerflagbind(plugin, tmpdir): """ a simple command in container with bindings, @@ -276,7 +264,6 @@ def test_docker_3_dockerflagbind(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_4(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -302,7 +289,6 @@ def test_docker_4(plugin, tmpdir): assert res.output.return_code == 0 -@no_win @need_docker def test_docker_4_dockerflag(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -331,7 +317,6 @@ def test_docker_4_dockerflag(plugin, tmpdir): # tests with State -@no_win @need_docker def test_docker_st_1(plugin): """ commands without arguments in container @@ -355,7 +340,6 @@ def test_docker_st_1(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 -@no_win @need_docker def test_docker_st_2(plugin): """ command with arguments in docker, checking the distribution @@ -379,7 +363,6 @@ def test_docker_st_2(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 -@no_win @need_docker def test_docker_st_3(plugin): """ outer splitter image and executable @@ -397,7 +380,6 @@ def test_docker_st_3(plugin): assert "Ubuntu" in res[3].output.stdout -@no_win @need_docker def test_docker_st_4(plugin): """ outer splitter image and executable, combining with images @@ -439,7 +421,6 @@ def test_docker_st_4(plugin): # tests with workflows -@no_win @need_docker def test_wf_docker_1(plugin, tmpdir): """ a workflow with two connected task @@ -483,7 +464,6 @@ def test_wf_docker_1(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" -@no_win @need_docker def test_wf_docker_1_dockerflag(plugin, tmpdir): """ a workflow with two connected task @@ -523,7 +503,6 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" -@no_win @need_docker def test_wf_docker_2pre(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -544,7 +523,6 @@ def test_wf_docker_2pre(plugin, tmpdir): assert res.output.stdout == "/outputs/tmp.txt" -@no_win @need_docker def test_wf_docker_2(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -584,7 +562,6 @@ def test_wf_docker_2(plugin, tmpdir): assert res.output.out == "Hello!" -@no_win @need_docker def test_wf_docker_3(plugin, tmpdir): """ a workflow with two connected task @@ -627,7 +604,6 @@ def test_wf_docker_3(plugin, tmpdir): # tests with customized output_spec -@no_win @need_docker def test_docker_outputspec_1(plugin, tmpdir): """ @@ -655,7 +631,6 @@ def test_docker_outputspec_1(plugin, tmpdir): # tests with customised input_spec -@no_win @need_docker def test_docker_inputspec_1(plugin, tmpdir): """ a simple customized input spec for docker task """ @@ -697,7 +672,6 @@ def test_docker_inputspec_1(plugin, tmpdir): assert res.output.stdout == "hello from pydra" -@no_win @need_docker def test_docker_inputspec_1a(plugin, tmpdir): """ a simple customized input spec for docker task @@ -736,7 +710,6 @@ def test_docker_inputspec_1a(plugin, tmpdir): assert res.output.stdout == "hello from pydra" -@no_win @need_docker def test_docker_inputspec_2(plugin, tmpdir): """ a customized input spec with two fields for docker task """ @@ -793,7 +766,6 @@ def test_docker_inputspec_2(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker def test_docker_inputspec_2a_except(plugin, tmpdir): """ a customized input spec with two fields @@ -853,7 +825,6 @@ def test_docker_inputspec_2a_except(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker def test_docker_inputspec_2a(plugin, tmpdir): """ a customized input spec with two fields @@ -913,7 +884,6 @@ def test_docker_inputspec_2a(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker def test_docker_inputspec_3(plugin, tmpdir): """ input file is in the container, so metadata["container_path"]: True, @@ -957,7 +927,6 @@ def test_docker_inputspec_3(plugin, tmpdir): assert cmdline == docky.cmdline -@no_win @need_docker def test_docker_inputspec_3a(plugin, tmpdir): """ input file does not exist in the local file system, @@ -1001,7 +970,6 @@ def test_docker_inputspec_3a(plugin, tmpdir): assert "use field.metadata['container_path']=True" in str(excinfo.value) -@no_win @need_docker def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): """ shelltask changes a file in place, @@ -1064,7 +1032,6 @@ def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): assert "hello from pydra\n" == f.read() -@no_win @need_docker def test_docker_inputspec_state_1(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1113,7 +1080,6 @@ def test_docker_inputspec_state_1(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" -@no_win @need_docker def test_docker_inputspec_state_1b(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1163,7 +1129,6 @@ def test_docker_inputspec_state_1b(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" -@no_win @need_docker def test_docker_wf_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks """ @@ -1215,7 +1180,6 @@ def test_docker_wf_inputspec_1(plugin, tmpdir): assert res.output.out == "hello from pydra" -@no_win @need_docker def test_docker_wf_state_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks that has a state""" @@ -1273,7 +1237,6 @@ def test_docker_wf_state_inputspec_1(plugin, tmpdir): assert res[1].output.out == "have a nice one" -@no_win @need_docker def test_docker_wf_ndst_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks with states""" From 74ead3740bb3453e10e0efac9f5bea65634df5e0 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 17:58:58 +0800 Subject: [PATCH 057/271] fix codecov ignore paths in github actions --- .github/workflows/testpydra.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 6b3e438428..b1eac92f2e 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -8,7 +8,6 @@ jobs: build: runs-on: ${{ matrix.os }} - continue-on-error: true strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] @@ -31,7 +30,7 @@ jobs: python -m pip install .[test] - name: Pytest env: - IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # docker tests run in a seperate action for now - run: pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now + run: pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests From 63b0aa57600e8960ec7f5a3379cbac3ef8fb7278 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 18:05:13 +0800 Subject: [PATCH 058/271] rename github action yaml file --- .github/workflows/pythonpackage.yml | 38 ---------------------- .github/workflows/testslurm.yml | 49 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/pythonpackage.yml create mode 100644 .github/workflows/testslurm.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml deleted file mode 100644 index 5c9d7ec94e..0000000000 --- a/.github/workflows/pythonpackage.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - - -name: Python package - -on: [push, pull_request] - -jobs: - build: - - runs-on: ${{ matrix.os }} - continue-on-error: true - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Update build tools - run: | - python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - - name: Install dependencies - run: | - python -m pip install -r min-requirements.txt - - name: Install pydra - run: | - python -m pip install .[test] - - name: Pytest tests - env: - IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # not running containers in this action - run: | - pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml new file mode 100644 index 0000000000..5af4070f63 --- /dev/null +++ b/.github/workflows/testslurm.yml @@ -0,0 +1,49 @@ +# use containers in github actions +# https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container +# https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions +# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 + +# using go and docker in github actions +# https://steele.blue/tiny-github-actions/ + +name: Run SLURM tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + continue-on-error: true + env: + DOCKER_IMAGE: mgxd/slurm:19.05.1 + + steps: + - uses: actions/checkout@v2 + - name: Pull docker image + run: | + docker pull $DOCKER_IMAGE + # Have image running in background + docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE + - name: Display previous jobs with sacct + run: | + echo "Allowing ports/daemons time to start" && sleep 10 + docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ + && supervisorctl restart slurmdbd \ + && supervisorctl restart slurmctld \ + && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" + docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null + if [ $? -ne 0 ]; then + echo "Slurm docker image error" + exit 1 + fi + - name: Setup Python + run: | + docker exec slurm bash -c "ls -la && echo list top level dir" + docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" + docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" + - name: Run pytest + run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" + - name: Upload to codecov + run: | + docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" + docker rm -f slurm From 92de1a0f171f24af1f1dbdeb918271488a4cfabe Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 18:16:30 +0800 Subject: [PATCH 059/271] remove old github actions --- .github/workflows/slurm-ci.yml | 49 ---------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/slurm-ci.yml diff --git a/.github/workflows/slurm-ci.yml b/.github/workflows/slurm-ci.yml deleted file mode 100644 index 5af4070f63..0000000000 --- a/.github/workflows/slurm-ci.yml +++ /dev/null @@ -1,49 +0,0 @@ -# use containers in github actions -# https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container -# https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions -# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 - -# using go and docker in github actions -# https://steele.blue/tiny-github-actions/ - -name: Run SLURM tests - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - continue-on-error: true - env: - DOCKER_IMAGE: mgxd/slurm:19.05.1 - - steps: - - uses: actions/checkout@v2 - - name: Pull docker image - run: | - docker pull $DOCKER_IMAGE - # Have image running in background - docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE - - name: Display previous jobs with sacct - run: | - echo "Allowing ports/daemons time to start" && sleep 10 - docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ - && supervisorctl restart slurmdbd \ - && supervisorctl restart slurmctld \ - && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" - docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null - if [ $? -ne 0 ]; then - echo "Slurm docker image error" - exit 1 - fi - - name: Setup Python - run: | - docker exec slurm bash -c "ls -la && echo list top level dir" - docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" - - name: Run pytest - run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" - - name: Upload to codecov - run: | - docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" - docker rm -f slurm From 5541b388335c9d7f665f52ddcb8ca4ba1a11dd93 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 18:40:38 +0800 Subject: [PATCH 060/271] xfail test_docker_inputspec_3 and fix pytest command in testdockertask --- ...testpydradocker.yml => testdockertask.yml} | 3 +- .github/workflows/testpydra.yml | 37 ------------------- pydra/engine/tests/test_dockertask.py | 1 + 3 files changed, 2 insertions(+), 39 deletions(-) rename .github/workflows/{testpydradocker.yml => testdockertask.yml} (89%) delete mode 100644 .github/workflows/testpydra.yml diff --git a/.github/workflows/testpydradocker.yml b/.github/workflows/testdockertask.yml similarity index 89% rename from .github/workflows/testpydradocker.yml rename to .github/workflows/testdockertask.yml index f9bb6d92b8..6c71fe45eb 100644 --- a/.github/workflows/testpydradocker.yml +++ b/.github/workflows/testdockertask.yml @@ -8,7 +8,6 @@ jobs: build: runs-on: ${{ matrix.os }} - continue-on-error: true strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] @@ -31,7 +30,7 @@ jobs: python -m pip install .[test] - name: Pytest run: | - pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml - name: Upload to codecov run: | codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml deleted file mode 100644 index 6b3e438428..0000000000 --- a/.github/workflows/testpydra.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions - -name: Test Pydra - -on: [push, pull_request] - -jobs: - build: - - runs-on: ${{ matrix.os }} - continue-on-error: true - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Update build tools - run: | - python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - - name: Install dependencies - run: | - python -m pip install -r min-requirements.txt - - name: Install pydra - run: | - python -m pip install .[test] - - name: Pytest - env: - IGNORE_TESTS: 'pydra/engine/tests/test_dockertask.py' # docker tests run in a seperate action for now - run: pytest --ignore=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 1b2060361d..3a1721ea41 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -885,6 +885,7 @@ def test_docker_inputspec_2a(plugin, tmpdir): @need_docker +@pytest.mark.xfail(reason="'docker' not in /proc/1/cgroup on ubuntu; TODO") def test_docker_inputspec_3(plugin, tmpdir): """ input file is in the container, so metadata["container_path"]: True, the input will be treated as a str """ From 8bd47d0cd5d128302cda422b5bd1af3a0bad9576 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 19:22:07 +0800 Subject: [PATCH 061/271] add singularity github actions CI framework --- .github/workflows/testpydra.yml | 38 -------------------- .github/workflows/testsingularity.yaml | 50 ++++++++++++++++++++++++++ .github/workflows/testslurm.yml | 49 ------------------------- 3 files changed, 50 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/testpydra.yml create mode 100644 .github/workflows/testsingularity.yaml delete mode 100644 .github/workflows/testslurm.yml diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml deleted file mode 100644 index 6a0c438de6..0000000000 --- a/.github/workflows/testpydra.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions - -name: Test Pydra - -on: [push, pull_request] - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Update build tools - run: | - python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - - name: Install dependencies - run: | - python -m pip install -r min-requirements.txt - - name: Install pydra - run: | - python -m pip install .[test] - - name: Pytest - env: - IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: | - pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - - name: Upload to codecov - run: | - codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml new file mode 100644 index 0000000000..23d18137c0 --- /dev/null +++ b/.github/workflows/testsingularity.yaml @@ -0,0 +1,50 @@ +#https://github.com/eWaterCycle/singularity-versions/blob/master/.github/workflows/dist.yml + +name: Test singularity + +on: [push, pull_request] + +jobs: + build: + name: Build + runs-on: ubuntu-latest + continue-on-error: true + + steps: + - name: Install OS deps + run: | + sudo apt-get update; + sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ + libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y; + - name: Set env + run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/v} + - uses: actions/checkout@v2 + with: + repository: hpcng/singularity + ref: v${{ env.RELEASE_VERSION }} + - name: Setup GO + uses: actions/setup-go@v2 + with: + go-version: '^1.13' + - name: Build + run: | + ./mconfig; + make -C ./builddir; + sudo make -C ./builddir install; + cd -; + - name: Echo version + run: | + echo ${{ github.ref }} + ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version + - name: Install miniconda + run: | + python setup.py develop + python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install + + + +#- name: Build +# ./mconfig --without-suid -p ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64 +# cd builddir +# make -j 2 +# ake install diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml deleted file mode 100644 index 5af4070f63..0000000000 --- a/.github/workflows/testslurm.yml +++ /dev/null @@ -1,49 +0,0 @@ -# use containers in github actions -# https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container -# https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions -# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 - -# using go and docker in github actions -# https://steele.blue/tiny-github-actions/ - -name: Run SLURM tests - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - continue-on-error: true - env: - DOCKER_IMAGE: mgxd/slurm:19.05.1 - - steps: - - uses: actions/checkout@v2 - - name: Pull docker image - run: | - docker pull $DOCKER_IMAGE - # Have image running in background - docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE - - name: Display previous jobs with sacct - run: | - echo "Allowing ports/daemons time to start" && sleep 10 - docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ - && supervisorctl restart slurmdbd \ - && supervisorctl restart slurmctld \ - && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" - docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null - if [ $? -ne 0 ]; then - echo "Slurm docker image error" - exit 1 - fi - - name: Setup Python - run: | - docker exec slurm bash -c "ls -la && echo list top level dir" - docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" - - name: Run pytest - run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" - - name: Upload to codecov - run: | - docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" - docker rm -f slurm From e34d11b69caf662bf1a6856fb426210e36130ec2 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 19:30:58 +0800 Subject: [PATCH 062/271] checkout specific version of singularity from hpcng/singularity --- .github/workflows/testsingularity.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index 23d18137c0..678ea2b514 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -16,12 +16,12 @@ jobs: sudo apt-get update; sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y; - - name: Set env - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/v} + #- name: Set env + # run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/v} - uses: actions/checkout@v2 with: repository: hpcng/singularity - ref: v${{ env.RELEASE_VERSION }} + ref: 'v3.5.0' - name: Setup GO uses: actions/setup-go@v2 with: From 27116933a9fec7686aaa5e058b3bf0145cb4e091 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 19:58:55 +0800 Subject: [PATCH 063/271] fix singularity build command in CI --- .github/workflows/testsingularity.yaml | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index 678ea2b514..515a1eafea 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -11,13 +11,10 @@ jobs: continue-on-error: true steps: - - name: Install OS deps + - name: Set env run: | - sudo apt-get update; - sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ - libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y; - #- name: Set env - # run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/v} + echo ::set-env name=RELEASE_VERSION::v3.5.0 + mkdir singularity && cd singularity - uses: actions/checkout@v2 with: repository: hpcng/singularity @@ -26,17 +23,27 @@ jobs: uses: actions/setup-go@v2 with: go-version: '^1.13' + - name: Install OS deps + run: | + sudo apt-get update + sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ + libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y - name: Build run: | - ./mconfig; - make -C ./builddir; - sudo make -C ./builddir install; - cd -; + ./mconfig --without-suid -p ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64 + make -C ./builddir + sudo make -C ./builddir install + cd .. - name: Echo version run: | echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version - name: Install miniconda + run: | + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda + eval "$($HOME/miniconda/bin/conda shell.bash hook)" + - name: Install Python run: | python setup.py develop python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install From 674d77c96a9584a2244015221cecd365c833a943 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 20:10:51 +0800 Subject: [PATCH 064/271] checkout repo, run pytest and codecov in singularity CI --- .github/workflows/testsingularity.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index 515a1eafea..dcf0c9f572 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -43,15 +43,19 @@ jobs: wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda eval "$($HOME/miniconda/bin/conda shell.bash hook)" - - name: Install Python + - uses: actions/checkout@v2 + - name: Set up dev env run: | python setup.py develop python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install - - - -#- name: Build -# ./mconfig --without-suid -p ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64 -# cd builddir -# make -j 2 -# ake install + - name: Install pydra + run: | + pip install -e ".[test]" + - name: Pytest + env: + IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now + run: | + pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml + - name: Upload to codecov + run: | + codecov --root /pydra -f /pydra/cov.xml -F unittests From 72608de425f71e55e9ee5cd8f217968a9352f13c Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 20:27:59 +0800 Subject: [PATCH 065/271] update miniconda install in singularity CI --- .github/workflows/testsingularity.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index dcf0c9f572..ff42384a10 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -38,11 +38,17 @@ jobs: run: | echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version - - name: Install miniconda + - uses: goanpeca/setup-miniconda@v1 + - name: Conda info + shell: bash -l {0} run: | - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda - eval "$($HOME/miniconda/bin/conda shell.bash hook)" + conda info + conda list + #- name: Install miniconda + #run: | + # wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + # bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda + # eval "$($HOME/miniconda/bin/conda shell.bash hook)" - uses: actions/checkout@v2 - name: Set up dev env run: | @@ -50,7 +56,7 @@ jobs: python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install - name: Install pydra run: | - pip install -e ".[test]" + python -m pip install -e .[test] - name: Pytest env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now From 6845cd0d383cc0ecd37a355cc7ddaf77d22463c4 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 17 Aug 2020 21:18:41 +0800 Subject: [PATCH 066/271] fix checkout repo actions in singularity CI --- .github/workflows/testsingularity.yaml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index ff42384a10..a81c7f47ee 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -14,11 +14,12 @@ jobs: - name: Set env run: | echo ::set-env name=RELEASE_VERSION::v3.5.0 - mkdir singularity && cd singularity - - uses: actions/checkout@v2 + - name: Setup Singularity + uses: actions/checkout@v2 with: repository: hpcng/singularity ref: 'v3.5.0' + path: 'singularity' - name: Setup GO uses: actions/setup-go@v2 with: @@ -30,11 +31,12 @@ jobs: libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y - name: Build run: | + cd singularity ./mconfig --without-suid -p ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64 make -C ./builddir sudo make -C ./builddir install cd .. - - name: Echo version + - name: Echo singularity version run: | echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version @@ -44,12 +46,15 @@ jobs: run: | conda info conda list - #- name: Install miniconda - #run: | - # wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - # bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda - # eval "$($HOME/miniconda/bin/conda shell.bash hook)" - - uses: actions/checkout@v2 + which python + echo you are here + pwd + eval "$(conda shell.bash hook)" + conda activate test + - name: Setup Pydra + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} - name: Set up dev env run: | python setup.py develop From 7610abd8a441b2efb4288420a13287ea8d8306cf Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 18 Aug 2020 01:12:56 +0800 Subject: [PATCH 067/271] fix miniconda --- .github/workflows/testsingularity.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index a81c7f47ee..b3eb15ca05 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -40,7 +40,12 @@ jobs: run: | echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version - - uses: goanpeca/setup-miniconda@v1 + #- uses: goanpeca/setup-miniconda@v1 + - name: Set up miniconda + run: | + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda + eval "$($HOME/miniconda/bin/conda shell.bash hook)" - name: Conda info shell: bash -l {0} run: | @@ -49,8 +54,9 @@ jobs: which python echo you are here pwd - eval "$(conda shell.bash hook)" - conda activate test + echo PATH=$PATH + #eval "$(conda shell.bash hook)" + #conda activate test - name: Setup Pydra uses: actions/checkout@v2 with: From 6d54f455c435761ad3ae2c3f38ee045e40ffe37e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 17 Aug 2020 20:31:42 -0400 Subject: [PATCH 068/271] cleaning output_spec part in FunctionTask.__init__, adding tests --- pydra/engine/task.py | 38 +++++------ pydra/mark/tests/test_functions.py | 104 ++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 21 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 60eaa54c9d..9a7a12076c 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -148,6 +148,8 @@ def __init__( ) else: return_info = func.__annotations__["return"] + # e.g. python annotation: fun() -> ty.NamedTuple("Output", [("out", float)]) + # or pydra decorator: @pydra.mark.annotate({"return": ty.NamedTuple(...)}) if hasattr(return_info, "__name__") and hasattr( return_info, "__annotations__" ): @@ -156,35 +158,31 @@ def __init__( fields=list(return_info.__annotations__.items()), bases=(BaseSpec,), ) - # Objects like int, float, list, tuple, and dict do not have __name__ attribute. - elif hasattr(return_info, "__annotations__"): + # e.g. python annotation: fun() -> {"out": int} + # or pydra decorator: @pydra.mark.annotate({"return": {"out": int}}) + elif isinstance(return_info, dict): output_spec = SpecInfo( name="Output", - fields=list(return_info.__annotations__.items()), + fields=list(return_info.items()), bases=(BaseSpec,), ) - elif isinstance(return_info, dict): + # e.g. python annotation: fun() -> (int, int) + # or pydra decorator: @pydra.mark.annotate({"return": (int, int)}) + elif isinstance(return_info, tuple): output_spec = SpecInfo( name="Output", - fields=list(return_info.items()), + fields=[ + ("out{}".format(n + 1), t) + for n, t in enumerate(return_info) + ], bases=(BaseSpec,), ) + # e.g. python annotation: fun() -> int + # or pydra decorator: @pydra.mark.annotate({"return": int}) else: - if not isinstance(return_info, tuple): - output_spec = SpecInfo( - name="Output", - fields=[("out", return_info)], - bases=(BaseSpec,), - ) - else: - output_spec = SpecInfo( - name="Output", - fields=[ - ("out{}".format(n + 1), t) - for n, t in enumerate(return_info) - ], - bases=(BaseSpec,), - ) + output_spec = SpecInfo( + name="Output", fields=[("out", return_info)], bases=(BaseSpec,) + ) elif "return" in func.__annotations__: raise NotImplementedError("Branch not implemented") self.output_spec = output_spec diff --git a/pydra/mark/tests/test_functions.py b/pydra/mark/tests/test_functions.py index 7a80f4595c..8826c92ec9 100644 --- a/pydra/mark/tests/test_functions.py +++ b/pydra/mark/tests/test_functions.py @@ -1,5 +1,6 @@ import pytest import random +import typing as ty from ..functions import task, annotate from ...engine.task import FunctionTask @@ -29,7 +30,9 @@ def addtwo(a): assert c_res.output.hash == d2_res.output.hash -def test_annotation_equivalence(): +def test_annotation_equivalence_1(): + """ testing various ways of annotation: one output, only types provided""" + def direct(a: int) -> int: return a + 2 @@ -41,6 +44,63 @@ def partial(a: int): def indirect(a): return a + 2 + # checking if the annotations are equivalent + assert direct.__annotations__ == partial.__annotations__ + assert direct.__annotations__ == indirect.__annotations__ + + # Run functions to ensure behavior is unaffected + a = random.randint(0, (1 << 32) - 3) + assert direct(a) == partial(a) + assert direct(a) == indirect(a) + + # checking if the annotation is properly converted to output_spec if used in task + task_direct = task(direct)() + assert task_direct.output_spec.fields[0] == ("out", int) + + +def test_annotation_equivalence_2(): + """ testing various ways of annotation: multiple outputs, using a tuple for output annot.""" + + def direct(a: int) -> (int, float): + return a + 2, a + 2.0 + + @annotate({"return": (int, float)}) + def partial(a: int): + return a + 2, a + 2.0 + + @annotate({"a": int, "return": (int, float)}) + def indirect(a): + return a + 2, a + 2.0 + + # checking if the annotations are equivalent + assert direct.__annotations__ == partial.__annotations__ + assert direct.__annotations__ == indirect.__annotations__ + + # Run functions to ensure behavior is unaffected + a = random.randint(0, (1 << 32) - 3) + assert direct(a) == partial(a) + assert direct(a) == indirect(a) + + # checking if the annotation is properly converted to output_spec if used in task + task_direct = task(direct)() + assert task_direct.output_spec.fields == [("out1", int), ("out2", float)] + + +def test_annotation_equivalence_3(): + """ testing various ways of annotation: using dictionary for output annot.""" + + def direct(a: int) -> {"out1": int}: + return a + 2 + + @annotate({"return": {"out1": int}}) + def partial(a: int): + return a + 2 + + @annotate({"a": int, "return": {"out1": int}}) + def indirect(a): + return a + 2 + + # checking if the annotations are equivalent assert direct.__annotations__ == partial.__annotations__ assert direct.__annotations__ == indirect.__annotations__ @@ -49,6 +109,48 @@ def indirect(a): assert direct(a) == partial(a) assert direct(a) == indirect(a) + # checking if the annotation is properly converted to output_spec if used in task + task_direct = task(direct)() + assert task_direct.output_spec.fields[0] == ("out1", int) + + +def test_annotation_equivalence_4(): + """ testing various ways of annotation: using ty.NamedTuple for the output""" + + def direct(a: int) -> ty.NamedTuple("Output", [("sum", int), ("sub", int)]): + return a + 2, a - 2 + + @annotate({"return": ty.NamedTuple("Output", [("sum", int), ("sub", int)])}) + def partial(a: int): + return a + 2, a - 2 + + @annotate( + {"a": int, "return": ty.NamedTuple("Output", [("sum", int), ("sub", int)])} + ) + def indirect(a): + return a + 2, a - 2 + + # checking if the annotations are equivalent + assert ( + direct.__annotations__["return"].__annotations__ + == partial.__annotations__["return"].__annotations__ + == indirect.__annotations__["return"].__annotations__ + ) + assert ( + direct.__annotations__["return"].__name__ + == partial.__annotations__["return"].__name__ + == indirect.__annotations__["return"].__name__ + ) + + # Run functions to ensure behavior is unaffected + a = random.randint(0, (1 << 32) - 3) + assert direct(a) == partial(a) + assert direct(a) == indirect(a) + + # checking if the annotation is properly converted to output_spec if used in task + task_direct = task(direct)() + assert task_direct.output_spec.fields == [("sum", int), ("sub", int)] + def test_annotation_override(): @annotate({"a": float, "return": float}) From e2daf6f6a60843167c1b750dfd85f9379d3a811c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 17 Aug 2020 23:07:40 -0400 Subject: [PATCH 069/271] adding tests for FunctionTask with output_spec (using BaseSpec) --- pydra/engine/task.py | 2 - pydra/engine/tests/test_task.py | 101 +++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 9a7a12076c..f4e732aa51 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -183,8 +183,6 @@ def __init__( output_spec = SpecInfo( name="Output", fields=[("out", return_info)], bases=(BaseSpec,) ) - elif "return" in func.__annotations__: - raise NotImplementedError("Branch not implemented") self.output_spec = output_spec def _run_task(self): diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index a0e20f875e..d0835d06e9 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -7,7 +7,7 @@ from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator -from ..specs import MultiInputObj, SpecInfo, FunctionSpec +from ..specs import MultiInputObj, SpecInfo, FunctionSpec, BaseSpec no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -639,6 +639,26 @@ def testfunc(a: int): assert getattr(funky.inputs, "a") == 3.5 +def test_input_spec_func_2a(use_validator): + """ the function with annotation, and the task has input_spec, + input_spec changes the type of the input (so error is not raised) + using the shorter syntax + """ + + @mark.task + def testfunc(a: int): + return a + + my_input_spec = SpecInfo( + name="Input", + fields=[("a", float, {"help_string": "input a"})], + bases=(FunctionSpec,), + ) + + funky = testfunc(a=3.5, input_spec=my_input_spec) + assert getattr(funky.inputs, "a") == 3.5 + + def test_input_spec_func_3(use_validator): """ the function w/o annotated, but input_spec is used additional keys (allowed_values) are used in metadata @@ -777,6 +797,85 @@ def testfunc(a): assert res.output.out == 1 +def test_output_spec_func_1(use_validator): + """ the function w/o annotated, but output_spec is used """ + + @mark.task + def testfunc(a): + return a + + my_output_spec = SpecInfo( + name="Output", + fields=[("out1", attr.ib(type=float, metadata={"help_string": "output"}))], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, output_spec=my_output_spec) + res = funky() + assert res.output.out1 == 3.5 + + +def test_output_spec_func_1a_except(use_validator): + """ the function w/o annotated, but output_spec is used + float returned instead of int - TypeError + """ + + @mark.task + def testfunc(a): + return a + + my_output_spec = SpecInfo( + name="Output", + fields=[("out1", attr.ib(type=int, metadata={"help_string": "output"}))], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, output_spec=my_output_spec) + with pytest.raises(TypeError): + res = funky() + + +def test_output_spec_func_2(use_validator): + """ the function w/o annotated, but output_spec is used + output_spec changes the type of the output (so error is not raised) + """ + + @mark.task + def testfunc(a) -> int: + return a + + my_output_spec = SpecInfo( + name="Output", + fields=[("out1", attr.ib(type=float, metadata={"help_string": "output"}))], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, output_spec=my_output_spec) + res = funky() + assert res.output.out1 == 3.5 + + +def test_output_spec_func_2a(use_validator): + """ the function w/o annotated, but output_spec is used + output_spec changes the type of the output (so error is not raised) + using a shorter syntax + """ + + @mark.task + def testfunc(a) -> int: + return a + + my_output_spec = SpecInfo( + name="Output", + fields=[("out1", float, {"help_string": "output"})], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, output_spec=my_output_spec) + res = funky() + assert res.output.out1 == 3.5 + + def test_exception_func(): @mark.task def raise_exception(c, d): From 31cc4004b10cc3412bd1fd264e866a53800e72d2 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 17 Aug 2020 23:59:33 -0400 Subject: [PATCH 070/271] adding MultiOutputObj; not validation for MultiOutputObj and MultiInputObj --- pydra/engine/helpers.py | 13 +++++++-- pydra/engine/specs.py | 13 ++++++++- pydra/engine/tests/test_task.py | 52 ++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index e7ca69f90c..6791ffe9f8 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -18,7 +18,16 @@ import warnings -from .specs import Runtime, File, Directory, attr_fields, Result, LazyField +from .specs import ( + Runtime, + File, + Directory, + attr_fields, + Result, + LazyField, + MultiOutputObj, + MultiInputObj, +) from .helpers_file import hash_file, hash_dir, copyfile, is_existing_file @@ -293,7 +302,7 @@ def custom_validator(instance, attribute, value): or value is None or attribute.name.startswith("_") # e.g. _func or isinstance(value, LazyField) - or tp_attr in [ty.Any, inspect._empty] + or tp_attr in [ty.Any, inspect._empty, MultiOutputObj, MultiInputObj] ): check_type = False # no checking of the type elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 9328d24cb2..f52beccd61 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -20,7 +20,7 @@ class Directory: class MultiInputObj: - """An :obj: ty.List[`os.pathlike`] object""" + """A ty.List[ty.Any] object, converter changes a single values to a list""" @classmethod def converter(cls, value): @@ -29,6 +29,17 @@ def converter(cls, value): return ensure_list(value) +class MultiOutputObj: + """A ty.List[ty.Any] object, converter changes an 1-el list to the single value""" + + @classmethod + def converter(cls, value): + if isinstance(value, list) and len(value) == 1: + return value[0] + else: + return value + + @attr.s(auto_attribs=True, kw_only=True) class SpecInfo: """Base data structure for metadata of specifications.""" diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index d0835d06e9..5d209a831d 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -7,7 +7,7 @@ from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator -from ..specs import MultiInputObj, SpecInfo, FunctionSpec, BaseSpec +from ..specs import MultiInputObj, MultiOutputObj, SpecInfo, FunctionSpec, BaseSpec no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -876,6 +876,56 @@ def testfunc(a) -> int: assert res.output.out1 == 3.5 +def test_output_spec_func_3(use_validator): + """ the function w/o annotated, but output_spec is used + MultiOutputObj is used, output is a 2-el list, so converter doesn't do anything + """ + + @mark.task + def testfunc(a, b): + return [a, b] + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out_list", + attr.ib(type=MultiOutputObj, metadata={"help_string": "output"}), + ) + ], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, b=1, output_spec=my_output_spec) + res = funky() + assert res.output.out_list == [3.5, 1] + + +def test_output_spec_func_4(use_validator): + """ the function w/o annotated, but output_spec is used + MultiOutputObj is used, output is a 1el list, so converter return the element + """ + + @mark.task + def testfunc(a): + return [a] + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out_1el", + attr.ib(type=MultiOutputObj, metadata={"help_string": "output"}), + ) + ], + bases=(BaseSpec,), + ) + + funky = testfunc(a=3.5, output_spec=my_output_spec) + res = funky() + assert res.output.out_1el == 3.5 + + def test_exception_func(): @mark.task def raise_exception(c, d): From fb105bdd993ed87e55a83edf1505aa7631ac7d05 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Aug 2020 00:25:06 -0400 Subject: [PATCH 071/271] checking splits for 2d numpy arrays --- pydra/engine/tests/test_node_task.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index ccb16ba4fc..6a5d8e9772 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -904,9 +904,13 @@ def test_task_state_3(plugin): assert nn.output_dir == [] -def test_task_state_4(plugin): +@pytest.mark.parametrize("input_type", ["list", "array"]) +def test_task_state_4(plugin, input_type): """ task with a list as an input, and a simple splitter """ - nn = moment(name="NA", n=3, lst=[[2, 3, 4], [1, 2, 3]]).split(splitter="lst") + lst_in = [[2, 3, 4], [1, 2, 3]] + if input_type == "array": + lst_in = np.array(lst_in) + nn = moment(name="NA", n=3, lst=lst_in).split(splitter="lst") assert np.allclose(nn.inputs.n, 3) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == "NA.lst" @@ -914,6 +918,14 @@ def test_task_state_4(plugin): with Submitter(plugin=plugin) as sub: sub(nn) + # checking that split is done across dim 0 + el_0 = nn.state.states_val[0]["NA.lst"] + if input_type == "list": + assert el_0 == [2, 3, 4] + elif input_type == "array": + assert isinstance(el_0, np.ndarray) + assert (el_0 == [2, 3, 4]).all() + # checking the results results = nn.result() for i, expected in enumerate([33, 12]): From bf243dde73499aa7ca62a5f2c3257bf782d5e89a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Aug 2020 12:43:22 -0400 Subject: [PATCH 072/271] asking for pytest-xdist< 2.0, i believe this might cause the travis error (travis has to update pytest-cov) --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f9948810f8..8e4eacb3a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ test_requires = pytest >= 4.4.0 pytest-cov pytest-env - pytest-xdist + pytest-xdist < 2.0 pytest-rerunfailures codecov numpy @@ -64,7 +64,7 @@ test = pytest >= 4.4.0 pytest-cov pytest-env - pytest-xdist + pytest-xdist < 2.0 pytest-rerunfailures codecov numpy From 090438e319601c0584daa965b552d96728ef723d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 19 Aug 2020 16:19:03 -0400 Subject: [PATCH 073/271] reverting changes to input_spec in TaskBase.__init__: bring back to steps - init and evolve --- pydra/engine/core.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 2a280f9793..3a091d4071 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -135,26 +135,30 @@ def __init__( if not self.input_spec: raise Exception("No input_spec in class: %s" % self.__class__.__name__) klass = make_klass(self.input_spec) - inp_dict = {} - for f in attr.fields(klass): - if f.name.startswith("_"): + + self.inputs = klass( + **{ # in attrs names that starts with "_" could be set when name provided w/o "_" - name = f.name[1:] - else: - name = f.name - if name in inputs: - inp_dict[name] = inputs[name] - else: - # if the field not in inputs, the default value should be used - inp_dict[name] = f.default - self.inputs = klass(**inp_dict) - # checking if metadata is set properly - self.inputs.check_metadata() + (f.name[1:] if f.name.startswith("_") else f.name): f.default + for f in attr.fields(klass) + } + ) + self.input_names = [ field.name for field in attr.fields(klass) if field.name not in ["_func", "_graph_checksums"] ] + + # selecting items that are in input_names (ignoring fields that are not in input_spec) + if isinstance(inputs, dict): + inputs = {k: v for k, v in inputs.items() if k in self.input_names} + + self.inputs = attr.evolve(self.inputs, **inputs) + + # checking if metadata is set properly + self.inputs.check_metadata() + self.state_inputs = inputs # dictionary to save the connections with lazy fields self.inp_lf = {} self.state = None @@ -164,7 +168,6 @@ def __init__( self._done = False if self._input_sets is None: self._input_sets = {} - self.state_inputs = {k: v for k, v in inputs.items() if k in self.input_names} self.audit = Audit( audit_flags=audit_flags, From 22f14d8a4b74793a5a6046ef32a9fb57f1af6f70 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 19 Aug 2020 17:14:05 -0400 Subject: [PATCH 074/271] reverting some changes for inputs in TaskBase's init --- pydra/engine/core.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 3a091d4071..26c67ad1a1 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -130,8 +130,6 @@ def __init__( if name in dir(self): raise ValueError("Cannot use names of attributes or methods as task name") self.name = name - if not inputs: - inputs = {} if not self.input_spec: raise Exception("No input_spec in class: %s" % self.__class__.__name__) klass = make_klass(self.input_spec) @@ -150,9 +148,18 @@ def __init__( if field.name not in ["_func", "_graph_checksums"] ] - # selecting items that are in input_names (ignoring fields that are not in input_spec) - if isinstance(inputs, dict): - inputs = {k: v for k, v in inputs.items() if k in self.input_names} + if inputs: + if isinstance(inputs, dict): + # selecting items that are in input_names (ignoring fields that are not in input_spec) + inputs = {k: v for k, v in inputs.items() if k in self.input_names} + # TODO: this needs to finished and tested after #305 + elif Path(inputs).is_file(): + inputs = json.loads(Path(inputs).read_text()) + # TODO: this needs to finished and tested after #305 + elif isinstance(inputs, str): + if self._input_sets is None or inputs not in self._input_sets: + raise ValueError(f"Unknown input set {inputs!r}") + inputs = self._input_sets[inputs] self.inputs = attr.evolve(self.inputs, **inputs) From 4bd238e69ad0665e31f896f962260e0cc1172680 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 17:17:19 +0800 Subject: [PATCH 075/271] test different ways of installing pydra on CI --- .github/workflows/testpydra.yml | 71 ++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 6a0c438de6..3b9ba56e4b 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -7,27 +7,80 @@ on: [push, pull_request] jobs: build: - runs-on: ${{ matrix.os }} strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] + install: [pip, install, develop, sdist, wheel] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update build tools - run: | - python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - - name: Install dependencies + + #- name: Update build tools + # run: | + # python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel + #- name: Install dependencies + # run: | + # python -m pip install -r min-requirements.txt + #- name: Install pydra + # run: | + # python -m pip install .[test] + +# https://www.edwardthomson.com/blog/github_actions_14_conditionals_with_a_matrix.html + - name: Install dependencies (pip) + if: matrix.install == 'pip' + run: pip install $PIP_ARGS . + + - name: Install dependencies (setup.py install) + if: matrix.install == 'install' + run: python setup.py install + + - name: Install dependencies (setup.py develop) + if: matrix.install == 'develop' + run: python setup.py develop + + - name: Install dependencies (sdist) + if: matrix.install == 'sdist' run: | - python -m pip install -r min-requirements.txt - - name: Install pydra + python setup.py sdist + pip install dist/*.whl + + - name: Install dependencies (wheel) + if: matrix.install == 'wheel' run: | - python -m pip install .[test] + python setup.py bdist_wheel + pip install dist/*.whl + + + - name: Verify Python import + run: python -c 'import pydra; print(pydra.__version__)' + + + - name: Install Pydra (pip or setup.py install) + if: matrix.install == 'pip' || matrix.install == 'install' + run: pip install ".[test]" + + - name: Install Pydra (setup.py develop) + if: matrix.install == 'develop' + run: pip install -e ".[test]" + + - name: Install Pydra (sdist) + if: matrix.install == 'sdist' + run: pip install "$( ls dist/pydra*.tar.gz )[test]" + + - name: Install Pydra (wheel) + if: matrix.install == 'wheel' + run: pip install "$( ls dist/pydra*.whl )[test]" + + + - name: Pytest env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now From fb2b3b2c876e8b8a939214e2d9e94c41fcf7f171 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 18:01:58 +0800 Subject: [PATCH 076/271] update build tools and fix sdist build on CI --- .github/workflows/testpydra.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 3b9ba56e4b..8913562271 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -4,6 +4,9 @@ name: Test Pydra on: [push, pull_request] +env: + install: pip + jobs: build: @@ -11,7 +14,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [pip, install, develop, sdist, wheel] + install: [min-req, install, develop, sdist, wheel] fail-fast: false runs-on: ${{ matrix.os }} @@ -22,21 +25,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Update build tools + run: python -m pip install --upgrade pip setuptools - #- name: Update build tools - # run: | - # python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - #- name: Install dependencies - # run: | - # python -m pip install -r min-requirements.txt - #- name: Install pydra - # run: | - # python -m pip install .[test] # https://www.edwardthomson.com/blog/github_actions_14_conditionals_with_a_matrix.html - - name: Install dependencies (pip) - if: matrix.install == 'pip' - run: pip install $PIP_ARGS . + - name: Install dependencies (min-requirements.txt) + if: matrix.install == 'min-req' + run: pip install -r min-requirements.txt - name: Install dependencies (setup.py install) if: matrix.install == 'install' @@ -50,7 +46,7 @@ jobs: if: matrix.install == 'sdist' run: | python setup.py sdist - pip install dist/*.whl + pip install dist/*.tar.gz - name: Install dependencies (wheel) if: matrix.install == 'wheel' @@ -63,8 +59,8 @@ jobs: run: python -c 'import pydra; print(pydra.__version__)' - - name: Install Pydra (pip or setup.py install) - if: matrix.install == 'pip' || matrix.install == 'install' + - name: Install Pydra (min-requirements.txt or setup.py install) + if: matrix.install == 'min-req' || matrix.install == 'install' run: pip install ".[test]" - name: Install Pydra (setup.py develop) From e1649f4e286be204d5711e0fc1e24075d3c09c7b Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 18:14:08 +0800 Subject: [PATCH 077/271] update cloudpickle for python 3.8 to work --- min-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/min-requirements.txt b/min-requirements.txt index d3dfcc4bb1..09b237e48c 100644 --- a/min-requirements.txt +++ b/min-requirements.txt @@ -1,5 +1,5 @@ # Auto-generated by tools/update_min_requirements.py attrs -cloudpickle == 0.8.0 +cloudpickle >= 1.2.2 filelock == 3.0.0 etelemetry == 0.2.0 From 92c5926333b7ed65ee5ce78ea594a89aaa19261a Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 20:29:09 +0800 Subject: [PATCH 078/271] install dependencies in windows with bash --- .github/workflows/testpydra.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 8913562271..0914e92359 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -4,9 +4,6 @@ name: Test Pydra on: [push, pull_request] -env: - install: pip - jobs: build: @@ -29,7 +26,6 @@ jobs: run: python -m pip install --upgrade pip setuptools -# https://www.edwardthomson.com/blog/github_actions_14_conditionals_with_a_matrix.html - name: Install dependencies (min-requirements.txt) if: matrix.install == 'min-req' run: pip install -r min-requirements.txt @@ -47,16 +43,14 @@ jobs: run: | python setup.py sdist pip install dist/*.tar.gz + shell: bash - name: Install dependencies (wheel) if: matrix.install == 'wheel' run: | python setup.py bdist_wheel pip install dist/*.whl - - - - name: Verify Python import - run: python -c 'import pydra; print(pydra.__version__)' + shell: bash - name: Install Pydra (min-requirements.txt or setup.py install) @@ -76,7 +70,6 @@ jobs: run: pip install "$( ls dist/pydra*.whl )[test]" - - name: Pytest env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now From 5fce9e6c47b4997c4a6223b14e5caed08e2d856a Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 20:39:12 +0800 Subject: [PATCH 079/271] add test to check style --- .github/workflows/testpydra.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 0914e92359..db3ebf9e5b 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -11,7 +11,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [min-req, install, develop, sdist, wheel] + install: [min-req, install, develop, sdist, wheel, style] fail-fast: false runs-on: ${{ matrix.os }} @@ -70,11 +70,20 @@ jobs: run: pip install "$( ls dist/pydra*.whl )[test]" + - name: Check Style + if: matrix.install == 'style' + run: | + pip install black==19.3b0 + black --check pydra tools setup.py + + - name: Pytest + if: matrix.install != 'style' env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now run: | pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + - name: Upload to codecov run: | codecov --root /pydra -f /pydra/cov.xml -F unittests From dd438bfb92af6b8c575c8966f95bd97450cb2c12 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 20:47:28 +0800 Subject: [PATCH 080/271] reformat test_workflow to pass style test --- pydra/engine/tests/test_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 50b03e066a..69a25b282a 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -2228,7 +2228,6 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): assert t1 > 2 assert t2 < max(1, t1 - 1) - # checking if the second wf didn't run again assert wf1.output_dir.exists() assert wf2.output_dir.exists() @@ -3110,7 +3109,7 @@ def test_wf_ndstate_cachelocations(plugin, tmpdir): # checking the execution time assert t1 > 2 assert t2 < max(1, t1 - 1) - + # checking all directories assert wf1.output_dir.exists() @@ -3402,6 +3401,7 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): assert t1 > 2 assert t2 < max(1, t1 - 1) + @pytest.fixture def create_tasks(): wf = Workflow(name="wf", input_spec=["x"]) From f2c921a5b5f4b8447deac41b124be73410f86373 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 23:04:12 +0800 Subject: [PATCH 081/271] run test_dockertask on all OSs --- .github/workflows/testpydra.yml | 12 ++++---- pydra/engine/tests/test_dockertask.py | 40 ++------------------------- 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index db3ebf9e5b..4bf612077d 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -1,4 +1,5 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# Reference +# https://hynek.me/articles/python-github-actions/ name: Test Pydra @@ -79,11 +80,8 @@ jobs: - name: Pytest if: matrix.install != 'style' - env: - IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: | - pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + - name: Upload to codecov - run: | - codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 3472e4d061..3a1721ea41 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -6,10 +6,9 @@ from ..submitter import Submitter from ..core import Workflow from ..specs import ShellOutSpec, SpecInfo, File, DockerSpec -from .utils import no_win, need_docker +from .utils import need_docker -@no_win @need_docker def test_docker_1_nosubm(): """ simple command in a container, a default bindings and working directory is added @@ -31,7 +30,6 @@ def test_docker_1_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1(plugin): """ simple command in a container, a default bindings and working directory is added @@ -50,7 +48,6 @@ def test_docker_1(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1_dockerflag(plugin): """ simple command in a container, a default bindings and working directory is added @@ -71,7 +68,6 @@ def test_docker_1_dockerflag(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_1_dockerflag_exception(plugin): """using ShellComandTask with container_info=("docker"), no image provided""" @@ -83,7 +79,6 @@ def test_docker_1_dockerflag_exception(plugin): assert "container_info has to have 2 or 3 elements" in str(excinfo.value) -@no_win @need_docker def test_docker_2_nosubm(): """ a command with arguments, cmd and args given as executable @@ -103,7 +98,6 @@ def test_docker_2_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2(plugin): """ a command with arguments, cmd and args given as executable @@ -125,7 +119,6 @@ def test_docker_2(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2_dockerflag(plugin): """ a command with arguments, cmd and args given as executable @@ -149,7 +142,6 @@ def test_docker_2_dockerflag(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2a_nosubm(): """ a command with arguments, using executable and args @@ -174,7 +166,6 @@ def test_docker_2a_nosubm(): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_2a(plugin): """ a command with arguments, using executable and args @@ -201,7 +192,6 @@ def test_docker_2a(plugin): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3(plugin, tmpdir): """ a simple command in container with bindings, @@ -224,7 +214,6 @@ def test_docker_3(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3_dockerflag(plugin, tmpdir): """ a simple command in container with bindings, @@ -250,7 +239,6 @@ def test_docker_3_dockerflag(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_3_dockerflagbind(plugin, tmpdir): """ a simple command in container with bindings, @@ -276,7 +264,6 @@ def test_docker_3_dockerflagbind(plugin, tmpdir): assert "Unable to find image" in res.output.stderr -@no_win @need_docker def test_docker_4(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -302,7 +289,6 @@ def test_docker_4(plugin, tmpdir): assert res.output.return_code == 0 -@no_win @need_docker def test_docker_4_dockerflag(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -331,7 +317,6 @@ def test_docker_4_dockerflag(plugin, tmpdir): # tests with State -@no_win @need_docker def test_docker_st_1(plugin): """ commands without arguments in container @@ -355,7 +340,6 @@ def test_docker_st_1(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 -@no_win @need_docker def test_docker_st_2(plugin): """ command with arguments in docker, checking the distribution @@ -379,7 +363,6 @@ def test_docker_st_2(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 -@no_win @need_docker def test_docker_st_3(plugin): """ outer splitter image and executable @@ -397,7 +380,6 @@ def test_docker_st_3(plugin): assert "Ubuntu" in res[3].output.stdout -@no_win @need_docker def test_docker_st_4(plugin): """ outer splitter image and executable, combining with images @@ -439,7 +421,6 @@ def test_docker_st_4(plugin): # tests with workflows -@no_win @need_docker def test_wf_docker_1(plugin, tmpdir): """ a workflow with two connected task @@ -483,7 +464,6 @@ def test_wf_docker_1(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" -@no_win @need_docker def test_wf_docker_1_dockerflag(plugin, tmpdir): """ a workflow with two connected task @@ -523,7 +503,6 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" -@no_win @need_docker def test_wf_docker_2pre(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -544,7 +523,6 @@ def test_wf_docker_2pre(plugin, tmpdir): assert res.output.stdout == "/outputs/tmp.txt" -@no_win @need_docker def test_wf_docker_2(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -584,7 +562,6 @@ def test_wf_docker_2(plugin, tmpdir): assert res.output.out == "Hello!" -@no_win @need_docker def test_wf_docker_3(plugin, tmpdir): """ a workflow with two connected task @@ -627,7 +604,6 @@ def test_wf_docker_3(plugin, tmpdir): # tests with customized output_spec -@no_win @need_docker def test_docker_outputspec_1(plugin, tmpdir): """ @@ -655,7 +631,6 @@ def test_docker_outputspec_1(plugin, tmpdir): # tests with customised input_spec -@no_win @need_docker def test_docker_inputspec_1(plugin, tmpdir): """ a simple customized input spec for docker task """ @@ -697,7 +672,6 @@ def test_docker_inputspec_1(plugin, tmpdir): assert res.output.stdout == "hello from pydra" -@no_win @need_docker def test_docker_inputspec_1a(plugin, tmpdir): """ a simple customized input spec for docker task @@ -736,7 +710,6 @@ def test_docker_inputspec_1a(plugin, tmpdir): assert res.output.stdout == "hello from pydra" -@no_win @need_docker def test_docker_inputspec_2(plugin, tmpdir): """ a customized input spec with two fields for docker task """ @@ -793,7 +766,6 @@ def test_docker_inputspec_2(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker def test_docker_inputspec_2a_except(plugin, tmpdir): """ a customized input spec with two fields @@ -853,7 +825,6 @@ def test_docker_inputspec_2a_except(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker def test_docker_inputspec_2a(plugin, tmpdir): """ a customized input spec with two fields @@ -913,8 +884,8 @@ def test_docker_inputspec_2a(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" -@no_win @need_docker +@pytest.mark.xfail(reason="'docker' not in /proc/1/cgroup on ubuntu; TODO") def test_docker_inputspec_3(plugin, tmpdir): """ input file is in the container, so metadata["container_path"]: True, the input will be treated as a str """ @@ -957,7 +928,6 @@ def test_docker_inputspec_3(plugin, tmpdir): assert cmdline == docky.cmdline -@no_win @need_docker def test_docker_inputspec_3a(plugin, tmpdir): """ input file does not exist in the local file system, @@ -1001,7 +971,6 @@ def test_docker_inputspec_3a(plugin, tmpdir): assert "use field.metadata['container_path']=True" in str(excinfo.value) -@no_win @need_docker def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): """ shelltask changes a file in place, @@ -1064,7 +1033,6 @@ def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): assert "hello from pydra\n" == f.read() -@no_win @need_docker def test_docker_inputspec_state_1(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1113,7 +1081,6 @@ def test_docker_inputspec_state_1(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" -@no_win @need_docker def test_docker_inputspec_state_1b(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1163,7 +1130,6 @@ def test_docker_inputspec_state_1b(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" -@no_win @need_docker def test_docker_wf_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks """ @@ -1215,7 +1181,6 @@ def test_docker_wf_inputspec_1(plugin, tmpdir): assert res.output.out == "hello from pydra" -@no_win @need_docker def test_docker_wf_state_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks that has a state""" @@ -1273,7 +1238,6 @@ def test_docker_wf_state_inputspec_1(plugin, tmpdir): assert res[1].output.out == "have a nice one" -@no_win @need_docker def test_docker_wf_ndst_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks with states""" From 513d4914e366ee8b1b2c299ecb1439f67300a2c6 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 23:14:06 +0800 Subject: [PATCH 082/271] add codecov install in style check test --- .github/workflows/testpydra.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 4bf612077d..69481b387a 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -74,7 +74,7 @@ jobs: - name: Check Style if: matrix.install == 'style' run: | - pip install black==19.3b0 + pip install black==19.3b0 codecov black --check pydra tools setup.py @@ -85,3 +85,4 @@ jobs: - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests + shell: bash From cda58dc9d3be7baf4105f46a3713789ad691a4f4 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 20 Aug 2020 23:56:17 +0800 Subject: [PATCH 083/271] mark flaky test in test_workflow --- pydra/engine/tests/test_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 69a25b282a..ba7b2a1d29 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -2173,6 +2173,7 @@ def test_wf_nostate_cachelocations_a(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_b(plugin, tmpdir): """ the same as previous test, but the 2nd workflows has two outputs From 217de2d7ba99942216dcb659b346ccb7f91e0ddc Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 00:16:36 +0800 Subject: [PATCH 084/271] add dask and old pip version installation builds --- .github/workflows/testallowfail.yaml | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/testallowfail.yaml diff --git a/.github/workflows/testallowfail.yaml b/.github/workflows/testallowfail.yaml new file mode 100644 index 0000000000..cb790cda85 --- /dev/null +++ b/.github/workflows/testallowfail.yaml @@ -0,0 +1,62 @@ + +name: Test Pydra (allow failures) + +on: [push, pull_request] + +jobs: + build: + + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: [3.7, 3.8] + install: [dask, old-pip] + fail-fast: false + runs-on: ${{ matrix.os }} + + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update build tools + if: matrix.install != 'old-pip' + run: python -m pip install --upgrade pip setuptools + + - name: Update build tools (pip 10.0.1) + if: matrix.install == 'old-pip' + run: python -m pip install pip==10.0.1 setuptools==30.3.0 + + + - name: Install dependencies (setup.py develop) + if: matrix.install == 'dask' + run: python setup.py develop + + - name: Install dependencies (min-requirements.txt) + if: matrix.install == 'old-pip' + run: pip install -r min-requirements.txt + + + - name: Install Pydra (dask) + if: matrix.install == 'dask' + run: pip install -e ".[dask]" + + - name: Install Pydra (test) + if: matrix.install != 'dask' + run: pip install -e ".[test]" + + + - name: Pytest + if: matrix.install == 'old-pip' + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + + - name: Pytest (dask) + if: matrix.install == 'dask' + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules --dask pydra/engine + + + - name: Upload to codecov + run: codecov --root /pydra -f /pydra/cov.xml -F unittests From d5b2ff0de432afcf60059bf0d8a19729c64b6cda Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 15:57:37 +0800 Subject: [PATCH 085/271] remove dask from CI (to be added later) --- .github/workflows/testallowfail.yaml | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/testallowfail.yaml b/.github/workflows/testallowfail.yaml index cb790cda85..e485e55d3c 100644 --- a/.github/workflows/testallowfail.yaml +++ b/.github/workflows/testallowfail.yaml @@ -10,7 +10,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [dask, old-pip] + install: [old-pip] fail-fast: false runs-on: ${{ matrix.os }} @@ -22,41 +22,18 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Update build tools - if: matrix.install != 'old-pip' - run: python -m pip install --upgrade pip setuptools - - name: Update build tools (pip 10.0.1) if: matrix.install == 'old-pip' run: python -m pip install pip==10.0.1 setuptools==30.3.0 - - - - name: Install dependencies (setup.py develop) - if: matrix.install == 'dask' - run: python setup.py develop - - name: Install dependencies (min-requirements.txt) if: matrix.install == 'old-pip' run: pip install -r min-requirements.txt - - - - name: Install Pydra (dask) - if: matrix.install == 'dask' - run: pip install -e ".[dask]" - - name: Install Pydra (test) - if: matrix.install != 'dask' + if: matrix.install == 'old-pip' run: pip install -e ".[test]" - - name: Pytest if: matrix.install == 'old-pip' run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - - - name: Pytest (dask) - if: matrix.install == 'dask' - run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules --dask pydra/engine - - - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests From fcdac6f2d25eafe443028c00e3f2181e37edc31c Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 16:00:01 +0800 Subject: [PATCH 086/271] remove test_dockertask from main github actions build before windows fix --- .github/workflows/testpydra.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 69481b387a..fe39972055 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -80,9 +80,11 @@ jobs: - name: Pytest if: matrix.install != 'style' - run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + env: + IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now + run: | + pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests - shell: bash From a26cb23453ba01a1ac581746e209a91161d5088a Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 16:01:53 +0800 Subject: [PATCH 087/271] clean up code in testdockertask.yml --- .github/workflows/testdockertask.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testdockertask.yml b/.github/workflows/testdockertask.yml index 6c71fe45eb..8fc7a06171 100644 --- a/.github/workflows/testdockertask.yml +++ b/.github/workflows/testdockertask.yml @@ -20,17 +20,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update build tools - run: | - python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel + run: python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - name: Install dependencies - run: | - python -m pip install -r min-requirements.txt + run: pip install -r min-requirements.txt - name: Install pydra - run: | - python -m pip install .[test] + run: pip install .[test] - name: Pytest - run: | - pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml + run: pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml - name: Upload to codecov - run: | - codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov --root /pydra -f /pydra/cov.xml -F unittests From 57ca8b5db2d9eb184447f39517d43460614d1b38 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 16:16:56 +0800 Subject: [PATCH 088/271] remove miniconda & clean code --- .github/workflows/testsingularity.yaml | 32 +++++--------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index b3eb15ca05..c3055259ff 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -40,39 +40,19 @@ jobs: run: | echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version - #- uses: goanpeca/setup-miniconda@v1 - - name: Set up miniconda - run: | - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda - eval "$($HOME/miniconda/bin/conda shell.bash hook)" - - name: Conda info - shell: bash -l {0} - run: | - conda info - conda list - which python - echo you are here - pwd - echo PATH=$PATH - #eval "$(conda shell.bash hook)" - #conda activate test + - name: Setup Pydra uses: actions/checkout@v2 with: repository: ${{ github.repository }} - name: Set up dev env - run: | - python setup.py develop - python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install + run: python setup.py develop - name: Install pydra - run: | - python -m pip install -e .[test] + run: python -m pip install -e .[test] + - name: Pytest env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: | - pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml + run: pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml - name: Upload to codecov - run: | - codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov --root /pydra -f /pydra/cov.xml -F unittests From e89aa5d629b6a822701a70195d5b05178cc252ed Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 16:28:01 +0800 Subject: [PATCH 089/271] test both versions of python --- .github/workflows/testsingularity.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index c3055259ff..c958f458a5 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -8,7 +8,9 @@ jobs: build: name: Build runs-on: ubuntu-latest - continue-on-error: true + strategy: + matrix: + python-version: [3.7, 3.8] steps: - name: Set env @@ -41,6 +43,10 @@ jobs: echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - name: Setup Pydra uses: actions/checkout@v2 with: From 192e34538fe9a09e3bed982a6e36e922f05d6aaf Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 17:04:57 +0800 Subject: [PATCH 090/271] fix pytest command on singularity CI --- .github/workflows/testsingularity.yaml | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index c958f458a5..13fa5de1a5 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -11,6 +11,7 @@ jobs: strategy: matrix: python-version: [3.7, 3.8] + fail-fast: False steps: - name: Set env @@ -43,22 +44,28 @@ jobs: echo ${{ github.ref }} ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version + + - name: Checkout Pydra repo + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Setup Pydra - uses: actions/checkout@v2 - with: - repository: ${{ github.repository }} - - name: Set up dev env - run: python setup.py develop - - name: Install pydra - run: python -m pip install -e .[test] + - name: Update build tools + run: python -m pip install --upgrade pip setuptools + - name: Install dependencies (setup.py develop) + run: | + python setup.py develop + python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install + - name: Install pydra (test) + run: pip install -e ".[test]" + - name: Pytest env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml + run: pytest --ignore-glob=$IGNORE_TESTS -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov -f /pydra/cov.xml -F unittests From aa99456918ae533bacdd5ea46494fe681b57c46c Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 17:15:53 +0800 Subject: [PATCH 091/271] run test_dockertask on singularity CI --- .github/workflows/testsingularity.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yaml index 13fa5de1a5..5211895ed5 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yaml @@ -45,16 +45,18 @@ jobs: ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version - - name: Checkout Pydra repo - uses: actions/checkout@v2 - with: - repository: ${{ github.repository }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Update build tools run: python -m pip install --upgrade pip setuptools + + + - name: Checkout Pydra repo + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} - name: Install dependencies (setup.py develop) run: | python setup.py develop @@ -64,8 +66,6 @@ jobs: - name: Pytest - env: - IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: pytest --ignore-glob=$IGNORE_TESTS -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov run: codecov -f /pydra/cov.xml -F unittests From 60a0daa928fdf2ed98e112c69be93cdd2f4f4524 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 17:43:31 +0800 Subject: [PATCH 092/271] use bash as default shell for all OSs --- .github/workflows/testallowfail.yaml | 4 ++++ .github/workflows/testdockertask.yml | 4 ++++ .github/workflows/testpydra.yml | 8 +++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testallowfail.yaml b/.github/workflows/testallowfail.yaml index e485e55d3c..b64feabafb 100644 --- a/.github/workflows/testallowfail.yaml +++ b/.github/workflows/testallowfail.yaml @@ -3,6 +3,10 @@ name: Test Pydra (allow failures) on: [push, pull_request] +defaults: + run: + shell: bash + jobs: build: diff --git a/.github/workflows/testdockertask.yml b/.github/workflows/testdockertask.yml index 72678912d9..502a76b598 100644 --- a/.github/workflows/testdockertask.yml +++ b/.github/workflows/testdockertask.yml @@ -5,6 +5,10 @@ name: Test Dockertask on: [push, pull_request] +defaults: + run: + shell: bash + jobs: build: diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index fe39972055..a93763d268 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -5,9 +5,12 @@ name: Test Pydra on: [push, pull_request] +defaults: + run: + shell: bash + jobs: build: - strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] @@ -82,8 +85,7 @@ jobs: if: matrix.install != 'style' env: IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: | - pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov From c1d5ec28debecbf1c684c9474abbe451f103fcbf Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 21 Aug 2020 17:51:10 +0800 Subject: [PATCH 093/271] fix bug in dockertask action --- .github/workflows/testdockertask.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testdockertask.yml b/.github/workflows/testdockertask.yml index 502a76b598..2c318ededc 100644 --- a/.github/workflows/testdockertask.yml +++ b/.github/workflows/testdockertask.yml @@ -19,7 +19,6 @@ jobs: python-version: [3.7, 3.8] install: [min-req, install, develop, sdist, wheel, style] fail-fast: false - runs-on: ${{ matrix.os }} steps: From 889efb0dc6508d849fe1ad046f4c8518154d7242 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Sat, 22 Aug 2020 11:35:39 +0800 Subject: [PATCH 094/271] mark timed cache-related tests as flaky --- pydra/engine/tests/test_workflow.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index ba7b2a1d29..82a4fa7f2c 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -252,7 +252,7 @@ def test_wf_2d_outpasdict(plugin): assert wf.output_dir.exists() -@pytest.mark.flaky(reruns=2) # when dask +@pytest.mark.flaky(reruns=3) # when dask def test_wf_3(plugin_dask_opt): """ testing None value for an input""" wf = Workflow(name="wf_3", input_spec=["x", "y"]) @@ -952,7 +952,7 @@ def test_wf_3sernd_ndst_1(plugin): # workflows with structures A -> C, B -> C -@pytest.mark.flaky(reruns=2) # when dask +@pytest.mark.flaky(reruns=3) # when dask def test_wf_3nd_st_1(plugin_dask_opt): """ workflow with three tasks, third one connected to two previous tasks, splitter on the workflow level @@ -979,7 +979,7 @@ def test_wf_3nd_st_1(plugin_dask_opt): assert odir.exists() -@pytest.mark.flaky(reruns=2) # when dask +@pytest.mark.flaky(reruns=3) # when dask def test_wf_3nd_ndst_1(plugin_dask_opt): """ workflow with three tasks, third one connected to two previous tasks, splitter on the tasks levels @@ -2013,6 +2013,7 @@ def test_wfasnd_wfst_3(plugin): # Testing caching +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachedir(plugin, tmpdir): """ wf with provided cache_dir using pytest tmpdir""" cache_dir = tmpdir.mkdir("test_wf_cache_1") @@ -2035,6 +2036,7 @@ def test_wf_nostate_cachedir(plugin, tmpdir): shutil.rmtree(cache_dir) +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachedir_relativepath(tmpdir, plugin): """ wf with provided cache_dir as relative path""" tmpdir.chdir() @@ -2059,6 +2061,7 @@ def test_wf_nostate_cachedir_relativepath(tmpdir, plugin): shutil.rmtree(cache_dir) +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations(plugin, tmpdir): """ Two identical wfs with provided cache_dir; @@ -2115,6 +2118,7 @@ def test_wf_nostate_cachelocations(plugin, tmpdir): assert not wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_a(plugin, tmpdir): """ the same as previous test, but workflows names differ; @@ -2535,6 +2539,7 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): assert len(list(Path(cache_dir2).glob("F*"))) == 0 +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_taskrerun_wfrerun_propagateFalse(plugin, tmpdir): """ Two identical wfs with provided cache_dir, and cache_locations for the second wf; @@ -2597,6 +2602,7 @@ def test_wf_nostate_cachelocations_taskrerun_wfrerun_propagateFalse(plugin, tmpd assert t2 > 2 +@pytest.mark.flaky(reruns=3) def test_wf_nostate_nodecachelocations(plugin, tmpdir): """ Two wfs with different input, but the second node has the same input; @@ -2645,6 +2651,7 @@ def test_wf_nostate_nodecachelocations(plugin, tmpdir): assert len(list(Path(cache_dir2).glob("F*"))) == 1 +@pytest.mark.flaky(reruns=3) def test_wf_nostate_nodecachelocations_upd(plugin, tmpdir): """ Two wfs with different input, but the second node has the same input; @@ -2757,6 +2764,7 @@ def test_wf_state_cachelocations(plugin, tmpdir): assert not odir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_state_cachelocations_forcererun(plugin, tmpdir): """ Two identical wfs (with states) with provided cache_dir; @@ -2894,6 +2902,7 @@ def test_wf_state_cachelocations_updateinp(plugin, tmpdir): assert not odir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_state_n_nostate_cachelocations(plugin, tmpdir): """ Two wfs with provided cache_dir, the first one has no state, the second has; @@ -3004,6 +3013,7 @@ def test_wf_nostate_cachelocations_updated(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_cachelocations_recompute(plugin, tmpdir): """ Two wfs with the same inputs but slightly different graph; @@ -3056,6 +3066,7 @@ def test_wf_nostate_cachelocations_recompute(plugin, tmpdir): assert len(list(Path(cache_dir2).glob("F*"))) == 1 +@pytest.mark.flaky(reruns=3) def test_wf_ndstate_cachelocations(plugin, tmpdir): """ Two wfs with identical inputs and node states; @@ -3119,6 +3130,7 @@ def test_wf_ndstate_cachelocations(plugin, tmpdir): assert not wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_ndstate_cachelocations_forcererun(plugin, tmpdir): """ Two wfs with identical inputs and node states; @@ -3182,6 +3194,7 @@ def test_wf_ndstate_cachelocations_forcererun(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_ndstate_cachelocations_updatespl(plugin, tmpdir): """ Two wfs with identical inputs and node state (that is set after adding the node!); @@ -3245,6 +3258,7 @@ def test_wf_ndstate_cachelocations_updatespl(plugin, tmpdir): assert not wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_ndstate_cachelocations_recompute(plugin, tmpdir): """ Two wfs (with nodes with states) with provided cache_dir; @@ -3308,6 +3322,7 @@ def test_wf_ndstate_cachelocations_recompute(plugin, tmpdir): assert wf2.output_dir.exists() +@pytest.mark.flaky(reruns=3) def test_wf_nostate_runtwice_usecache(plugin, tmpdir): """ running worflow (without state) twice, From 0e952aec2964b96714a31c3a835b74926fad72a1 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Sat, 22 Aug 2020 12:11:39 +0800 Subject: [PATCH 095/271] allow failure on testallowfail and testdockertask, yaml-> yml --- .../{testallowfail.yaml => testallowfail.yml} | 12 +++--------- .github/workflows/testdockertask.yml | 6 +++--- 2 files changed, 6 insertions(+), 12 deletions(-) rename .github/workflows/{testallowfail.yaml => testallowfail.yml} (84%) diff --git a/.github/workflows/testallowfail.yaml b/.github/workflows/testallowfail.yml similarity index 84% rename from .github/workflows/testallowfail.yaml rename to .github/workflows/testallowfail.yml index b64feabafb..521f5475e2 100644 --- a/.github/workflows/testallowfail.yaml +++ b/.github/workflows/testallowfail.yml @@ -1,3 +1,4 @@ +# Test on an older version of pip name: Test Pydra (allow failures) @@ -10,14 +11,13 @@ defaults: jobs: build: + runs-on: ${{ matrix.os }} + continue-on-error: true strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [old-pip] fail-fast: false - runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v2 @@ -25,19 +25,13 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update build tools (pip 10.0.1) - if: matrix.install == 'old-pip' run: python -m pip install pip==10.0.1 setuptools==30.3.0 - name: Install dependencies (min-requirements.txt) - if: matrix.install == 'old-pip' run: pip install -r min-requirements.txt - name: Install Pydra (test) - if: matrix.install == 'old-pip' run: pip install -e ".[test]" - - name: Pytest - if: matrix.install == 'old-pip' run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/.github/workflows/testdockertask.yml b/.github/workflows/testdockertask.yml index 2c318ededc..038931286c 100644 --- a/.github/workflows/testdockertask.yml +++ b/.github/workflows/testdockertask.yml @@ -1,5 +1,4 @@ -# Reference -# https://hynek.me/articles/python-github-actions/ +# Test docker-related tasks on linux & windows name: Test Dockertask @@ -13,9 +12,10 @@ jobs: build: runs-on: ${{ matrix.os }} + continue-on-error: true # until we fix tests on windows strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest] # docker not available on macOS python-version: [3.7, 3.8] install: [min-req, install, develop, sdist, wheel, style] fail-fast: false From ec3f3c55e0f329d964f3e203a528b8d8b12877f1 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Sat, 22 Aug 2020 12:12:41 +0800 Subject: [PATCH 096/271] remove install dependencies step, yaml->yml --- .../workflows/{testsingularity.yaml => testsingularity.yml} | 4 ---- 1 file changed, 4 deletions(-) rename .github/workflows/{testsingularity.yaml => testsingularity.yml} (91%) diff --git a/.github/workflows/testsingularity.yaml b/.github/workflows/testsingularity.yml similarity index 91% rename from .github/workflows/testsingularity.yaml rename to .github/workflows/testsingularity.yml index 5211895ed5..10eadfd74b 100644 --- a/.github/workflows/testsingularity.yaml +++ b/.github/workflows/testsingularity.yml @@ -57,10 +57,6 @@ jobs: uses: actions/checkout@v2 with: repository: ${{ github.repository }} - - name: Install dependencies (setup.py develop) - run: | - python setup.py develop - python -c 'import pydra; print(pydra.__version__)' # Verify import with bare install - name: Install pydra (test) run: pip install -e ".[test]" From 4c08bf5cae169a6d0f6ade0b8cfa23cdec2e16f3 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 25 Aug 2020 11:15:06 -0400 Subject: [PATCH 097/271] adding more tests for hooks --- pydra/engine/tests/test_task.py | 77 ++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 67c4c4b6e9..de9c426c4a 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -663,7 +663,7 @@ def test_functask_callable(tmpdir): assert foo2.plugin == "cf" -def test_taskhooks(tmpdir, capsys): +def test_taskhooks_1(tmpdir, capsys): foo = funaddtwo(name="foo", a=1, cache_dir=tmpdir) assert foo.hooks # ensure all hooks are defined @@ -715,3 +715,78 @@ def myhook(task, *args): for attr in ("pre_run", "post_run", "pre_run_task", "post_run_task"): hook = getattr(foo.hooks, attr) assert hook() is None + + +def test_taskhooks_2(tmpdir, capsys): + """checking order of the hooks; using task's attributes""" + foo = funaddtwo(name="foo", a=1, cache_dir=tmpdir) + + def myhook_prerun(task, *args): + print(f"i. prerun hook was called from {task.name}") + + def myhook_prerun_task(task, *args): + print(f"ii. prerun task hook was called {task.name}") + + def myhook_postrun_task(task, *args): + print(f"iii. postrun task hook was called {task.name}") + + def myhook_postrun(task, *args): + print(f"iv. postrun hook was called {task.name}") + + foo.hooks.pre_run = myhook_prerun + foo.hooks.post_run = myhook_postrun + foo.hooks.pre_run_task = myhook_prerun_task + foo.hooks.post_run_task = myhook_postrun_task + foo() + + captured = capsys.readouterr() + hook_messages = captured.out.strip().split("\n") + # checking the order of the hooks + assert "i. prerun hook" in hook_messages[0] + assert "ii. prerun task hook" in hook_messages[1] + assert "iii. postrun task hook" in hook_messages[2] + assert "iv. postrun hook" in hook_messages[3] + + +def test_taskhooks_3(tmpdir, capsys): + """checking results in the post run hooks""" + foo = funaddtwo(name="foo", a=1, cache_dir=tmpdir) + + def myhook_postrun_task(task, result, *args): + print(f"postrun task hook, the result is {result.output.out}") + + def myhook_postrun(task, result, *args): + print(f"postrun hook, the result is {result.output.out}") + + foo.hooks.post_run = myhook_postrun + foo.hooks.post_run_task = myhook_postrun_task + foo() + + captured = capsys.readouterr() + hook_messages = captured.out.strip().split("\n") + # checking that the postrun hooks have access to results + assert "postrun task hook, the result is 3" in hook_messages[0] + assert "postrun hook, the result is 3" in hook_messages[1] + + +def test_taskhooks_4(tmpdir, capsys): + """task raises an error: postrun task should be called, postrun shouldn't be called""" + foo = funaddtwo(name="foo", a="one", cache_dir=tmpdir) + + def myhook_postrun_task(task, result, *args): + print(f"postrun task hook was called, result object is {result}") + + def myhook_postrun(task, result, *args): + print(f"postrun hook should not be called") + + foo.hooks.post_run = myhook_postrun + foo.hooks.post_run_task = myhook_postrun_task + + with pytest.raises(Exception): + foo() + + captured = capsys.readouterr() + hook_messages = captured.out.strip().split("\n") + # only post run task hook should be called + assert len(hook_messages) == 1 + assert "postrun task hook was called" in hook_messages[0] From 5717492396fbb674dd389f77230b534ad7d47a75 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 26 Aug 2020 01:07:22 -0400 Subject: [PATCH 098/271] adding require field to output_spec; simplifying functions in helpers --- pydra/engine/core.py | 7 ++--- pydra/engine/helpers.py | 42 +++++++++++----------------- pydra/engine/specs.py | 8 ++++-- pydra/engine/tests/test_shelltask.py | 29 +++++++++++++++++++ 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 26c67ad1a1..de0aedefd6 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -35,7 +35,6 @@ record_error, hash_function, output_from_inputfields, - output_names_from_inputfields, ) from .helpers_file import copyfile_input, template_update from .graph import DiGraph @@ -312,10 +311,8 @@ def set_state(self, splitter, combiner=None): @property def output_names(self): - """Get the names of the parameters generated by the task.""" - output_spec_names = [f.name for f in attr.fields(make_klass(self.output_spec))] - from_input_spec_names = output_names_from_inputfields(self.inputs) - return output_spec_names + from_input_spec_names + """Get the names of the outputs generated by the task.""" + return output_from_inputfields(self.output_spec, self.inputs, names_only=True) @property def can_resume(self): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index da44246769..c1a30bf8de 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -672,30 +672,11 @@ def hash_value(value, tp=None, metadata=None): return value -def output_names_from_inputfields(inputs): - """ - Collect outputs from input fields with output_file_template. - - Parameters - ---------- - inputs : - TODO - - """ - output_names = [] - for fld in attr_fields(inputs): - if "output_file_template" in fld.metadata: - if "output_field_name" in fld.metadata: - field_name = fld.metadata["output_field_name"] - else: - field_name = fld.name - output_names.append(field_name) - return output_names - - -def output_from_inputfields(output_spec, inputs): +def output_from_inputfields(output_spec, inputs, names_only=False): """ Collect values from output from input fields. + If names_only is False, the output_spec is updated, + if names_only is True only the names are returned Parameters ---------- @@ -705,6 +686,8 @@ def output_from_inputfields(output_spec, inputs): TODO """ + current_output_spec_names = [f.name for f in attr.fields(make_klass(output_spec))] + new_fields = [] for fld in attr_fields(inputs): if "output_file_template" in fld.metadata: value = getattr(inputs, fld.name) @@ -712,10 +695,17 @@ def output_from_inputfields(output_spec, inputs): field_name = fld.metadata["output_field_name"] else: field_name = fld.name - output_spec.fields.append( - (field_name, attr.ib(type=File, metadata={"value": value})) - ) - return output_spec + # not adding if the field already in teh output_spec + if field_name not in current_output_spec_names: + new_fields.append( + (field_name, attr.ib(type=File, metadata={"value": value})) + ) + if names_only: + new_names = [el[0] for el in new_fields] + return current_output_spec_names + new_names + else: + output_spec.fields += new_fields + return output_spec def get_available_cpus(): diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index f52beccd61..862e47e844 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -416,7 +416,7 @@ def collect_additional_outputs(self, inputs, output_dir): fld, inputs, output_dir ) else: - raise Exception("not implemented") + raise Exception("not implemented (collect_additional_output)") return additional_out def _field_defaultvalue(self, fld, output_dir): @@ -449,6 +449,10 @@ def _field_defaultvalue(self, fld, output_dir): def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" + if "requires" in fld.metadata: + for inp in fld.metadata["requires"]: + if getattr(inputs, inp) in [attr.NOTHING, None, False]: + return attr.NOTHING if "value" in fld.metadata: return output_dir / fld.metadata["value"] # this block is only run if "output_file_template" is provided in output_spec @@ -461,7 +465,7 @@ def _field_metadata(self, fld, inputs, output_dir): elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: - raise Exception("not implemented") + raise Exception("not implemented (_field_metadata)") @attr.s(auto_attribs=True, kw_only=True) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 7db1c5f77a..0f1c6e0260 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2377,6 +2377,35 @@ def test_shell_cmd_outputspec_5(plugin, results_function): assert res.output.out1.exists() +def test_shell_cmd_outputspec_5a(): + """ + providing output name by providing output_file_template + (using shorter syntax) + """ + cmd = "touch" + args = "newfile_tmp.txt" + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out1", + File, + {"output_file_template": "{args}", "help_string": "output file"}, + ) + ], + bases=(ShellOutSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd, args=args, output_spec=my_output_spec + ) + + res = shelly() + assert res.output.stdout == "" + assert res.output.out1.exists() + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function): """ From 822b972202e1457d28ef93d55916024db288fefd Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 28 Aug 2020 14:44:21 -0400 Subject: [PATCH 099/271] fixing the requires in the output fields, using fields from output_file_template if requires is not avaliable; adding tests --- pydra/engine/specs.py | 30 ++- pydra/engine/task.py | 10 +- pydra/engine/tests/test_shelltask.py | 292 +++++++++++++++++++++++++++ 3 files changed, 322 insertions(+), 10 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 862e47e844..83df314a2a 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -3,6 +3,7 @@ from pathlib import Path import typing as ty import inspect +import re from .helpers_file import template_update_single @@ -450,9 +451,34 @@ def _field_defaultvalue(self, fld, output_dir): def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" if "requires" in fld.metadata: - for inp in fld.metadata["requires"]: - if getattr(inputs, inp) in [attr.NOTHING, None, False]: + field_required = fld.metadata["requires"] + # if the output field doesn't have requires field, we use fields from output_file_template + elif "output_file_template" in fld.metadata: + inp_fields = re.findall("{\w+}", fld.metadata["output_file_template"]) + field_required = [el[1:-1] for el in inp_fields] + else: + field_required = [] + for inp in field_required: + if isinstance(inp, str): # name of the input field + if not hasattr(inputs, inp): + raise Exception( + f"{inp} is not a valid input field, can't be used in requires" + ) + elif getattr(inputs, inp) in [attr.NOTHING, None]: return attr.NOTHING + elif isinstance(inp, tuple): # (name, allowed values) + inp, allowed_val = inp + if not hasattr(inputs, inp): + raise Exception( + f"{inp} is not a valid input field, can't be used in requires" + ) + elif getattr(inputs, inp) not in allowed_val: + return attr.NOTHING + else: + raise Exception( + f"each element of the requires should be a string or a tuple, " + f"but {inp} is found" + ) if "value" in fld.metadata: return output_dir / fld.metadata["value"] # this block is only run if "output_file_template" is provided in output_spec diff --git a/pydra/engine/task.py b/pydra/engine/task.py index f4e732aa51..5eacb38252 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -388,14 +388,8 @@ def _command_pos_args(self, field, state_ind, ind): """ argstr = field.metadata.get("argstr", None) if argstr is None: - if "output_file_template" in field.metadata: - # assuming that input that has output_file_template and no arstr - # are not used in the command - return None - else: - raise Exception( - f"{field.name} doesn't have argstr field in the metadata" - ) + # assuming that input that has no arstr is not used in the command + return None pos = field.metadata.get("position", None) if pos is None: # position will be set at the end diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 0f1c6e0260..7f131269ed 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2479,6 +2479,298 @@ def test_shell_cmd_outputspec_wf_1(plugin): assert res.output.newfile.parent == wf.output_dir +def test_shell_cmd_inputspec_outputspec_1(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in templates + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + {"output_file_template": "{file1}", "help_string": "newfile 1"}, + ), + ( + "newfile2", + File, + {"output_file_template": "{file2}", "help_string": "newfile 2"}, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.file2 = "new_file_2.txt" + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + assert res.output.newfile2.exists() + + +def test_shell_cmd_inputspec_outputspec_1a(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in templates, + file2 is used in a template for newfile2, but it is not provided, so newfile2 is set to NOTHING + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + {"output_file_template": "{file1}", "help_string": "newfile 1"}, + ), + ( + "newfile2", + File, + {"output_file_template": "{file2}", "help_string": "newfile 2"}, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + # newfile2 is not created, since file2 is not provided + assert res.output.newfile2 is attr.NOTHING + + +def test_shell_cmd_inputspec_outputspec_2(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1"], + }, + ), + ( + "newfile2", + File, + { + "output_file_template": "{file2}", + "help_string": "newfile 1", + "requires": ["file1", "file2"], + }, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.file2 = "new_file_2.txt" + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + assert res.output.newfile2.exists() + + +def test_shell_cmd_inputspec_outputspec_2a(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1"], + }, + ), + ( + "newfile2", + File, + { + "output_file_template": "{file2}", + "help_string": "newfile 1", + "requires": ["file1", "file2"], + }, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + assert res.output.newfile2 is attr.NOTHING + + +def test_shell_cmd_inputspec_outputspec_3(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + adding one additional input that is not in the template, but in the requires field, + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ("additional_inp", str, {"help_string": "additional inp"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1"], + }, + ), + ( + "newfile2", + File, + { + "output_file_template": "{file2}", + "help_string": "newfile 1", + "requires": ["file1", "file2", "additional_inp"], + }, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.file2 = "new_file_2.txt" + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + # additional input not provided so no newfile2 set (even if the file was created) + assert res.output.newfile2 is attr.NOTHING + + def no_fsl(): if "FSLDIR" not in os.environ: return True From f60e2192a36c8ad9e55ea740100b94d9018025c0 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 30 Aug 2020 21:00:37 -0400 Subject: [PATCH 100/271] Update pydra/engine/specs.py Co-authored-by: Satrajit Ghosh --- pydra/engine/specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 83df314a2a..4f02ef8965 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -491,7 +491,7 @@ def _field_metadata(self, fld, inputs, output_dir): elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: - raise Exception("not implemented (_field_metadata)") + raise Exception("(_field_metadata) is not a current valid metadata key.") @attr.s(auto_attribs=True, kw_only=True) From c1f2fe7ef30fc262501167a867e8ea2b8270509c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 30 Aug 2020 23:23:51 -0400 Subject: [PATCH 101/271] adding more tests for output spec with requires --- pydra/engine/tests/test_shelltask.py | 163 +++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 7f131269ed..db4fdc37a1 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2731,6 +2731,71 @@ def test_shell_cmd_inputspec_outputspec_3(): bases=(ShellSpec,), ) + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1"], + }, + ), + ( + "newfile2", + File, + { + "output_file_template": "{file2}", + "help_string": "newfile 1", + "requires": ["file1", "file2", "additional_inp"], + }, + ), + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.file2 = "new_file_2.txt" + shelly.inputs.additional_inp = 2 + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + assert res.output.newfile2.exists() + + +def test_shell_cmd_inputspec_outputspec_3a(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + adding one additional input that is not in the template, but in the requires field, + the additional input not provided, so the output is NOTHING + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ( + "file2", + str, + {"help_string": "2nd creadted file", "argstr": "", "position": 2}, + ), + ("additional_inp", str, {"help_string": "additional inp"}), + ], + bases=(ShellSpec,), + ) + my_output_spec = SpecInfo( name="Output", fields=[ @@ -2771,6 +2836,104 @@ def test_shell_cmd_inputspec_outputspec_3(): assert res.output.newfile2 is attr.NOTHING +def test_shell_cmd_inputspec_outputspec_4(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + adding one additional input to the requires together with a list of the allowed values, + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp", str, {"help_string": "additional inp"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1", ("additional_inp", [2, 3])], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.additional_inp = 2 + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + + +def test_shell_cmd_inputspec_outputspec_4a(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed + adding one additional input to the requires together with a list of the allowed values, + the input is set to a value that is not in the list, so output is NOTHING + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp", str, {"help_string": "additional inp"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + "requires": ["file1", ("additional_inp", [2, 3])], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + # the value is not in the list from requires + shelly.inputs.additional_inp = 1 + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1 is attr.NOTHING + + def no_fsl(): if "FSLDIR" not in os.environ: return True From 2c63b9c16be2143ccd0a11c6cb8983ce3f4360b2 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 30 Aug 2020 23:41:19 -0400 Subject: [PATCH 102/271] always adding all fields from output_file_template to requires --- pydra/engine/specs.py | 11 +++++++---- pydra/engine/tests/test_shelltask.py | 16 ++++------------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 83df314a2a..6f430083e4 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -452,12 +452,15 @@ def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" if "requires" in fld.metadata: field_required = fld.metadata["requires"] - # if the output field doesn't have requires field, we use fields from output_file_template - elif "output_file_template" in fld.metadata: - inp_fields = re.findall("{\w+}", fld.metadata["output_file_template"]) - field_required = [el[1:-1] for el in inp_fields] else: field_required = [] + # if the output has output_file_template field, adding all input fields from the template to requires + if "output_file_template" in fld.metadata: + inp_fields = re.findall("{\w+}", fld.metadata["output_file_template"]) + field_required += [ + el[1:-1] for el in inp_fields if el[1:-1] not in field_required + ] + # checking if the input fields from requires have set values for inp in field_required: if isinstance(inp, str): # name of the input field if not hasattr(inputs, inp): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index db4fdc37a1..e7fd91d684 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2737,11 +2737,7 @@ def test_shell_cmd_inputspec_outputspec_3(): ( "newfile1", File, - { - "output_file_template": "{file1}", - "help_string": "newfile 1", - "requires": ["file1"], - }, + {"output_file_template": "{file1}", "help_string": "newfile 1"}, ), ( "newfile2", @@ -2749,7 +2745,7 @@ def test_shell_cmd_inputspec_outputspec_3(): { "output_file_template": "{file2}", "help_string": "newfile 1", - "requires": ["file1", "file2", "additional_inp"], + "requires": ["file1", "additional_inp"], }, ), ], @@ -2802,11 +2798,7 @@ def test_shell_cmd_inputspec_outputspec_3a(): ( "newfile1", File, - { - "output_file_template": "{file1}", - "help_string": "newfile 1", - "requires": ["file1"], - }, + {"output_file_template": "{file1}", "help_string": "newfile 1"}, ), ( "newfile2", @@ -2814,7 +2806,7 @@ def test_shell_cmd_inputspec_outputspec_3a(): { "output_file_template": "{file2}", "help_string": "newfile 1", - "requires": ["file1", "file2", "additional_inp"], + "requires": ["file1", "additional_inp"], }, ), ], From a0ccc77ef17a8ebc33d765ca07f8c58bd2fcaeed Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 16:43:52 +0800 Subject: [PATCH 103/271] merge testpydra.yml and testdockertask.yml --- .github/workflows/testdockertask.yml | 39 --------------------------- .github/workflows/testpydra.yml | 4 +-- pydra/engine/tests/test_dockertask.py | 39 ++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/testdockertask.yml diff --git a/.github/workflows/testdockertask.yml b/.github/workflows/testdockertask.yml deleted file mode 100644 index 038931286c..0000000000 --- a/.github/workflows/testdockertask.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Test docker-related tasks on linux & windows - -name: Test Dockertask - -on: [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - build: - - runs-on: ${{ matrix.os }} - continue-on-error: true # until we fix tests on windows - strategy: - matrix: - os: [ubuntu-latest, windows-latest] # docker not available on macOS - python-version: [3.7, 3.8] - install: [min-req, install, develop, sdist, wheel, style] - fail-fast: false - - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Update build tools - run: python -m pip install --upgrade pip==20.2.1 setuptools==30.2.1 wheel - - name: Install dependencies - run: pip install -r min-requirements.txt - - name: Install pydra - run: pip install .[test] - - name: Pytest - run: pytest pydra/engine/tests/test_dockertask.py -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml - - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index a93763d268..691111981d 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -83,9 +83,7 @@ jobs: - name: Pytest if: matrix.install != 'style' - env: - IGNORE_TESTS: '*dockertask.py' # docker tests run in a seperate action for now - run: pytest --ignore-glob=$IGNORE_TESTS -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 3a1721ea41..e9e8e09237 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -6,9 +6,10 @@ from ..submitter import Submitter from ..core import Workflow from ..specs import ShellOutSpec, SpecInfo, File, DockerSpec -from .utils import need_docker +from .utils import no_win, need_docker +@no_win @need_docker def test_docker_1_nosubm(): """ simple command in a container, a default bindings and working directory is added @@ -30,6 +31,7 @@ def test_docker_1_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_1(plugin): """ simple command in a container, a default bindings and working directory is added @@ -48,6 +50,7 @@ def test_docker_1(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_1_dockerflag(plugin): """ simple command in a container, a default bindings and working directory is added @@ -68,6 +71,7 @@ def test_docker_1_dockerflag(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_1_dockerflag_exception(plugin): """using ShellComandTask with container_info=("docker"), no image provided""" @@ -79,6 +83,7 @@ def test_docker_1_dockerflag_exception(plugin): assert "container_info has to have 2 or 3 elements" in str(excinfo.value) +@no_win @need_docker def test_docker_2_nosubm(): """ a command with arguments, cmd and args given as executable @@ -98,6 +103,7 @@ def test_docker_2_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_2(plugin): """ a command with arguments, cmd and args given as executable @@ -119,6 +125,7 @@ def test_docker_2(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_2_dockerflag(plugin): """ a command with arguments, cmd and args given as executable @@ -142,6 +149,7 @@ def test_docker_2_dockerflag(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_2a_nosubm(): """ a command with arguments, using executable and args @@ -166,6 +174,7 @@ def test_docker_2a_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_2a(plugin): """ a command with arguments, using executable and args @@ -192,6 +201,7 @@ def test_docker_2a(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_3(plugin, tmpdir): """ a simple command in container with bindings, @@ -214,6 +224,7 @@ def test_docker_3(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_3_dockerflag(plugin, tmpdir): """ a simple command in container with bindings, @@ -239,6 +250,7 @@ def test_docker_3_dockerflag(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_3_dockerflagbind(plugin, tmpdir): """ a simple command in container with bindings, @@ -264,6 +276,7 @@ def test_docker_3_dockerflagbind(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_4(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -289,6 +302,7 @@ def test_docker_4(plugin, tmpdir): assert res.output.return_code == 0 +@no_win @need_docker def test_docker_4_dockerflag(plugin, tmpdir): """ task reads the file that is bounded to the container @@ -317,6 +331,7 @@ def test_docker_4_dockerflag(plugin, tmpdir): # tests with State +@no_win @need_docker def test_docker_st_1(plugin): """ commands without arguments in container @@ -340,6 +355,7 @@ def test_docker_st_1(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 +@no_win @need_docker def test_docker_st_2(plugin): """ command with arguments in docker, checking the distribution @@ -363,6 +379,7 @@ def test_docker_st_2(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 +@no_win @need_docker def test_docker_st_3(plugin): """ outer splitter image and executable @@ -380,6 +397,7 @@ def test_docker_st_3(plugin): assert "Ubuntu" in res[3].output.stdout +@no_win @need_docker def test_docker_st_4(plugin): """ outer splitter image and executable, combining with images @@ -421,6 +439,7 @@ def test_docker_st_4(plugin): # tests with workflows +@no_win @need_docker def test_wf_docker_1(plugin, tmpdir): """ a workflow with two connected task @@ -464,6 +483,7 @@ def test_wf_docker_1(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" +@no_win @need_docker def test_wf_docker_1_dockerflag(plugin, tmpdir): """ a workflow with two connected task @@ -503,6 +523,7 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" +@no_win @need_docker def test_wf_docker_2pre(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -523,6 +544,7 @@ def test_wf_docker_2pre(plugin, tmpdir): assert res.output.stdout == "/outputs/tmp.txt" +@no_win @need_docker def test_wf_docker_2(plugin, tmpdir): """ a workflow with two connected task that run python scripts @@ -562,6 +584,7 @@ def test_wf_docker_2(plugin, tmpdir): assert res.output.out == "Hello!" +@no_win @need_docker def test_wf_docker_3(plugin, tmpdir): """ a workflow with two connected task @@ -604,6 +627,7 @@ def test_wf_docker_3(plugin, tmpdir): # tests with customized output_spec +@no_win @need_docker def test_docker_outputspec_1(plugin, tmpdir): """ @@ -631,6 +655,7 @@ def test_docker_outputspec_1(plugin, tmpdir): # tests with customised input_spec +@no_win @need_docker def test_docker_inputspec_1(plugin, tmpdir): """ a simple customized input spec for docker task """ @@ -672,6 +697,7 @@ def test_docker_inputspec_1(plugin, tmpdir): assert res.output.stdout == "hello from pydra" +@no_win @need_docker def test_docker_inputspec_1a(plugin, tmpdir): """ a simple customized input spec for docker task @@ -710,6 +736,7 @@ def test_docker_inputspec_1a(plugin, tmpdir): assert res.output.stdout == "hello from pydra" +@no_win @need_docker def test_docker_inputspec_2(plugin, tmpdir): """ a customized input spec with two fields for docker task """ @@ -766,6 +793,7 @@ def test_docker_inputspec_2(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker def test_docker_inputspec_2a_except(plugin, tmpdir): """ a customized input spec with two fields @@ -825,6 +853,7 @@ def test_docker_inputspec_2a_except(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker def test_docker_inputspec_2a(plugin, tmpdir): """ a customized input spec with two fields @@ -884,6 +913,7 @@ def test_docker_inputspec_2a(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker @pytest.mark.xfail(reason="'docker' not in /proc/1/cgroup on ubuntu; TODO") def test_docker_inputspec_3(plugin, tmpdir): @@ -928,6 +958,7 @@ def test_docker_inputspec_3(plugin, tmpdir): assert cmdline == docky.cmdline +@no_win @need_docker def test_docker_inputspec_3a(plugin, tmpdir): """ input file does not exist in the local file system, @@ -971,6 +1002,7 @@ def test_docker_inputspec_3a(plugin, tmpdir): assert "use field.metadata['container_path']=True" in str(excinfo.value) +@no_win @need_docker def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): """ shelltask changes a file in place, @@ -1033,6 +1065,7 @@ def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): assert "hello from pydra\n" == f.read() +@no_win @need_docker def test_docker_inputspec_state_1(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1081,6 +1114,7 @@ def test_docker_inputspec_state_1(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" +@no_win @need_docker def test_docker_inputspec_state_1b(plugin, tmpdir): """ a customised input spec for a docker file with a splitter, @@ -1130,6 +1164,7 @@ def test_docker_inputspec_state_1b(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" +@no_win @need_docker def test_docker_wf_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks """ @@ -1181,6 +1216,7 @@ def test_docker_wf_inputspec_1(plugin, tmpdir): assert res.output.out == "hello from pydra" +@no_win @need_docker def test_docker_wf_state_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks that has a state""" @@ -1238,6 +1274,7 @@ def test_docker_wf_state_inputspec_1(plugin, tmpdir): assert res[1].output.out == "have a nice one" +@no_win @need_docker def test_docker_wf_ndst_inputspec_1(plugin, tmpdir): """ a customized input spec for workflow with docker tasks with states""" From bc980f070d63580fd65a2fadec378ddce566e174 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 16:44:56 +0800 Subject: [PATCH 104/271] remove sdist from testpydra --- .github/workflows/testpydra.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 691111981d..0a643c1c2a 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -15,7 +15,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [min-req, install, develop, sdist, wheel, style] + install: [min-req, install, develop, wheel, style] fail-fast: false runs-on: ${{ matrix.os }} @@ -42,13 +42,6 @@ jobs: if: matrix.install == 'develop' run: python setup.py develop - - name: Install dependencies (sdist) - if: matrix.install == 'sdist' - run: | - python setup.py sdist - pip install dist/*.tar.gz - shell: bash - - name: Install dependencies (wheel) if: matrix.install == 'wheel' run: | @@ -65,10 +58,6 @@ jobs: if: matrix.install == 'develop' run: pip install -e ".[test]" - - name: Install Pydra (sdist) - if: matrix.install == 'sdist' - run: pip install "$( ls dist/pydra*.tar.gz )[test]" - - name: Install Pydra (wheel) if: matrix.install == 'wheel' run: pip install "$( ls dist/pydra*.whl )[test]" From ade81339965e944ecaf72ca48cb94b0a1623e4f1 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 16:52:04 +0800 Subject: [PATCH 105/271] check style in a separate github action --- .github/workflows/testpydra.yml | 13 +---------- .github/workflows/testsingularity.yml | 4 +--- .github/workflows/testslurm.yml | 10 +-------- .github/workflows/teststyle.yaml | 31 +++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/teststyle.yaml diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 0a643c1c2a..ad1ebaeae0 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -1,6 +1,3 @@ -# Reference -# https://hynek.me/articles/python-github-actions/ - name: Test Pydra on: [push, pull_request] @@ -15,7 +12,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [min-req, install, develop, wheel, style] + install: [min-req, install, develop, wheel] fail-fast: false runs-on: ${{ matrix.os }} @@ -63,15 +60,7 @@ jobs: run: pip install "$( ls dist/pydra*.whl )[test]" - - name: Check Style - if: matrix.install == 'style' - run: | - pip install black==19.3b0 codecov - black --check pydra tools setup.py - - - name: Pytest - if: matrix.install != 'style' run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 10eadfd74b..34c53ae42a 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -1,6 +1,4 @@ -#https://github.com/eWaterCycle/singularity-versions/blob/master/.github/workflows/dist.yml - -name: Test singularity +name: Test Pydra (singularity) on: [push, pull_request] diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 5af4070f63..ae8fad5c48 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -1,12 +1,4 @@ -# use containers in github actions -# https://stackoverflow.com/questions/58930529/github-action-how-do-i-run-commands-inside-a-docker-container -# https://stackoverflow.com/questions/58476228/how-to-use-a-variable-docker-image-in-github-actions -# https://github.community/t/confused-with-runs-on-and-container-options/16258/2 - -# using go and docker in github actions -# https://steele.blue/tiny-github-actions/ - -name: Run SLURM tests +name: Test Pydra (SLURM) on: [push, pull_request] diff --git a/.github/workflows/teststyle.yaml b/.github/workflows/teststyle.yaml new file mode 100644 index 0000000000..806ea096d3 --- /dev/null +++ b/.github/workflows/teststyle.yaml @@ -0,0 +1,31 @@ +name: Test Pydra (Style) + +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + python-version: [3.7, 3.8] + fail-fast: false + runs-on: ubuntu-latest + + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} on ubuntu-latest + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Update build tools + run: python -m pip install --upgrade pip setuptools + + - name: Install dependencies + run: pip install -r min-requirements.txt + - name: Install Pydra + run: pip install ".[test]" + + - name: Check Style + run: | + pip install black==19.3b0 codecov + black --check pydra tools setup.py From b405c1a1bfa28b9596342a1244abc747ba4f6829 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 17:48:19 +0800 Subject: [PATCH 106/271] fix setup.py develop test --- .github/workflows/testpydra.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index ad1ebaeae0..70c2603481 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -35,10 +35,6 @@ jobs: if: matrix.install == 'install' run: python setup.py install - - name: Install dependencies (setup.py develop) - if: matrix.install == 'develop' - run: python setup.py develop - - name: Install dependencies (wheel) if: matrix.install == 'wheel' run: | From 2f6d811974bc32e78126ca9933552a333dc6927f Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 18:35:03 +0800 Subject: [PATCH 107/271] distinguish between github action workflows in codecov --- .github/workflows/testallowfail.yml | 2 +- .github/workflows/testpydra.yml | 2 +- .github/workflows/testsingularity.yml | 2 +- .github/workflows/{teststyle.yaml => teststyle.yml} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{teststyle.yaml => teststyle.yml} (100%) diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml index 521f5475e2..7f6cdcda56 100644 --- a/.github/workflows/testallowfail.yml +++ b/.github/workflows/testallowfail.yml @@ -34,4 +34,4 @@ jobs: - name: Pytest run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov --root /pydra -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 70c2603481..0225b43897 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -61,4 +61,4 @@ jobs: - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests + run: codecov --root /pydra -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 34c53ae42a..92f1bee47e 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -62,4 +62,4 @@ jobs: - name: Pytest run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov - run: codecov -f /pydra/cov.xml -F unittests + run: codecov -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW diff --git a/.github/workflows/teststyle.yaml b/.github/workflows/teststyle.yml similarity index 100% rename from .github/workflows/teststyle.yaml rename to .github/workflows/teststyle.yml From 6551d34f4768227e38c1e98e4583e8ac3557d2cb Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Mon, 31 Aug 2020 19:14:52 +0800 Subject: [PATCH 108/271] fix codecov xml upload --- .github/workflows/testallowfail.yml | 2 +- .github/workflows/testpydra.yml | 2 +- .github/workflows/testsingularity.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml index 7f6cdcda56..ba8d414546 100644 --- a/.github/workflows/testallowfail.yml +++ b/.github/workflows/testallowfail.yml @@ -32,6 +32,6 @@ jobs: - name: Install Pydra (test) run: pip install -e ".[test]" - name: Pytest - run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules pydra - name: Upload to codecov run: codecov --root /pydra -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 0225b43897..4a3803b772 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -61,4 +61,4 @@ jobs: - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW + run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 92f1bee47e..b553cd9556 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -62,4 +62,4 @@ jobs: - name: Pytest run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov - run: codecov -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW + run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW From da47bf66cb68b4e87e112d6b2f8508a5be21c690 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 31 Aug 2020 23:22:02 -0400 Subject: [PATCH 109/271] [wip] changing the path for outputs that have output_file_template - they should be created in the output_dir; some tests from test_shelltask_inputspec.py are failing - will be solved after deciding the relation between requires and fields from the template in the output fields --- pydra/engine/core.py | 4 ++- pydra/engine/helpers_file.py | 12 +++++-- pydra/engine/specs.py | 6 ++-- pydra/engine/task.py | 2 +- pydra/engine/tests/test_shelltask.py | 6 +++- .../engine/tests/test_shelltask_inputspec.py | 36 ++++++++++++------- 6 files changed, 45 insertions(+), 21 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 26c67ad1a1..30d8ab94c6 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -403,7 +403,9 @@ def _run(self, rerun=False, **kwargs): odir.mkdir(parents=False, exist_ok=True if self.can_resume else False) orig_inputs = attr.asdict(self.inputs) map_copyfiles = copyfile_input(self.inputs, self.output_dir) - modified_inputs = template_update(self.inputs, map_copyfiles) + modified_inputs = template_update( + self.inputs, self.output_dir, map_copyfiles=map_copyfiles + ) if modified_inputs: self.inputs = attr.evolve(self.inputs, **modified_inputs) self.audit.start_audit(odir) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index c0d68735b6..06106d0640 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -499,7 +499,7 @@ def copyfile_input(inputs, output_dir): # not sure if this might be useful for Function Task -def template_update(inputs, map_copyfiles=None): +def template_update(inputs, output_dir, map_copyfiles=None): """ Update all templates that are present in the input spec. @@ -521,7 +521,9 @@ def template_update(inputs, map_copyfiles=None): f"fields with output_file_template" "has to be a string or Union[str, bool]" ) - dict_[fld.name] = template_update_single(field=fld, inputs_dict=dict_) + dict_[fld.name] = template_update_single( + field=fld, inputs_dict=dict_, output_dir=output_dir + ) # using is and == so it covers list and numpy arrays updated_templ_dict = { k: v @@ -531,7 +533,7 @@ def template_update(inputs, map_copyfiles=None): return updated_templ_dict -def template_update_single(field, inputs_dict, spec_type="input"): +def template_update_single(field, inputs_dict, output_dir=None, spec_type="input"): """Update a single template from the input_spec or output_spec based on the value from inputs_dict (checking the types of the fields, that have "output_file_template)" @@ -560,6 +562,7 @@ def template_update_single(field, inputs_dict, spec_type="input"): ) else: raise Exception(f"spec_type can be input or output, but {spec_type} provided") + # for inputs that the value is set (so the template is ignored) if spec_type == "input" and isinstance(inputs_dict[field.name], str): return inputs_dict[field.name] elif spec_type == "input" and inputs_dict[field.name] is False: @@ -572,6 +575,9 @@ def template_update_single(field, inputs_dict, spec_type="input"): value = _template_formatting( template, inputs_dict, keep_extension=keep_extension ) + # changing path so it is in the output_dir + if output_dir: + value = output_dir / Path(value).name return value diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index f52beccd61..ba2c1c693a 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -456,8 +456,10 @@ def _field_metadata(self, fld, inputs, output_dir): # than the field already should have value elif "output_file_template" in fld.metadata: inputs_templ = attr.asdict(inputs) - value = template_update_single(fld, inputs_templ, spec_type="output") - return output_dir / value + value = template_update_single( + fld, inputs_templ, output_dir=output_dir, spec_type="output" + ) + return value elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: diff --git a/pydra/engine/task.py b/pydra/engine/task.py index f4e732aa51..761e8262e8 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -471,7 +471,7 @@ def cmdline(self): # checking the inputs fields before returning the command line self.inputs.check_fields_input_spec() orig_inputs = attr.asdict(self.inputs) - modified_inputs = template_update(self.inputs) + modified_inputs = template_update(self.inputs, output_dir=self.output_dir) if modified_inputs is not None: self.inputs = attr.evolve(self.inputs, **modified_inputs) if isinstance(self, ContainerTask): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 7db1c5f77a..ba00890035 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -914,6 +914,8 @@ def test_shell_cmd_inputspec_7(plugin, results_function): res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() + # checking if the file is created in a good place + assert shelly.output_dir == res.output.out1.parent assert res.output.out1.name == "newfile_tmp.txt" @@ -1117,7 +1119,7 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): the template has a suffix, the extension of the file will be moved to the end """ cmd = "cp" - file = tmpdir.join("file.txt") + file = tmpdir.mkdir("data_inp").join("file.txt") file.write("content") my_input_spec = SpecInfo( @@ -1153,6 +1155,8 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file_copy.txt" + # checking if it's created in a good place + assert shelly.output_dir == res.output.file_copy.parent @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 061eec0220..8f2f30fab4 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -570,7 +570,8 @@ def test_shell_cmd_inputs_template_1(): executable="executable", input_spec=my_input_spec, inpA="inpA" ) # outA has argstr in the metadata fields, so it's a part of the command line - assert shelly.cmdline == "executable inpA -o inpA_out" + # the full path will be use din the command line + assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" # checking if outA in the output fields assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] @@ -610,9 +611,10 @@ def test_shell_cmd_inputs_template_1a(): executable="executable", input_spec=my_input_spec, inpA="inpA" ) # outA has no argstr in metadata, so it's not a part of the command line - assert shelly.cmdline == "executable inpA" + assert shelly.cmdline == f"executable inpA" +# TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_template_2(): """ additional inputs, one uses output_file_template (and argstr, but input not provided)""" my_input_spec = SpecInfo( @@ -719,7 +721,10 @@ def test_shell_cmd_inputs_template_3(): executable="executable", input_spec=my_input_spec, inpA="inpA", inpB="inpB" ) # using syntax from the outAB field - assert shelly.cmdline == "executable inpA inpB -o inpA_out inpB_out" + assert ( + shelly.cmdline + == f"executable inpA inpB -o {str(shelly.output_dir / 'inpA_out')} {str(shelly.output_dir / 'inpB_out')}" + ) # checking if outA and outB in the output fields (outAB should not be) assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] @@ -796,11 +801,15 @@ def test_shell_cmd_inputs_template_3a(): executable="executable", input_spec=my_input_spec, inpA="inpA", inpB="inpB" ) # using syntax from the outAB field - assert shelly.cmdline == "executable inpA inpB -o inpA_out inpB_out" + assert ( + shelly.cmdline + == f"executable inpA inpB -o {str(shelly.output_dir / 'inpA_out')} {str(shelly.output_dir / 'inpB_out')}" + ) # checking if outA and outB in the output fields (outAB should not be) assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] +# TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_template_4(): """ additional inputs with output_file_template and an additional read-only fields that combine two outputs together in the command line @@ -868,7 +877,7 @@ def test_shell_cmd_inputs_template_4(): executable="executable", input_spec=my_input_spec, inpA="inpA" ) # inpB is not provided so outB not in the command line - assert shelly.cmdline == "executable inpA -o inpA_out" + assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] @@ -943,7 +952,7 @@ def test_shell_cmd_inputs_template_6(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA" ) - assert shelly.cmdline == "executable inpA -o inpA_out" + assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" # a string is provided for outA, so this should be used as the outA value shelly = ShellCommandTask( @@ -955,7 +964,7 @@ def test_shell_cmd_inputs_template_6(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True ) - assert shelly.cmdline == "executable inpA -o inpA_out" + assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" # False is provided for outA, so the outA shouldn't be used shelly = ShellCommandTask( @@ -1017,7 +1026,7 @@ def test_shell_cmd_inputs_template_6a(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True ) - assert shelly.cmdline == "executable inpA -o inpA_out" + assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" # False is provided for outA, so the outA shouldn't be used shelly = ShellCommandTask( @@ -1070,7 +1079,7 @@ def test_shell_cmd_inputs_template_7(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out.txt')}" ) @@ -1119,7 +1128,7 @@ def test_shell_cmd_inputs_template_7a(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out.txt')}" ) @@ -1168,7 +1177,7 @@ def test_shell_cmd_inputs_template_7b(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {tmpdir.join('a_file_out')}" + == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out')}" ) @@ -1214,10 +1223,11 @@ def test_shell_cmd_inputs_template_8(tmpdir): # outA should be formatted in a way that inpA extension is removed and the template extension is used assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.t')} {tmpdir.join('a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.t')} {str(shelly.output_dir / 'a_file_out.txt')}" ) +# TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_di(tmpdir, use_validator): """ example from #279 """ my_input_spec = SpecInfo( @@ -1413,7 +1423,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{str(shelly.output_dir / 'a_file_out.ext')}]" ) # input file name, noiseImage is set to True, so template is used in the utput From bcb1f1924b6a97e984be0013ee6f9098b6ae63fe Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 1 Sep 2020 20:05:57 -0400 Subject: [PATCH 110/271] adding an option for OR in requires field for output (list of list can be provided instead of list of strings/tuples) --- pydra/engine/specs.py | 100 +++++++++---- pydra/engine/tests/test_shelltask.py | 209 +++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 32 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 9899d11498..1246a1d073 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -450,38 +450,9 @@ def _field_defaultvalue(self, fld, output_dir): def _field_metadata(self, fld, inputs, output_dir): """Collect output file if metadata specified.""" - if "requires" in fld.metadata: - field_required = fld.metadata["requires"] - else: - field_required = [] - # if the output has output_file_template field, adding all input fields from the template to requires - if "output_file_template" in fld.metadata: - inp_fields = re.findall("{\w+}", fld.metadata["output_file_template"]) - field_required += [ - el[1:-1] for el in inp_fields if el[1:-1] not in field_required - ] - # checking if the input fields from requires have set values - for inp in field_required: - if isinstance(inp, str): # name of the input field - if not hasattr(inputs, inp): - raise Exception( - f"{inp} is not a valid input field, can't be used in requires" - ) - elif getattr(inputs, inp) in [attr.NOTHING, None]: - return attr.NOTHING - elif isinstance(inp, tuple): # (name, allowed values) - inp, allowed_val = inp - if not hasattr(inputs, inp): - raise Exception( - f"{inp} is not a valid input field, can't be used in requires" - ) - elif getattr(inputs, inp) not in allowed_val: - return attr.NOTHING - else: - raise Exception( - f"each element of the requires should be a string or a tuple, " - f"but {inp} is found" - ) + if self._check_requires(fld, inputs) is False: + return attr.NOTHING + if "value" in fld.metadata: return output_dir / fld.metadata["value"] # this block is only run if "output_file_template" is provided in output_spec @@ -496,6 +467,71 @@ def _field_metadata(self, fld, inputs, output_dir): else: raise Exception("(_field_metadata) is not a current valid metadata key.") + def _check_requires(self, fld, inputs): + """ checking if all fields from the requires and template are set in the input + if requires is a list of list, checking if at least one list has all elements set + """ + if "requires" in fld.metadata: + # if requires is a list of list it is treated as el[0] OR el[1] OR... + if all([isinstance(el, list) for el in fld.metadata["requires"]]): + field_required_OR = fld.metadata["requires"] + # if requires is a list of tuples/strings - I'm creating a 1-el nested list + elif all([isinstance(el, (str, tuple)) for el in fld.metadata["requires"]]): + field_required_OR = [fld.metadata["requires"]] + else: + raise Exception( + f"requires field can be a list of list, or a list " + f"of strings/tuples, but {fld.metadata['requires']} " + f"provided for {fld.name}" + ) + else: + field_required_OR = [[]] + + for field_required in field_required_OR: + # if the output has output_file_template field, adding all input fields from the template to requires + if "output_file_template" in fld.metadata: + inp_fields = re.findall("{\w+}", fld.metadata["output_file_template"]) + field_required += [ + el[1:-1] for el in inp_fields if el[1:-1] not in field_required + ] + + # it's a flag, of the field from the list is not in input it will be changed to False + required_found = True + for field_required in field_required_OR: + required_found = True + # checking if the input fields from requires have set values + for inp in field_required: + if isinstance(inp, str): # name of the input field + if not hasattr(inputs, inp): + raise Exception( + f"{inp} is not a valid input field, can't be used in requires" + ) + elif getattr(inputs, inp) in [attr.NOTHING, None]: + required_found = False + break + elif isinstance(inp, tuple): # (name, allowed values) + inp, allowed_val = inp + if not hasattr(inputs, inp): + raise Exception( + f"{inp} is not a valid input field, can't be used in requires" + ) + elif getattr(inputs, inp) not in allowed_val: + required_found = False + break + else: + raise Exception( + f"each element of the requires element should be a string or a tuple, " + f"but {inp} is found in {field_required}" + ) + # if the specific list from field_required_OR has all elements set, no need to check more + if required_found: + break + + if required_found: + return True + else: + return False + @attr.s(auto_attribs=True, kw_only=True) class ContainerSpec(ShellSpec): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index e7fd91d684..38456b307d 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2926,6 +2926,215 @@ def test_shell_cmd_inputspec_outputspec_4a(): assert res.output.newfile1 is attr.NOTHING +def test_shell_cmd_inputspec_outputspec_5(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires + requires is a list of list so it is treated as OR list (i.e. el[0] OR el[1] OR...) + the firs element of the requires list has all the fields set + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp_A", str, {"help_string": "additional inp A"}), + ("additional_inp_B", str, {"help_string": "additional inp B"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + "requires": [ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.additional_inp_A = 2 + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + + +def test_shell_cmd_inputspec_outputspec_5a(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires + requires is a list of list so it is treated as OR list (i.e. el[0] OR el[1] OR...) + the second element of the requires list (i.e. additional_inp_B) has all the fields set + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp_A", str, {"help_string": "additional inp A"}), + ("additional_inp_B", str, {"help_string": "additional inp B"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + "requires": [ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + shelly.inputs.additional_inp_B = 2 + + res = shelly() + assert res.output.stdout == "" + assert res.output.newfile1.exists() + + +def test_shell_cmd_inputspec_outputspec_5b(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires + requires is a list of list so it is treated as OR list (i.e. el[0] OR el[1] OR...) + neither of the list from requirements has all the fields set, so the output is NOTHING + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp_A", str, {"help_string": "additional inp A"}), + ("additional_inp_B", str, {"help_string": "additional inp B"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + "requires": [ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + + res = shelly() + assert res.output.stdout == "" + # neither additional_inp_A nor additional_inp_B is set, so newfile1 is NOTHING + assert res.output.newfile1 is attr.NOTHING + + +def test_shell_cmd_inputspec_outputspec_6_except(): + """ + customised input_spec and output_spec, output_spec uses input_spec fields in the requires + requires has invalid syntax - exception is raised + """ + cmd = ["touch", "newfile_tmp.txt"] + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file1", + str, + {"help_string": "1st creadted file", "argstr": "", "position": 1}, + ), + ("additional_inp_A", str, {"help_string": "additional inp A"}), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "newfile1", + File, + { + "output_file_template": "{file1}", + "help_string": "newfile 1", + # requires has invalid syntax + "requires": [["file1", "additional_inp_A"], "file1"], + }, + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + ) + shelly.inputs.file1 = "new_file_1.txt" + + with pytest.raises(Exception, match="requires field can be"): + res = shelly() + + def no_fsl(): if "FSLDIR" not in os.environ: return True From f3eba8774f24174052f5aa72b978cf151ffe8c43 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 2 Sep 2020 00:54:41 -0400 Subject: [PATCH 111/271] fixing types, fixing tests --- pydra/engine/helpers_file.py | 8 +++++--- pydra/engine/specs.py | 2 +- pydra/engine/tests/test_shelltask.py | 12 +++++++++++- pydra/engine/tests/test_shelltask_inputspec.py | 8 ++++---- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 06106d0640..09e69badd9 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -576,9 +576,11 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input template, inputs_dict, keep_extension=keep_extension ) # changing path so it is in the output_dir - if output_dir: - value = output_dir / Path(value).name - return value + if output_dir and value is not attr.NOTHING: + # should be converted to str, it is also used for input fields that should be str + return str(output_dir / Path(value).name) + else: + return value def _template_formatting(template, inputs_dict, keep_extension=True): diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 5dbcb8e4ef..190a297845 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -463,7 +463,7 @@ def _field_metadata(self, fld, inputs, output_dir): value = template_update_single( fld, inputs_templ, output_dir=output_dir, spec_type="output" ) - return value + return Path(value) elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 74114f9d91..b8938910a7 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -954,6 +954,9 @@ def test_shell_cmd_inputspec_7a(plugin, results_function): res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1_changed.exists() + # checking if the file is created in a good place + assert shelly.output_dir == res.output.out1_changed.parent + assert res.output.out1_changed.name == "newfile_tmp.txt" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -1250,6 +1253,7 @@ def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file" + assert res.output.file_copy.parent == shelly.output_dir @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -1623,6 +1627,7 @@ def test_shell_cmd_inputspec_state_2(plugin, results_function): for i in range(len(args)): assert res[i].output.stdout == "" assert res[i].output.out1.exists() + assert res[i].output.out1.parent == shelly.output_dir[i] @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -1780,6 +1785,7 @@ def test_wf_shell_cmd_2(plugin_dask_opt): res = wf.result() assert res.output.out == "" assert res.output.out_f.exists() + assert res.output.out_f.parent == wf.output_dir def test_wf_shell_cmd_2a(plugin): @@ -1917,8 +1923,10 @@ def test_wf_shell_cmd_3(plugin): res = wf.result() assert res.output.out1 == "" assert res.output.touch_file.exists() + assert res.output.touch_file.parent == wf.output_dir assert res.output.out2 == "" assert res.output.cp_file.exists() + assert res.output.cp_file.parent == wf.output_dir def test_wf_shell_cmd_3a(plugin): @@ -2103,11 +2111,13 @@ def test_wf_shell_cmd_state_1(plugin): wf(submitter=sub) res_l = wf.result() - for res in res_l: + for i, res in enumerate(res_l): assert res.output.out1 == "" assert res.output.touch_file.exists() + assert res.output.touch_file.parent == wf.output_dir[i] assert res.output.out2 == "" assert res.output.cp_file.exists() + assert res.output.cp_file.parent == wf.output_dir[i] def test_wf_shell_cmd_ndst_1(plugin): diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 8f2f30fab4..e43cb445cf 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1434,8 +1434,8 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): noiseImage=True, ) assert ( - shelly.cmdline - == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}, {tmpdir.join('a_file_noise.ext')}]" + shelly.cmdline == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 " + f"-o [{str(shelly.output_dir / 'a_file_out.ext')}, {str(shelly.output_dir / 'a_file_noise.ext')}]" ) # input file name and help_short @@ -1447,7 +1447,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -h -o [{tmpdir.join('a_file_out.ext')}]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -h -o [{str(shelly.output_dir / 'a_file_out.ext')}]" ) assert shelly.output_names == [ @@ -1467,7 +1467,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -d 2 -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{tmpdir.join('a_file_out.ext')}]" + == f"DenoiseImage -d 2 -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{str(shelly.output_dir / 'a_file_out.ext')}]" ) # adding image_dimensionality that has allowed_values [2, 3, 4] and providing 5 - exception should be raised From dfe8e6d8a59a5dbce3537a2fc7deac20e0722b12 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 3 Sep 2020 21:53:44 +0800 Subject: [PATCH 112/271] fix pytest cov.xml save path in testallowfail.yml --- .github/workflows/testallowfail.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml index ba8d414546..9f96e09c28 100644 --- a/.github/workflows/testallowfail.yml +++ b/.github/workflows/testallowfail.yml @@ -32,6 +32,6 @@ jobs: - name: Install Pydra (test) run: pip install -e ".[test]" - name: Pytest - run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules pydra + run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - name: Upload to codecov - run: codecov --root /pydra -f /pydra/cov.xml -F unittests -e GITHUB_WORKFLOW + run: codecov --root /pydra -f cov.xml -F unittests -e GITHUB_WORKFLOW From 5d615fc21b01867cfc5fe07bcddf3aee2e178787 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 3 Sep 2020 22:34:15 +0800 Subject: [PATCH 113/271] rename github action workflows --- .github/workflows/testallowfail.yml | 2 +- .github/workflows/testpydra.yml | 3 +-- .github/workflows/testsingularity.yml | 2 +- .github/workflows/testslurm.yml | 3 +-- .github/workflows/teststyle.yml | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml index 9f96e09c28..a9dd5d95f8 100644 --- a/.github/workflows/testallowfail.yml +++ b/.github/workflows/testallowfail.yml @@ -1,6 +1,6 @@ # Test on an older version of pip -name: Test Pydra (allow failures) +name: Allow failure on: [push, pull_request] diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 4a3803b772..115be2f79b 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -1,4 +1,4 @@ -name: Test Pydra +name: Pydra on: [push, pull_request] @@ -40,7 +40,6 @@ jobs: run: | python setup.py bdist_wheel pip install dist/*.whl - shell: bash - name: Install Pydra (min-requirements.txt or setup.py install) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index b553cd9556..b4f1b17cb6 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -1,4 +1,4 @@ -name: Test Pydra (singularity) +name: Singularity on: [push, pull_request] diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index ae8fad5c48..67acb9c92d 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -1,11 +1,10 @@ -name: Test Pydra (SLURM) +name: SLURM on: [push, pull_request] jobs: build: runs-on: ubuntu-latest - continue-on-error: true env: DOCKER_IMAGE: mgxd/slurm:19.05.1 diff --git a/.github/workflows/teststyle.yml b/.github/workflows/teststyle.yml index 806ea096d3..e8df9fb7a9 100644 --- a/.github/workflows/teststyle.yml +++ b/.github/workflows/teststyle.yml @@ -1,4 +1,4 @@ -name: Test Pydra (Style) +name: Style on: [push, pull_request] From 475910d685577250ecd85c9efca4ca1a219bb0f5 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Sat, 5 Sep 2020 18:49:31 +0800 Subject: [PATCH 114/271] add action to publish on pypi --- .github/workflows/publish.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..8e1d277e8d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,34 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload to PyPI + +on: + release: + types: [created] + push: + tags: + - '*' + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py bdist_wheel + twine upload --repository testpypi dist/* From 3145173921b9556b57fb0c625844a100b89854b6 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 10 Sep 2020 18:01:49 +0800 Subject: [PATCH 115/271] publish to pypi instead of test pypi --- .github/workflows/publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e1d277e8d..bc2335f5f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,9 +6,6 @@ name: Upload to PyPI on: release: types: [created] - push: - tags: - - '*' jobs: deploy: @@ -31,4 +28,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py bdist_wheel - twine upload --repository testpypi dist/* + twine upload dist/* From 93fe37bca2a5df7524456d6a128866271eb63759 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 10 Sep 2020 21:58:45 +0800 Subject: [PATCH 116/271] delete /ci folder --- ci/none.sh | 64 ------------------------------ ci/osx.sh | 39 ------------------ ci/singularity.sh | 36 ----------------- ci/slurm.sh | 39 ------------------ ci/slurm/Dockerfile | 13 ------ ci/slurm/slurm.conf | 96 --------------------------------------------- ci/win37.sh | 37 ----------------- 7 files changed, 324 deletions(-) delete mode 100644 ci/none.sh delete mode 100644 ci/osx.sh delete mode 100644 ci/singularity.sh delete mode 100644 ci/slurm.sh delete mode 100644 ci/slurm/Dockerfile delete mode 100644 ci/slurm/slurm.conf delete mode 100644 ci/win37.sh diff --git a/ci/none.sh b/ci/none.sh deleted file mode 100644 index 01fbb20dab..0000000000 --- a/ci/none.sh +++ /dev/null @@ -1,64 +0,0 @@ -# local build environment - -function travis_before_install { - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh); - travis_retry python -m pip install --upgrade $INSTALL_DEPENDS -} - -function travis_install { - if [ "$CHECK_TYPE" = "test" ]; then - if [ "$INSTALL_TYPE" = "pip" ]; then - pip install $PIP_ARGS . - elif [ "$INSTALL_TYPE" = "install" ]; then - python setup.py install - elif [ "$INSTALL_TYPE" = "develop" ]; then - python setup.py develop - elif [ "$INSTALL_TYPE" = "sdist" ]; then - python setup.py sdist - pip install dist/*.tar.gz - elif [ "$INSTALL_TYPE" = "wheel" ]; then - python setup.py bdist_wheel - pip install dist/*.whl - fi - # Verify import with bare install - python -c 'import pydra; print(pydra.__version__)' - fi -} - -function travis_before_script { - if [ "$CHECK_TYPE" = "test" ]; then - # Install test dependencies using similar methods... - # Extras are interpreted by pip, not setup.py, so develop becomes editable - # and install just becomes pip - if [ "$INSTALL_TYPE" = "develop" ]; then - pip install -e ".[test]" - elif [ "$INSTALL_TYPE" = "sdist" ]; then - pip install "$( ls dist/pydra*.tar.gz )[test]" - elif [ "$INSTALL_TYPE" = "wheel" ]; then - pip install "$( ls dist/pydra*.whl )[test]" - else - # extras don't seem possible with setup.py install, so switch to pip - pip install ".[test]" - fi - elif [ "$CHECK_TYPE" = "test_dask" ]; then - if [ "$INSTALL_TYPE" = "develop" ]; then - pip install -e ".[dask]" - fi - elif [ "$CHECK_TYPE" = "style" ]; then - pip install black==19.3b0 - fi -} - -function travis_script { - if [ "$CHECK_TYPE" = "test" ]; then - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - elif [ "$CHECK_TYPE" = "test_dask" ]; then - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules --dask pydra/engine - elif [ "$CHECK_TYPE" = "style" ]; then - black --check pydra tools setup.py - fi -} - -function travis_after_script { - codecov --file cov.xml --flags unittests -e TRAVIS_JOB_NUMBER -} diff --git a/ci/osx.sh b/ci/osx.sh deleted file mode 100644 index 69d88203e2..0000000000 --- a/ci/osx.sh +++ /dev/null @@ -1,39 +0,0 @@ -# local build environment - -function travis_before_install { - wget https://raw.githubusercontent.com/mjirik/discon/master/tools/install_conda.sh && source install_conda.sh; - conda config --add channels conda-forge; - conda update -q conda; - conda create --yes -n travis python=$CONDA_VERSION; - source activate travis; - travis_retry python -m pip install --upgrade $INSTALL_DEPENDS - echo "my python pip" - echo $(python --version) - echo $(pip --version) -} - -function travis_install { - if [ "$CHECK_TYPE" = "test" ]; then - echo "Hello from pip" - pip install $PIP_ARGS . - # Verify import with bare install - python -c 'import pydra; print(pydra.__version__)' - fi -} - -function travis_before_script { - if [ "$CHECK_TYPE" = "test" ]; then - # Install test dependencies using similar methods... - pip install ".[test]" - fi -} - -function travis_script { - if [ "$CHECK_TYPE" = "test" ]; then - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - fi -} - -function travis_after_script { - codecov --file cov.xml --flags unittests -e TRAVIS_JOB_NUMBER -} diff --git a/ci/singularity.sh b/ci/singularity.sh deleted file mode 100644 index 86e0dea131..0000000000 --- a/ci/singularity.sh +++ /dev/null @@ -1,36 +0,0 @@ -# local build environment - -function travis_before_install { - sudo apt-get update; - sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ - libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y; - export VERSION=3.5.0; - travis_retry wget -q https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz; - tar -xzf singularity-${VERSION}.tar.gz; - cd singularity; - ./mconfig; - make -C ./builddir; - sudo make -C ./builddir install; - cd -; - travis_retry wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh; - bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda; - eval "$($HOME/miniconda/bin/conda shell.bash hook)" -} - -function travis_install { - python setup.py develop - # Verify import with bare install - python -c 'import pydra; print(pydra.__version__)' -} - -function travis_before_script { - pip install -e ".[test]" -} - -function travis_script { - pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra -} - -function travis_after_script { - codecov --file cov.xml --flags unittests -e TRAVIS_JOB_NUMBER -} diff --git a/ci/slurm.sh b/ci/slurm.sh deleted file mode 100644 index 6f00ed8dc1..0000000000 --- a/ci/slurm.sh +++ /dev/null @@ -1,39 +0,0 @@ -# docker environment for slurm worker testing - -export DOCKER_IMAGE="mgxd/slurm:19.05.1" - -function travis_before_install { - CI_ENV=`bash <(curl -s https://codecov.io/env)` - docker pull ${DOCKER_IMAGE} - # have image running in background - docker run $CI_ENV -itd -h ernie --name slurm -v `pwd`:/pydra ${DOCKER_IMAGE} - echo "Allowing ports/daemons time to start" && sleep 10 - # ensure sacct displays previous jobs - # https://github.com/giovtorres/docker-centos7-slurm/issues/3 - docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ - && supervisorctl restart slurmdbd \ - && supervisorctl restart slurmctld \ - && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" - docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null - if [ $? -ne 0 ]; then - echo "Slurm docker image error" - exit 1 - fi -} - -function travis_install { - docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" -} - -function travis_before_script { - : -} - -function travis_script { - docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" -} - -function travis_after_script { - docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" - docker rm -f slurm -} diff --git a/ci/slurm/Dockerfile b/ci/slurm/Dockerfile deleted file mode 100644 index 090b357414..0000000000 --- a/ci/slurm/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM giovtorres/docker-centos7-slurm:19.05.1 - -ENV CONDAPATH="~/miniconda" -# install miniconda3 -RUN wget -q http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ - && bash ~/miniconda.sh -b -p ${CONDAPATH} -ENV PATH="${CONDAPATH}/bin:${PATH}" -RUN conda update -yq --all conda \ - && conda clean -tipy \ - && conda config --set always_yes yes \ - && conda config --add channels conda-forge \ - && rm ~/miniconda.sh -COPY ./slurm.conf /etc/slurm/slurm.conf diff --git a/ci/slurm/slurm.conf b/ci/slurm/slurm.conf deleted file mode 100644 index 5baed79d1a..0000000000 --- a/ci/slurm/slurm.conf +++ /dev/null @@ -1,96 +0,0 @@ -# slurm.conf -# -# See the slurm.conf man page for more information. -# -ClusterName=linux -ControlMachine=ernie -#ControlAddr= -#BackupController= -#BackupAddr= -# -SlurmUser=slurm -#SlurmdUser=root -SlurmctldPort=6817 -SlurmdPort=6818 -AuthType=auth/munge -#JobCredentialPrivateKey= -#JobCredentialPublicCertificate= -StateSaveLocation=/var/lib/slurmd -SlurmdSpoolDir=/var/spool/slurmd -SwitchType=switch/none -MpiDefault=none -SlurmctldPidFile=/var/run/slurmd/slurmctld.pid -SlurmdPidFile=/var/run/slurmd/slurmd.pid -ProctrackType=proctrack/pgid -#PluginDir= -CacheGroups=0 -#FirstJobId= -ReturnToService=0 -#MaxJobCount= -#PlugStackConfig= -#PropagatePrioProcess= -#PropagateResourceLimits= -#PropagateResourceLimitsExcept= -#Prolog= -#Epilog= -#SrunProlog= -#SrunEpilog= -#TaskProlog= -#TaskEpilog= -#TaskPlugin= -#TrackWCKey=no -#TreeWidth=50 -#TmpFS= -#UsePAM= -# -# TIMERS -SlurmctldTimeout=300 -SlurmdTimeout=300 -InactiveLimit=0 -MinJobAge=300 -KillWait=30 -Waittime=0 -# -# SCHEDULING -SchedulerType=sched/backfill -#SchedulerAuth= -#SchedulerPort= -#SchedulerRootFilter= -SelectType=select/cons_res -SelectTypeParameters=CR_CPU_Memory -FastSchedule=1 -#PriorityType=priority/multifactor -#PriorityDecayHalfLife=14-0 -#PriorityUsageResetPeriod=14-0 -#PriorityWeightFairshare=100000 -#PriorityWeightAge=1000 -#PriorityWeightPartition=10000 -#PriorityWeightJobSize=1000 -#PriorityMaxAge=1-0 -# -# LOGGING -SlurmctldDebug=3 -SlurmctldLogFile=/var/log/slurm/slurmctld.log -SlurmdDebug=3 -SlurmdLogFile=/var/log/slurm/slurmd.log -JobCompType=jobcomp/none -#JobCompLoc= -# -# ACCOUNTING -#JobAcctGatherType=jobacct_gather/linux -#JobAcctGatherFrequency=30 -# -AccountingStorageType=accounting_storage/slurmdbd -#AccountingStorageHost=localhost -#AccountingStorageLoc= -#AccountingStoragePass= -#AccountingStorageUser= -# -# COMPUTE NODES -GresTypes=gpu -NodeName=c[1-40] NodeHostName=localhost NodeAddr=127.0.0.1 RealMemory=1000 -NodeName=c[41-44] NodeHostName=localhost NodeAddr=127.0.0.1 RealMemory=1000 Gres=gpu:titanxp:1 -# -# PARTITIONS -PartitionName=normal Default=yes Nodes=c[1-40] Priority=50 DefMemPerCPU=500 Shared=NO MaxNodes=5 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP -PartitionName=debug Nodes=c[41-44] Priority=50 DefMemPerCPU=500 Shared=NO MaxNodes=5 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP diff --git a/ci/win37.sh b/ci/win37.sh deleted file mode 100644 index a4e1ec3dee..0000000000 --- a/ci/win37.sh +++ /dev/null @@ -1,37 +0,0 @@ -# local build environment - -function travis_before_install { - choco install python --version 3.7; - export PATH="/c/Python37:/c/Python37/Scripts:$PATH"; - python -m pip install --upgrade pip; - pip install --upgrade pip; - virtualenv $HOME/venv; - source $HOME/venv/Scripts/activate; - travis_retry python -m pip install --upgrade $INSTALL_DEPENDS -} - -function travis_install { - if [ "$CHECK_TYPE" = "test" ]; then - echo "Hello from pip" - pip install $PIP_ARGS . - # Verify import with bare install - python -c 'import pydra; print(pydra.__version__)' - fi -} - -function travis_before_script { - if [ "$CHECK_TYPE" = "test" ]; then - # Install test dependencies using similar methods... - pip install ".[test]" - fi -} - -function travis_script { - if [ "$CHECK_TYPE" = "test" ]; then - pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - fi -} - -function travis_after_script { - codecov --file cov.xml --flags unittests -e TRAVIS_JOB_NUMBER -} From 946a2f813cc5dcccc08b47a2910a7cc23d1dc6de Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 12 Sep 2020 10:57:49 -0400 Subject: [PATCH 117/271] setting full output_spec during task initialization, using task output_spec to set proper workflow output_spec in wf.set_output --- pydra/engine/core.py | 27 ++++++++++++++++++++------- pydra/engine/helpers.py | 18 +++++++----------- pydra/engine/task.py | 9 ++++++++- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index b04a50eafe..90bd3d3451 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -34,7 +34,6 @@ ensure_list, record_error, hash_function, - output_from_inputfields, ) from .helpers_file import copyfile_input, template_update from .graph import DiGraph @@ -312,7 +311,7 @@ def set_state(self, splitter, combiner=None): @property def output_names(self): """Get the names of the outputs generated by the task.""" - return output_from_inputfields(self.output_spec, self.inputs, names_only=True) + return [f.name for f in attr.fields(make_klass(self.output_spec))] @property def can_resume(self): @@ -411,7 +410,7 @@ def _run(self, rerun=False, **kwargs): try: self.audit.monitor() self._run_task() - result.output = self._collect_outputs() + result.output = self._collect_outputs(output_dir=odir) except Exception as e: record_error(self.output_dir, e) result.errored = True @@ -429,12 +428,11 @@ def _run(self, rerun=False, **kwargs): self.hooks.post_run(self, result) return result - def _collect_outputs(self): + def _collect_outputs(self, output_dir): run_output = self.output_ - self.output_spec = output_from_inputfields(self.output_spec, self.inputs) output_klass = make_klass(self.output_spec) output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) - other_output = output.collect_additional_outputs(self.inputs, self.output_dir) + other_output = output.collect_additional_outputs(self.inputs, output_dir) return attr.evolve(output, **run_output, **other_output) def split(self, splitter, overwrite=False, **kwargs): @@ -996,7 +994,22 @@ def set_output(self, connections): ) self._connections += new_connections - fields = [(name, ty.Any) for name, _ in self._connections] + fields = [] + for con in self._connections: + wf_out_nm, lf = con + task_nm, task_out_nm = lf.name, lf.field + if task_out_nm == "all_": + help_string = f"all outputs from {task_nm}" + fields.append((wf_out_nm, dict, {"help_string": help_string})) + else: + # getting information about the output field from the task output_spec + # providing proper type and some help string + task_output_spec = getattr(self, task_nm).output_spec + out_fld = attr.fields_dict(make_klass(task_output_spec))[task_out_nm] + help_string = ( + f"{out_fld.metadata.get('help_string', '')} (from {task_nm})" + ) + fields.append((wf_out_nm, out_fld.type, {"help_string": help_string})) self.output_spec = SpecInfo(name="Output", fields=fields, bases=(BaseSpec,)) logger.info("Added %s to %s", self.output_spec, self) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index c1a30bf8de..a4decbb118 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -672,7 +672,7 @@ def hash_value(value, tp=None, metadata=None): return value -def output_from_inputfields(output_spec, inputs, names_only=False): +def output_from_inputfields(output_spec, input_spec): """ Collect values from output from input fields. If names_only is False, the output_spec is updated, @@ -682,30 +682,26 @@ def output_from_inputfields(output_spec, inputs, names_only=False): ---------- output_spec : TODO - inputs : + input_spec : TODO """ current_output_spec_names = [f.name for f in attr.fields(make_klass(output_spec))] new_fields = [] - for fld in attr_fields(inputs): + for fld in attr.fields(make_klass(input_spec)): if "output_file_template" in fld.metadata: - value = getattr(inputs, fld.name) if "output_field_name" in fld.metadata: field_name = fld.metadata["output_field_name"] else: field_name = fld.name # not adding if the field already in teh output_spec if field_name not in current_output_spec_names: + # TODO: should probably remove some of the keys new_fields.append( - (field_name, attr.ib(type=File, metadata={"value": value})) + (field_name, attr.ib(type=File, metadata=fld.metadata)) ) - if names_only: - new_names = [el[0] for el in new_fields] - return current_output_spec_names + new_names - else: - output_spec.fields += new_fields - return output_spec + output_spec.fields += new_fields + return output_spec def get_available_cpus(): diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 3a5f0d7d02..f86c91721f 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -57,7 +57,13 @@ SingularitySpec, attr_fields, ) -from .helpers import ensure_list, execute, position_adjustment, argstr_formatting +from .helpers import ( + ensure_list, + execute, + position_adjustment, + argstr_formatting, + output_from_inputfields, +) from .helpers_file import template_update, is_local_file @@ -276,6 +282,7 @@ def __init__( output_spec = SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,)) self.output_spec = output_spec + self.output_spec = output_from_inputfields(self.output_spec, self.input_spec) super().__init__( name=name, From 2bdb0a87a37ab24428dc49a2d90a4ccae13a478d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 12 Sep 2020 12:21:05 -0400 Subject: [PATCH 118/271] adding extra check to copyfile_workflow, adding tests for function with numpy output --- pydra/engine/helpers.py | 10 ++++++++-- pydra/engine/tests/test_numpy_examples.py | 16 ++++++++++++++++ pydra/engine/tests/utils.py | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index a4decbb118..975ef8e58d 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -155,8 +155,14 @@ def copyfile_workflow(wf_path, result): """ if file in the wf results, the file will be copied to the workflow directory""" for field in attr_fields(result.output): value = getattr(result.output, field.name) - new_value = _copyfile_single_value(wf_path=wf_path, value=value) - if new_value != value: + # if the field is a path or it can contain a path _copyfile_single_value is run + # to move all files and directories to the workflow directory + if field.type in [File, Directory, MultiOutputObj] or type(value) in [ + list, + tuple, + dict, + ]: + new_value = _copyfile_single_value(wf_path=wf_path, value=value) setattr(result.output, field.name, new_value) return result diff --git a/pydra/engine/tests/test_numpy_examples.py b/pydra/engine/tests/test_numpy_examples.py index 55884ac257..35b8972319 100644 --- a/pydra/engine/tests/test_numpy_examples.py +++ b/pydra/engine/tests/test_numpy_examples.py @@ -19,6 +19,22 @@ def arrayout(val): def test_multiout(plugin): """ testing a simple function that returns a numpy array""" + wf = Workflow("wf", input_spec=["val"], val=2) + wf.add(arrayout(name="mo", val=wf.lzin.val)) + + wf.set_output([("array", wf.mo.lzout.b)]) + + with Submitter(plugin=plugin, n_procs=2) as sub: + sub(runnable=wf) + + results = wf.result(return_inputs=True) + + assert results[0] == {"wf.val": 2} + assert np.array_equal(results[1].output.array, np.array([2, 2])) + + +def test_multiout_st(plugin): + """ testing a simple function that returns a numpy array, adding splitter""" wf = Workflow("wf", input_spec=["val"], val=[0, 1, 2]) wf.add(arrayout(name="mo", val=wf.lzin.val)) wf.mo.split("val").combine("val") diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index ad0878861b..b2fbdab762 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -164,7 +164,7 @@ def fun_dict(d): @mark.task -def fun_write_file(filename: ty.Union[str, File, Path], text="hello"): +def fun_write_file(filename: ty.Union[str, File, Path], text="hello") -> File: with open(filename, "w") as f: f.write(text) return Path(filename).absolute() From e040b155bab8a00582a27d7aec9677297fa300c7 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 13 Sep 2020 15:43:27 -0400 Subject: [PATCH 119/271] fixing tests with messengers, adding test that sets messenger for wf --- pydra/engine/tests/test_task.py | 89 ++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 9b28df1341..e75857a489 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -4,6 +4,7 @@ import pytest from ... import mark +from ..core import Workflow from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator @@ -972,15 +973,99 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] # saving the audit message into the file funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) + funky.cache_dir = tmpdir + funky() + # this should be the default loctaion message_path = tmpdir / funky.checksum / "messages" + assert (tmpdir / funky.checksum / "messages").exists() + + collect_messages(tmpdir / funky.checksum, message_path, ld_op="compact") + assert (tmpdir / funky.checksum / "messages.jsonld").exists() + + +def test_audit_prov_messdir_1(tmpdir, use_validator): + """customized messenger dir""" + + @mark.task + def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): + return a + b + + # printing the audit message + funky = testfunc(a=1, audit_flags=AuditFlag.PROV, messengers=PrintMessenger()) + funky.cache_dir = tmpdir + funky() + + # saving the audit message into the file + funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) + # user defined path + message_path = tmpdir / funky.checksum / "my_messages" funky.cache_dir = tmpdir - funky.messenger_args = dict(message_dir=message_path) + # providing messenger_dir for audit + funky.audit.messenger_args = dict(message_dir=message_path) funky() + assert (tmpdir / funky.checksum / "my_messages").exists() collect_messages(tmpdir / funky.checksum, message_path, ld_op="compact") assert (tmpdir / funky.checksum / "messages.jsonld").exists() +def test_audit_prov_messdir_2(tmpdir, use_validator): + """customized messenger dir in init""" + + @mark.task + def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): + return a + b + + # printing the audit message + funky = testfunc(a=1, audit_flags=AuditFlag.PROV, messengers=PrintMessenger()) + funky.cache_dir = tmpdir + funky() + + # user defined path (doesnt depend on checksum, can be defined before init) + message_path = tmpdir / "my_messages" + # saving the audit message into the file + funky = testfunc( + a=2, + audit_flags=AuditFlag.PROV, + messengers=FileMessenger(), + messenger_args=dict(message_dir=message_path), + ) + funky.cache_dir = tmpdir + # providing messenger_dir for audit + funky() + assert (tmpdir / "my_messages").exists() + + collect_messages(tmpdir, message_path, ld_op="compact") + assert (tmpdir / "messages.jsonld").exists() + + +def test_audit_prov_wf(tmpdir, use_validator): + """FileMessenger for wf""" + + @mark.task + def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): + return a + b + + wf = Workflow( + name="wf", + input_spec=["x"], + cache_dir=tmpdir, + audit_flags=AuditFlag.PROV, + messengers=FileMessenger(), + ) + wf.add(testfunc(name="testfunc", a=wf.lzin.x)) + wf.set_output([("out", wf.testfunc.lzout.out)]) + wf.inputs.x = 2 + + wf(plugin="cf") + # default path + message_path = tmpdir / wf.checksum / "messages" + assert message_path.exists() + + collect_messages(tmpdir / wf.checksum, message_path, ld_op="compact") + assert (tmpdir / wf.checksum / "messages.jsonld").exists() + + def test_audit_all(tmpdir, use_validator): @mark.task def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)]): @@ -989,7 +1074,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] funky = testfunc(a=2, audit_flags=AuditFlag.ALL, messengers=FileMessenger()) message_path = tmpdir / funky.checksum / "messages" funky.cache_dir = tmpdir - funky.messenger_args = dict(message_dir=message_path) + funky.audit.messenger_args = dict(message_dir=message_path) funky() from glob import glob From 727792048e8fe057c46776c06cef57e0d0e67f01 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 17 Sep 2020 22:07:47 +0800 Subject: [PATCH 120/271] update etelemetry to >=0.2.2 in setup.cfg --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 57ec811ebd..46910f819d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ install_requires = attrs >= 19.1.0 cloudpickle >= 1.2.2 filelock >= 3.0.0 - etelemetry >= 0.2.0 + etelemetry >= 0.2.2 test_requires = pytest >= 4.4.0 From 2e335d58a92f2320e8a050713a80020c44067d6e Mon Sep 17 00:00:00 2001 From: Nicole Lo Date: Fri, 18 Sep 2020 13:53:18 +0800 Subject: [PATCH 121/271] Update README.rst --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index b8eb9b033f..07cd101c0c 100644 --- a/README.rst +++ b/README.rst @@ -55,6 +55,14 @@ Installation pip install pydra +Note that installation fails with older versions of pip on Windows. Upgrade pip before installing: + +:: + + pip install –upgrade pip + pip install pydra + + Developer installation ====================== From ed13cabc273b3bda398dccbf62b56fd404cf9d29 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 18 Sep 2020 22:28:28 -0400 Subject: [PATCH 122/271] Update README.rst Adding GitHub Action Badge --- README.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 07cd101c0c..5e658866ad 100644 --- a/README.rst +++ b/README.rst @@ -5,15 +5,11 @@ A simple dataflow engine with scalable semantics. -|Build Status| -.. |Build Status| image:: https://travis-ci.org/nipype/pydra.svg?branch=master - :alt: Build Status +.. image:: https://github.com/nipype/pydra/workflows/Pydra/badge.svg + :alt: GitHub Actions CI + :target: https://github.com/nipype/Pydra/actions -|CircleCI| - -.. |CircleCI| image:: https://circleci.com/gh/nipype/pydra.svg?style=svg - :alt: CircleCI |codecov| From 7ea94774c2e541467b14de03bcc2a0d74443d1cf Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 18 Sep 2020 22:32:05 -0400 Subject: [PATCH 123/271] Update README.rst bringing back circleci badge --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 5e658866ad..a6b5bc3a8b 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,11 @@ A simple dataflow engine with scalable semantics. :alt: GitHub Actions CI :target: https://github.com/nipype/Pydra/actions +|CircleCI| + +.. |CircleCI| image:: https://circleci.com/gh/nipype/pydra.svg?style=svg + :alt: CircleCI + |codecov| From ce02168a88d8b093e272a985b83ae1c6d00e4439 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 19 Sep 2020 01:40:35 -0400 Subject: [PATCH 124/271] changing ShellCommandTask, so the child classes can have a class attribute for input/output_specs and executable/args; adding a new test file with tests related to the nipype1 interf conversion --- pydra/engine/task.py | 32 +++++-- pydra/engine/tests/test_nipype1_convert.py | 104 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 pydra/engine/tests/test_nipype1_convert.py diff --git a/pydra/engine/task.py b/pydra/engine/task.py index f86c91721f..ace102c029 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -43,6 +43,7 @@ import inspect import typing as ty from pathlib import Path +import warnings from .core import TaskBase, is_lazy from ..utils.messenger import AuditFlag @@ -275,15 +276,32 @@ def __init__( TODO """ - if input_spec is None: - input_spec = SpecInfo(name="Inputs", fields=[], bases=(ShellSpec,)) - self.input_spec = input_spec - if output_spec is None: - output_spec = SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,)) - - self.output_spec = output_spec + if input_spec is not None: + self.input_spec = input_spec + elif not hasattr(self, "input_spec"): + self.input_spec = SpecInfo(name="Inputs", fields=[], bases=(ShellSpec,)) + else: + # changing class attribute to instance attribute, so it is part of __dict__ + # (used in TaskBase.__set/get-state__) + self.input_spec = self.input_spec + if output_spec is not None: + self.output_spec = output_spec + elif not hasattr(self, "output_spec"): + self.output_spec = SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,)) + else: + # changing class attribute to instance attribute, so it is part of __dict__ + self.output_spec = self.output_spec self.output_spec = output_from_inputfields(self.output_spec, self.input_spec) + for special_inp in ["executable", "args"]: + if hasattr(self, special_inp): + if special_inp not in kwargs: + kwargs[special_inp] = getattr(self, special_inp) + elif kwargs[special_inp] != getattr(self, special_inp): + warnings.warn( + f"you are changing the executable from {getattr(self, special_inp)} to {kwargs[special_inp]}" + ) + super().__init__( name=name, inputs=kwargs, diff --git a/pydra/engine/tests/test_nipype1_convert.py b/pydra/engine/tests/test_nipype1_convert.py new file mode 100644 index 0000000000..2439049c31 --- /dev/null +++ b/pydra/engine/tests/test_nipype1_convert.py @@ -0,0 +1,104 @@ +import attr +import typing as ty +import os, sys +import pytest +from pathlib import Path + + +from ..task import ShellCommandTask +from ..submitter import Submitter +from ..core import Workflow +from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File +from .utils import result_no_submitter, result_submitter, use_validator + +interf_input_spec = SpecInfo( + name="Input", fields=[("test", ty.Any, {"help_string": "test"})], bases=(ShellSpec,) +) +interf_output_spec = SpecInfo( + name="Output", fields=[("test_out", File, "*.txt")], bases=(ShellOutSpec,) +) + + +class Interf_1(ShellCommandTask): + """class with customized input/output specs""" + + input_spec = interf_input_spec + output_spec = interf_output_spec + + +class Interf_2(ShellCommandTask): + """class with customized input/output specs and executables""" + + input_spec = interf_input_spec + output_spec = interf_output_spec + executable = "testing command" + + +class TouchInterf(ShellCommandTask): + """class with customized input and executables""" + + input_spec = SpecInfo( + name="Input", + fields=[ + ( + "new_file", + str, + { + "help_string": "new_file", + "argstr": "", + "output_file_template": "{new_file}", + }, + ) + ], + bases=(ShellSpec,), + ) + executable = "touch" + + +def test_interface_specs_1(): + """testing if class input/output spec are set properly""" + task = Interf_1(executable="ls") + assert task.input_spec == interf_input_spec + assert task.output_spec == interf_output_spec + + +def test_interface_specs_2(): + """testing if class input/output spec are overwritten properly by the user's specs""" + my_input_spec = SpecInfo( + name="Input", + fields=[("my_inp", ty.Any, {"help_string": "my inp"})], + bases=(ShellSpec,), + ) + my_output_spec = SpecInfo( + name="Output", fields=[("my_out", File, "*.txt")], bases=(ShellOutSpec,) + ) + task = Interf_1(input_spec=my_input_spec, output_spec=my_output_spec) + assert task.input_spec == my_input_spec + assert task.output_spec == my_output_spec + + +def test_interface_executable_1(): + """testing if the class executable is properly set and used in the command line""" + task = Interf_2() + assert task.executable == "testing command" + assert task.inputs.executable == "testing command" + assert task.cmdline == "testing command" + + +def test_interface_executable_2(): + """testing if the class executable is overwritten by the user's input (and if the warning is raised)""" + # warning that the user changes the executable from the one that is set as a class attribute + with pytest.warns(UserWarning, match="changing the executable"): + task = Interf_2(executable="i want a different command") + assert task.executable == "testing command" + # task.executable stays the same, but input.executable is changed, so the cmd is changed + assert task.inputs.executable == "i want a different command" + assert task.cmdline == "i want a different command" + + +def test_interface_run_1(): + """testing execution of a simple interf with customized input and executable""" + task = TouchInterf(new_file="hello.txt") + assert task.cmdline == "touch hello.txt" + res = task() + assert res.output.new_file.exists() From 1b7b4c81444b049f235e6197f0b38086139c2296 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 20 Sep 2020 22:49:29 -0400 Subject: [PATCH 125/271] improving syntax for input/output_spec --- pydra/engine/task.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index ace102c029..1d17b577f7 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -217,6 +217,9 @@ def _run_task(self): class ShellCommandTask(TaskBase): """Wrap a shell command as a task element.""" + input_spec = None + output_spec = None + def __new__(cls, container_info=None, *args, **kwargs): if not container_info: return super().__new__(cls) @@ -276,21 +279,18 @@ def __init__( TODO """ - if input_spec is not None: - self.input_spec = input_spec - elif not hasattr(self, "input_spec"): - self.input_spec = SpecInfo(name="Inputs", fields=[], bases=(ShellSpec,)) - else: - # changing class attribute to instance attribute, so it is part of __dict__ - # (used in TaskBase.__set/get-state__) - self.input_spec = self.input_spec - if output_spec is not None: - self.output_spec = output_spec - elif not hasattr(self, "output_spec"): - self.output_spec = SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,)) - else: - # changing class attribute to instance attribute, so it is part of __dict__ - self.output_spec = self.output_spec + + # using provided spec, class attribute or setting the default SpecInfo + self.input_spec = ( + input_spec + or self.input_spec + or SpecInfo(name="Inputs", fields=[], bases=(ShellSpec,)) + ) + self.output_spec = ( + output_spec + or self.output_spec + or SpecInfo(name="Output", fields=[], bases=(ShellOutSpec,)) + ) self.output_spec = output_from_inputfields(self.output_spec, self.input_spec) for special_inp in ["executable", "args"]: From e469555f7136b1543bd6fa366a37f23c0427496d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 21 Sep 2020 17:39:23 -0400 Subject: [PATCH 126/271] fixing xor and allowed_val checks --- pydra/engine/specs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 190a297845..1c950931f5 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -135,7 +135,7 @@ def check_fields_input_spec(self): # checking if fields meet the xor and requires are if "xor" in mdata: - if [el for el in mdata["xor"] if el in names]: + if [el for el in mdata["xor"] if (el in names and el != fld.name)]: raise AttributeError( f"{fld.name} is mutually exclusive with {mdata['xor']}" ) @@ -473,6 +473,8 @@ def _check_requires(self, fld, inputs): """ checking if all fields from the requires and template are set in the input if requires is a list of list, checking if at least one list has all elements set """ + from .helpers import ensure_list + if "requires" in fld.metadata: # if requires is a list of list it is treated as el[0] OR el[1] OR... if all([isinstance(el, list) for el in fld.metadata["requires"]]): @@ -512,7 +514,7 @@ def _check_requires(self, fld, inputs): required_found = False break elif isinstance(inp, tuple): # (name, allowed values) - inp, allowed_val = inp + inp, allowed_val = inp[0], ensure_list(inp[1]) if not hasattr(inputs, inp): raise Exception( f"{inp} is not a valid input field, can't be used in requires" From 0b0b68c3d68fe333fc831fe220dfde72356bff01 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 21 Sep 2020 23:58:08 -0400 Subject: [PATCH 127/271] editing input_spec, adding a short output_spec section --- docs/input_spec.rst | 30 +++++++++++++--- docs/output_spec.rst | 81 ++++++++++++++++++++++++++++++++++++++++++++ docs/user_guide.rst | 1 + 3 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 docs/output_spec.rst diff --git a/docs/input_spec.rst b/docs/input_spec.rst index 3fc8ac8216..a8454b3df4 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -35,14 +35,14 @@ Let's start from the previous example: In order to create an input specification, a new `SpecInfo` object has to be created. -The field `name` specifiest the typo of the spec and it should be always "Input" for +The field `name` specifies the type of the spec and it should be always "Input" for the input specification. The field `bases` specifies the "base specification" you want to use (can think about it as a `parent class`) and it will usually contains `ShellSpec` only, unless you want to build on top of your other specification (this will not be cover in this section). The part that should be always customised is the `fields` part. Each element of the `fields` is a separate input field that is added to the specification. -In this example, a three-elements tuples - with name, type and dictionary with additional +In this example, three-elements tuples - with name, type and dictionary with additional information - are used. But this is only one of the supported syntax, more options will be described below. @@ -86,9 +86,25 @@ However, we allow for shorter syntax, that does not include `attr.ib`: Each of the shorter versions will be converted to the `(name, attr.ib(...)`. + +Types +----- + Type can be provided as a simple python type (e.g. `str`, `int`, `float`, etc.) or can be more complex by using `typing.List`, `typing.Dict` and `typing.Union`. +There are also special types provided by Pydra: + +- `File` and `Directory` - should be used in `input_spec` if the field is an existing file + or directory. + Pydra checks if the file or directory exists, and returns an error if it doesn't exist. + + +- `MultiInputObj` - a special type that takes a any value and if the value is not a list it + converts value to a 1-element list (it could be used together with `MultiOutputObj` + in the `output_spec` to reverse the conversion of the output values). + + Metadata -------- @@ -126,9 +142,6 @@ In the example we used multiple keys in the metadata dictionary including `help_ `xor` (`list`): List of field names that are mutually exclusive with the field. -`keep_extension` (`bool`, default: `True`): - A flag that specifies if the file extension should be removed from the field value. - `copyfile` (`bool`, default: `False`): If `True`, a hard link is created for the input file in the output directory. If hard link not possible, the file is copied to the output directory. @@ -139,9 +152,16 @@ In the example we used multiple keys in the metadata dictionary including `help_ `output_file_template` (`str`): If provided, the field is treated also as an output field and it is added to the output spec. The template can use other fields, e.g. `{file1}`. + Used in order to create an output specification. `output_field_name` (`str`, used together with `output_file_template`) If provided the field is added to the output spec with changed name. + Used in order to create an output specification. + +`keep_extension` (`bool`, default: `True`): + A flag that specifies if the file extension should be removed from the field value. + Used in order to create an output specification. + `readonly` (`bool`, default: `False`): If `True` the input field can't be provided by the user but it aggregates other input fields diff --git a/docs/output_spec.rst b/docs/output_spec.rst new file mode 100644 index 0000000000..af0eacc366 --- /dev/null +++ b/docs/output_spec.rst @@ -0,0 +1,81 @@ +.. _Output Specification section: + +Output Specification +==================== + +As it was mentioned in :ref:`shell_command_task`, the user can customize the input and output +for the `ShellCommandTask`. +In this section, the output specification will be covered. + + +Instead of using field with `output_file_template` in the customized `input_spec` to specify an output field, +a customized `output_spec` can be used, e.g.: + + +.. code-block:: python + + output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out1", + attr.ib( + type=File, + metadata={ + "output_file_template": "{inp1}", + "help_string": "output file", + "requires": ["inp1", "inp2"] + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + ShellCommandTask(executable=executable, + output_spec=output_spec) + + + +Similarly as for `input_spec`, in order to create an output specification, +a new `SpecInfo` object has to be created. +The field `name` specifies the type of the spec and it should be always "Output" for +the output specification. +The field `bases` specifies the "base specification" you want to use (can think about it as a +`parent class`) and it will usually contains `ShellOutSpec` only, unless you want to build on top of +your other specification (this will not be cover in this section). +The part that should be always customised is the `fields` part. +Each element of the `fields` is a separate output field that is added to the specification. +In this example, a three-elements tuple - with name, type and dictionary with additional +information - is used. +See :ref:`Input Specification section` for other recognized syntax for specification's fields +and possible types. + + + +Metadata +-------- + +The metadata dictionary for `output_spec` can include: + +`help_string` (`str`, mandatory): + A short description of the input field. The same as in `input_spec`. + +`output_file_template` (`str`): + If provided, the field is treated also as an output field and it is added to the output spec. + The template can use other fields, e.g. `{file1}`. The same as in `input_spec`. + +`output_field_name` (`str`, used together with `output_file_template`) + If provided the field is added to the output spec with changed name. + The same as in `input_spec`. + +`keep_extension` (`bool`, default: `True`): + A flag that specifies if the file extension should be removed from the field value. + The same as in `input_spec`. + + +`requires` (`list`): + List of field names that are required to create a specific output. + The fields do not have to be a part of the `output_file_template` and + if any field from the list is not provided in the input, a `NOTHING` is returned for the specific output. + This has a different meaning than the `requires` form the `input_spec`. diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 642b85252a..bf48a9a8a9 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -9,3 +9,4 @@ User Guide state combiner input_spec + output_spec From 20b21b2fae860b5abe857279251749a0f5d3bddc Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 22 Sep 2020 15:34:48 +0800 Subject: [PATCH 128/271] disable etelemetry in testslurm and testsingularity --- .github/workflows/testsingularity.yml | 1 + .github/workflows/testslurm.yml | 5 ++++- pydra/__init__.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index b4f1b17cb6..7d17f7702b 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -15,6 +15,7 @@ jobs: - name: Set env run: | echo ::set-env name=RELEASE_VERSION::v3.5.0 + echo ::set-env name=NO_ET::TRUE - name: Setup Singularity uses: actions/checkout@v2 with: diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 67acb9c92d..1497b5b545 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -9,12 +9,14 @@ jobs: DOCKER_IMAGE: mgxd/slurm:19.05.1 steps: + - name: Disable etelemetry + run: echo ::set-env name=NO_ET::TRUE - uses: actions/checkout@v2 - name: Pull docker image run: | docker pull $DOCKER_IMAGE # Have image running in background - docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra $DOCKER_IMAGE + docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE - name: Display previous jobs with sacct run: | echo "Allowing ports/daemons time to start" && sleep 10 @@ -29,6 +31,7 @@ jobs: fi - name: Setup Python run: | + docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" diff --git a/pydra/__init__.py b/pydra/__init__.py index 233b2058bd..37e24116e5 100644 --- a/pydra/__init__.py +++ b/pydra/__init__.py @@ -22,9 +22,15 @@ def check_latest_version(): - import etelemetry + import os - return etelemetry.check_available_version("nipype/pydra", __version__, lgr=logger) + if "NO_ET" not in os.environ: + + import etelemetry + + return etelemetry.check_available_version( + "nipype/pydra", __version__, lgr=logger + ) # Run telemetry on import for interactive sessions, such as IPython, Jupyter notebooks, Python REPL From bcb647b1ec6f9dc20a55ea9dfe5b7da747a70b28 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 23 Sep 2020 02:02:40 -0400 Subject: [PATCH 129/271] editing checksum_states so it doesnt always run prepare_states (much faster), small edit to result so it ask for one checksum at a time --- pydra/engine/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 90bd3d3451..b9a57a7f36 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -262,8 +262,6 @@ def checksum_states(self, state_index=None): TODO """ - self.state.prepare_states(self.inputs) - self.state.prepare_inputs() if state_index is not None: inputs_copy = deepcopy(self.inputs) for key, ind in self.state.inputs_ind[state_index].items(): @@ -284,6 +282,9 @@ def checksum_states(self, state_index=None): return checksum_ind else: checksum_list = [] + if not hasattr(self.state, "inputs_ind"): + self.state.prepare_states(self.inputs) + self.state.prepare_inputs() for ind in range(len(self.state.inputs_ind)): checksum_list.append(self.checksum_states(state_index=ind)) return checksum_list @@ -611,7 +612,8 @@ def result(self, state_index=None, return_inputs=False): return self._combined_output(return_inputs=return_inputs) else: results = [] - for checksum in self.checksum_states(): + for ind in range(len(self.state.inputs_ind)): + checksum = self.checksum_states(state_index=ind) result = load_result(checksum, self.cache_locations) if result is None: return None From 5fbc228f19ef81ce29591d8a526322220934a024 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 23 Sep 2020 12:53:15 -0400 Subject: [PATCH 130/271] changing action version --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bc2335f5f1..0fabe4041d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '3.x' - name: Install dependencies From 12a43ca723593520de99c0b30539cd69a83fc3a3 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 23 Sep 2020 16:48:21 -0400 Subject: [PATCH 131/271] adding methods to ShellOutputSpec and TaskBase that allows to predic the generated output names before running the task --- pydra/engine/core.py | 20 +++++++++++++- pydra/engine/specs.py | 30 ++++++++++++++++++++- pydra/engine/tests/test_shelltask.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index b9a57a7f36..db650c2e5f 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -311,9 +311,27 @@ def set_state(self, splitter, combiner=None): @property def output_names(self): - """Get the names of the outputs generated by the task.""" + """Get the names of the outputs from the task's output_spec + (not everything has to be generated, see generated_output_names). + """ return [f.name for f in attr.fields(make_klass(self.output_spec))] + @property + def generated_output_names(self): + """ Get the names of the outputs generated by the task. + If the spec doesn't have generated_output_names method, + it uses output_names. + The results depends on the input provided to the task + """ + output_klass = make_klass(self.output_spec) + if hasattr(output_klass, "generated_output_names"): + output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) + return output.generated_output_names( + inputs=self.inputs, output_dir=self.output_dir + ) + else: + return self.output_names + @property def can_resume(self): """Whether the task accepts checkpoint-restart.""" diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 1c950931f5..08cf4d2419 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -408,7 +408,7 @@ def collect_additional_outputs(self, inputs, output_dir): raise AttributeError( "File has to have default value or metadata" ) - elif not fld.default == attr.NOTHING: + elif fld.default != attr.NOTHING: additional_out[fld.name] = self._field_defaultvalue( fld, output_dir ) @@ -420,6 +420,34 @@ def collect_additional_outputs(self, inputs, output_dir): raise Exception("not implemented (collect_additional_output)") return additional_out + def generated_output_names(self, inputs, output_dir): + """ Returns a list of all outputs that will be generated by the task. + Takes into account the task input and the requires list for the output fields. + TODO: should be in all Output specs? + """ + output_names = ["return_code", "stdout", "stderr"] + for fld in attr_fields(self): + if fld.name not in ["return_code", "stdout", "stderr"]: + if fld.type is File: + # assuming that field should have either default or metadata, but not both + if ( + fld.default is None or fld.default == attr.NOTHING + ) and not fld.metadata: # TODO: is it right? + raise AttributeError( + "File has to have default value or metadata" + ) + elif fld.default != attr.NOTHING: + output_names.append(fld.name) + elif ( + fld.metadata + and self._field_metadata(fld, inputs, output_dir) + != attr.NOTHING + ): + output_names.append(fld.name) + else: + raise Exception("not implemented (collect_additional_output)") + return output_names + def _field_defaultvalue(self, fld, output_dir): """Collect output file if the default value specified.""" if not isinstance(fld.default, (str, Path)): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index b8938910a7..740e22d78c 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2654,6 +2654,12 @@ def test_shell_cmd_inputspec_outputspec_2(): ) shelly.inputs.file1 = "new_file_1.txt" shelly.inputs.file2 = "new_file_2.txt" + # all fileds from output_spec should be in output_names and generated_output_names + assert ( + shelly.output_names + == shelly.generated_output_names + == ["return_code", "stdout", "stderr", "newfile1", "newfile2"] + ) res = shelly() assert res.output.stdout == "" @@ -2714,6 +2720,20 @@ def test_shell_cmd_inputspec_outputspec_2a(): output_spec=my_output_spec, ) shelly.inputs.file1 = "new_file_1.txt" + # generated_output_names shoule know that newfile2 will not be generated + assert shelly.output_names == [ + "return_code", + "stdout", + "stderr", + "newfile1", + "newfile2", + ] + assert shelly.generated_output_names == [ + "return_code", + "stdout", + "stderr", + "newfile1", + ] res = shelly() assert res.output.stdout == "" @@ -2834,6 +2854,20 @@ def test_shell_cmd_inputspec_outputspec_3a(): ) shelly.inputs.file1 = "new_file_1.txt" shelly.inputs.file2 = "new_file_2.txt" + # generated_output_names shoule know that newfile2 will not be generated + assert shelly.output_names == [ + "return_code", + "stdout", + "stderr", + "newfile1", + "newfile2", + ] + assert shelly.generated_output_names == [ + "return_code", + "stdout", + "stderr", + "newfile1", + ] res = shelly() assert res.output.stdout == "" @@ -2884,6 +2918,12 @@ def test_shell_cmd_inputspec_outputspec_4(): ) shelly.inputs.file1 = "new_file_1.txt" shelly.inputs.additional_inp = 2 + # generated_output_names should be the same as output_names + assert ( + shelly.output_names + == shelly.generated_output_names + == ["return_code", "stdout", "stderr", "newfile1"] + ) res = shelly() assert res.output.stdout == "" From b672213f14e2ba4ce47c6848ffff224036c05368 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 23 Sep 2020 21:18:06 -0400 Subject: [PATCH 132/271] small fix to generated_output_names --- pydra/engine/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index db650c2e5f..266788908c 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -326,8 +326,14 @@ def generated_output_names(self): output_klass = make_klass(self.output_spec) if hasattr(output_klass, "generated_output_names"): output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) + # using updated input (after filing the templates) + _inputs = deepcopy(self.inputs) + modified_inputs = template_update(_inputs, self.output_dir) + if modified_inputs: + _inputs = attr.evolve(_inputs, **modified_inputs) + return output.generated_output_names( - inputs=self.inputs, output_dir=self.output_dir + inputs=_inputs, output_dir=self.output_dir ) else: return self.output_names From 8c828fec7300c3b9cad22ba779b309c524359392 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 24 Sep 2020 19:12:15 +0800 Subject: [PATCH 133/271] trigger publish.yml when new tag is published --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0fabe4041d..6f80ac01f5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,7 @@ name: Upload to PyPI on: release: - types: [created] + types: [published] jobs: deploy: From 6441ae79082a4aad3aff96dc90700f5d9d0e147b Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:02:53 -0400 Subject: [PATCH 134/271] fix: readme for pypi --- README.rst | 25 ++++++++++++++----------- setup.cfg | 7 ++++--- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index a6b5bc3a8b..00bc4f68aa 100644 --- a/README.rst +++ b/README.rst @@ -1,22 +1,25 @@ .. image:: https://raw.githubusercontent.com/nipype/pydra/master/docs/logo/pydra_logo.jpg - :width: 50 + :width: 50px :alt: pydra logo +====================== +Pydra: Dataflow Engine +====================== + A simple dataflow engine with scalable semantics. +Pydra is a rewrite of the Nipype engine with mapping and joining as +first-class operations. It forms the core of the Nipype 2.0 ecosystem. -.. image:: https://github.com/nipype/pydra/workflows/Pydra/badge.svg +|GHAction| |CircleCI| |codecov| + +.. |GHAction| image:: https://github.com/nipype/pydra/workflows/Pydra/badge.svg :alt: GitHub Actions CI :target: https://github.com/nipype/Pydra/actions -|CircleCI| - -.. |CircleCI| image:: https://circleci.com/gh/nipype/pydra.svg?style=svg - :alt: CircleCI - - -|codecov| +.. |CircleCI| image:: https://circleci.com/gh/nipype/pydra.svg?style=svg + :alt: CircleCI .. |codecov| image:: https://codecov.io/gh/nipype/pydra/branch/master/graph/badge.svg :alt: codecov @@ -59,10 +62,10 @@ Installation Note that installation fails with older versions of pip on Windows. Upgrade pip before installing: :: - + pip install –upgrade pip pip install pydra - + Developer installation ====================== diff --git a/setup.cfg b/setup.cfg index 1bddd0ac5a..bc0ce81abe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,11 @@ [metadata] url = https://github.com/nipype/pydra -author = nipype developers +author = Nipype developers author_email = neuroimaging@python.org -maintainer = nipype developers +maintainer = Nipype developers maintainer_email = neuroimaging@python.org description = Pydra dataflow engine -long_description = file:long_description.rst +long_description = file:README.rst long_description_content_type = text/x-rst; charset=UTF-8 license = Apache License, 2.0 provides = @@ -18,6 +18,7 @@ classifiers = Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Topic :: Scientific/Engineering [options] From e42257bd0b237f8ae1aad7e0179df837cbd1e1ca Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:05:03 -0400 Subject: [PATCH 135/271] PR: move acknowledgments to the top of the PR --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b2cb3a760e..cd17d053fd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,6 @@ +## Acknowledgment +- [ ] I acknowledge that this contribution will be available under the Apache 2 license. + ## Types of changes - [ ] Bug fix (non-breaking change which fixes an issue) @@ -16,6 +19,3 @@ (we are using `black`: you can `pip install pre-commit`, run `pre-commit install` in the `pydra` directory and `black` will be run automatically with each commit) - -## Acknowledgment -- [ ] I acknowledge that this contribution will be available under the Apache 2 license. From b2812aa8ffbd9adfb701c1e9497e00fa16d9f7ff Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:11:25 -0400 Subject: [PATCH 136/271] update readme --- README.rst | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 00bc4f68aa..eadc677ffc 100644 --- a/README.rst +++ b/README.rst @@ -1,18 +1,10 @@ +|GHAction| |CircleCI| |codecov| -.. image:: https://raw.githubusercontent.com/nipype/pydra/master/docs/logo/pydra_logo.jpg - :width: 50px - :alt: pydra logo - -====================== -Pydra: Dataflow Engine -====================== - -A simple dataflow engine with scalable semantics. - -Pydra is a rewrite of the Nipype engine with mapping and joining as -first-class operations. It forms the core of the Nipype 2.0 ecosystem. +|Pydralogo| -|GHAction| |CircleCI| |codecov| +.. |Pydralogo| image:: https://raw.githubusercontent.com/nipype/pydra/master/docs/logo/pydra_logo.jpg + :width: 200px + :alt: pydra logo .. |GHAction| image:: https://github.com/nipype/pydra/workflows/Pydra/badge.svg :alt: GitHub Actions CI @@ -24,7 +16,17 @@ first-class operations. It forms the core of the Nipype 2.0 ecosystem. .. |codecov| image:: https://codecov.io/gh/nipype/pydra/branch/master/graph/badge.svg :alt: codecov -The goal of pydra is to provide a lightweight Python dataflow engine for DAG construction, manipulation, and distributed execution. +====================== +Pydra: Dataflow Engine +====================== + +A simple dataflow engine with scalable semantics. + +Pydra is a rewrite of the Nipype engine with mapping and joining as +first-class operations. It forms the core of the Nipype 2.0 ecosystem. + +The goal of pydra is to provide a lightweight Python dataflow engine for DAG +construction, manipulation, and distributed execution. Feature list: ============= From 654d019602d12f99cf099f5c4d555a3279e2f1cd Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:28:49 -0400 Subject: [PATCH 137/271] try normal pytest-xdist --- pydra/schema/context.jsonld | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/schema/context.jsonld b/pydra/schema/context.jsonld index 3e99fb614f..7e01b96ccd 100644 --- a/pydra/schema/context.jsonld +++ b/pydra/schema/context.jsonld @@ -10,7 +10,7 @@ }, "https://raw.githubusercontent.com/openprov/prov-jsonld/69964ed16818f78dc5f71bdf97add026288291d4/context.json", { - "pydra": "https://s.pydra.org/", + "pydra": "http://s.pydra.org/", "uid": "pydra:id/", "task": { "@id": "pydra:task", diff --git a/setup.cfg b/setup.cfg index bc0ce81abe..99a4c0657d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,7 +65,7 @@ test = pytest >= 4.4.0 pytest-cov pytest-env - pytest-xdist < 2.0 + pytest-xdist pytest-rerunfailures codecov numpy From 8345ac84cbd57e29e443b497be4b1d3727711afd Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:32:02 -0400 Subject: [PATCH 138/271] switch back to long_description --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 99a4c0657d..f2fe2aa677 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ author_email = neuroimaging@python.org maintainer = Nipype developers maintainer_email = neuroimaging@python.org description = Pydra dataflow engine -long_description = file:README.rst +long_description = file:long_description.rst long_description_content_type = text/x-rst; charset=UTF-8 license = Apache License, 2.0 provides = From 765b184272fef3696ec19ce0cb872ff23d9b8d28 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 15:39:04 -0400 Subject: [PATCH 139/271] fix requirements --- min-requirements.txt | 6 +++--- setup.cfg | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/min-requirements.txt b/min-requirements.txt index 09b237e48c..2983b38651 100644 --- a/min-requirements.txt +++ b/min-requirements.txt @@ -1,5 +1,5 @@ # Auto-generated by tools/update_min_requirements.py -attrs -cloudpickle >= 1.2.2 +attrs == 19.1.0 +cloudpickle == 1.2.2 filelock == 3.0.0 -etelemetry == 0.2.0 +etelemetry == 0.2.2 diff --git a/setup.cfg b/setup.cfg index f2fe2aa677..a2f23f517f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ test_requires = pytest >= 4.4.0 pytest-cov pytest-env - pytest-xdist < 2.0 + pytest-xdist pytest-rerunfailures codecov numpy From 121b214746591d44125bad5c6489c4169737ce06 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 16:08:13 -0400 Subject: [PATCH 140/271] fix: limit pytest < 6.0.0 --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index a2f23f517f..9f8b38b043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ author_email = neuroimaging@python.org maintainer = Nipype developers maintainer_email = neuroimaging@python.org description = Pydra dataflow engine -long_description = file:long_description.rst +long_description = file:README.rst long_description_content_type = text/x-rst; charset=UTF-8 license = Apache License, 2.0 provides = @@ -30,7 +30,7 @@ install_requires = etelemetry >= 0.2.2 test_requires = - pytest >= 4.4.0 + pytest >= 4.4.0, < 6.0.0 pytest-cov pytest-env pytest-xdist @@ -62,7 +62,7 @@ doc = docs = %(doc)s test = - pytest >= 4.4.0 + pytest >= 4.4.0, < 6.0.0 pytest-cov pytest-env pytest-xdist From aa6cae9a9c1ca9de9d0e235c9f2c3ca50e33e4c8 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Sat, 26 Sep 2020 16:11:10 -0400 Subject: [PATCH 141/271] fix: limit xdist --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9f8b38b043..029433c26a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ test_requires = pytest >= 4.4.0, < 6.0.0 pytest-cov pytest-env - pytest-xdist + pytest-xdist < 2.0 pytest-rerunfailures codecov numpy @@ -65,7 +65,7 @@ test = pytest >= 4.4.0, < 6.0.0 pytest-cov pytest-env - pytest-xdist + pytest-xdist < 2.0 pytest-rerunfailures codecov numpy From 4d3a33267e416481b26980f2fe1348a60ca87f48 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 30 Sep 2020 11:20:29 +0800 Subject: [PATCH 142/271] fix check_latest_version in __init__.py --- pydra/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pydra/__init__.py b/pydra/__init__.py index 37e24116e5..fa944f6c79 100644 --- a/pydra/__init__.py +++ b/pydra/__init__.py @@ -22,15 +22,10 @@ def check_latest_version(): - import os - if "NO_ET" not in os.environ: + import etelemetry - import etelemetry - - return etelemetry.check_available_version( - "nipype/pydra", __version__, lgr=logger - ) + return etelemetry.check_available_version("nipype/pydra", __version__, lgr=logger) # Run telemetry on import for interactive sessions, such as IPython, Jupyter notebooks, Python REPL From 3ee05c1cb48d4c97b69a851bf27e80c048893249 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 30 Sep 2020 11:33:07 +0800 Subject: [PATCH 143/271] add NO_ET to all testing workflows --- .github/workflows/testallowfail.yml | 2 ++ .github/workflows/testpydra.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml index a9dd5d95f8..608817dc5f 100644 --- a/.github/workflows/testallowfail.yml +++ b/.github/workflows/testallowfail.yml @@ -21,6 +21,8 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Disable etelemetry + run: echo ::set-env name=NO_ET::TRUE - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} uses: actions/setup-python@v2 with: diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 115be2f79b..a4e18c0bdc 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -19,6 +19,8 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Disable etelemetry + run: echo ::set-env name=NO_ET::TRUE - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} uses: actions/setup-python@v2 with: From bdbce8ebdcfaa3a5cdb27f6b005d8f30da90cd64 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 30 Sep 2020 00:33:10 -0400 Subject: [PATCH 144/271] adding input fields check to generated_output_names in order to catch the incomplete input before running --- pydra/engine/helpers_file.py | 2 +- pydra/engine/specs.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 09e69badd9..b1e55aca3b 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -519,7 +519,7 @@ def template_update(inputs, output_dir, map_copyfiles=None): if fld.type not in [str, ty.Union[str, bool]]: raise Exception( f"fields with output_file_template" - "has to be a string or Union[str, bool]" + " has to be a string or Union[str, bool]" ) dict_[fld.name] = template_update_single( field=fld, inputs_dict=dict_, output_dir=output_dir diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 08cf4d2419..874a304e43 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -425,6 +425,8 @@ def generated_output_names(self, inputs, output_dir): Takes into account the task input and the requires list for the output fields. TODO: should be in all Output specs? """ + # checking the input (if all mandatory fields are provided, etc.) + inputs.check_fields_input_spec() output_names = ["return_code", "stdout", "stderr"] for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: From 61e14a03cbb9d2717b4b78d1cf4956a0b003a826 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 1 Oct 2020 22:32:42 +0800 Subject: [PATCH 145/271] fix: only run tests in test_singularity.py in singularity CI --- .github/workflows/testsingularity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 7d17f7702b..3341c75265 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -61,6 +61,6 @@ jobs: - name: Pytest - run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py - name: Upload to codecov run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW From 177402c9ffda8c3a7f4245f6c3e81a3f639f1cfe Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 2 Oct 2020 02:02:00 -0400 Subject: [PATCH 146/271] checking sbatch args provided to the submitter; passing sbatch errors if the options are invalid --- pydra/engine/tests/test_submitter.py | 28 ++++++++++++++++++++++++++++ pydra/engine/workers.py | 8 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 2dbda71c21..9f7d6022a3 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -233,3 +233,31 @@ def test_slurm_max_jobs(tmpdir): prev = et continue assert (prev - et).seconds >= 2 + + +@pytest.mark.skipif(not slurm_available, reason="slurm not installed") +def test_slurm_args_1(tmpdir): + """ testing sbatch_args provided to the submitter""" + task = sleep_add_one(x=1) + task.cache_dir = tmpdir + # submit workflow and every task as slurm job + with Submitter("slurm", sbatch_args="-N1") as sub: + sub(task) + + res = task.result() + assert res.output.out == 2 + script_dir = tmpdir / "SlurmWorker_scripts" + assert script_dir.exists() + + +@pytest.mark.skipif(not slurm_available, reason="slurm not installed") +def test_slurm_args_2(tmpdir): + """ testing sbatch_args provided to the submitter + exception should be raised for invalid options + """ + task = sleep_add_one(x=1) + task.cache_dir = tmpdir + # submit workflow and every task as slurm job + with pytest.raises(RuntimeError, match="Error returned from sbatch:"): + with Submitter("slurm", sbatch_args="-N1 --invalid") as sub: + sub(task) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 2b04fd3b63..82b73717f5 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -280,9 +280,13 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): error_file = None sargs.append(str(batchscript)) # TO CONSIDER: add random sleep to avoid overloading calls - _, stdout, _ = await read_and_display_async("sbatch", *sargs, hide_display=True) + rc, stdout, stderr = await read_and_display_async( + "sbatch", *sargs, hide_display=True + ) jobid = re.search(r"\d+", stdout) - if not jobid: + if rc: + raise RuntimeError(f"Error returned from sbatch: {stderr}") + elif not jobid: raise RuntimeError("Could not extract job ID") jobid = jobid.group() if error_file: From a9cf5925241eaa55d6444fd4f52e009cad9090fd Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 2 Oct 2020 02:08:51 -0400 Subject: [PATCH 147/271] adding requeue for slurm when the job is cancelled --- pydra/engine/workers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 82b73717f5..4fd7d06763 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -1,6 +1,6 @@ """Execution workers.""" import asyncio -import sys +import sys, os import re from tempfile import gettempdir from pathlib import Path @@ -300,7 +300,12 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): # Exception: Polling / job failure done = await self._poll_job(jobid) if done: - return True + if done == "CANCELLED": + os.remove(cache_dir / f"{checksum}.lock") + cmd_re = ("scontrol", "requeue", jobid) + await read_and_display_async(*cmd_re, hide_display=True) + else: + return True await asyncio.sleep(self.poll_delay) async def _poll_job(self, jobid): @@ -321,7 +326,9 @@ async def _verify_exit_code(self, jobid): m = self._sacct_re.search(stdout) error_file = self.error[jobid] if int(m.group("exit_code")) != 0 or m.group("status") != "COMPLETED": - if m.group("status") in ["RUNNING", "PENDING"]: + if m.group("status") == "CANCELLED": + return "CANCELLED" + elif m.group("status") in ["RUNNING", "PENDING"]: return False # TODO: potential for requeuing # parsing the error message From 0b9734c8340cbd31aaad9d7523c0bf5493c2f053 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 2 Oct 2020 10:28:26 -0400 Subject: [PATCH 148/271] Update pydra/engine/workers.py Co-authored-by: Mathias Goncalves --- pydra/engine/workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 4fd7d06763..bf4bdc4d1e 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -301,7 +301,7 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): done = await self._poll_job(jobid) if done: if done == "CANCELLED": - os.remove(cache_dir / f"{checksum}.lock") + (cache_dir / f"{checksum}.lock").unlink(missing_ok=True) cmd_re = ("scontrol", "requeue", jobid) await read_and_display_async(*cmd_re, hide_display=True) else: From f6152fd773e7a80461d5a1a500b8a2364b431e9e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 4 Oct 2020 21:02:15 -0400 Subject: [PATCH 149/271] removing ci with wol version of pip, 10.0.1 (removing testallowfail) --- .github/workflows/testallowfail.yml | 39 ----------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/testallowfail.yml diff --git a/.github/workflows/testallowfail.yml b/.github/workflows/testallowfail.yml deleted file mode 100644 index 608817dc5f..0000000000 --- a/.github/workflows/testallowfail.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Test on an older version of pip - -name: Allow failure - -on: [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - build: - - runs-on: ${{ matrix.os }} - continue-on-error: true - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] - fail-fast: false - - steps: - - uses: actions/checkout@v2 - - name: Disable etelemetry - run: echo ::set-env name=NO_ET::TRUE - - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Update build tools (pip 10.0.1) - run: python -m pip install pip==10.0.1 setuptools==30.3.0 - - name: Install dependencies (min-requirements.txt) - run: pip install -r min-requirements.txt - - name: Install Pydra (test) - run: pip install -e ".[test]" - - name: Pytest - run: pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra - - name: Upload to codecov - run: codecov --root /pydra -f cov.xml -F unittests -e GITHUB_WORKFLOW From d07be70580646bb77d8505ee647cfdd273fd1cd4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 10:52:29 -0400 Subject: [PATCH 150/271] adding test for slurm rescheduling --- pydra/engine/tests/test_submitter.py | 56 ++++++++++++++++++++++++++++ pydra/engine/workers.py | 10 +++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 9f7d6022a3..005ec5205c 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -8,6 +8,7 @@ from .utils import gen_basic_wf from ..core import Workflow +from ..task import ShellCommandTask from ..submitter import Submitter from ... import mark @@ -261,3 +262,58 @@ def test_slurm_args_2(tmpdir): with pytest.raises(RuntimeError, match="Error returned from sbatch:"): with Submitter("slurm", sbatch_args="-N1 --invalid") as sub: sub(task) + + +@pytest.mark.skipif(not slurm_available, reason="slurm not installed") +def test_slurm_cancel_rerun(tmpdir): + """ testing that tasks run with slurm is re-queue + Running wf with 2 tasks, one sleeps and the other trying to get + job_id of the first task and cancel it. + The first job should be re-queue and finish without problem. + (possibly has to be improved, in theory cancel job might finish before cancel) + """ + + @mark.task + def sleep(x): + time.sleep(x) + return x + + @mark.task + def cancel(job_name_part): + import subprocess as sp + + # getting the job_id of the first job that sleeps + job_id = "" + while job_id == "": + time.sleep(1) + id_p1 = sp.Popen(["squeue"], stdout=sp.PIPE) + id_p2 = sp.Popen( + ["grep", job_name_part], stdin=id_p1.stdout, stdout=sp.PIPE + ) + id_p3 = sp.Popen(["awk", "{print $1}"], stdin=id_p2.stdout, stdout=sp.PIPE) + job_id = id_p3.communicate()[0].decode("utf-8").strip() + + # # canceling the job + proc1 = sp.run(["scancel", job_id]) + # checking the status of jobs with the name; returning the last item + proc2 = sp.run(["sacct", "-j", job_id], stdout=sp.PIPE, stderr=sp.PIPE) + return proc2.stdout.decode("utf-8").strip() # .split("\n")[-1] + + wf = Workflow(name="wf", input_spec=["x", "job_name"], cache_dir=tmpdir) + wf.add(sleep(name="sleep", x=wf.lzin.x)) + wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name)) + # this is a job name for x=10, if x is different checksum and jobname would have to be updated + wf.inputs.x = 20 + wf.inputs.job_name = "sleep" + + wf.set_output([("out", wf.sleep.lzout.out), ("canc_out", wf.cancel.lzout.out)]) + with Submitter("slurm") as sub: + sub(wf) + + res = wf.result() + assert res.output.out == 20 + # checking if indeed the sleep-task job was cancelled by cancel-task + assert "CANCELLED" in res.output.canc_out + breakpoint() + script_dir = tmpdir / "SlurmWorker_scripts" + assert script_dir.exists() diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index bf4bdc4d1e..b6c37a309f 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -300,8 +300,10 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): # Exception: Polling / job failure done = await self._poll_job(jobid) if done: - if done == "CANCELLED": - (cache_dir / f"{checksum}.lock").unlink(missing_ok=True) + if done in ["CANCELLED", "TIMEOUT", "PREEMPTED"]: + if (cache_dir / f"{checksum}.lock").exists(): + # for pyt3.8 we could you missing_ok=True + (cache_dir / f"{checksum}.lock").unlink() cmd_re = ("scontrol", "requeue", jobid) await read_and_display_async(*cmd_re, hide_display=True) else: @@ -326,8 +328,8 @@ async def _verify_exit_code(self, jobid): m = self._sacct_re.search(stdout) error_file = self.error[jobid] if int(m.group("exit_code")) != 0 or m.group("status") != "COMPLETED": - if m.group("status") == "CANCELLED": - return "CANCELLED" + if m.group("status") in ["CANCELLED", "TIMEOUT", "PREEMPTED"]: + return m.group("status") elif m.group("status") in ["RUNNING", "PENDING"]: return False # TODO: potential for requeuing From 0d50498a1d89c7c2282c537f58bb1f6ac22e8f16 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 15:12:44 -0400 Subject: [PATCH 151/271] fixing test for rerunning jobs - should be more rebust; checking --no-requeue option --- pydra/engine/tests/test_submitter.py | 106 ++++++++++++++++++--------- pydra/engine/workers.py | 5 +- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 005ec5205c..72205acee9 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -264,56 +264,90 @@ def test_slurm_args_2(tmpdir): sub(task) +@mark.task +def sleep(x, job_name_part): + time.sleep(x) + import subprocess as sp + + # getting the job_id of the first job that sleeps + job_id = 999 + while job_id != "": + time.sleep(3) + id_p1 = sp.Popen(["squeue"], stdout=sp.PIPE) + id_p2 = sp.Popen(["grep", job_name_part], stdin=id_p1.stdout, stdout=sp.PIPE) + id_p3 = sp.Popen(["awk", "{print $1}"], stdin=id_p2.stdout, stdout=sp.PIPE) + job_id = id_p3.communicate()[0].decode("utf-8").strip() + + return x + + +@mark.task +def cancel(job_name_part): + import subprocess as sp + + # getting the job_id of the first job that sleeps + job_id = "" + while job_id == "": + time.sleep(1) + id_p1 = sp.Popen(["squeue"], stdout=sp.PIPE) + id_p2 = sp.Popen(["grep", job_name_part], stdin=id_p1.stdout, stdout=sp.PIPE) + id_p3 = sp.Popen(["awk", "{print $1}"], stdin=id_p2.stdout, stdout=sp.PIPE) + job_id = id_p3.communicate()[0].decode("utf-8").strip() + + # # canceling the job + proc1 = sp.run(["scancel", job_id]) + # checking the status of jobs with the name; returning the last item + proc2 = sp.run(["sacct", "-j", job_id], stdout=sp.PIPE, stderr=sp.PIPE) + return proc2.stdout.decode("utf-8").strip() + + @pytest.mark.skipif(not slurm_available, reason="slurm not installed") -def test_slurm_cancel_rerun(tmpdir): +def test_slurm_cancel_rerun_1(tmpdir): """ testing that tasks run with slurm is re-queue Running wf with 2 tasks, one sleeps and the other trying to get job_id of the first task and cancel it. The first job should be re-queue and finish without problem. (possibly has to be improved, in theory cancel job might finish before cancel) """ - - @mark.task - def sleep(x): - time.sleep(x) - return x - - @mark.task - def cancel(job_name_part): - import subprocess as sp - - # getting the job_id of the first job that sleeps - job_id = "" - while job_id == "": - time.sleep(1) - id_p1 = sp.Popen(["squeue"], stdout=sp.PIPE) - id_p2 = sp.Popen( - ["grep", job_name_part], stdin=id_p1.stdout, stdout=sp.PIPE - ) - id_p3 = sp.Popen(["awk", "{print $1}"], stdin=id_p2.stdout, stdout=sp.PIPE) - job_id = id_p3.communicate()[0].decode("utf-8").strip() - - # # canceling the job - proc1 = sp.run(["scancel", job_id]) - # checking the status of jobs with the name; returning the last item - proc2 = sp.run(["sacct", "-j", job_id], stdout=sp.PIPE, stderr=sp.PIPE) - return proc2.stdout.decode("utf-8").strip() # .split("\n")[-1] - - wf = Workflow(name="wf", input_spec=["x", "job_name"], cache_dir=tmpdir) - wf.add(sleep(name="sleep", x=wf.lzin.x)) - wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name)) - # this is a job name for x=10, if x is different checksum and jobname would have to be updated - wf.inputs.x = 20 - wf.inputs.job_name = "sleep" + wf = Workflow( + name="wf", + input_spec=["x", "job_name_cancel", "job_name_resqueue"], + cache_dir=tmpdir, + ) + wf.add(sleep(name="sleep", x=wf.lzin.x, job_name_part=wf.lzin.job_name_cancel)) + wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name_resqueue)) + wf.inputs.x = 10 + wf.inputs.job_name_resqueue = "sleep" + wf.inputs.job_name_cancel = "cancel" wf.set_output([("out", wf.sleep.lzout.out), ("canc_out", wf.cancel.lzout.out)]) with Submitter("slurm") as sub: sub(wf) res = wf.result() - assert res.output.out == 20 + assert res.output.out == 10 # checking if indeed the sleep-task job was cancelled by cancel-task assert "CANCELLED" in res.output.canc_out - breakpoint() script_dir = tmpdir / "SlurmWorker_scripts" assert script_dir.exists() + + +@pytest.mark.skipif(not slurm_available, reason="slurm not installed") +def test_slurm_cancel_rerun_2(tmpdir): + """ testing that tasks run with slurm is re-queue + Running wf with 2 tasks, one sleeps and the other trying to get + job_id of the first task and cancel it. + The first job should be re-queue and finish without problem. + (possibly has to be improved, in theory cancel job might finish before cancel) + """ + wf = Workflow(name="wf", input_spec=["x", "job_name"], cache_dir=tmpdir) + wf.add(sleep(name="sleep", x=wf.lzin.x)) + wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name)) + + wf.inputs.x = 10 + wf.inputs.job_name = "sleep" + + wf.set_output([("out", wf.sleep.lzout.out), ("canc_out", wf.cancel.lzout.out)]) + with pytest.raises(Exception): + with Submitter("slurm", sbatch_args="--no-requeue") as sub: + sub(wf) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index b6c37a309f..540739b4e7 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -300,7 +300,10 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): # Exception: Polling / job failure done = await self._poll_job(jobid) if done: - if done in ["CANCELLED", "TIMEOUT", "PREEMPTED"]: + if ( + done in ["CANCELLED", "TIMEOUT", "PREEMPTED"] + and "--no-requeue" not in self.sbatch_args + ): if (cache_dir / f"{checksum}.lock").exists(): # for pyt3.8 we could you missing_ok=True (cache_dir / f"{checksum}.lock").unlink() From e94a10c5b130b021a238e94be90e60f71f6e6e0d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 16:19:38 -0400 Subject: [PATCH 152/271] small edit of tests --- pydra/engine/tests/test_submitter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 72205acee9..3f450495a3 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -334,11 +334,10 @@ def test_slurm_cancel_rerun_1(tmpdir): @pytest.mark.skipif(not slurm_available, reason="slurm not installed") def test_slurm_cancel_rerun_2(tmpdir): - """ testing that tasks run with slurm is re-queue - Running wf with 2 tasks, one sleeps and the other trying to get + """ testing that tasks run with slurm that has --no-requeue + Running wf with 2 tasks, one sleeps and the other gets job_id of the first task and cancel it. - The first job should be re-queue and finish without problem. - (possibly has to be improved, in theory cancel job might finish before cancel) + The first job is not able t be rescheduled and the error is returned. """ wf = Workflow(name="wf", input_spec=["x", "job_name"], cache_dir=tmpdir) wf.add(sleep(name="sleep", x=wf.lzin.x)) From 1a8fee63bf18489b052eaf67581ee38c4d185b0a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 17:46:20 -0400 Subject: [PATCH 153/271] removing min-requirements; changing orders in GA pydra testing - installing tests requirements first --- .github/workflows/testpydra.yml | 19 ++++++------------- min-requirements.txt | 5 ----- tools/update_min_requirements.py | 17 ----------------- 3 files changed, 6 insertions(+), 35 deletions(-) delete mode 100644 min-requirements.txt delete mode 100755 tools/update_min_requirements.py diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index a4e18c0bdc..f859781253 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -12,7 +12,7 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: [3.7, 3.8] - install: [min-req, install, develop, wheel] + install: [install, develop, wheel] fail-fast: false runs-on: ${{ matrix.os }} @@ -29,9 +29,10 @@ jobs: run: python -m pip install --upgrade pip setuptools - - name: Install dependencies (min-requirements.txt) - if: matrix.install == 'min-req' - run: pip install -r min-requirements.txt + - name: Install Pydra tests dependencies (develop or setup.py install) + if: matrix.install == 'develop' || matrix.install == 'install' + run: pip install ".[test]" + - name: Install dependencies (setup.py install) if: matrix.install == 'install' @@ -44,15 +45,7 @@ jobs: pip install dist/*.whl - - name: Install Pydra (min-requirements.txt or setup.py install) - if: matrix.install == 'min-req' || matrix.install == 'install' - run: pip install ".[test]" - - - name: Install Pydra (setup.py develop) - if: matrix.install == 'develop' - run: pip install -e ".[test]" - - - name: Install Pydra (wheel) + - name: Install Pydra tests dependencies (wheel) if: matrix.install == 'wheel' run: pip install "$( ls dist/pydra*.whl )[test]" diff --git a/min-requirements.txt b/min-requirements.txt deleted file mode 100644 index 2983b38651..0000000000 --- a/min-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Auto-generated by tools/update_min_requirements.py -attrs == 19.1.0 -cloudpickle == 1.2.2 -filelock == 3.0.0 -etelemetry == 0.2.2 diff --git a/tools/update_min_requirements.py b/tools/update_min_requirements.py deleted file mode 100755 index d0d25c4799..0000000000 --- a/tools/update_min_requirements.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -from configparser import ConfigParser -from pathlib import Path - -repo_root = Path(__file__).parent.parent -setup_cfg = repo_root / "setup.cfg" -min_reqs = repo_root / "min-requirements.txt" - -config = ConfigParser() -config.read(setup_cfg) -requirements = config.get("options", "install_requires").strip().splitlines() - -lines = ["# Auto-generated by tools/update_min_requirements.py", ""] - -lines[1:-1] = [req.replace(">=", "==").replace("~=", "==") for req in requirements] - -min_reqs.write_text("\n".join(lines)) From 5c0d2c4fc80b964c358d39e88b4c9ce95b7a5876 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 17:53:39 -0400 Subject: [PATCH 154/271] fixing style check --- .github/workflows/teststyle.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/teststyle.yml b/.github/workflows/teststyle.yml index e8df9fb7a9..29e6ccfd95 100644 --- a/.github/workflows/teststyle.yml +++ b/.github/workflows/teststyle.yml @@ -20,10 +20,8 @@ jobs: - name: Update build tools run: python -m pip install --upgrade pip setuptools - - name: Install dependencies - run: pip install -r min-requirements.txt - name: Install Pydra - run: pip install ".[test]" + run: pip install ".[dev]" - name: Check Style run: | From 25639d273ab64a472620cdebab122319907f7c18 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 6 Oct 2020 17:59:40 -0400 Subject: [PATCH 155/271] small fix --- .github/workflows/teststyle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/teststyle.yml b/.github/workflows/teststyle.yml index 29e6ccfd95..9b8d76a9e1 100644 --- a/.github/workflows/teststyle.yml +++ b/.github/workflows/teststyle.yml @@ -26,4 +26,4 @@ jobs: - name: Check Style run: | pip install black==19.3b0 codecov - black --check pydra tools setup.py + black --check pydra setup.py From f39f27fdb761ccfa84919d937b1d419afe1c934e Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 8 Oct 2020 23:11:54 +0800 Subject: [PATCH 156/271] add tmpdir to tests in test_workflow.py and test_subtmitter.py --- pydra/engine/tests/test_submitter.py | 23 ++- pydra/engine/tests/test_workflow.py | 269 +++++++++++++++++---------- 2 files changed, 187 insertions(+), 105 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 3f450495a3..0d40b7f1b0 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -21,8 +21,10 @@ def sleep_add_one(x): return x + 1 -def test_callable_wf(plugin): +def test_callable_wf(plugin, tmpdir): wf = gen_basic_wf() + wf.cache_dir = tmpdir + with pytest.raises(NotImplementedError): wf() @@ -36,7 +38,7 @@ def test_callable_wf(plugin): assert res.output.out == 9 -def test_concurrent_wf(plugin): +def test_concurrent_wf(plugin, tmpdir): # concurrent workflow # A --> C # B --> D @@ -48,6 +50,8 @@ def test_concurrent_wf(plugin): wf.add(sleep_add_one(name="taskc", x=wf.taska.lzout.out)) wf.add(sleep_add_one(name="taskd", x=wf.taskb.lzout.out)) wf.set_output([("out1", wf.taskc.lzout.out), ("out2", wf.taskd.lzout.out)]) + wf.cache_dir = tmpdir + with Submitter(plugin) as sub: sub(wf) @@ -56,7 +60,7 @@ def test_concurrent_wf(plugin): assert res.output.out2 == 12 -def test_concurrent_wf_nprocs(): +def test_concurrent_wf_nprocs(tmpdir): # concurrent workflow # setting n_procs in Submitter that is passed to the worker # A --> C @@ -69,8 +73,8 @@ def test_concurrent_wf_nprocs(): wf.add(sleep_add_one(name="taskc", x=wf.taska.lzout.out)) wf.add(sleep_add_one(name="taskd", x=wf.taskb.lzout.out)) wf.set_output([("out1", wf.taskc.lzout.out), ("out2", wf.taskd.lzout.out)]) - # wf.plugin = 'cf' - # res = wf.run() + wf.cache_dir = tmpdir + with Submitter("cf", n_procs=2) as sub: sub(wf) @@ -79,7 +83,7 @@ def test_concurrent_wf_nprocs(): assert res.output.out2 == 12 -def test_wf_in_wf(plugin): +def test_wf_in_wf(plugin, tmpdir): """WF(A --> SUBWF(A --> B) --> B)""" wf = Workflow(name="wf_in_wf", input_spec=["x"]) wf.inputs.x = 3 @@ -96,6 +100,7 @@ def test_wf_in_wf(plugin): wf.add(sleep_add_one(name="wf_b", x=wf.sub_wf.lzout.out)) wf.set_output([("out", wf.wf_b.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin) as sub: sub(wf) @@ -105,7 +110,7 @@ def test_wf_in_wf(plugin): @pytest.mark.flaky(reruns=2) # when dask -def test_wf2(plugin_dask_opt): +def test_wf2(plugin_dask_opt, tmpdir): """ workflow as a node workflow-node with one task and no splitter """ @@ -117,6 +122,7 @@ def test_wf2(plugin_dask_opt): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -126,7 +132,7 @@ def test_wf2(plugin_dask_opt): @pytest.mark.flaky(reruns=2) # when dask -def test_wf_with_state(plugin_dask_opt): +def test_wf_with_state(plugin_dask_opt, tmpdir): wf = Workflow(name="wf_with_state", input_spec=["x"]) wf.add(sleep_add_one(name="taska", x=wf.lzin.x)) wf.add(sleep_add_one(name="taskb", x=wf.taska.lzout.out)) @@ -134,6 +140,7 @@ def test_wf_with_state(plugin_dask_opt): wf.inputs.x = [1, 2, 3] wf.split("x") wf.set_output([("out", wf.taskb.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 085f656c33..50265d3eb9 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -48,7 +48,7 @@ def test_wf_name_conflict2(): assert "Another task named task_name is already added" in str(excinfo.value) -def test_wf_no_output(plugin): +def test_wf_no_output(plugin, tmpdir): """ Raise error when output isn't set with set_output""" wf = Workflow(name="wf_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -60,12 +60,13 @@ def test_wf_no_output(plugin): assert "Workflow output cannot be None" in str(excinfo.value) -def test_wf_1(plugin): +def test_wf_1(plugin, tmpdir): """ workflow with one task and no splitter""" wf = Workflow(name="wf_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -77,7 +78,7 @@ def test_wf_1(plugin): assert wf.output_dir.exists() -def test_wf_1a_outpastuple(plugin): +def test_wf_1a_outpastuple(plugin, tmpdir): """ workflow with one task and no splitter set_output takes a tuple """ @@ -86,6 +87,7 @@ def test_wf_1a_outpastuple(plugin): wf.set_output(("out", wf.add2.lzout.out)) wf.inputs.x = 2 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -95,12 +97,13 @@ def test_wf_1a_outpastuple(plugin): assert wf.output_dir.exists() -def test_wf_1_call_subm(plugin): +def test_wf_1_call_subm(plugin, tmpdir): """using wf.__call_ with submitter""" wf = Workflow(name="wf_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: wf(submitter=sub) @@ -110,13 +113,14 @@ def test_wf_1_call_subm(plugin): assert wf.output_dir.exists() -def test_wf_1_call_plug(plugin): +def test_wf_1_call_plug(plugin, tmpdir): """using wf.__call_ with plugin""" wf = Workflow(name="wf_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.plugin = plugin + wf.cache_dir = tmpdir wf(plugin=plugin) @@ -125,13 +129,14 @@ def test_wf_1_call_plug(plugin): assert wf.output_dir.exists() -def test_wf_1_call_exception(plugin): +def test_wf_1_call_exception(plugin, tmpdir): """using wf.__call_ with plugin and submitter - should raise an exception""" wf = Workflow(name="wf_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: with pytest.raises(Exception) as e: @@ -139,7 +144,7 @@ def test_wf_1_call_exception(plugin): assert "Specify submitter OR plugin" in str(e.value) -def test_wf_2(plugin): +def test_wf_2(plugin, tmpdir): """ workflow with 2 tasks, no splitter""" wf = Workflow(name="wf_2", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -157,7 +162,7 @@ def test_wf_2(plugin): assert 8 == results.output.out -def test_wf_2a(plugin): +def test_wf_2a(plugin, tmpdir): """ workflow with 2 tasks, no splitter creating add2_task first (before calling add method), """ @@ -170,6 +175,7 @@ def test_wf_2a(plugin): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -179,7 +185,7 @@ def test_wf_2a(plugin): assert wf.output_dir.exists() -def test_wf_2b(plugin): +def test_wf_2b(plugin, tmpdir): """ workflow with 2 tasks, no splitter creating add2_task first (before calling add method), adding inputs.x after add method @@ -193,6 +199,7 @@ def test_wf_2b(plugin): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -203,7 +210,7 @@ def test_wf_2b(plugin): assert wf.output_dir.exists() -def test_wf_2c_multoutp(plugin): +def test_wf_2c_multoutp(plugin, tmpdir): """ workflow with 2 tasks, no splitter setting multiple outputs for the workflow """ @@ -217,6 +224,7 @@ def test_wf_2c_multoutp(plugin): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -228,7 +236,7 @@ def test_wf_2c_multoutp(plugin): assert wf.output_dir.exists() -def test_wf_2d_outpasdict(plugin): +def test_wf_2d_outpasdict(plugin, tmpdir): """ workflow with 2 tasks, no splitter setting multiple outputs using a dictionary """ @@ -242,6 +250,7 @@ def test_wf_2d_outpasdict(plugin): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -262,6 +271,7 @@ def test_wf_3(plugin_dask_opt): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = None + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -272,7 +282,7 @@ def test_wf_3(plugin_dask_opt): @pytest.mark.xfail(reason="the task error doesn't propagate") -def test_wf_3a_exception(plugin): +def test_wf_3a_exception(plugin, tmpdir): """ testinh wf without set input, attr.NOTHING should be set and the function should raise an exception """ @@ -283,6 +293,7 @@ def test_wf_3a_exception(plugin): wf.inputs.x = 2 wf.inputs.y = attr.NOTHING wf.plugin = plugin + wf.cache_dir = tmpdir with pytest.raises(TypeError) as excinfo: with Submitter(plugin=plugin) as sub: @@ -290,7 +301,7 @@ def test_wf_3a_exception(plugin): assert "unsupported" in str(excinfo.value) -def test_wf_4(plugin): +def test_wf_4(plugin, tmpdir): """wf with a task that doesn't set one input and use the function default value""" wf = Workflow(name="wf_4", input_spec=["x", "y"]) wf.add(fun_addvar_default(name="addvar", a=wf.lzin.x)) @@ -298,6 +309,7 @@ def test_wf_4(plugin): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -307,7 +319,7 @@ def test_wf_4(plugin): assert 5 == results.output.out -def test_wf_4a(plugin): +def test_wf_4a(plugin, tmpdir): """ wf with a task that doesn't set one input, the unset input is send to the task input, so the task should use the function default value @@ -318,6 +330,7 @@ def test_wf_4a(plugin): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -327,13 +340,14 @@ def test_wf_4a(plugin): assert 5 == results.output.out -def test_wf_5(plugin): +def test_wf_5(plugin, tmpdir): """ wf with two outputs connected to the task outputs one set_output """ wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) wf.add(fun_addsubvar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) wf.set_output([("out_sum", wf.addsub.lzout.sum), ("out_sub", wf.addsub.lzout.sub)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -343,7 +357,7 @@ def test_wf_5(plugin): assert 1 == results.output.out_sub -def test_wf_5a(plugin): +def test_wf_5a(plugin, tmpdir): """ wf with two outputs connected to the task outputs, set_output set twice """ @@ -351,6 +365,7 @@ def test_wf_5a(plugin): wf.add(fun_addsubvar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) wf.set_output([("out_sum", wf.addsub.lzout.sum)]) wf.set_output([("out_sub", wf.addsub.lzout.sub)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -365,13 +380,14 @@ def test_wf_5b_exception(): wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) wf.add(fun_addsubvar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) wf.set_output([("out", wf.addsub.lzout.sum)]) + wf.cache_dir = tmpdir with pytest.raises(Exception) as excinfo: wf.set_output([("out", wf.addsub.lzout.sub)]) assert "is already set" in str(excinfo.value) -def test_wf_6(plugin): +def test_wf_6(plugin, tmpdir): """ wf with two tasks and two outputs connected to both tasks, one set_output """ @@ -379,6 +395,7 @@ def test_wf_6(plugin): wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) wf.add(add2(name="add2", x=wf.mult.lzout.out)) wf.set_output([("out1", wf.mult.lzout.out), ("out2", wf.add2.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -389,7 +406,7 @@ def test_wf_6(plugin): assert 8 == results.output.out2 -def test_wf_6a(plugin): +def test_wf_6a(plugin, tmpdir): """ wf with two tasks and two outputs connected to both tasks, set_output used twice """ @@ -398,6 +415,7 @@ def test_wf_6a(plugin): wf.add(add2(name="add2", x=wf.mult.lzout.out)) wf.set_output([("out1", wf.mult.lzout.out)]) wf.set_output([("out2", wf.add2.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -408,7 +426,7 @@ def test_wf_6a(plugin): assert 8 == results.output.out2 -def test_wf_st_1(plugin): +def test_wf_st_1(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -417,6 +435,7 @@ def test_wf_st_1(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -433,7 +452,7 @@ def test_wf_st_1(plugin): assert odir.exists() -def test_wf_st_1_call_subm(plugin): +def test_wf_st_1_call_subm(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -442,6 +461,7 @@ def test_wf_st_1_call_subm(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: wf(submitter=sub) @@ -456,7 +476,7 @@ def test_wf_st_1_call_subm(plugin): assert odir.exists() -def test_wf_st_1_call_plug(plugin): +def test_wf_st_1_call_plug(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -465,6 +485,7 @@ def test_wf_st_1_call_plug(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir wf(plugin=plugin) @@ -478,7 +499,7 @@ def test_wf_st_1_call_plug(plugin): assert odir.exists() -def test_wf_st_noinput_1(plugin): +def test_wf_st_noinput_1(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -487,6 +508,7 @@ def test_wf_st_noinput_1(plugin): wf.inputs.x = [] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -499,13 +521,14 @@ def test_wf_st_noinput_1(plugin): assert wf.output_dir == [] -def test_wf_ndst_1(plugin): +def test_wf_ndst_1(plugin, tmpdir): """ workflow with one task, a splitter on the task level""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x).split("x")) wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -518,7 +541,7 @@ def test_wf_ndst_1(plugin): assert wf.output_dir.exists() -def test_wf_ndst_updatespl_1(plugin): +def test_wf_ndst_updatespl_1(plugin, tmpdir): """ workflow with one task, a splitter on the task level is added *after* calling add """ @@ -527,6 +550,7 @@ def test_wf_ndst_updatespl_1(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir wf.add2.split("x") with Submitter(plugin=plugin) as sub: @@ -540,7 +564,7 @@ def test_wf_ndst_updatespl_1(plugin): assert wf.output_dir.exists() -def test_wf_ndst_updatespl_1a(plugin): +def test_wf_ndst_updatespl_1a(plugin, tmpdir): """ workflow with one task (initialize before calling add), a splitter on the task level is added *after* calling add """ @@ -551,6 +575,7 @@ def test_wf_ndst_updatespl_1a(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -563,7 +588,7 @@ def test_wf_ndst_updatespl_1a(plugin): assert wf.output_dir.exists() -def test_wf_ndst_updateinp_1(plugin): +def test_wf_ndst_updateinp_1(plugin, tmpdir): """ workflow with one task, a splitter on the task level, updating input of the task after calling add @@ -576,6 +601,7 @@ def test_wf_ndst_updateinp_1(plugin): wf.plugin = plugin wf.add2.split("x") wf.add2.inputs.x = wf.lzin.y + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -587,13 +613,14 @@ def test_wf_ndst_updateinp_1(plugin): assert wf.output_dir.exists() -def test_wf_ndst_noinput_1(plugin): +def test_wf_ndst_noinput_1(plugin, tmpdir): """ workflow with one task, a splitter on the task level""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x).split("x")) wf.inputs.x = [] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -606,7 +633,7 @@ def test_wf_ndst_noinput_1(plugin): assert wf.output_dir.exists() -def test_wf_st_2(plugin): +def test_wf_st_2(plugin, tmpdir): """ workflow with one task, splitters and combiner for workflow""" wf = Workflow(name="wf_st_2", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) @@ -615,6 +642,7 @@ def test_wf_st_2(plugin): wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -629,13 +657,14 @@ def test_wf_st_2(plugin): assert odir.exists() -def test_wf_ndst_2(plugin): +def test_wf_ndst_2(plugin, tmpdir): """ workflow with one task, splitters and combiner on the task level""" wf = Workflow(name="wf_ndst_2", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x).split("x").combine(combiner="x")) wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -649,7 +678,7 @@ def test_wf_ndst_2(plugin): # workflows with structures A -> B -def test_wf_st_3(plugin): +def test_wf_st_3(plugin, tmpdir): """ workflow with 2 tasks, splitter on wf level""" wf = Workflow(name="wfst_3", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -659,6 +688,7 @@ def test_wf_st_3(plugin): wf.split(("x", "y")) wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -696,7 +726,7 @@ def test_wf_st_3(plugin): assert odir.exists() -def test_wf_ndst_3(plugin): +def test_wf_ndst_3(plugin, tmpdir): """Test workflow with 2 tasks, splitter on a task level""" wf = Workflow(name="wf_ndst_3", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split(("x", "y"))) @@ -705,6 +735,7 @@ def test_wf_ndst_3(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -716,7 +747,7 @@ def test_wf_ndst_3(plugin): assert wf.output_dir.exists() -def test_wf_st_4(plugin): +def test_wf_st_4(plugin, tmpdir): """ workflow with two tasks, scalar splitter and combiner for the workflow""" wf = Workflow(name="wf_st_4", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -726,6 +757,7 @@ def test_wf_st_4(plugin): wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -742,7 +774,7 @@ def test_wf_st_4(plugin): assert odir.exists() -def test_wf_ndst_4(plugin): +def test_wf_ndst_4(plugin, tmpdir): """ workflow with two tasks, scalar splitter and combiner on tasks level""" wf = Workflow(name="wf_ndst_4", input_spec=["a", "b"]) wf.add(multiply(name="mult", x=wf.lzin.a, y=wf.lzin.b).split(("x", "y"))) @@ -750,6 +782,7 @@ def test_wf_ndst_4(plugin): wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir wf.inputs.a = [1, 2] wf.inputs.b = [11, 12] @@ -765,7 +798,7 @@ def test_wf_ndst_4(plugin): assert wf.output_dir.exists() -def test_wf_st_5(plugin): +def test_wf_st_5(plugin, tmpdir): """ workflow with two tasks, outer splitter and no combiner""" wf = Workflow(name="wf_st_5", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -773,6 +806,7 @@ def test_wf_st_5(plugin): wf.split(["x", "y"], x=[1, 2], y=[11, 12]) wf.set_output([("out", wf.add2.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -788,7 +822,7 @@ def test_wf_st_5(plugin): assert odir.exists() -def test_wf_ndst_5(plugin): +def test_wf_ndst_5(plugin, tmpdir): """ workflow with two tasks, outer splitter on tasks level and no combiner""" wf = Workflow(name="wf_ndst_5", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split(["x", "y"])) @@ -796,6 +830,7 @@ def test_wf_ndst_5(plugin): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -809,7 +844,7 @@ def test_wf_ndst_5(plugin): assert wf.output_dir.exists() -def test_wf_st_6(plugin): +def test_wf_st_6(plugin, tmpdir): """ workflow with two tasks, outer splitter and combiner for the workflow""" wf = Workflow(name="wf_st_6", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -819,6 +854,7 @@ def test_wf_st_6(plugin): wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -836,7 +872,7 @@ def test_wf_st_6(plugin): assert odir.exists() -def test_wf_ndst_6(plugin): +def test_wf_ndst_6(plugin, tmpdir): """ workflow with two tasks, outer splitter and combiner on tasks level""" wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split(["x", "y"])) @@ -845,6 +881,7 @@ def test_wf_ndst_6(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -857,7 +894,7 @@ def test_wf_ndst_6(plugin): assert wf.output_dir.exists() -def test_wf_ndst_7(plugin): +def test_wf_ndst_7(plugin, tmpdir): """ workflow with two tasks, outer splitter and (full) combiner for first node only""" wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split("x").combine("x")) @@ -866,6 +903,7 @@ def test_wf_ndst_7(plugin): wf.inputs.y = 11 wf.set_output([("out", wf.iden.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -877,7 +915,7 @@ def test_wf_ndst_7(plugin): assert wf.output_dir.exists() -def test_wf_ndst_8(plugin): +def test_wf_ndst_8(plugin, tmpdir): """ workflow with two tasks, outer splitter and (partial) combiner for first task only""" wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) wf.add( @@ -888,6 +926,7 @@ def test_wf_ndst_8(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.iden.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -900,7 +939,7 @@ def test_wf_ndst_8(plugin): assert wf.output_dir.exists() -def test_wf_ndst_9(plugin): +def test_wf_ndst_9(plugin, tmpdir): """ workflow with two tasks, outer splitter and (full) combiner for first task only""" wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) wf.add( @@ -913,6 +952,7 @@ def test_wf_ndst_9(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.iden.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -927,7 +967,7 @@ def test_wf_ndst_9(plugin): # workflows with structures A -> B -> C -def test_wf_3sernd_ndst_1(plugin): +def test_wf_3sernd_ndst_1(plugin, tmpdir): """ workflow with three "serial" tasks, checking if the splitter is propagating""" wf = Workflow(name="wf_3sernd_ndst_1", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split(["x", "y"])) @@ -937,6 +977,7 @@ def test_wf_3sernd_ndst_1(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2_2nd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -965,6 +1006,7 @@ def test_wf_3nd_st_1(plugin_dask_opt): wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]) wf.set_output([("out", wf.mult.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -992,6 +1034,7 @@ def test_wf_3nd_ndst_1(plugin_dask_opt): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -1003,7 +1046,7 @@ def test_wf_3nd_ndst_1(plugin_dask_opt): assert wf.output_dir.exists() -def test_wf_3nd_st_2(plugin): +def test_wf_3nd_st_2(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner on the workflow level """ @@ -1015,6 +1058,7 @@ def test_wf_3nd_st_2(plugin): wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1033,7 +1077,7 @@ def test_wf_3nd_st_2(plugin): assert odir.exists() -def test_wf_3nd_ndst_2(plugin): +def test_wf_3nd_ndst_2(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner on the tasks levels """ @@ -1049,6 +1093,7 @@ def test_wf_3nd_ndst_2(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1061,7 +1106,7 @@ def test_wf_3nd_ndst_2(plugin): assert wf.output_dir.exists() -def test_wf_3nd_st_3(plugin): +def test_wf_3nd_st_3(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner (from the second task) on the workflow level """ @@ -1072,6 +1117,7 @@ def test_wf_3nd_st_3(plugin): wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("y") wf.set_output([("out", wf.mult.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1090,7 +1136,7 @@ def test_wf_3nd_st_3(plugin): assert odir.exists() -def test_wf_3nd_ndst_3(plugin): +def test_wf_3nd_ndst_3(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner (from the second task) on the tasks levels """ @@ -1105,6 +1151,7 @@ def test_wf_3nd_ndst_3(plugin): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1118,7 +1165,7 @@ def test_wf_3nd_ndst_3(plugin): assert wf.output_dir.exists() -def test_wf_3nd_st_4(plugin): +def test_wf_3nd_st_4(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and full combiner on the workflow level """ @@ -1129,6 +1176,7 @@ def test_wf_3nd_st_4(plugin): wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine(["x", "y"]) wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1147,7 +1195,7 @@ def test_wf_3nd_st_4(plugin): assert odir.exists() -def test_wf_3nd_ndst_4(plugin): +def test_wf_3nd_ndst_4(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter and full combiner on the tasks levels """ @@ -1163,6 +1211,7 @@ def test_wf_3nd_ndst_4(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1175,7 +1224,7 @@ def test_wf_3nd_ndst_4(plugin): assert wf.output_dir.exists() -def test_wf_3nd_st_5(plugin): +def test_wf_3nd_st_5(plugin, tmpdir): """ workflow with three tasks (A->C, B->C) and three fields in the splitter, splitter and partial combiner (from the second task) on the workflow level """ @@ -1191,6 +1240,7 @@ def test_wf_3nd_st_5(plugin): wf.set_output([("out", wf.addvar.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1212,7 +1262,7 @@ def test_wf_3nd_st_5(plugin): assert odir.exists() -def test_wf_3nd_ndst_5(plugin): +def test_wf_3nd_ndst_5(plugin, tmpdir): """ workflow with three tasks (A->C, B->C) and three fields in the splitter, all tasks have splitters and the last one has a partial combiner (from the 2nd) """ @@ -1232,6 +1282,7 @@ def test_wf_3nd_ndst_5(plugin): wf.set_output([("out", wf.addvar.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1247,7 +1298,7 @@ def test_wf_3nd_ndst_5(plugin): assert wf.output_dir.exists() -def test_wf_3nd_ndst_6(plugin): +def test_wf_3nd_ndst_6(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, the third one uses scalar splitter from the previous ones and a combiner """ @@ -1263,6 +1314,7 @@ def test_wf_3nd_ndst_6(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1276,7 +1328,7 @@ def test_wf_3nd_ndst_6(plugin): # workflows with Left and Right part in splitters A -> B (L&R parts of the splitter) -def test_wf_ndstLR_1(plugin): +def test_wf_ndstLR_1(plugin, tmpdir): """ Test workflow with 2 tasks, splitters on tasks levels The second task has its own simple splitter and the Left part from the first task should be added @@ -1288,6 +1340,7 @@ def test_wf_ndstLR_1(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1304,7 +1357,7 @@ def test_wf_ndstLR_1(plugin): assert wf.output_dir.exists() -def test_wf_ndstLR_1a(plugin): +def test_wf_ndstLR_1a(plugin, tmpdir): """ Test workflow with 2 tasks, splitters on tasks levels The second task has splitter that has Left part (from previous state) and the Right part (it's onw splitter) @@ -1318,6 +1371,7 @@ def test_wf_ndstLR_1a(plugin): wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1334,7 +1388,7 @@ def test_wf_ndstLR_1a(plugin): assert wf.output_dir.exists() -def test_wf_ndstLR_2(plugin): +def test_wf_ndstLR_2(plugin, tmpdir): """ Test workflow with 2 tasks, splitters on tasks levels The second task has its own outer splitter and the Left part from the first task should be added @@ -1351,6 +1405,7 @@ def test_wf_ndstLR_2(plugin): wf.inputs.z = [100, 200] wf.set_output([("out", wf.addvar.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1383,7 +1438,7 @@ def test_wf_ndstLR_2(plugin): assert wf.output_dir.exists() -def test_wf_ndstLR_2a(plugin): +def test_wf_ndstLR_2a(plugin, tmpdir): """ Test workflow with 2 tasks, splitters on tasks levels The second task has splitter that has Left part (from previous state) and the Right part (it's onw outer splitter) @@ -1400,6 +1455,7 @@ def test_wf_ndstLR_2a(plugin): wf.inputs.z = [100, 200] wf.set_output([("out", wf.addvar.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1435,7 +1491,7 @@ def test_wf_ndstLR_2a(plugin): # workflows with inner splitters A -> B (inner spl) -def test_wf_ndstinner_1(plugin): +def test_wf_ndstinner_1(plugin, tmpdir): """ workflow with 2 tasks, the second task has inner splitter """ @@ -1445,6 +1501,7 @@ def test_wf_ndstinner_1(plugin): wf.inputs.x = 1 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1459,7 +1516,7 @@ def test_wf_ndstinner_1(plugin): assert wf.output_dir.exists() -def test_wf_ndstinner_2(plugin): +def test_wf_ndstinner_2(plugin, tmpdir): """ workflow with 2 tasks, the second task has two inputs and inner splitter from one of the input """ @@ -1470,6 +1527,7 @@ def test_wf_ndstinner_2(plugin): wf.inputs.y = 10 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1484,7 +1542,7 @@ def test_wf_ndstinner_2(plugin): assert wf.output_dir.exists() -def test_wf_ndstinner_3(plugin): +def test_wf_ndstinner_3(plugin, tmpdir): """ workflow with 2 tasks, the second task has two inputs and outer splitter that includes an inner field """ @@ -1495,6 +1553,7 @@ def test_wf_ndstinner_3(plugin): wf.inputs.y = [10, 100] wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1509,7 +1568,7 @@ def test_wf_ndstinner_3(plugin): assert wf.output_dir.exists() -def test_wf_ndstinner_4(plugin): +def test_wf_ndstinner_4(plugin, tmpdir): """ workflow with 3 tasks, the second task has two inputs and inner splitter from one of the input, the third task has no its own splitter @@ -1522,6 +1581,7 @@ def test_wf_ndstinner_4(plugin): wf.inputs.y = 10 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1541,7 +1601,7 @@ def test_wf_ndstinner_4(plugin): # workflow that have some single values as the input -def test_wf_st_singl_1(plugin): +def test_wf_st_singl_1(plugin, tmpdir): """ workflow with two tasks, only one input is in the splitter and combiner""" wf = Workflow(name="wf_st_5", input_spec=["x", "y"]) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) @@ -1551,6 +1611,7 @@ def test_wf_st_singl_1(plugin): wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1564,7 +1625,7 @@ def test_wf_st_singl_1(plugin): assert odir.exists() -def test_wf_ndst_singl_1(plugin): +def test_wf_ndst_singl_1(plugin, tmpdir): """ workflow with two tasks, outer splitter and combiner on tasks level; only one input is part of the splitter, the other is a single value """ @@ -1575,6 +1636,7 @@ def test_wf_ndst_singl_1(plugin): wf.inputs.y = 11 wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1585,7 +1647,7 @@ def test_wf_ndst_singl_1(plugin): assert wf.output_dir.exists() -def test_wf_st_singl_2(plugin): +def test_wf_st_singl_2(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter on the workflow level only one input is part of the splitter, the other is a single value @@ -1598,6 +1660,7 @@ def test_wf_st_singl_2(plugin): wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1613,7 +1676,7 @@ def test_wf_st_singl_2(plugin): assert odir.exists() -def test_wf_ndst_singl_2(plugin): +def test_wf_ndst_singl_2(plugin, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter on the tasks levels only one input is part of the splitter, the other is a single value @@ -1626,6 +1689,7 @@ def test_wf_ndst_singl_2(plugin): wf.inputs.y = 11 wf.set_output([("out", wf.mult.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1640,7 +1704,7 @@ def test_wf_ndst_singl_2(plugin): # workflows with structures wf(A) -def test_wfasnd_1(plugin): +def test_wfasnd_1(plugin, tmpdir): """ workflow as a node workflow-node with one task and no splitter """ @@ -1653,6 +1717,7 @@ def test_wfasnd_1(plugin): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1663,7 +1728,7 @@ def test_wfasnd_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfinp_1(plugin): +def test_wfasnd_wfinp_1(plugin, tmpdir): """ workflow as a node workflow-node with one task and no splitter input set for the main workflow @@ -1677,6 +1742,7 @@ def test_wfasnd_wfinp_1(plugin): wf.inputs.x = 2 wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -1689,7 +1755,7 @@ def test_wfasnd_wfinp_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfndupdate(plugin): +def test_wfasnd_wfndupdate(plugin, tmpdir): """ workflow as a node workflow-node with one task and no splitter wfasnode input is updated to use the main workflow input @@ -1704,6 +1770,7 @@ def test_wfasnd_wfndupdate(plugin): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1713,7 +1780,7 @@ def test_wfasnd_wfndupdate(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfndupdate_rerun(plugin): +def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): """ workflow as a node workflow-node with one task and no splitter wfasnode is run first and later is @@ -1734,6 +1801,7 @@ def test_wfasnd_wfndupdate_rerun(plugin): wf.wfnd.inputs.x = wf.lzin.x wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1757,7 +1825,7 @@ def test_wfasnd_wfndupdate_rerun(plugin): assert wf_o.output_dir.exists() -def test_wfasnd_st_1(plugin): +def test_wfasnd_st_1(plugin, tmpdir): """ workflow as a node workflow-node with one task, splitter for wfnd @@ -1772,6 +1840,7 @@ def test_wfasnd_st_1(plugin): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir checksum_before = wf.checksum with Submitter(plugin=plugin) as sub: @@ -1784,7 +1853,7 @@ def test_wfasnd_st_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_st_updatespl_1(plugin): +def test_wfasnd_st_updatespl_1(plugin, tmpdir): """ workflow as a node workflow-node with one task, splitter for wfnd is set after add @@ -1799,6 +1868,7 @@ def test_wfasnd_st_updatespl_1(plugin): wfnd.split("x") wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1809,7 +1879,7 @@ def test_wfasnd_st_updatespl_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_ndst_1(plugin): +def test_wfasnd_ndst_1(plugin, tmpdir): """ workflow as a node workflow-node with one task, splitter for node @@ -1825,6 +1895,7 @@ def test_wfasnd_ndst_1(plugin): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1835,7 +1906,7 @@ def test_wfasnd_ndst_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_ndst_updatespl_1(plugin): +def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): """ workflow as a node workflow-node with one task, splitter for node added after add @@ -1852,6 +1923,7 @@ def test_wfasnd_ndst_updatespl_1(plugin): wfnd.add2.split("x") wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1862,7 +1934,7 @@ def test_wfasnd_ndst_updatespl_1(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfst_1(plugin): +def test_wfasnd_wfst_1(plugin, tmpdir): """ workflow as a node workflow-node with one task, splitter for the main workflow @@ -1893,7 +1965,7 @@ def test_wfasnd_wfst_1(plugin): # workflows with structures wf(A) -> B -def test_wfasnd_st_2(plugin): +def test_wfasnd_st_2(plugin, tmpdir): """ workflow as a node, the main workflow has two tasks, splitter for wfnd @@ -1910,6 +1982,7 @@ def test_wfasnd_st_2(plugin): wf.add(add2(name="add2", x=wf.wfnd.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1920,7 +1993,7 @@ def test_wfasnd_st_2(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfst_2(plugin): +def test_wfasnd_wfst_2(plugin, tmpdir): """ workflow as a node, the main workflow has two tasks, splitter for the main workflow @@ -1953,7 +2026,7 @@ def test_wfasnd_wfst_2(plugin): # workflows with structures A -> wf(B) -def test_wfasnd_ndst_3(plugin): +def test_wfasnd_ndst_3(plugin, tmpdir): """ workflow as the second node, the main workflow has two tasks, splitter for the first task @@ -1970,6 +2043,7 @@ def test_wfasnd_ndst_3(plugin): wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -1980,7 +2054,7 @@ def test_wfasnd_ndst_3(plugin): assert wf.output_dir.exists() -def test_wfasnd_wfst_3(plugin): +def test_wfasnd_wfst_3(plugin, tmpdir): """ workflow as the second node, the main workflow has two tasks, splitter for the main workflow @@ -1998,6 +2072,7 @@ def test_wfasnd_wfst_3(plugin): wf.set_output([("out", wf.wfnd.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3497,7 +3572,7 @@ def test_workflow_combine2(tmpdir): # testing lzout.all to collect all of the results and let FunctionTask deal with it -def test_wf_lzoutall_1(plugin): +def test_wf_lzoutall_1(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_sub2_res function by using lzout.all syntax @@ -3518,7 +3593,7 @@ def test_wf_lzoutall_1(plugin): assert 8 == results.output.out -def test_wf_lzoutall_1a(plugin): +def test_wf_lzoutall_1a(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_res function by using lzout.all syntax in the node connections and for wf output @@ -3539,7 +3614,7 @@ def test_wf_lzoutall_1a(plugin): assert results.output.out_all == {"out_add": 8, "out_sub": 4} -def test_wf_lzoutall_st_1(plugin): +def test_wf_lzoutall_st_1(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_res function by using lzout.all syntax @@ -3560,7 +3635,7 @@ def test_wf_lzoutall_st_1(plugin): assert results.output.out_add == [8, 62, 62, 602] -def test_wf_lzoutall_st_1a(plugin): +def test_wf_lzoutall_st_1a(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_res function by using lzout.all syntax @@ -3586,7 +3661,7 @@ def test_wf_lzoutall_st_1a(plugin): ] -def test_wf_lzoutall_st_2(plugin): +def test_wf_lzoutall_st_2(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_res function by using lzout.all syntax @@ -3610,7 +3685,7 @@ def test_wf_lzoutall_st_2(plugin): assert results.output.out_add[1] == [62, 602] -def test_wf_lzoutall_st_2a(plugin): +def test_wf_lzoutall_st_2a(plugin, tmpdir): """ workflow with 2 tasks, no splitter passing entire result object to add2_res function by using lzout.all syntax @@ -3639,7 +3714,7 @@ def test_wf_lzoutall_st_2a(plugin): # worfklows that have files in the result, the files should be copied to the wf dir -def test_wf_resultfile_1(plugin): +def test_wf_resultfile_1(plugin, tmpdir): """ workflow with a file in the result, file should be copied to the wf dir""" wf = Workflow(name="wf_file_1", input_spec=["x"]) wf.add(fun_write_file(name="writefile", filename=wf.lzin.x)) @@ -3656,7 +3731,7 @@ def test_wf_resultfile_1(plugin): assert results.output.wf_out == wf.output_dir / "file_1.txt" -def test_wf_resultfile_2(plugin): +def test_wf_resultfile_2(plugin, tmpdir): """ workflow with a list of files in the wf result, all files should be copied to the wf dir """ @@ -3677,7 +3752,7 @@ def test_wf_resultfile_2(plugin): assert file == wf.output_dir / file_list[ii] -def test_wf_resultfile_3(plugin): +def test_wf_resultfile_3(plugin, tmpdir): """ workflow with a dictionaries of files in the wf result, all files should be copied to the wf dir """ @@ -3702,7 +3777,7 @@ def test_wf_resultfile_3(plugin): assert val == wf.output_dir / file_list[ii] -def test_wf_upstream_error1(plugin): +def test_wf_upstream_error1(plugin, tmpdir): """ workflow with two tasks, task2 dependent on an task1 which raised an error""" wf = Workflow(name="wf", input_spec=["x"]) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) @@ -3718,7 +3793,7 @@ def test_wf_upstream_error1(plugin): assert "raised an error" in str(excinfo.value) -def test_wf_upstream_error2(plugin): +def test_wf_upstream_error2(plugin, tmpdir): """ task2 dependent on task1, task1 errors, workflow-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ @@ -3737,7 +3812,7 @@ def test_wf_upstream_error2(plugin): assert "raised an error" in str(excinfo.value) -def test_wf_upstream_error3(plugin): +def test_wf_upstream_error3(plugin, tmpdir): """ task2 dependent on task1, task1 errors, task-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ @@ -3756,7 +3831,7 @@ def test_wf_upstream_error3(plugin): assert "raised an error" in str(excinfo.value) -def test_wf_upstream_error4(plugin): +def test_wf_upstream_error4(plugin, tmpdir): """ workflow with one task, which raises an error""" wf = Workflow(name="wf", input_spec=["x"]) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) @@ -3771,7 +3846,7 @@ def test_wf_upstream_error4(plugin): assert "addvar1" in str(excinfo.value) -def test_wf_upstream_error5(plugin): +def test_wf_upstream_error5(plugin, tmpdir): """ nested workflow with one task, which raises an error""" wf_main = Workflow(name="wf_main", input_spec=["x"]) wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) @@ -3791,7 +3866,7 @@ def test_wf_upstream_error5(plugin): assert "raised an error" in str(excinfo.value) -def test_wf_upstream_error6(plugin): +def test_wf_upstream_error6(plugin, tmpdir): """ nested workflow with two tasks, the first one raises an error""" wf_main = Workflow(name="wf_main", input_spec=["x"]) wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) @@ -3812,7 +3887,7 @@ def test_wf_upstream_error6(plugin): assert "raised an error" in str(excinfo.value) -def test_wf_upstream_error7(plugin): +def test_wf_upstream_error7(plugin, tmpdir): """ workflow with three sequential tasks, the first task raises an error the last task is set as the workflow output @@ -3834,7 +3909,7 @@ def test_wf_upstream_error7(plugin): assert wf.addvar2._errored == wf.addvar3._errored == ["addvar1"] -def test_wf_upstream_error7a(plugin): +def test_wf_upstream_error7a(plugin, tmpdir): """ workflow with three sequential tasks, the first task raises an error the second task is set as the workflow output @@ -3856,7 +3931,7 @@ def test_wf_upstream_error7a(plugin): assert wf.addvar2._errored == wf.addvar3._errored == ["addvar1"] -def test_wf_upstream_error7b(plugin): +def test_wf_upstream_error7b(plugin, tmpdir): """ workflow with three sequential tasks, the first task raises an error the second and the third tasks are set as the workflow output @@ -3878,7 +3953,7 @@ def test_wf_upstream_error7b(plugin): assert wf.addvar2._errored == wf.addvar3._errored == ["addvar1"] -def test_wf_upstream_error8(plugin): +def test_wf_upstream_error8(plugin, tmpdir): """ workflow with three tasks, the first one raises an error, so 2 others are removed""" wf = Workflow(name="wf", input_spec=["x"]) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) @@ -3898,7 +3973,7 @@ def test_wf_upstream_error8(plugin): assert wf.addvar2._errored == wf.addtwo._errored == ["addvar1"] -def test_wf_upstream_error9(plugin): +def test_wf_upstream_error9(plugin, tmpdir): """ workflow with five tasks with two "branches", one branch has an error, the second is fine @@ -3924,7 +3999,7 @@ def test_wf_upstream_error9(plugin): assert wf.follow_err._errored == ["err"] -def test_wf_upstream_error9a(plugin): +def test_wf_upstream_error9a(plugin, tmpdir): """ workflow with five tasks with two "branches", one branch has an error, the second is fine @@ -3948,7 +4023,7 @@ def test_wf_upstream_error9a(plugin): assert wf.follow_err._errored == ["err"] -def test_wf_upstream_error9b(plugin): +def test_wf_upstream_error9b(plugin, tmpdir): """ workflow with five tasks with two "branches", one branch has an error, the second is fine From 153140698fa94553cd88eb98f9a41dda00dc43da Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 8 Oct 2020 23:22:18 +0800 Subject: [PATCH 157/271] fix workflow cache_dir in test_submitter.py --- pydra/engine/tests/test_submitter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 0d40b7f1b0..e420294d3e 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -23,7 +23,6 @@ def sleep_add_one(x): def test_callable_wf(plugin, tmpdir): wf = gen_basic_wf() - wf.cache_dir = tmpdir with pytest.raises(NotImplementedError): wf() @@ -33,6 +32,8 @@ def test_callable_wf(plugin, tmpdir): del wf, res wf = gen_basic_wf() + wf.cache_dir = tmpdir + sub = Submitter(plugin) res = wf(submitter=sub) assert res.output.out == 9 @@ -96,8 +97,9 @@ def test_wf_in_wf(plugin, tmpdir): subwf.set_output([("out", subwf.sub_b.lzout.out)]) # connect, then add subwf.inputs.x = wf.wf_a.lzout.out - wf.add(subwf) + subwf.cache_dir = tmpdir + wf.add(subwf) wf.add(sleep_add_one(name="wf_b", x=wf.sub_wf.lzout.out)) wf.set_output([("out", wf.wf_b.lzout.out)]) wf.cache_dir = tmpdir @@ -118,6 +120,7 @@ def test_wf2(plugin_dask_opt, tmpdir): wfnd.add(sleep_add_one(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = 2 + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) From 83cf8eaeb5febe5a13128ac9efe7310987428d10 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 8 Oct 2020 23:37:41 +0800 Subject: [PATCH 158/271] fix workflow cache_dir in test_workflow.py --- pydra/engine/tests/test_workflow.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 50265d3eb9..c34e004d2d 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -153,6 +153,7 @@ def test_wf_2(plugin, tmpdir): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -263,7 +264,7 @@ def test_wf_2d_outpasdict(plugin, tmpdir): @pytest.mark.flaky(reruns=3) # when dask -def test_wf_3(plugin_dask_opt): +def test_wf_3(plugin_dask_opt, tmpdir): """ testing None value for an input""" wf = Workflow(name="wf_3", input_spec=["x", "y"]) wf.add(fun_addvar_none(name="addvar", a=wf.lzin.x, b=wf.lzin.y)) @@ -375,7 +376,7 @@ def test_wf_5a(plugin, tmpdir): assert 1 == results.output.out_sub -def test_wf_5b_exception(): +def test_wf_5b_exception(tmpdir): """ set_output used twice with the same name - exception should be raised """ wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) wf.add(fun_addsubvar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) @@ -995,7 +996,7 @@ def test_wf_3sernd_ndst_1(plugin, tmpdir): @pytest.mark.flaky(reruns=3) # when dask -def test_wf_3nd_st_1(plugin_dask_opt): +def test_wf_3nd_st_1(plugin_dask_opt, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter on the workflow level """ @@ -1023,7 +1024,7 @@ def test_wf_3nd_st_1(plugin_dask_opt): @pytest.mark.flaky(reruns=3) # when dask -def test_wf_3nd_ndst_1(plugin_dask_opt): +def test_wf_3nd_ndst_1(plugin_dask_opt, tmpdir): """ workflow with three tasks, third one connected to two previous tasks, splitter on the tasks levels """ @@ -1712,6 +1713,7 @@ def test_wfasnd_1(plugin, tmpdir): wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = 2 + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1737,6 +1739,7 @@ def test_wfasnd_wfinp_1(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir wf.add(wfnd) wf.inputs.x = 2 @@ -1764,6 +1767,7 @@ def test_wfasnd_wfndupdate(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=2) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"], x=3) wfnd.inputs.x = wf.lzin.x @@ -1790,6 +1794,7 @@ def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=2) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wfnd) @@ -1835,6 +1840,7 @@ def test_wfasnd_st_1(plugin, tmpdir): wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.split("x") wfnd.inputs.x = [2, 4] + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1862,6 +1868,7 @@ def test_wfasnd_st_updatespl_1(plugin, tmpdir): wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = [2, 4] + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1890,6 +1897,7 @@ def test_wfasnd_ndst_1(plugin, tmpdir): # TODO: without this the test is failing wfnd.plugin = plugin wfnd.inputs.x = [2, 4] + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1917,6 +1925,7 @@ def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): # TODO: without this the test is failing wfnd.plugin = plugin wfnd.inputs.x = [2, 4] + wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1943,6 +1952,7 @@ def test_wfasnd_wfst_1(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir wf.add(wfnd) wf.split("x") @@ -1976,6 +1986,7 @@ def test_wfasnd_st_2(plugin, tmpdir): wfnd.split(("x", "y")) wfnd.inputs.x = [2, 4] wfnd.inputs.y = [1, 10] + wfnd.cache_dir = tmpdir wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) wf.add(wfnd) @@ -2002,6 +2013,7 @@ def test_wfasnd_wfst_2(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x", "y"], x=wf.lzin.x, y=wf.lzin.y) wfnd.add(multiply(name="mult", x=wfnd.lzin.x, y=wfnd.lzin.y)) wfnd.set_output([("out", wfnd.mult.lzout.out)]) + wfnd.cache_dir = tmpdir wf.add(wfnd) wf.add(add2(name="add2", x=wf.wfnd.lzout.out)) @@ -2010,6 +2022,7 @@ def test_wfasnd_wfst_2(plugin, tmpdir): wf.inputs.y = [1, 10] wf.set_output([("out", wf.add2.lzout.out)]) wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -2039,6 +2052,7 @@ def test_wfasnd_ndst_3(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) @@ -2068,6 +2082,7 @@ def test_wfasnd_wfst_3(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) + wfnd.cache_dir = tmpdir wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) From 4a608b1010ae9f62e2997006ac259492032b1dd1 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 9 Oct 2020 00:02:10 +0800 Subject: [PATCH 159/271] add tmpdir to tests in test_workflow.py --- pydra/engine/tests/test_node_task.py | 74 +++++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index 518cf95a02..34ec3c11ca 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -365,9 +365,10 @@ def test_odir_init(): @pytest.mark.flaky(reruns=2) # when dask -def test_task_nostate_1(plugin_dask_opt): +def test_task_nostate_1(plugin_dask_opt, tmpdir): """ task without splitter""" nn = fun_addtwo(name="NA", a=3) + nn.cache_dir = tmpdir assert np.allclose(nn.inputs.a, [3]) assert nn.state is None @@ -405,9 +406,10 @@ def test_task_nostate_1_call(): @pytest.mark.flaky(reruns=2) # when dask -def test_task_nostate_1_call_subm(plugin_dask_opt): +def test_task_nostate_1_call_subm(plugin_dask_opt, tmpdir): """ task without splitter""" nn = fun_addtwo(name="NA", a=3) + nn.cache_dir = tmpdir assert np.allclose(nn.inputs.a, [3]) assert nn.state is None @@ -422,9 +424,10 @@ def test_task_nostate_1_call_subm(plugin_dask_opt): @pytest.mark.flaky(reruns=2) # when dask -def test_task_nostate_1_call_plug(plugin_dask_opt): +def test_task_nostate_1_call_plug(plugin_dask_opt, tmpdir): """ task without splitter""" nn = fun_addtwo(name="NA", a=3) + nn.cache_dir = tmpdir assert np.allclose(nn.inputs.a, [3]) assert nn.state is None @@ -450,9 +453,10 @@ def test_task_nostate_1_call_updateinp(): assert nn.output_dir.exists() -def test_task_nostate_2(plugin): +def test_task_nostate_2(plugin, tmpdir): """ task with a list as an input, but no splitter""" nn = moment(name="NA", n=3, lst=[2, 3, 4]) + nn.cache_dir = tmpdir assert np.allclose(nn.inputs.n, [3]) assert np.allclose(nn.inputs.lst, [2, 3, 4]) assert nn.state is None @@ -467,9 +471,10 @@ def test_task_nostate_2(plugin): assert nn.output_dir.exists() -def test_task_nostate_3(plugin): +def test_task_nostate_3(plugin, tmpdir): """ task with a dictionary as an input""" nn = fun_dict(name="NA", d={"a": "ala", "b": "bala"}) + nn.cache_dir = tmpdir assert nn.inputs.d == {"a": "ala", "b": "bala"} with Submitter(plugin=plugin) as sub: @@ -489,6 +494,7 @@ def test_task_nostate_4(plugin, tmpdir): f.write("hello from pydra\n") nn = fun_file(name="NA", filename=file1) + nn.cache_dir = tmpdir with Submitter(plugin) as sub: sub(nn) @@ -719,13 +725,14 @@ def test_task_nostate_cachelocations_updated(plugin, tmpdir): @pytest.mark.flaky(reruns=2) # when dask @pytest.mark.parametrize("input_type", ["list", "array"]) -def test_task_state_1(plugin_dask_opt, input_type): +def test_task_state_1(plugin_dask_opt, input_type, tmpdir): """ task with the simplest splitter""" a_in = [3, 5] if input_type == "array": a_in = np.array(a_in) nn = fun_addtwo(name="NA").split(splitter="a", a=a_in) + nn.cache_dir = tmpdir assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] @@ -761,11 +768,12 @@ def test_task_state_1(plugin_dask_opt, input_type): assert odir.exists() -def test_task_state_1a(plugin): +def test_task_state_1a(plugin, tmpdir): """ task with the simplest splitter (inputs set separately)""" nn = fun_addtwo(name="NA") nn.split(splitter="a") nn.inputs.a = [3, 5] + nn.cache_dir = tmpdir assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] @@ -781,11 +789,12 @@ def test_task_state_1a(plugin): assert results[i].output.out == res[1] -def test_task_state_singl_1(plugin): +def test_task_state_singl_1(plugin, tmpdir): """ Tasks with two inputs and a splitter (no combiner) one input is a single value, the other is in the splitter and combiner """ nn = fun_addvar(name="NA").split(splitter="a", a=[3, 5], b=10) + nn.cache_dir = tmpdir assert nn.inputs.a == [3, 5] assert nn.inputs.b == 10 @@ -839,7 +848,14 @@ def test_task_state_singl_1(plugin): ) @pytest.mark.parametrize("input_type", ["list", "array", "mixed"]) def test_task_state_2( - plugin, splitter, state_splitter, state_rpn, expected, expected_ind, input_type + plugin, + splitter, + state_splitter, + state_rpn, + expected, + expected_ind, + input_type, + tmpdir, ): """ Tasks with two inputs and a splitter (no combiner)""" a_in, b_in = [3, 5], [10, 20] @@ -848,6 +864,8 @@ def test_task_state_2( elif input_type == "mixed": a_in = np.array(a_in) nn = fun_addvar(name="NA").split(splitter=splitter, a=a_in, b=b_in) + nn.cache_dir = tmpdir + assert (nn.inputs.a == np.array([3, 5])).all() assert (nn.inputs.b == np.array([10, 20])).all() assert nn.state.splitter == state_splitter @@ -883,9 +901,10 @@ def test_task_state_2( assert odir.exists() -def test_task_state_3(plugin): +def test_task_state_3(plugin, tmpdir): """ task with the simplest splitter, the input is an empty list""" nn = fun_addtwo(name="NA").split(splitter="a", a=[]) + nn.cache_dir = tmpdir assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] @@ -904,12 +923,14 @@ def test_task_state_3(plugin): @pytest.mark.parametrize("input_type", ["list", "array"]) -def test_task_state_4(plugin, input_type): +def test_task_state_4(plugin, input_type, tmpdir): """ task with a list as an input, and a simple splitter """ lst_in = [[2, 3, 4], [1, 2, 3]] if input_type == "array": lst_in = np.array(lst_in) nn = moment(name="NA", n=3, lst=lst_in).split(splitter="lst") + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, 3) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == "NA.lst" @@ -935,9 +956,11 @@ def test_task_state_4(plugin, input_type): assert odir.exists() -def test_task_state_4a(plugin): +def test_task_state_4a(plugin, tmpdir): """ task with a tuple as an input, and a simple splitter """ nn = moment(name="NA", n=3, lst=[(2, 3, 4), (1, 2, 3)]).split(splitter="lst") + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, 3) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == "NA.lst" @@ -955,11 +978,13 @@ def test_task_state_4a(plugin): assert odir.exists() -def test_task_state_5(plugin): +def test_task_state_5(plugin, tmpdir): """ task with a list as an input, and the variable is part of the scalar splitter""" nn = moment(name="NA", n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]]).split( splitter=("n", "lst") ) + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, [1, 3]) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == ("NA.n", "NA.lst") @@ -977,13 +1002,15 @@ def test_task_state_5(plugin): assert odir.exists() -def test_task_state_5_exception(plugin): +def test_task_state_5_exception(plugin, tmpdir): """ task with a list as an input, and the variable is part of the scalar splitter the shapes are not matching, so exception should be raised """ nn = moment(name="NA", n=[1, 3, 3], lst=[[2, 3, 4], [1, 2, 3]]).split( splitter=("n", "lst") ) + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, [1, 3, 3]) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == ("NA.n", "NA.lst") @@ -994,11 +1021,13 @@ def test_task_state_5_exception(plugin): assert "shape" in str(excinfo.value) -def test_task_state_6(plugin): +def test_task_state_6(plugin, tmpdir): """ ask with a list as an input, and the variable is part of the outer splitter """ nn = moment(name="NA", n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]]).split( splitter=["n", "lst"] ) + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, [1, 3]) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == ["NA.n", "NA.lst"] @@ -1016,11 +1045,13 @@ def test_task_state_6(plugin): assert odir.exists() -def test_task_state_6a(plugin): +def test_task_state_6a(plugin, tmpdir): """ ask with a tuple as an input, and the variable is part of the outer splitter """ nn = moment(name="NA", n=[1, 3], lst=[(2, 3, 4), (1, 2, 3)]).split( splitter=["n", "lst"] ) + nn.cache_dir = tmpdir + assert np.allclose(nn.inputs.n, [1, 3]) assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) assert nn.state.splitter == ["NA.n", "NA.lst"] @@ -1039,9 +1070,10 @@ def test_task_state_6a(plugin): @pytest.mark.flaky(reruns=2) # when dask -def test_task_state_comb_1(plugin_dask_opt): +def test_task_state_comb_1(plugin_dask_opt, tmpdir): """ task with the simplest splitter and combiner""" nn = fun_addtwo(name="NA").split(a=[3, 5], splitter="a").combine(combiner="a") + nn.cache_dir = tmpdir assert (nn.inputs.a == np.array([3, 5])).all() @@ -1173,6 +1205,7 @@ def test_task_state_comb_2( state_rpn_final, expected, expected_val, + tmpdir, ): """ Tasks with scalar and outer splitters and partial or full combiners""" nn = ( @@ -1180,6 +1213,7 @@ def test_task_state_comb_2( .split(a=[3, 5], b=[10, 20], splitter=splitter) .combine(combiner=combiner) ) + nn.cache_dir = tmpdir assert (nn.inputs.a == np.array([3, 5])).all() @@ -1219,11 +1253,12 @@ def test_task_state_comb_2( assert odir.exists() -def test_task_state_comb_singl_1(plugin): +def test_task_state_comb_singl_1(plugin, tmpdir): """ Tasks with two inputs; one input is a single value, the other is in the splitter and combiner """ nn = fun_addvar(name="NA").split(splitter="a", a=[3, 5], b=10).combine(combiner="a") + nn.cache_dir = tmpdir assert nn.inputs.a == [3, 5] assert nn.inputs.b == 10 @@ -1248,9 +1283,10 @@ def test_task_state_comb_singl_1(plugin): assert odir.exists() -def test_task_state_comb_3(plugin): +def test_task_state_comb_3(plugin, tmpdir): """ task with the simplest splitter, the input is an empty list""" nn = fun_addtwo(name="NA").split(splitter="a", a=[]).combine(combiner=["a"]) + nn.cache_dir = tmpdir assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] From ec3e92b1ec34c5b8d4de26002802b5d69cfc4d74 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 9 Oct 2020 00:12:27 +0800 Subject: [PATCH 160/271] add tmpdir to tests in test_shelltask.py --- pydra/engine/tests/test_shelltask.py | 142 +++++++++++++-------------- pydra/engine/tests/utils.py | 5 +- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 740e22d78c..38b1e19dc2 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -17,20 +17,20 @@ @pytest.mark.flaky(reruns=2) # when dask @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_1(plugin_dask_opt, results_function): +def test_shell_cmd_1(plugin_dask_opt, results_function, tmpdir): """ simple command, no arguments """ cmd = ["pwd"] shelly = ShellCommandTask(name="shelly", executable=cmd) assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin=plugin_dask_opt) + res = results_function(shelly, plugin=plugin_dask_opt, tmpdir=tmpdir) assert Path(res.output.stdout.rstrip()) == shelly.output_dir assert res.output.return_code == 0 assert res.output.stderr == "" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_1_strip(plugin, results_function): +def test_shell_cmd_1_strip(plugin, results_function, tmpdir): """ simple command, no arguments strip option to remove \n at the end os stdout """ @@ -38,27 +38,27 @@ def test_shell_cmd_1_strip(plugin, results_function): shelly = ShellCommandTask(name="shelly", executable=cmd, strip=True) assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert Path(res.output.stdout) == Path(shelly.output_dir) assert res.output.return_code == 0 assert res.output.stderr == "" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_2(plugin, results_function): +def test_shell_cmd_2(plugin, results_function, tmpdir): """ a command with arguments, cmd and args given as executable """ cmd = ["echo", "hail", "pydra"] shelly = ShellCommandTask(name="shelly", executable=cmd) assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout.strip() == " ".join(cmd[1:]) assert res.output.return_code == 0 assert res.output.stderr == "" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_2a(plugin, results_function): +def test_shell_cmd_2a(plugin, results_function, tmpdir): """ a command with arguments, using executable and args """ cmd_exec = "echo" cmd_args = ["hail", "pydra"] @@ -67,14 +67,14 @@ def test_shell_cmd_2a(plugin, results_function): assert shelly.inputs.executable == "echo" assert shelly.cmdline == "echo " + " ".join(cmd_args) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout.strip() == " ".join(cmd_args) assert res.output.return_code == 0 assert res.output.stderr == "" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_2b(plugin, results_function): +def test_shell_cmd_2b(plugin, results_function, tmpdir): """ a command with arguments, using strings executable and args """ cmd_exec = "echo" cmd_args = "pydra" @@ -83,7 +83,7 @@ def test_shell_cmd_2b(plugin, results_function): assert shelly.inputs.executable == "echo" assert shelly.cmdline == "echo pydra" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "pydra\n" assert res.output.return_code == 0 assert res.output.stderr == "" @@ -250,7 +250,7 @@ def test_wf_shell_cmd_1(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_1(plugin, results_function, use_validator): +def test_shell_cmd_inputspec_1(plugin, results_function, use_validator, tmpdir): """ a command with executable, args and one command opt, using a customized input_spec to add the opt to the command in the right place that is specified in metadata["cmd_pos"] @@ -284,12 +284,12 @@ def test_shell_cmd_inputspec_1(plugin, results_function, use_validator): assert shelly.inputs.args == cmd_args assert shelly.cmdline == "echo -n hello from pydra" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "hello from pydra" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_2(plugin, results_function, use_validator): +def test_shell_cmd_inputspec_2(plugin, results_function, use_validator, tmpdir): """ a command with executable, args and two command options, using a customized input_spec to add the opt to the command in the right place that is specified in metadata["cmd_pos"] @@ -331,12 +331,12 @@ def test_shell_cmd_inputspec_2(plugin, results_function, use_validator): assert shelly.inputs.executable == cmd_exec assert shelly.inputs.args == cmd_args assert shelly.cmdline == "echo -n HELLO from pydra" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "HELLO from pydra" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_3(plugin, results_function): +def test_shell_cmd_inputspec_3(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided """ cmd_exec = "echo" hello = "HELLO" @@ -365,12 +365,12 @@ def test_shell_cmd_inputspec_3(plugin, results_function): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "HELLO\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_3a(plugin, results_function): +def test_shell_cmd_inputspec_3a(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided using shorter syntax for input spec (no attr.ib) """ @@ -394,12 +394,12 @@ def test_shell_cmd_inputspec_3a(plugin, results_function): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "HELLO\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_3b(plugin, results_function): +def test_shell_cmd_inputspec_3b(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided after init""" cmd_exec = "echo" hello = "HELLO" @@ -429,7 +429,7 @@ def test_shell_cmd_inputspec_3b(plugin, results_function): shelly.inputs.text = hello assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "HELLO\n" @@ -464,7 +464,7 @@ def test_shell_cmd_inputspec_3c_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_3c(plugin, results_function): +def test_shell_cmd_inputspec_3c(plugin, results_function, tmpdir): """ mandatory=False, so tasks runs fine even without the value """ cmd_exec = "echo" my_input_spec = SpecInfo( @@ -493,12 +493,12 @@ def test_shell_cmd_inputspec_3c(plugin, results_function): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_4(plugin, results_function): +def test_shell_cmd_inputspec_4(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided """ cmd_exec = "echo" my_input_spec = SpecInfo( @@ -524,12 +524,12 @@ def test_shell_cmd_inputspec_4(plugin, results_function): assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "Hello\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_4a(plugin, results_function): +def test_shell_cmd_inputspec_4a(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided using shorter syntax for input spec (no attr.ib) """ @@ -550,12 +550,12 @@ def test_shell_cmd_inputspec_4a(plugin, results_function): assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "Hello\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_4b(plugin, results_function): +def test_shell_cmd_inputspec_4b(plugin, results_function, tmpdir): """ mandatory field added to fields, value provided """ cmd_exec = "echo" my_input_spec = SpecInfo( @@ -581,7 +581,7 @@ def test_shell_cmd_inputspec_4b(plugin, results_function): assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hi" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "Hi\n" @@ -654,7 +654,7 @@ def test_shell_cmd_inputspec_4d_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_5_nosubm(plugin, results_function): +def test_shell_cmd_inputspec_5_nosubm(plugin, results_function, tmpdir): """ checking xor in metadata: task should work fine, since only one option is True""" cmd_exec = "ls" cmd_t = True @@ -695,7 +695,7 @@ def test_shell_cmd_inputspec_5_nosubm(plugin, results_function): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -t" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) def test_shell_cmd_inputspec_5a_exception(plugin): @@ -747,7 +747,7 @@ def test_shell_cmd_inputspec_5a_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_6(plugin, results_function): +def test_shell_cmd_inputspec_6(plugin, results_function, tmpdir): """ checking requires in metadata: the required field is set in the init, so the task works fine """ @@ -790,7 +790,7 @@ def test_shell_cmd_inputspec_6(plugin, results_function): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) def test_shell_cmd_inputspec_6a_exception(plugin): @@ -834,7 +834,7 @@ def test_shell_cmd_inputspec_6a_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_6b(plugin, results_function): +def test_shell_cmd_inputspec_6b(plugin, results_function, tmpdir): """ checking requires in metadata: the required field set after the init """ @@ -878,11 +878,11 @@ def test_shell_cmd_inputspec_6b(plugin, results_function): shelly.inputs.opt_l = cmd_l assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_7(plugin, results_function): +def test_shell_cmd_inputspec_7(plugin, results_function, tmpdir): """ providing output name using input_spec, using name_tamplate in metadata @@ -911,7 +911,7 @@ def test_shell_cmd_inputspec_7(plugin, results_function): name="shelly", executable=cmd, args=args, input_spec=my_input_spec ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1.exists() # checking if the file is created in a good place @@ -920,7 +920,7 @@ def test_shell_cmd_inputspec_7(plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_7a(plugin, results_function): +def test_shell_cmd_inputspec_7a(plugin, results_function, tmpdir): """ providing output name using input_spec, using name_tamplate in metadata @@ -951,7 +951,7 @@ def test_shell_cmd_inputspec_7a(plugin, results_function): name="shelly", executable=cmd, args=args, input_spec=my_input_spec ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1_changed.exists() # checking if the file is created in a good place @@ -960,7 +960,7 @@ def test_shell_cmd_inputspec_7a(plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_7b(plugin, results_function): +def test_shell_cmd_inputspec_7b(plugin, results_function, tmpdir): """ providing new file and output name using input_spec, using name_template in metadata @@ -998,7 +998,7 @@ def test_shell_cmd_inputspec_7b(plugin, results_function): input_spec=my_input_spec, ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1054,7 +1054,7 @@ def test_shell_cmd_inputspec_8(plugin, results_function, tmpdir): input_spec=my_input_spec, ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1110,7 +1110,7 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmpdir): input_spec=my_input_spec, ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1154,7 +1154,7 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file_copy.txt" @@ -1202,7 +1202,7 @@ def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file_copy" @@ -1249,7 +1249,7 @@ def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file" @@ -1295,7 +1295,7 @@ def test_shell_cmd_inputspec_10(plugin, results_function, tmpdir): ) assert shelly.inputs.executable == cmd_exec - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "hello from boston" @@ -1345,7 +1345,7 @@ def test_shell_cmd_inputspec_copyfile_1(plugin, results_function, tmpdir): name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is copied, and than it is changed in place @@ -1403,7 +1403,7 @@ def test_shell_cmd_inputspec_copyfile_1a(plugin, results_function, tmpdir): name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is uses a soft link, but it creates and an extra copy before modifying @@ -1475,7 +1475,7 @@ def test_shell_cmd_inputspec_copyfile_1b(plugin, results_function, tmpdir): name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is not copied, it is changed in place @@ -1485,7 +1485,7 @@ def test_shell_cmd_inputspec_copyfile_1b(plugin, results_function, tmpdir): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_state_1(plugin, results_function): +def test_shell_cmd_inputspec_state_1(plugin, results_function, tmpdir): """ adding state to the input from input_spec """ cmd_exec = "echo" hello = ["HELLO", "hi"] @@ -1515,7 +1515,7 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function): assert shelly.inputs.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res[0].output.stdout == "HELLO\n" assert res[1].output.stdout == "hi\n" @@ -1565,7 +1565,7 @@ def test_shell_cmd_inputspec_typeval_2(use_validator): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_state_1a(plugin, results_function): +def test_shell_cmd_inputspec_state_1a(plugin, results_function, tmpdir): """ adding state to the input from input_spec using shorter syntax for input_spec (without default) """ @@ -1589,13 +1589,13 @@ def test_shell_cmd_inputspec_state_1a(plugin, results_function): ).split("text") assert shelly.inputs.executable == cmd_exec - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res[0].output.stdout == "HELLO\n" assert res[1].output.stdout == "hi\n" @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_state_2(plugin, results_function): +def test_shell_cmd_inputspec_state_2(plugin, results_function, tmpdir): """ adding splitter to input tha is used in the output_file_tamplate """ @@ -1623,7 +1623,7 @@ def test_shell_cmd_inputspec_state_2(plugin, results_function): name="shelly", executable=cmd, args=args, input_spec=my_input_spec ).split("args") - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) for i in range(len(args)): assert res[i].output.stdout == "" assert res[i].output.out1.exists() @@ -1670,7 +1670,7 @@ def test_shell_cmd_inputspec_state_3(plugin, results_function, tmpdir): assert shelly.inputs.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res[0].output.stdout == "hello from pydra" assert res[1].output.stdout == "have a nice one" @@ -1725,7 +1725,7 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmpdir): ).split("orig_file") txt_l = ["from pydra", "world"] - res_l = results_function(shelly, plugin) + res_l = results_function(shelly, plugin, tmpdir) for i, res in enumerate(res_l): assert res.output.stdout == "" assert res.output.out_file.exists() @@ -2218,7 +2218,7 @@ def test_wf_shell_cmd_ndst_1(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_1(plugin, results_function): +def test_shell_cmd_outputspec_1(plugin, results_function, tmpdir): """ customised output_spec, adding files to the output, providing specific pathname """ @@ -2230,13 +2230,13 @@ def test_shell_cmd_outputspec_1(plugin, results_function): ) shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.newfile.exists() @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_1a(plugin, results_function): +def test_shell_cmd_outputspec_1a(plugin, results_function, tmpdir): """ customised output_spec, adding files to the output, providing specific pathname """ @@ -2248,7 +2248,7 @@ def test_shell_cmd_outputspec_1a(plugin, results_function): ) shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.newfile.exists() @@ -2272,7 +2272,7 @@ def test_shell_cmd_outputspec_1b_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_2(plugin, results_function): +def test_shell_cmd_outputspec_2(plugin, results_function, tmpdir): """ customised output_spec, adding files to the output, using a wildcard in default @@ -2285,7 +2285,7 @@ def test_shell_cmd_outputspec_2(plugin, results_function): ) shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.newfile.exists() @@ -2310,7 +2310,7 @@ def test_shell_cmd_outputspec_2a_exception(plugin): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_3(plugin, results_function): +def test_shell_cmd_outputspec_3(plugin, results_function, tmpdir): """ customised output_spec, adding files to the output, using a wildcard in default, should collect two files @@ -2323,7 +2323,7 @@ def test_shell_cmd_outputspec_3(plugin, results_function): ) shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" # newfile is a list assert len(res.output.newfile) == 2 @@ -2331,7 +2331,7 @@ def test_shell_cmd_outputspec_3(plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_4(plugin, results_function): +def test_shell_cmd_outputspec_4(plugin, results_function, tmpdir): """ customised output_spec, adding files to the output, using a function to collect output, the function is saved in the field metadata @@ -2349,7 +2349,7 @@ def gather_output(keyname, output_dir): ) shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" # newfile is a list assert len(res.output.newfile) == 2 @@ -2357,7 +2357,7 @@ def gather_output(keyname, output_dir): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_5(plugin, results_function): +def test_shell_cmd_outputspec_5(plugin, results_function, tmpdir): """ providing output name by providing output_file_template (similar to the previous example, but not touching input_spec) @@ -2386,7 +2386,7 @@ def test_shell_cmd_outputspec_5(plugin, results_function): name="shelly", executable=cmd, args=args, output_spec=my_output_spec ) - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.out1.exists() @@ -2421,7 +2421,7 @@ def test_shell_cmd_outputspec_5a(): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_state_outputspec_1(plugin, results_function): +def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): """ providing output name by providing output_file_template splitter for a field that is used in the template @@ -2450,7 +2450,7 @@ def test_shell_cmd_state_outputspec_1(plugin, results_function): name="shelly", executable=cmd, args=args, output_spec=my_output_spec ).split("args") - res = results_function(shelly, plugin) + res = results_function(shelly, plugin, tmpdir) for i in range(len(args)): assert res[i].output.stdout == "" assert res[i].output.out1.exists() diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index b2fbdab762..edc2533c28 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -23,15 +23,16 @@ ) -def result_no_submitter(shell_task, plugin=None): +def result_no_submitter(shell_task, plugin=None, tmpdir=None): """ helper function to return result when running without submitter """ return shell_task() -def result_submitter(shell_task, plugin): +def result_submitter(shell_task, plugin, tmpdir): """ helper function to return result when running with submitter with specific plugin """ + shell_task.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: shell_task(submitter=sub) return shell_task.result() From d10e9c83ef4f788e0892d2590371c3a406b8437e Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 13 Oct 2020 15:11:16 +0800 Subject: [PATCH 161/271] add tmpdir to test_boutiques.py --- pydra/engine/tests/test_boutiques.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_boutiques.py b/pydra/engine/tests/test_boutiques.py index b2ea022b77..5399b2e88a 100644 --- a/pydra/engine/tests/test_boutiques.py +++ b/pydra/engine/tests/test_boutiques.py @@ -26,12 +26,12 @@ "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_boutiques_1(maskfile, plugin, results_function): +def test_boutiques_1(maskfile, plugin, results_function, tmpdir): """ simple task to run fsl.bet using BoshTask""" btask = BoshTask(name="NA", zenodo_id="1482743") btask.inputs.infile = Infile btask.inputs.maskfile = maskfile - res = results_function(btask, plugin) + res = results_function(btask, plugin, tmpdir) assert res.output.return_code == 0 From 6a8874f34145f320ef0cb52cdd5236e6bf5c3ce0 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 13 Oct 2020 04:24:00 -0400 Subject: [PATCH 162/271] add tmpdir to tests in test_numpy_examples.py --- pydra/engine/tests/test_numpy_examples.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_numpy_examples.py b/pydra/engine/tests/test_numpy_examples.py index 35b8972319..572d8707a2 100644 --- a/pydra/engine/tests/test_numpy_examples.py +++ b/pydra/engine/tests/test_numpy_examples.py @@ -17,12 +17,13 @@ def arrayout(val): return np.array([val, val]) -def test_multiout(plugin): +def test_multiout(plugin, tmpdir): """ testing a simple function that returns a numpy array""" wf = Workflow("wf", input_spec=["val"], val=2) wf.add(arrayout(name="mo", val=wf.lzin.val)) wf.set_output([("array", wf.mo.lzout.b)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin, n_procs=2) as sub: sub(runnable=wf) @@ -33,13 +34,14 @@ def test_multiout(plugin): assert np.array_equal(results[1].output.array, np.array([2, 2])) -def test_multiout_st(plugin): +def test_multiout_st(plugin, tmpdir): """ testing a simple function that returns a numpy array, adding splitter""" wf = Workflow("wf", input_spec=["val"], val=[0, 1, 2]) wf.add(arrayout(name="mo", val=wf.lzin.val)) wf.mo.split("val").combine("val") wf.set_output([("array", wf.mo.lzout.b)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin, n_procs=2) as sub: sub(runnable=wf) From 28d174297508a96dde07c6a43c11f0caab0c0d6b Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 13 Oct 2020 16:53:05 +0800 Subject: [PATCH 163/271] add tmpdir to tests in test_shelltask.py --- pydra/engine/tests/test_shelltask.py | 81 ++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 38b1e19dc2..bf93cbd085 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -93,7 +93,7 @@ def test_shell_cmd_2b(plugin, results_function, tmpdir): @pytest.mark.flaky(reruns=2) -def test_shell_cmd_3(plugin_dask_opt): +def test_shell_cmd_3(plugin_dask_opt, tmpdir): """ commands without arguments splitter = executable """ @@ -101,6 +101,8 @@ def test_shell_cmd_3(plugin_dask_opt): # all args given as executable shelly = ShellCommandTask(name="shelly", executable=cmd).split("executable") + shelly.cache_dir = tmpdir + assert shelly.cmdline == ["pwd", "whoami"] res = shelly(plugin=plugin_dask_opt) assert Path(res[0].output.stdout.rstrip()) == shelly.output_dir[0] @@ -113,7 +115,7 @@ def test_shell_cmd_3(plugin_dask_opt): assert res[0].output.stderr == res[1].output.stderr == "" -def test_shell_cmd_4(plugin): +def test_shell_cmd_4(plugin, tmpdir): """ a command with arguments, using executable and args splitter=args """ @@ -123,6 +125,8 @@ def test_shell_cmd_4(plugin): shelly = ShellCommandTask(name="shelly", executable=cmd_exec, args=cmd_args).split( splitter="args" ) + shelly.cache_dir = tmpdir + assert shelly.inputs.executable == "echo" assert shelly.inputs.args == ["nipype", "pydra"] assert shelly.cmdline == ["echo nipype", "echo pydra"] @@ -135,7 +139,7 @@ def test_shell_cmd_4(plugin): assert res[0].output.stderr == res[1].output.stderr == "" -def test_shell_cmd_5(plugin): +def test_shell_cmd_5(plugin, tmpdir): """ a command with arguments using splitter and combiner for args """ @@ -147,6 +151,8 @@ def test_shell_cmd_5(plugin): .split(splitter="args") .combine("args") ) + shelly.cache_dir = tmpdir + assert shelly.inputs.executable == "echo" assert shelly.inputs.args == ["nipype", "pydra"] assert shelly.cmdline == ["echo nipype", "echo pydra"] @@ -156,7 +162,7 @@ def test_shell_cmd_5(plugin): assert res[1].output.stdout == "pydra\n" -def test_shell_cmd_6(plugin): +def test_shell_cmd_6(plugin, tmpdir): """ a command with arguments, outer splitter for executable and args """ @@ -166,6 +172,8 @@ def test_shell_cmd_6(plugin): shelly = ShellCommandTask(name="shelly", executable=cmd_exec, args=cmd_args).split( splitter=["executable", "args"] ) + shelly.cache_dir = tmpdir + assert shelly.inputs.executable == ["echo", ["echo", "-n"]] assert shelly.inputs.args == ["nipype", "pydra"] assert shelly.cmdline == [ @@ -197,7 +205,7 @@ def test_shell_cmd_6(plugin): ) -def test_shell_cmd_7(plugin): +def test_shell_cmd_7(plugin, tmpdir): """ a command with arguments, outer splitter for executable and args, and combiner=args """ @@ -209,6 +217,8 @@ def test_shell_cmd_7(plugin): .split(splitter=["executable", "args"]) .combine("args") ) + shelly.cache_dir = tmpdir + assert shelly.inputs.executable == ["echo", ["echo", "-n"]] assert shelly.inputs.args == ["nipype", "pydra"] @@ -224,7 +234,7 @@ def test_shell_cmd_7(plugin): # tests with workflows -def test_wf_shell_cmd_1(plugin): +def test_wf_shell_cmd_1(plugin, tmpdir): """ a workflow with two connected commands""" wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"]) wf.inputs.cmd1 = "pwd" @@ -237,6 +247,7 @@ def test_wf_shell_cmd_1(plugin): ) wf.set_output([("out", wf.shelly_ls.lzout.stdout)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: wf(submitter=sub) @@ -427,13 +438,14 @@ def test_shell_cmd_inputspec_3b(plugin, results_function, tmpdir): name="shelly", executable=cmd_exec, input_spec=my_input_spec ) shelly.inputs.text = hello + assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "HELLO\n" -def test_shell_cmd_inputspec_3c_exception(plugin): +def test_shell_cmd_inputspec_3c_exception(plugin, tmpdir): """ mandatory field added to fields, value is not provided, so exception is raised """ cmd_exec = "echo" my_input_spec = SpecInfo( @@ -458,6 +470,8 @@ def test_shell_cmd_inputspec_3c_exception(plugin): shelly = ShellCommandTask( name="shelly", executable=cmd_exec, input_spec=my_input_spec ) + shelly.cache_dir = tmpdir + with pytest.raises(Exception) as excinfo: shelly() assert "mandatory" in str(excinfo.value) @@ -491,6 +505,8 @@ def test_shell_cmd_inputspec_3c(plugin, results_function, tmpdir): shelly = ShellCommandTask( name="shelly", executable=cmd_exec, input_spec=my_input_spec ) + shelly.cache_dir = tmpdir + assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo" res = results_function(shelly, plugin, tmpdir) @@ -520,6 +536,7 @@ def test_shell_cmd_inputspec_4(plugin, results_function, tmpdir): shelly = ShellCommandTask( name="shelly", executable=cmd_exec, input_spec=my_input_spec ) + shelly.cache_dir = tmpdir assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hello" @@ -1742,7 +1759,7 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmpdir): @pytest.mark.flaky(reruns=2) # when dask -def test_wf_shell_cmd_2(plugin_dask_opt): +def test_wf_shell_cmd_2(plugin_dask_opt, tmpdir): """ a workflow with input with defined output_file_template (str) that requires wf.lzin """ @@ -1750,6 +1767,7 @@ def test_wf_shell_cmd_2(plugin_dask_opt): wf.inputs.cmd = "touch" wf.inputs.args = "newfile.txt" + wf.cache_dir = tmpdir my_input_spec = SpecInfo( name="Input", @@ -1788,7 +1806,7 @@ def test_wf_shell_cmd_2(plugin_dask_opt): assert res.output.out_f.parent == wf.output_dir -def test_wf_shell_cmd_2a(plugin): +def test_wf_shell_cmd_2a(plugin, tmpdir): """ a workflow with input with defined output_file_template (tuple) that requires wf.lzin """ @@ -1796,6 +1814,7 @@ def test_wf_shell_cmd_2a(plugin): wf.inputs.cmd = "touch" wf.inputs.args = "newfile.txt" + wf.cache_dir = tmpdir my_input_spec = SpecInfo( name="Input", @@ -1833,7 +1852,7 @@ def test_wf_shell_cmd_2a(plugin): assert res.output.out_f.exists() -def test_wf_shell_cmd_3(plugin): +def test_wf_shell_cmd_3(plugin, tmpdir): """ a workflow with 2 tasks, first one has input with output_file_template (str, uses wf.lzin), that is passed to the second task @@ -1843,6 +1862,7 @@ def test_wf_shell_cmd_3(plugin): wf.inputs.cmd1 = "touch" wf.inputs.cmd2 = "cp" wf.inputs.args = "newfile.txt" + wf.cache_dir = tmpdir my_input_spec1 = SpecInfo( name="Input", @@ -1929,7 +1949,7 @@ def test_wf_shell_cmd_3(plugin): assert res.output.cp_file.parent == wf.output_dir -def test_wf_shell_cmd_3a(plugin): +def test_wf_shell_cmd_3a(plugin, tmpdir): """ a workflow with 2 tasks, first one has input with output_file_template (str, uses wf.lzin), that is passed to the second task @@ -1939,6 +1959,7 @@ def test_wf_shell_cmd_3a(plugin): wf.inputs.cmd1 = "touch" wf.inputs.cmd2 = "cp" wf.inputs.args = "newfile.txt" + wf.cache_dir = tmpdir my_input_spec1 = SpecInfo( name="Input", @@ -2120,7 +2141,7 @@ def test_wf_shell_cmd_state_1(plugin): assert res.output.cp_file.parent == wf.output_dir[i] -def test_wf_shell_cmd_ndst_1(plugin): +def test_wf_shell_cmd_ndst_1(plugin, tmpdir): """ a workflow with 2 tasks and a splitter on the node level, first one has input with output_file_template (str, uses wf.lzin), that is passed to the second task @@ -2130,6 +2151,7 @@ def test_wf_shell_cmd_ndst_1(plugin): wf.inputs.cmd1 = "touch" wf.inputs.cmd2 = "cp" wf.inputs.args = ["newfile_1.txt", "newfile_2.txt"] + wf.cache_dir = tmpdir my_input_spec1 = SpecInfo( name="Input", @@ -2228,7 +2250,9 @@ def test_shell_cmd_outputspec_1(plugin, results_function, tmpdir): fields=[("newfile", File, "newfile_tmp.txt")], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" @@ -2246,14 +2270,16 @@ def test_shell_cmd_outputspec_1a(plugin, results_function, tmpdir): fields=[("newfile", attr.ib(type=File, default="newfile_tmp.txt"))], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.newfile.exists() -def test_shell_cmd_outputspec_1b_exception(plugin): +def test_shell_cmd_outputspec_1b_exception(plugin, tmpdir): """ customised output_spec, adding files to the output, providing specific pathname """ @@ -2263,7 +2289,9 @@ def test_shell_cmd_outputspec_1b_exception(plugin): fields=[("newfile", File, "newfile_tmp_.txt")], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) with pytest.raises(Exception) as exinfo: with Submitter(plugin=plugin) as sub: @@ -2283,14 +2311,16 @@ def test_shell_cmd_outputspec_2(plugin, results_function, tmpdir): fields=[("newfile", File, "newfile_*.txt")], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" assert res.output.newfile.exists() -def test_shell_cmd_outputspec_2a_exception(plugin): +def test_shell_cmd_outputspec_2a_exception(plugin, tmpdir): """ customised output_spec, adding files to the output, using a wildcard in default @@ -2301,7 +2331,9 @@ def test_shell_cmd_outputspec_2a_exception(plugin): fields=[("newfile", File, "newfile_*K.txt")], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) with pytest.raises(Exception) as excinfo: with Submitter(plugin=plugin) as sub: @@ -2321,7 +2353,9 @@ def test_shell_cmd_outputspec_3(plugin, results_function, tmpdir): fields=[("newfile", File, "newfile_*.txt")], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" @@ -2347,7 +2381,9 @@ def gather_output(keyname, output_dir): fields=[("newfile", attr.ib(type=File, metadata={"callable": gather_output}))], bases=(ShellOutSpec,), ) - shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + shelly = ShellCommandTask( + name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir + ) res = results_function(shelly, plugin, tmpdir) assert res.output.stdout == "" @@ -2459,7 +2495,7 @@ def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): # customised output_spec for tasks in workflows -def test_shell_cmd_outputspec_wf_1(plugin): +def test_shell_cmd_outputspec_wf_1(plugin, tmpdir): """ customised output_spec for tasks within a Workflow, adding files to the output, providing specific pathname @@ -2468,6 +2504,7 @@ def test_shell_cmd_outputspec_wf_1(plugin): cmd = ["touch", "newfile_tmp.txt"] wf = Workflow(name="wf", input_spec=["cmd"]) wf.inputs.cmd = cmd + wf.cache_dir = tmpdir my_output_spec = SpecInfo( name="Output", From 6f8c3a7dff9dd26020d767ce31a007895d8f7756 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Tue, 13 Oct 2020 22:59:39 +0800 Subject: [PATCH 164/271] add tmpdir to tests on non-nested workflow in test_workflow.py --- pydra/engine/tests/test_workflow.py | 38 +++++++++++++++++------------ 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index c34e004d2d..7bfb35f7d4 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3599,6 +3599,7 @@ def test_wf_lzoutall_1(plugin, tmpdir): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3620,6 +3621,7 @@ def test_wf_lzoutall_1a(plugin, tmpdir): wf.inputs.x = 2 wf.inputs.y = 3 wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3641,6 +3643,7 @@ def test_wf_lzoutall_st_1(plugin, tmpdir): wf.inputs.x = [2, 20] wf.inputs.y = [3, 30] wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3662,6 +3665,7 @@ def test_wf_lzoutall_st_1a(plugin, tmpdir): wf.inputs.x = [2, 20] wf.inputs.y = [3, 30] wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3690,6 +3694,7 @@ def test_wf_lzoutall_st_2(plugin, tmpdir): wf.inputs.x = [2, 20] wf.inputs.y = [3, 30] wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3714,6 +3719,7 @@ def test_wf_lzoutall_st_2a(plugin, tmpdir): wf.inputs.x = [2, 20] wf.inputs.y = [3, 30] wf.plugin = plugin + wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf) @@ -3731,7 +3737,7 @@ def test_wf_lzoutall_st_2a(plugin, tmpdir): def test_wf_resultfile_1(plugin, tmpdir): """ workflow with a file in the result, file should be copied to the wf dir""" - wf = Workflow(name="wf_file_1", input_spec=["x"]) + wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_write_file(name="writefile", filename=wf.lzin.x)) wf.inputs.x = "file_1.txt" wf.plugin = plugin @@ -3750,7 +3756,7 @@ def test_wf_resultfile_2(plugin, tmpdir): """ workflow with a list of files in the wf result, all files should be copied to the wf dir """ - wf = Workflow(name="wf_file_1", input_spec=["x"]) + wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_write_file_list(name="writefile", filename_list=wf.lzin.x)) file_list = ["file_1.txt", "file_2.txt", "file_3.txt"] wf.inputs.x = file_list @@ -3771,7 +3777,7 @@ def test_wf_resultfile_3(plugin, tmpdir): """ workflow with a dictionaries of files in the wf result, all files should be copied to the wf dir """ - wf = Workflow(name="wf_file_1", input_spec=["x"]) + wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_write_file_list2dict(name="writefile", filename_list=wf.lzin.x)) file_list = ["file_1.txt", "file_2.txt", "file_3.txt"] wf.inputs.x = file_list @@ -3794,7 +3800,7 @@ def test_wf_resultfile_3(plugin, tmpdir): def test_wf_upstream_error1(plugin, tmpdir): """ workflow with two tasks, task2 dependent on an task1 which raised an error""" - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3812,7 +3818,7 @@ def test_wf_upstream_error2(plugin, tmpdir): """ task2 dependent on task1, task1 errors, workflow-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = [1, "hi"] # TypeError for adding str and int wf.split("x") # workflow-level split @@ -3831,7 +3837,7 @@ def test_wf_upstream_error3(plugin, tmpdir): """ task2 dependent on task1, task1 errors, task-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = [1, "hi"] # TypeError for adding str and int wf.addvar1.split("a") # task-level split @@ -3848,7 +3854,7 @@ def test_wf_upstream_error3(plugin, tmpdir): def test_wf_upstream_error4(plugin, tmpdir): """ workflow with one task, which raises an error""" - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3863,7 +3869,7 @@ def test_wf_upstream_error4(plugin, tmpdir): def test_wf_upstream_error5(plugin, tmpdir): """ nested workflow with one task, which raises an error""" - wf_main = Workflow(name="wf_main", input_spec=["x"]) + wf_main = Workflow(name="wf_main", input_spec=["x"], cache_dir=tmpdir) wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.plugin = plugin @@ -3883,7 +3889,7 @@ def test_wf_upstream_error5(plugin, tmpdir): def test_wf_upstream_error6(plugin, tmpdir): """ nested workflow with two tasks, the first one raises an error""" - wf_main = Workflow(name="wf_main", input_spec=["x"]) + wf_main = Workflow(name="wf_main", input_spec=["x"], cache_dir=tmpdir) wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.add(fun_addvar_default(name="addvar2", a=wf.addvar1.lzout.out)) @@ -3907,7 +3913,7 @@ def test_wf_upstream_error7(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the last task is set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3929,7 +3935,7 @@ def test_wf_upstream_error7a(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the second task is set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3951,7 +3957,7 @@ def test_wf_upstream_error7b(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the second and the third tasks are set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3970,7 +3976,7 @@ def test_wf_upstream_error7b(plugin, tmpdir): def test_wf_upstream_error8(plugin, tmpdir): """ workflow with three tasks, the first one raises an error, so 2 others are removed""" - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = "hi" # TypeError for adding str and int wf.plugin = plugin @@ -3994,7 +4000,7 @@ def test_wf_upstream_error9(plugin, tmpdir): one branch has an error, the second is fine the errored branch is connected to the workflow output """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = 2 wf.add(fun_addvar(name="err", a=wf.addvar1.lzout.out, b="hi")) @@ -4021,7 +4027,7 @@ def test_wf_upstream_error9a(plugin, tmpdir): the branch without error is connected to the workflow output so the workflow finished clean """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = 2 wf.add(fun_addvar(name="err", a=wf.addvar1.lzout.out, b="hi")) @@ -4044,7 +4050,7 @@ def test_wf_upstream_error9b(plugin, tmpdir): one branch has an error, the second is fine both branches are connected to the workflow output """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wf.add(fun_addvar_default(name="addvar1", a=wf.lzin.x)) wf.inputs.x = 2 wf.add(fun_addvar(name="err", a=wf.addvar1.lzout.out, b="hi")) From da6c32b1733099b2a046f416036f3abf5052bc78 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 14 Oct 2020 22:32:59 -0400 Subject: [PATCH 165/271] adding python 3.9 for GA --- .github/workflows/testpydra.yml | 2 +- .github/workflows/testsingularity.yml | 2 +- .github/workflows/teststyle.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index f859781253..7f2d9e8157 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9] install: [install, develop, wheel] fail-fast: false runs-on: ${{ matrix.os }} diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 3341c75265..61625d1ffe 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9] fail-fast: False steps: diff --git a/.github/workflows/teststyle.yml b/.github/workflows/teststyle.yml index 9b8d76a9e1..7d6819828b 100644 --- a/.github/workflows/teststyle.yml +++ b/.github/workflows/teststyle.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9] fail-fast: false runs-on: ubuntu-latest From 332d2ba22f81ea9a02eb936034053717595d0626 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 22 Oct 2020 19:18:30 +0800 Subject: [PATCH 166/271] add tests to test invalid file or directory paths as inputs --- pydra/engine/tests/test_shelltask.py | 50 ++++++++++ pydra/engine/tests/test_tasks_files.py | 122 ++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 740e22d78c..4bcdbba0c0 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1299,6 +1299,56 @@ def test_shell_cmd_inputspec_10(plugin, results_function, tmpdir): assert res.output.stdout == "hello from boston" +def test_shell_cmd_inputspec_10_err(tmpdir): + """ checking if the proper error is raised when broken symlink is provided + as a input field with File as a type + """ + + file_1 = tmpdir.join("file_1.txt") + with open(file_1, "w") as f: + f.write("hello") + file_2 = tmpdir.join("file_2.txt") + + # creating symlink and removing the original file + os.symlink(file_1, file_2) + os.remove(file_1) + + cmd_exec = "cat" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "files", + attr.ib( + type=File, + metadata={ + "position": 1, + "argstr": "", + "help_string": "a file", + "mandatory": True, + }, + ), + ) + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd_exec, files=file_2, input_spec=my_input_spec + ) + + # checking if the broken symlink error is raised when checksum is calculated + with pytest.raises(FileNotFoundError) as e: + checksum = shelly.checksum + assert "Broken symlink" in str(e.value) + + # checking if the broken symlink error is raised when the task is run + with pytest.raises(FileNotFoundError) as e: + res = shelly() + assert "Broken symlink" in str(e.value) + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_inputspec_copyfile_1(plugin, results_function, tmpdir): """ shelltask changes a file in place, diff --git a/pydra/engine/tests/test_tasks_files.py b/pydra/engine/tests/test_tasks_files.py index 6b5fd14fc6..6a15b2cfed 100644 --- a/pydra/engine/tests/test_tasks_files.py +++ b/pydra/engine/tests/test_tasks_files.py @@ -7,7 +7,17 @@ from ..submitter import Submitter from ..core import Workflow from ... import mark -from ..specs import File +from ..specs import File, Directory + + +@mark.task +def dir_count_file(dirpath): + return len(os.listdir(dirpath)) + + +@mark.task +def dir_count_file_annot(dirpath: Directory): + return len(os.listdir(dirpath)) @mark.task @@ -111,3 +121,113 @@ def test_file_annotation_1(tmpdir): results = nn.result() res = np.load(results.output.out) assert res == np.array([4]) + + +def test_broken_file(tmpdir): + """ task that takes file as an input""" + os.chdir(tmpdir) + file = os.path.join(os.getcwd(), "non_existent.npy") + + nn = file_add2(name="add2", file=file) + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn) + + nn2 = file_add2_annot(name="add2_annot", file=file) + # AttributeError: the file from the file input does not exist + with pytest.raises(AttributeError) as e: + with Submitter(plugin="cf") as sub: + sub(nn2) + + +def test_broken_file_link(tmpdir): + """ + Test how broken symlinks are handled during hashing + """ + os.chdir(tmpdir) + file = os.path.join(os.getcwd(), "arr.npy") + arr = np.array([2]) + np.save(file, arr) + + file_link = os.path.join(os.getcwd(), "link_to_arr.npy") + os.symlink(file, file_link) + os.remove(file) + + nn = file_add2(name="add2", file=file_link) + # raises error inside task + # unless variable is defined as a File pydra will treat it as a string + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn) + + # raises error before task is run + # AttributeError: the file from the file input does not exist + nn2 = file_add2_annot(name="add2_annot", file=file_link) + with pytest.raises(AttributeError) as e: + with Submitter(plugin="cf") as sub: + sub(nn2) + + +def test_broken_dir(): + """ Test how broken directories are handled during hashing""" + + # dirpath doesn't exist + nn = dir_count_file(name="listdir", dirpath="/broken_dir_path/") + # raises error inside task + # unless variable is defined as a File pydra will treat it as a string + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn) + + # raises error before task is run + nn2 = dir_count_file_annot(name="listdir", dirpath="/broken_dir_path/") + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn2) + + +def test_broken_dir_link1(tmpdir): + """ + Test how broken symlinks are hashed in hash_dir + """ + # broken symlink to dir path + dir1 = tmpdir.join("dir1") + os.mkdir(dir1) + dir1_link = tmpdir.join("dir1_link") + os.symlink(dir1, dir1_link) + os.rmdir(dir1) + + nn = dir_count_file(name="listdir", dirpath=dir1) + # raises error while running task + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn) + + nn2 = dir_count_file_annot(name="listdir", dirpath=dir1) + # raises error before task is run + with pytest.raises(FileNotFoundError) as e: + with Submitter(plugin="cf") as sub: + sub(nn2) + + +def test_broken_dir_link2(tmpdir): + # valid dirs with broken symlink(s) are hashed + dir2 = tmpdir.join("dir2") + os.mkdir(dir2) + file1 = dir2.join("file1") + file2 = dir2.join("file2") + file1.open("w+").close() + file2.open("w+").close() + + file1_link = dir2.join("file1_link") + os.symlink(file1, file1_link) + os.remove(file1) # file1_link is broken + + nn = dir_count_file(name="listdir", dirpath=dir2) + # does not raises error because pydra treats dirpath as a string + with Submitter(plugin="cf") as sub: + sub(nn) + + nn2 = dir_count_file_annot(name="listdir", dirpath=str(dir2)) + with Submitter(plugin="cf") as sub: + sub(nn2) From 96eaf45e55775ce2519da290166098714f51087f Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 22 Oct 2020 19:19:39 +0800 Subject: [PATCH 167/271] if input is a directory that contains broken symlinks, skip symlink hashing --- pydra/engine/helpers_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index b1e55aca3b..1f03313e40 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -139,8 +139,9 @@ def hash_dir( for filename in filenames: if ignore_hidden_files and filename.startswith("."): continue - this_hash = hash_file(dpath / filename) - file_hashes.append(this_hash) + if is_existing_file(filename): + this_hash = hash_file(dpath / filename) + file_hashes.append(this_hash) crypto_obj = crypto() for h in file_hashes: From 10cd9d9420b23825ef49ed0aca0e1a32be4d5631 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Thu, 22 Oct 2020 21:34:03 +0800 Subject: [PATCH 168/271] if directory input contains broken symlinks, hash string of broken filepath instead of skipping --- pydra/engine/helpers_file.py | 4 +++- pydra/engine/tests/test_shelltask.py | 9 +-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 1f03313e40..fa77262216 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -139,7 +139,9 @@ def hash_dir( for filename in filenames: if ignore_hidden_files and filename.startswith("."): continue - if is_existing_file(filename): + if not is_existing_file(dpath / filename): + file_hashes.append(str(filename)) + else: this_hash = hash_file(dpath / filename) file_hashes.append(this_hash) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 4bcdbba0c0..ba95c6af81 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1338,15 +1338,8 @@ def test_shell_cmd_inputspec_10_err(tmpdir): name="shelly", executable=cmd_exec, files=file_2, input_spec=my_input_spec ) - # checking if the broken symlink error is raised when checksum is calculated - with pytest.raises(FileNotFoundError) as e: - checksum = shelly.checksum - assert "Broken symlink" in str(e.value) - - # checking if the broken symlink error is raised when the task is run - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(AttributeError) as e: res = shelly() - assert "Broken symlink" in str(e.value) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) From 8a53aa4378293907aa3664becde33f46fb5bb6a0 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Fri, 23 Oct 2020 13:29:33 +0800 Subject: [PATCH 169/271] match AttributeError message in broken symlink tests --- pydra/engine/tests/test_tasks_files.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pydra/engine/tests/test_tasks_files.py b/pydra/engine/tests/test_tasks_files.py index 6a15b2cfed..3b11445821 100644 --- a/pydra/engine/tests/test_tasks_files.py +++ b/pydra/engine/tests/test_tasks_files.py @@ -129,13 +129,12 @@ def test_broken_file(tmpdir): file = os.path.join(os.getcwd(), "non_existent.npy") nn = file_add2(name="add2", file=file) - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn) nn2 = file_add2_annot(name="add2_annot", file=file) - # AttributeError: the file from the file input does not exist - with pytest.raises(AttributeError) as e: + with pytest.raises(AttributeError, match="file from the file input does not exist"): with Submitter(plugin="cf") as sub: sub(nn2) @@ -156,14 +155,13 @@ def test_broken_file_link(tmpdir): nn = file_add2(name="add2", file=file_link) # raises error inside task # unless variable is defined as a File pydra will treat it as a string - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn) # raises error before task is run - # AttributeError: the file from the file input does not exist nn2 = file_add2_annot(name="add2_annot", file=file_link) - with pytest.raises(AttributeError) as e: + with pytest.raises(AttributeError, match="file from the file input does not exist"): with Submitter(plugin="cf") as sub: sub(nn2) @@ -175,13 +173,13 @@ def test_broken_dir(): nn = dir_count_file(name="listdir", dirpath="/broken_dir_path/") # raises error inside task # unless variable is defined as a File pydra will treat it as a string - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn) # raises error before task is run nn2 = dir_count_file_annot(name="listdir", dirpath="/broken_dir_path/") - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn2) @@ -199,13 +197,13 @@ def test_broken_dir_link1(tmpdir): nn = dir_count_file(name="listdir", dirpath=dir1) # raises error while running task - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn) nn2 = dir_count_file_annot(name="listdir", dirpath=dir1) # raises error before task is run - with pytest.raises(FileNotFoundError) as e: + with pytest.raises(FileNotFoundError): with Submitter(plugin="cf") as sub: sub(nn2) From b7a8823f7ab94d13b019d314a20a87e9817133c9 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 28 Oct 2020 09:29:06 +0800 Subject: [PATCH 170/271] hash full path instead of filename for broken links in directory inputs --- pydra/engine/helpers_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index fa77262216..351e23610b 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -140,7 +140,7 @@ def hash_dir( if ignore_hidden_files and filename.startswith("."): continue if not is_existing_file(dpath / filename): - file_hashes.append(str(filename)) + file_hashes.append(str(dpath / filename)) else: this_hash = hash_file(dpath / filename) file_hashes.append(this_hash) From 1a17feab31cf7b8ef8ebe52ef47ee3b112066349 Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 28 Oct 2020 10:26:56 +0800 Subject: [PATCH 171/271] mark test_slurm_cancel_rerun tests in test_submitter as flaky --- pydra/engine/tests/test_submitter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 3f450495a3..70d3036188 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -301,6 +301,7 @@ def cancel(job_name_part): return proc2.stdout.decode("utf-8").strip() +@pytest.mark.flaky(reruns=3) @pytest.mark.skipif(not slurm_available, reason="slurm not installed") def test_slurm_cancel_rerun_1(tmpdir): """ testing that tasks run with slurm is re-queue @@ -332,6 +333,7 @@ def test_slurm_cancel_rerun_1(tmpdir): assert script_dir.exists() +@pytest.mark.flaky(reruns=3) @pytest.mark.skipif(not slurm_available, reason="slurm not installed") def test_slurm_cancel_rerun_2(tmpdir): """ testing that tasks run with slurm that has --no-requeue From 1b973003f15d7f60ef262d7adbe447a31a7b2c5a Mon Sep 17 00:00:00 2001 From: Nicol Lo Date: Wed, 28 Oct 2020 10:30:10 +0800 Subject: [PATCH 172/271] mark test_slurm_cancel_rerun tests in test_submitter as flaky --- pydra/engine/tests/test_boutiques.py | 7 +- pydra/engine/tests/test_shelltask.py | 228 ++++++++++++++++++--------- pydra/engine/tests/utils.py | 5 +- 3 files changed, 164 insertions(+), 76 deletions(-) diff --git a/pydra/engine/tests/test_boutiques.py b/pydra/engine/tests/test_boutiques.py index 5399b2e88a..6e503698dc 100644 --- a/pydra/engine/tests/test_boutiques.py +++ b/pydra/engine/tests/test_boutiques.py @@ -31,7 +31,8 @@ def test_boutiques_1(maskfile, plugin, results_function, tmpdir): btask = BoshTask(name="NA", zenodo_id="1482743") btask.inputs.infile = Infile btask.inputs.maskfile = maskfile - res = results_function(btask, plugin, tmpdir) + btask.cache_dir = tmpdir + res = results_function(btask, plugin) assert res.output.return_code == 0 @@ -102,6 +103,7 @@ def test_boutiques_wf_1(maskfile, plugin): wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile wf.inputs.infile = Infile + wf.cache_dir = tmpdir wf.add( BoshTask( @@ -128,11 +130,12 @@ def test_boutiques_wf_1(maskfile, plugin): @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) -def test_boutiques_wf_2(maskfile, plugin): +def test_boutiques_wf_2(maskfile, plugin, tmdpir): """ wf with two BoshTasks (fsl.bet and fsl.stats) and one ShellTask""" wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile wf.inputs.infile = Infile + wf.cache_dir = tmpdir wf.add( BoshTask( diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index bf93cbd085..e270fbbd1e 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -20,10 +20,11 @@ def test_shell_cmd_1(plugin_dask_opt, results_function, tmpdir): """ simple command, no arguments """ cmd = ["pwd"] - shelly = ShellCommandTask(name="shelly", executable=cmd) + shelly = ShellCommandTask(name="shelly", executable=cmd, cache_dir=tmpdir) + shelly.cache_dir = tmpdir assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin=plugin_dask_opt, tmpdir=tmpdir) + res = results_function(shelly, plugin=plugin_dask_opt) assert Path(res.output.stdout.rstrip()) == shelly.output_dir assert res.output.return_code == 0 assert res.output.stderr == "" @@ -36,9 +37,10 @@ def test_shell_cmd_1_strip(plugin, results_function, tmpdir): """ cmd = ["pwd"] shelly = ShellCommandTask(name="shelly", executable=cmd, strip=True) + shelly.cache_dir = tmpdir assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert Path(res.output.stdout) == Path(shelly.output_dir) assert res.output.return_code == 0 assert res.output.stderr == "" @@ -49,9 +51,10 @@ def test_shell_cmd_2(plugin, results_function, tmpdir): """ a command with arguments, cmd and args given as executable """ cmd = ["echo", "hail", "pydra"] shelly = ShellCommandTask(name="shelly", executable=cmd) + shelly.cache_dir = tmpdir assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout.strip() == " ".join(cmd[1:]) assert res.output.return_code == 0 assert res.output.stderr == "" @@ -64,10 +67,11 @@ def test_shell_cmd_2a(plugin, results_function, tmpdir): cmd_args = ["hail", "pydra"] # separate command into exec + args shelly = ShellCommandTask(name="shelly", executable=cmd_exec, args=cmd_args) + shelly.cache_dir = tmpdir assert shelly.inputs.executable == "echo" assert shelly.cmdline == "echo " + " ".join(cmd_args) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout.strip() == " ".join(cmd_args) assert res.output.return_code == 0 assert res.output.stderr == "" @@ -80,10 +84,11 @@ def test_shell_cmd_2b(plugin, results_function, tmpdir): cmd_args = "pydra" # separate command into exec + args shelly = ShellCommandTask(name="shelly", executable=cmd_exec, args=cmd_args) + shelly.cache_dir = tmpdir assert shelly.inputs.executable == "echo" assert shelly.cmdline == "echo pydra" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "pydra\n" assert res.output.return_code == 0 assert res.output.stderr == "" @@ -290,12 +295,14 @@ def test_shell_cmd_inputspec_1(plugin, results_function, use_validator, tmpdir): args=cmd_args, opt_n=cmd_opt, input_spec=my_input_spec, + cache_dir=tmpdir, ) + shelly.cache_dir = tmpdir assert shelly.inputs.executable == cmd_exec assert shelly.inputs.args == cmd_args assert shelly.cmdline == "echo -n hello from pydra" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "hello from pydra" @@ -338,11 +345,12 @@ def test_shell_cmd_inputspec_2(plugin, results_function, use_validator, tmpdir): opt_n=cmd_opt, opt_hello=cmd_opt_hello, input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec assert shelly.inputs.args == cmd_args assert shelly.cmdline == "echo -n HELLO from pydra" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "HELLO from pydra" @@ -372,11 +380,15 @@ def test_shell_cmd_inputspec_3(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, text=hello, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + text=hello, + input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "HELLO\n" @@ -401,11 +413,15 @@ def test_shell_cmd_inputspec_3a(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, text=hello, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + text=hello, + input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "HELLO\n" @@ -435,13 +451,13 @@ def test_shell_cmd_inputspec_3b(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) shelly.inputs.text = hello assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "HELLO\n" @@ -468,9 +484,8 @@ def test_shell_cmd_inputspec_3c_exception(plugin, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) - shelly.cache_dir = tmpdir with pytest.raises(Exception) as excinfo: shelly() @@ -503,13 +518,12 @@ def test_shell_cmd_inputspec_3c(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) - shelly.cache_dir = tmpdir assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "\n" @@ -534,14 +548,13 @@ def test_shell_cmd_inputspec_4(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) - shelly.cache_dir = tmpdir assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "Hello\n" @@ -561,13 +574,13 @@ def test_shell_cmd_inputspec_4a(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "Hello\n" @@ -592,13 +605,13 @@ def test_shell_cmd_inputspec_4b(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec + name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmpdir ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "echo Hi" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "Hi\n" @@ -708,14 +721,18 @@ def test_shell_cmd_inputspec_5_nosubm(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, opt_t=cmd_t, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + opt_t=cmd_t, + input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -t" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) -def test_shell_cmd_inputspec_5a_exception(plugin): +def test_shell_cmd_inputspec_5a_exception(plugin, tmpdir): """ checking xor in metadata: both options are True, so the task raises exception""" cmd_exec = "ls" cmd_t = True @@ -757,6 +774,7 @@ def test_shell_cmd_inputspec_5a_exception(plugin): opt_t=cmd_t, opt_S=cmd_S, input_spec=my_input_spec, + cache_dir=tmpdir, ) with pytest.raises(Exception) as excinfo: shelly() @@ -807,7 +825,7 @@ def test_shell_cmd_inputspec_6(plugin, results_function, tmpdir): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) def test_shell_cmd_inputspec_6a_exception(plugin): @@ -891,11 +909,12 @@ def test_shell_cmd_inputspec_6b(plugin, results_function, tmpdir): opt_t=cmd_t, # opt_l=cmd_l, input_spec=my_input_spec, + cache_dir=tmpdir, ) shelly.inputs.opt_l = cmd_l assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -925,10 +944,14 @@ def test_shell_cmd_inputspec_7(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, input_spec=my_input_spec + name="shelly", + executable=cmd, + args=args, + input_spec=my_input_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() # checking if the file is created in a good place @@ -965,10 +988,14 @@ def test_shell_cmd_inputspec_7a(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, input_spec=my_input_spec + name="shelly", + executable=cmd, + args=args, + input_spec=my_input_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1_changed.exists() # checking if the file is created in a good place @@ -1013,9 +1040,10 @@ def test_shell_cmd_inputspec_7b(plugin, results_function, tmpdir): executable=cmd, newfile="newfile_tmp.txt", input_spec=my_input_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1069,9 +1097,10 @@ def test_shell_cmd_inputspec_8(plugin, results_function, tmpdir): newfile="newfile_tmp.txt", time="02121010", input_spec=my_input_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1125,9 +1154,10 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmpdir): newfile="newfile_tmp.txt", time="02121010", input_spec=my_input_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() @@ -1168,10 +1198,14 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + name="shelly", + executable=cmd, + input_spec=my_input_spec, + file_orig=file, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file_copy.txt" @@ -1216,10 +1250,14 @@ def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + name="shelly", + executable=cmd, + input_spec=my_input_spec, + file_orig=file, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file_copy" @@ -1263,10 +1301,14 @@ def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + name="shelly", + executable=cmd, + input_spec=my_input_spec, + file_orig=file, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.file_copy.exists() assert res.output.file_copy.name == "file" @@ -1308,11 +1350,15 @@ def test_shell_cmd_inputspec_10(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, files=files_list, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + files=files_list, + input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "hello from boston" @@ -1359,10 +1405,14 @@ def test_shell_cmd_inputspec_copyfile_1(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) + name="shelly", + executable=cmd, + input_spec=my_input_spec, + orig_file=str(file), + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is copied, and than it is changed in place @@ -1417,10 +1467,14 @@ def test_shell_cmd_inputspec_copyfile_1a(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) + name="shelly", + executable=cmd, + input_spec=my_input_spec, + orig_file=str(file), + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is uses a soft link, but it creates and an extra copy before modifying @@ -1489,10 +1543,14 @@ def test_shell_cmd_inputspec_copyfile_1b(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=str(file) + name="shelly", + executable=cmd, + input_spec=my_input_spec, + orig_file=str(file), + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out_file.exists() # the file is not copied, it is changed in place @@ -1527,12 +1585,16 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, text=hello, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + text=hello, + input_spec=my_input_spec, + cache_dir=tmpdir, ).split("text") assert shelly.inputs.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res[0].output.stdout == "HELLO\n" assert res[1].output.stdout == "hi\n" @@ -1602,11 +1664,15 @@ def test_shell_cmd_inputspec_state_1a(plugin, results_function, tmpdir): # separate command into exec + args shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, text=hello, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + text=hello, + input_spec=my_input_spec, + cache_dir=tmpdir, ).split("text") assert shelly.inputs.executable == cmd_exec - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res[0].output.stdout == "HELLO\n" assert res[1].output.stdout == "hi\n" @@ -1637,10 +1703,14 @@ def test_shell_cmd_inputspec_state_2(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, input_spec=my_input_spec + name="shelly", + executable=cmd, + args=args, + input_spec=my_input_spec, + cache_dir=tmpdir, ).split("args") - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) for i in range(len(args)): assert res[i].output.stdout == "" assert res[i].output.out1.exists() @@ -1681,13 +1751,17 @@ def test_shell_cmd_inputspec_state_3(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, file=files, input_spec=my_input_spec + name="shelly", + executable=cmd_exec, + file=files, + input_spec=my_input_spec, + cache_dir=tmpdir, ).split("file") assert shelly.inputs.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res[0].output.stdout == "hello from pydra" assert res[1].output.stdout == "have a nice one" @@ -1738,11 +1812,15 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, input_spec=my_input_spec, orig_file=files + name="shelly", + executable=cmd, + input_spec=my_input_spec, + orig_file=files, + cache_dir=tmpdir, ).split("orig_file") txt_l = ["from pydra", "world"] - res_l = results_function(shelly, plugin, tmpdir) + res_l = results_function(shelly, plugin) for i, res in enumerate(res_l): assert res.output.stdout == "" assert res.output.out_file.exists() @@ -2254,7 +2332,7 @@ def test_shell_cmd_outputspec_1(plugin, results_function, tmpdir): name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.newfile.exists() @@ -2274,7 +2352,7 @@ def test_shell_cmd_outputspec_1a(plugin, results_function, tmpdir): name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.newfile.exists() @@ -2315,7 +2393,7 @@ def test_shell_cmd_outputspec_2(plugin, results_function, tmpdir): name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.newfile.exists() @@ -2357,7 +2435,7 @@ def test_shell_cmd_outputspec_3(plugin, results_function, tmpdir): name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" # newfile is a list assert len(res.output.newfile) == 2 @@ -2385,7 +2463,7 @@ def gather_output(keyname, output_dir): name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmpdir ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" # newfile is a list assert len(res.output.newfile) == 2 @@ -2419,10 +2497,14 @@ def test_shell_cmd_outputspec_5(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, output_spec=my_output_spec + name="shelly", + executable=cmd, + args=args, + output_spec=my_output_spec, + cache_dir=tmpdir, ) - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) assert res.output.stdout == "" assert res.output.out1.exists() @@ -2483,10 +2565,14 @@ def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): ) shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, output_spec=my_output_spec + name="shelly", + executable=cmd, + args=args, + output_spec=my_output_spec, + cache_dir=tmpdir, ).split("args") - res = results_function(shelly, plugin, tmpdir) + res = results_function(shelly, plugin) for i in range(len(args)): assert res[i].output.stdout == "" assert res[i].output.out1.exists() diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index edc2533c28..b2fbdab762 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -23,16 +23,15 @@ ) -def result_no_submitter(shell_task, plugin=None, tmpdir=None): +def result_no_submitter(shell_task, plugin=None): """ helper function to return result when running without submitter """ return shell_task() -def result_submitter(shell_task, plugin, tmpdir): +def result_submitter(shell_task, plugin): """ helper function to return result when running with submitter with specific plugin """ - shell_task.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: shell_task(submitter=sub) return shell_task.result() From 0c84519a76efe3f81291f52d101f10676951e322 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 1 Nov 2020 22:33:42 -0500 Subject: [PATCH 173/271] improving callable used in for output spec; adding additional tests --- pydra/engine/specs.py | 20 ++++++++++- pydra/engine/tests/test_shelltask.py | 54 ++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 874a304e43..9b9a8539fa 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -495,7 +495,25 @@ def _field_metadata(self, fld, inputs, output_dir): ) return Path(value) elif "callable" in fld.metadata: - return fld.metadata["callable"](fld.name, output_dir) + call_args = inspect.getargspec(fld.metadata["callable"]) + call_args_val = {} + for argnm in call_args.args: + if argnm == "field": + call_args_val[argnm] = fld + elif argnm == "output_dir": + call_args_val[argnm] = output_dir + elif argnm == "inputs": + call_args_val[argnm] = inputs + else: + try: + call_args_val[argnm] = getattr(inputs, argnm) + except AttributeError: + raise AttributeError( + f"arguments of the callable function from {fld.name} " + f"has to be in inputs or be field or output_dir, " + f"but {argnm} is used" + ) + return fld.metadata["callable"](**call_args_val) else: raise Exception("(_field_metadata) is not a current valid metadata key.") diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index ba95c6af81..81e692a556 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2378,11 +2378,12 @@ def test_shell_cmd_outputspec_4(plugin, results_function): """ customised output_spec, adding files to the output, using a function to collect output, the function is saved in the field metadata + and uses output_dir and the glob function """ cmd = ["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"] - def gather_output(keyname, output_dir): - if keyname == "newfile": + def gather_output(field, output_dir): + if field.name == "newfile": return list(Path(output_dir).expanduser().glob("newfile*.txt")) my_output_spec = SpecInfo( @@ -2399,6 +2400,55 @@ def gather_output(keyname, output_dir): assert all([file.exists for file in res.output.newfile]) +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_4a(plugin, results_function): + """ + customised output_spec, adding files to the output, + using a function to collect output, the function is saved in the field metadata + and uses output_dir and inputs element + """ + cmd = ["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"] + + def gather_output(executable, output_dir): + files = executable[1:] + return [Path(output_dir) / file for file in files] + + my_output_spec = SpecInfo( + name="Output", + fields=[("newfile", attr.ib(type=File, metadata={"callable": gather_output}))], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + # newfile is a list + assert len(res.output.newfile) == 2 + assert all([file.exists for file in res.output.newfile]) + + +def test_shell_cmd_outputspec_4b_error(): + """ + customised output_spec, adding files to the output, + using a function to collect output, the function is saved in the field metadata + with an argument that is not part of the inputs - error is raised + """ + cmd = ["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"] + + def gather_output(executable, output_dir, ble): + files = executable[1:] + return [Path(output_dir) / file for file in files] + + my_output_spec = SpecInfo( + name="Output", + fields=[("newfile", attr.ib(type=File, metadata={"callable": gather_output}))], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec) + with pytest.raises(AttributeError, match="ble"): + shelly() + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_5(plugin, results_function): """ From db3083f664e302b8e23c441c2b40d33c1fffab1e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 6 Nov 2020 22:11:51 -0500 Subject: [PATCH 174/271] removing state_inputs from Task class (was not used anymore), and fixing a memory issue for wf witha big splitter; adding a test for it together with an extra library for testing: pympler --- pydra/engine/core.py | 2 -- pydra/engine/tests/test_profiles.py | 39 +++++++++++++++++++++++++++++ setup.cfg | 2 ++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 pydra/engine/tests/test_profiles.py diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 266788908c..f5490cef83 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -163,7 +163,6 @@ def __init__( # checking if metadata is set properly self.inputs.check_metadata() - self.state_inputs = inputs # dictionary to save the connections with lazy fields self.inp_lf = {} self.state = None @@ -481,7 +480,6 @@ def split(self, splitter, overwrite=False, **kwargs): ) if kwargs: self.inputs = attr.evolve(self.inputs, **kwargs) - self.state_inputs = kwargs if not self.state or splitter != self.state.splitter: self.set_state(splitter) return self diff --git a/pydra/engine/tests/test_profiles.py b/pydra/engine/tests/test_profiles.py new file mode 100644 index 0000000000..dd758b8d0b --- /dev/null +++ b/pydra/engine/tests/test_profiles.py @@ -0,0 +1,39 @@ +from ..core import Workflow +from ..helpers import load_task +from ... import mark + +import numpy as np +from pympler import asizeof + + +def test_load_task_memory(): + """creating two workflow with relatively big splitter: 1000 and 4000 elements + testings if load_task for a single element returns tasks of a similar size + """ + + def generate_list(l): + return np.arange(l).tolist() + + @mark.task + def show_var(a): + return a + + def create_wf_pkl(size): + wf = Workflow(name="wf", input_spec=["x"]) + wf.split("x", x=generate_list(size)) + wf.add(show_var(name="show", a=wf.lzin.x)) + wf.set_output([("out", wf.show.lzout.out)]) + wf.state.prepare_states(wf.inputs) + wf.state.prepare_inputs() + wf_pkl = wf.pickle_task() + return wf_pkl + + wf_1000_pkl = create_wf_pkl(size=1000) + wf_1000_loaded = load_task(task_pkl=wf_1000_pkl, ind=1) + wf_1000_single_mem = asizeof.asizeof(wf_1000_loaded) + + wf_4000_pkl = create_wf_pkl(size=4000) + wf_4000_loaded = load_task(task_pkl=wf_4000_pkl, ind=1) + wf_4000_single_mem = asizeof.asizeof(wf_4000_loaded) + + assert abs(wf_1000_single_mem - wf_4000_single_mem) / wf_1000_single_mem < 0.1 diff --git a/setup.cfg b/setup.cfg index 029433c26a..a560fa5fe7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ test_requires = python-dateutil tornado boutiques + pympler packages = find: include_package_data = True @@ -74,6 +75,7 @@ test = python-dateutil tornado boutiques + pympler tests = %(test)s dev = From 37c005950df9a84368f0ac61b9e6efb572442d30 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 7 Nov 2020 09:57:33 -0500 Subject: [PATCH 175/271] adding additional test that checks the wf size --- pydra/engine/tests/test_profiles.py | 66 ++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/pydra/engine/tests/test_profiles.py b/pydra/engine/tests/test_profiles.py index dd758b8d0b..11c09f6c10 100644 --- a/pydra/engine/tests/test_profiles.py +++ b/pydra/engine/tests/test_profiles.py @@ -4,36 +4,62 @@ import numpy as np from pympler import asizeof +from pytest import approx -def test_load_task_memory(): - """creating two workflow with relatively big splitter: 1000 and 4000 elements - testings if load_task for a single element returns tasks of a similar size +def generate_list(l): + return np.arange(l).tolist() + + +@mark.task +def show_var(a): + return a + + +def create_wf(size): + wf = Workflow(name="wf", input_spec=["x"]) + wf.split("x", x=generate_list(size)) + wf.add(show_var(name="show", a=wf.lzin.x)) + wf.set_output([("out", wf.show.lzout.out)]) + wf.state.prepare_states(wf.inputs) + wf.state.prepare_inputs() + return wf + + +def test_wf_memory(): + """creating two workflow with relatively big splitter: 1000, 2000 and 4000 elements + testings if the size of workflow grows linearly """ - def generate_list(l): - return np.arange(l).tolist() + wf_1000 = create_wf(size=1000) + wf_1000_mem = asizeof.asizeof(wf_1000) + + wf_2000 = create_wf(size=2000) + wf_2000_mem = asizeof.asizeof(wf_2000) - @mark.task - def show_var(a): - return a + wf_4000 = create_wf(size=4000) + wf_4000_mem = asizeof.asizeof(wf_4000) + # checking if it's linear with the size of the splitter + # check print(asizeof.asized(wf_4000, detail=2).format()) in case of problems + assert wf_4000_mem / wf_2000_mem == approx(2, 0.05) + assert wf_2000_mem / wf_1000_mem == approx(2, 0.05) - def create_wf_pkl(size): - wf = Workflow(name="wf", input_spec=["x"]) - wf.split("x", x=generate_list(size)) - wf.add(show_var(name="show", a=wf.lzin.x)) - wf.set_output([("out", wf.show.lzout.out)]) - wf.state.prepare_states(wf.inputs) - wf.state.prepare_inputs() - wf_pkl = wf.pickle_task() - return wf_pkl - wf_1000_pkl = create_wf_pkl(size=1000) +def test_load_task_memory(): + """creating two workflow with relatively big splitter: 1000 and 4000 elements + testings if load_task for a single element returns tasks of a similar size + """ + + wf_1000 = create_wf(size=1000) + wf_1000_pkl = wf_1000.pickle_task() wf_1000_loaded = load_task(task_pkl=wf_1000_pkl, ind=1) wf_1000_single_mem = asizeof.asizeof(wf_1000_loaded) - wf_4000_pkl = create_wf_pkl(size=4000) + wf_4000 = create_wf(size=4000) + wf_4000_pkl = wf_4000.pickle_task() wf_4000_loaded = load_task(task_pkl=wf_4000_pkl, ind=1) wf_4000_single_mem = asizeof.asizeof(wf_4000_loaded) - assert abs(wf_1000_single_mem - wf_4000_single_mem) / wf_1000_single_mem < 0.1 + # checking if it doesn't change with size of the splitter + # check print(asizeof.asized(wf_4000_loaded, detail=2).format()) in case of problems + assert wf_1000_single_mem / wf_4000_single_mem == approx(1, 0.05) From 56785abab70471e75eb3e0a6f5627b6a4e022456 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 8 Nov 2020 19:45:55 -0500 Subject: [PATCH 176/271] fixing bosh tests --- pydra/engine/tests/test_boutiques.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_boutiques.py b/pydra/engine/tests/test_boutiques.py index 6e503698dc..4f6665bac3 100644 --- a/pydra/engine/tests/test_boutiques.py +++ b/pydra/engine/tests/test_boutiques.py @@ -98,7 +98,7 @@ def test_boutiques_spec_2(): @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) -def test_boutiques_wf_1(maskfile, plugin): +def test_boutiques_wf_1(maskfile, plugin, tmpdir): """ wf with one task that runs fsl.bet using BoshTask""" wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile @@ -130,7 +130,7 @@ def test_boutiques_wf_1(maskfile, plugin): @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) -def test_boutiques_wf_2(maskfile, plugin, tmdpir): +def test_boutiques_wf_2(maskfile, plugin, tmpdir): """ wf with two BoshTasks (fsl.bet and fsl.stats) and one ShellTask""" wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile From 7669b814f7d7a3618c08f19a00e2d864fa223ad8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 10 Nov 2020 18:43:26 -0500 Subject: [PATCH 177/271] fixing cache_dir settings for workflow used as a node; addressing issues with submitter/slurm that the task might finish, but theresults are not available right away (repeating request for avalibale tasks for another 60s); fixing some tests so they use tmpdir --- pydra/engine/submitter.py | 21 ++++- pydra/engine/tests/test_shelltask.py | 4 +- pydra/engine/tests/test_submitter.py | 1 - pydra/engine/tests/test_workflow.py | 120 ++++++++++++++++++++++----- 4 files changed, 118 insertions(+), 28 deletions(-) diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index cd8c61b84b..56d711de1c 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -1,5 +1,6 @@ """Handle execution backends.""" import asyncio +import time from .workers import SerialWorker, ConcurrentFuturesWorker, SlurmWorker, DaskWorker from .core import is_workflow from .helpers import get_open_loop, load_and_run_async @@ -150,15 +151,31 @@ async def _run_workflow(self, wf, rerun=False): The computed workflow """ + for nd in wf.graph.nodes: + if nd.allow_cache_override: + nd.cache_dir = wf.cache_dir + # creating a copy of the graph that will be modified # the copy contains new lists with original runnable objects graph_copy = wf.graph.copy() # keep track of pending futures task_futures = set() tasks, tasks_follow_errored = get_runnable_tasks(graph_copy) - while tasks or len(task_futures): + while tasks or task_futures or graph_copy.nodes: if not tasks and not task_futures: - raise Exception("Nothing queued or todo - something went wrong") + # it's possible that task_futures is empty, but not able to get any + # tasks from graph_copy (using get_runnable_tasks) + # this might be related to some delays saving the files + # so try to get_runnable_tasks for another minut + ii = 0 + while not tasks and graph_copy.nodes: + tasks, follow_err = get_runnable_tasks(graph_copy) + ii += 1 + time.sleep(1) + if ii > 60: + raise Exception( + "graph is not empty, but not able to get more tasks - something is wrong (e.g. with the filesystem)" + ) for task in tasks: # grab inputs if needed logger.debug(f"Retrieving inputs for {task}") diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 31ce8f32c9..bbcf637cb7 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -21,7 +21,6 @@ def test_shell_cmd_1(plugin_dask_opt, results_function, tmpdir): """ simple command, no arguments """ cmd = ["pwd"] shelly = ShellCommandTask(name="shelly", executable=cmd, cache_dir=tmpdir) - shelly.cache_dir = tmpdir assert shelly.cmdline == " ".join(cmd) res = results_function(shelly, plugin=plugin_dask_opt) @@ -297,7 +296,6 @@ def test_shell_cmd_inputspec_1(plugin, results_function, use_validator, tmpdir): input_spec=my_input_spec, cache_dir=tmpdir, ) - shelly.cache_dir = tmpdir assert shelly.inputs.executable == cmd_exec assert shelly.inputs.args == cmd_args assert shelly.cmdline == "echo -n hello from pydra" @@ -822,6 +820,7 @@ def test_shell_cmd_inputspec_6(plugin, results_function, tmpdir): opt_t=cmd_t, opt_l=cmd_l, input_spec=my_input_spec, + cache_dir=tmpdir, ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" @@ -1400,6 +1399,7 @@ def test_shell_cmd_inputspec_10_err(tmpdir): shelly = ShellCommandTask( name="shelly", executable=cmd_exec, files=file_2, input_spec=my_input_spec ) + shelly.cache_dir = tmpdir with pytest.raises(AttributeError) as e: res = shelly() diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 0b54cb5893..29e37a3c57 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -120,7 +120,6 @@ def test_wf2(plugin_dask_opt, tmpdir): wfnd.add(sleep_add_one(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = 2 - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 7bfb35f7d4..ecf84e5091 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -50,7 +50,7 @@ def test_wf_name_conflict2(): def test_wf_no_output(plugin, tmpdir): """ Raise error when output isn't set with set_output""" - wf = Workflow(name="wf_1", input_spec=["x"]) + wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) wf.add(add2(name="add2", x=wf.lzin.x)) wf.inputs.x = 2 @@ -1713,7 +1713,6 @@ def test_wfasnd_1(plugin, tmpdir): wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = 2 - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1739,7 +1738,6 @@ def test_wfasnd_wfinp_1(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir wf.add(wfnd) wf.inputs.x = 2 @@ -1767,7 +1765,6 @@ def test_wfasnd_wfndupdate(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=2) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"], x=3) wfnd.inputs.x = wf.lzin.x @@ -1821,6 +1818,7 @@ def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): wf_o.add(wf) wf_o.set_output([("out", wf_o.wf.lzout.out)]) wf_o.plugin = plugin + wf_o.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: sub(wf_o) @@ -1840,7 +1838,6 @@ def test_wfasnd_st_1(plugin, tmpdir): wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.split("x") wfnd.inputs.x = [2, 4] - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1868,7 +1865,6 @@ def test_wfasnd_st_updatespl_1(plugin, tmpdir): wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) wfnd.inputs.x = [2, 4] - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1897,7 +1893,6 @@ def test_wfasnd_ndst_1(plugin, tmpdir): # TODO: without this the test is failing wfnd.plugin = plugin wfnd.inputs.x = [2, 4] - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1925,7 +1920,6 @@ def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): # TODO: without this the test is failing wfnd.plugin = plugin wfnd.inputs.x = [2, 4] - wfnd.cache_dir = tmpdir wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) @@ -1948,11 +1942,10 @@ def test_wfasnd_wfst_1(plugin, tmpdir): workflow-node with one task, splitter for the main workflow """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir wf.add(wfnd) wf.split("x") @@ -1986,7 +1979,6 @@ def test_wfasnd_st_2(plugin, tmpdir): wfnd.split(("x", "y")) wfnd.inputs.x = [2, 4] wfnd.inputs.y = [1, 10] - wfnd.cache_dir = tmpdir wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) wf.add(wfnd) @@ -2013,7 +2005,6 @@ def test_wfasnd_wfst_2(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x", "y"], x=wf.lzin.x, y=wf.lzin.y) wfnd.add(multiply(name="mult", x=wfnd.lzin.x, y=wfnd.lzin.y)) wfnd.set_output([("out", wfnd.mult.lzout.out)]) - wfnd.cache_dir = tmpdir wf.add(wfnd) wf.add(add2(name="add2", x=wf.wfnd.lzout.out)) @@ -2052,7 +2043,6 @@ def test_wfasnd_ndst_3(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) @@ -2082,7 +2072,6 @@ def test_wfasnd_wfst_3(plugin, tmpdir): wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) @@ -2101,6 +2090,91 @@ def test_wfasnd_wfst_3(plugin, tmpdir): assert odir.exists() +# workflows with structures wfns(A->B) + + +def test_wfasnd_4(plugin, tmpdir): + """ workflow as a node + workflow-node with two tasks and no splitter + """ + wfnd = Workflow(name="wfnd", input_spec=["x"]) + wfnd.add(add2(name="add2_1st", x=wfnd.lzin.x)) + wfnd.add(add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) + wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) + wfnd.inputs.x = 2 + + wf = Workflow(name="wf", input_spec=["x"]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + wf.plugin = plugin + wf.cache_dir = tmpdir + + with Submitter(plugin=plugin) as sub: + sub(wf) + + results = wf.result() + assert results.output.out == 6 + # checking the output directory + assert wf.output_dir.exists() + + +def test_wfasnd_ndst_4(plugin, tmpdir): + """ workflow as a node + workflow-node with two tasks, + splitter for node + """ + wfnd = Workflow(name="wfnd", input_spec=["x"]) + wfnd.add(add2(name="add2_1st", x=wfnd.lzin.x).split("x")) + wfnd.add(add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) + wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) + # TODO: without this the test is failing + wfnd.plugin = plugin + wfnd.inputs.x = [2, 4] + + wf = Workflow(name="wf", input_spec=["x"]) + wf.add(wfnd) + wf.set_output([("out", wf.wfnd.lzout.out)]) + wf.plugin = plugin + wf.cache_dir = tmpdir + + with Submitter(plugin=plugin) as sub: + sub(wf) + + results = wf.result() + assert results.output.out == [6, 8] + # checking the output directory + assert wf.output_dir.exists() + + +def test_wfasnd_wfst_4(plugin, tmpdir): + """ workflow as a node + workflow-node with two tasks, + splitter for the main workflow + """ + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) + wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) + wfnd.add(add2(name="add2_1st", x=wfnd.lzin.x)) + wfnd.add(add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) + wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) + + wf.add(wfnd) + wf.split("x") + wf.inputs.x = [2, 4] + wf.set_output([("out", wf.wfnd.lzout.out)]) + wf.plugin = plugin + + with Submitter(plugin=plugin) as sub: + sub(wf) + # assert wf.output_dir.exists() + results = wf.result() + assert results[0].output.out == 6 + assert results[1].output.out == 8 + # checking all directories + assert wf.output_dir + for odir in wf.output_dir: + assert odir.exists() + + # Testing caching @@ -4098,7 +4172,7 @@ def exporting_graphs(wf, name): def test_graph_1(tmpdir): """creating a set of graphs, wf with two nodes""" - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) @@ -4139,7 +4213,7 @@ def test_graph_1st(tmpdir): """creating a set of graphs, wf with two nodes some nodes have splitters, should be marked with blue color """ - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y).split("x")) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) @@ -4180,7 +4254,7 @@ def test_graph_1st_cmb(tmpdir): the first one has a splitter, the second has a combiner, so the third one is stateless first two nodes should be blue and the arrow between them should be blue """ - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split("x")) wf.add(add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) wf.add(list_sum(name="sum", x=wf.add2.lzout.out)) @@ -4219,7 +4293,7 @@ def test_graph_1st_cmb(tmpdir): def test_graph_2(tmpdir): """creating a graph, wf with one worfklow as a node""" - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) @@ -4253,7 +4327,7 @@ def test_graph_2st(tmpdir): """creating a set of graphs, wf with one worfklow as a node the inner workflow has a state, so should be blue """ - wf = Workflow(name="wf", input_spec=["x"]) + wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x).split("x") wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) @@ -4287,7 +4361,7 @@ def test_graph_2st(tmpdir): def test_graph_3(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow)""" - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) @@ -4329,7 +4403,7 @@ def test_graph_3st(tmpdir): the first node has a state and it should be passed to the second node (blue node and a wfasnd, and blue arrow from the node to the wfasnd) """ - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y).split("x")) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) @@ -4370,7 +4444,7 @@ def test_graph_4(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the node to the inner workflow. """ - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) @@ -4413,7 +4487,7 @@ def test_graph_5(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the inner workflow to the node. """ - wf = Workflow(name="wf", input_spec=["x", "y"]) + wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) wfnd.add(add2(name="add2_a", x=wfnd.lzin.x)) wfnd.add(add2(name="add2_b", x=wfnd.add2_a.lzout.out)) From 635b505c7fb0f6910858ceb5920185f77f4b3109 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 10 Nov 2020 20:53:52 -0500 Subject: [PATCH 178/271] fixing/improving tests for requing tasks (I believe tests didn't work if the task was resubmitted to quickly or when both tests were running the same time with the same jon names) --- pydra/engine/tests/test_submitter.py | 59 ++++++++++++++++------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 70d3036188..a2ddabd59c 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -21,8 +21,9 @@ def sleep_add_one(x): return x + 1 -def test_callable_wf(plugin): +def test_callable_wf(plugin, tmpdir): wf = gen_basic_wf() + with pytest.raises(NotImplementedError): wf() @@ -31,12 +32,14 @@ def test_callable_wf(plugin): del wf, res wf = gen_basic_wf() + wf.cache_dir = tmpdir + sub = Submitter(plugin) res = wf(submitter=sub) assert res.output.out == 9 -def test_concurrent_wf(plugin): +def test_concurrent_wf(plugin, tmpdir): # concurrent workflow # A --> C # B --> D @@ -48,6 +51,8 @@ def test_concurrent_wf(plugin): wf.add(sleep_add_one(name="taskc", x=wf.taska.lzout.out)) wf.add(sleep_add_one(name="taskd", x=wf.taskb.lzout.out)) wf.set_output([("out1", wf.taskc.lzout.out), ("out2", wf.taskd.lzout.out)]) + wf.cache_dir = tmpdir + with Submitter(plugin) as sub: sub(wf) @@ -56,7 +61,7 @@ def test_concurrent_wf(plugin): assert res.output.out2 == 12 -def test_concurrent_wf_nprocs(): +def test_concurrent_wf_nprocs(tmpdir): # concurrent workflow # setting n_procs in Submitter that is passed to the worker # A --> C @@ -69,8 +74,8 @@ def test_concurrent_wf_nprocs(): wf.add(sleep_add_one(name="taskc", x=wf.taska.lzout.out)) wf.add(sleep_add_one(name="taskd", x=wf.taskb.lzout.out)) wf.set_output([("out1", wf.taskc.lzout.out), ("out2", wf.taskd.lzout.out)]) - # wf.plugin = 'cf' - # res = wf.run() + wf.cache_dir = tmpdir + with Submitter("cf", n_procs=2) as sub: sub(wf) @@ -79,7 +84,7 @@ def test_concurrent_wf_nprocs(): assert res.output.out2 == 12 -def test_wf_in_wf(plugin): +def test_wf_in_wf(plugin, tmpdir): """WF(A --> SUBWF(A --> B) --> B)""" wf = Workflow(name="wf_in_wf", input_spec=["x"]) wf.inputs.x = 3 @@ -92,10 +97,12 @@ def test_wf_in_wf(plugin): subwf.set_output([("out", subwf.sub_b.lzout.out)]) # connect, then add subwf.inputs.x = wf.wf_a.lzout.out - wf.add(subwf) + subwf.cache_dir = tmpdir + wf.add(subwf) wf.add(sleep_add_one(name="wf_b", x=wf.sub_wf.lzout.out)) wf.set_output([("out", wf.wf_b.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin) as sub: sub(wf) @@ -105,7 +112,7 @@ def test_wf_in_wf(plugin): @pytest.mark.flaky(reruns=2) # when dask -def test_wf2(plugin_dask_opt): +def test_wf2(plugin_dask_opt, tmpdir): """ workflow as a node workflow-node with one task and no splitter """ @@ -117,6 +124,7 @@ def test_wf2(plugin_dask_opt): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -126,7 +134,7 @@ def test_wf2(plugin_dask_opt): @pytest.mark.flaky(reruns=2) # when dask -def test_wf_with_state(plugin_dask_opt): +def test_wf_with_state(plugin_dask_opt, tmpdir): wf = Workflow(name="wf_with_state", input_spec=["x"]) wf.add(sleep_add_one(name="taska", x=wf.lzin.x)) wf.add(sleep_add_one(name="taskb", x=wf.taska.lzout.out)) @@ -134,6 +142,7 @@ def test_wf_with_state(plugin_dask_opt): wf.inputs.x = [1, 2, 3] wf.split("x") wf.set_output([("out", wf.taskb.lzout.out)]) + wf.cache_dir = tmpdir with Submitter(plugin=plugin_dask_opt) as sub: sub(wf) @@ -295,13 +304,12 @@ def cancel(job_name_part): job_id = id_p3.communicate()[0].decode("utf-8").strip() # # canceling the job - proc1 = sp.run(["scancel", job_id]) - # checking the status of jobs with the name; returning the last item - proc2 = sp.run(["sacct", "-j", job_id], stdout=sp.PIPE, stderr=sp.PIPE) - return proc2.stdout.decode("utf-8").strip() + proc = sp.run(["scancel", job_id, "--verbose"], stdout=sp.PIPE, stderr=sp.PIPE) + # cancelling the job returns message in the sterr + return proc.stderr.decode("utf-8").strip() -@pytest.mark.flaky(reruns=3) +@pytest.mark.flaky(reruns=1) @pytest.mark.skipif(not slurm_available, reason="slurm not installed") def test_slurm_cancel_rerun_1(tmpdir): """ testing that tasks run with slurm is re-queue @@ -315,25 +323,26 @@ def test_slurm_cancel_rerun_1(tmpdir): input_spec=["x", "job_name_cancel", "job_name_resqueue"], cache_dir=tmpdir, ) - wf.add(sleep(name="sleep", x=wf.lzin.x, job_name_part=wf.lzin.job_name_cancel)) - wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name_resqueue)) + wf.add(sleep(name="sleep1", x=wf.lzin.x, job_name_part=wf.lzin.job_name_cancel)) + wf.add(cancel(name="cancel1", job_name_part=wf.lzin.job_name_resqueue)) wf.inputs.x = 10 - wf.inputs.job_name_resqueue = "sleep" - wf.inputs.job_name_cancel = "cancel" + wf.inputs.job_name_resqueue = "sleep1" + wf.inputs.job_name_cancel = "cancel1" - wf.set_output([("out", wf.sleep.lzout.out), ("canc_out", wf.cancel.lzout.out)]) + wf.set_output([("out", wf.sleep1.lzout.out), ("canc_out", wf.cancel1.lzout.out)]) with Submitter("slurm") as sub: sub(wf) res = wf.result() assert res.output.out == 10 # checking if indeed the sleep-task job was cancelled by cancel-task - assert "CANCELLED" in res.output.canc_out + assert "Terminating" in res.output.canc_out + assert "Invalid" not in res.output.canc_out script_dir = tmpdir / "SlurmWorker_scripts" assert script_dir.exists() -@pytest.mark.flaky(reruns=3) +@pytest.mark.flaky(reruns=1) @pytest.mark.skipif(not slurm_available, reason="slurm not installed") def test_slurm_cancel_rerun_2(tmpdir): """ testing that tasks run with slurm that has --no-requeue @@ -342,13 +351,13 @@ def test_slurm_cancel_rerun_2(tmpdir): The first job is not able t be rescheduled and the error is returned. """ wf = Workflow(name="wf", input_spec=["x", "job_name"], cache_dir=tmpdir) - wf.add(sleep(name="sleep", x=wf.lzin.x)) - wf.add(cancel(nane="cancel", job_name_part=wf.lzin.job_name)) + wf.add(sleep(name="sleep2", x=wf.lzin.x)) + wf.add(cancel(name="cancel2", job_name_part=wf.lzin.job_name)) wf.inputs.x = 10 - wf.inputs.job_name = "sleep" + wf.inputs.job_name = "sleep2" - wf.set_output([("out", wf.sleep.lzout.out), ("canc_out", wf.cancel.lzout.out)]) + wf.set_output([("out", wf.sleep2.lzout.out), ("canc_out", wf.cancel2.lzout.out)]) with pytest.raises(Exception): with Submitter("slurm", sbatch_args="--no-requeue") as sub: sub(wf) From 937172f0858c3881b6609316279fa630c8c5a4a3 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 10 Nov 2020 21:39:13 -0500 Subject: [PATCH 179/271] fixing template formatting - didnt work well if the directory had a dot in the name --- pydra/engine/helpers_file.py | 8 +++-- pydra/engine/tests/test_shelltask.py | 52 ++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 351e23610b..eff8e20033 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -600,8 +600,12 @@ def _template_formatting(template, inputs_dict, keep_extension=True): fld_value = inputs_dict[fld_name] if fld_value is attr.NOTHING: return attr.NOTHING - fld_value = str(fld_value) # in case it's a path - filename, *ext = fld_value.split(".", maxsplit=1) + fld_value_parent = Path(fld_value).parent + fld_value_name = Path(fld_value).name + + name, *ext = fld_value_name.split(".", maxsplit=1) + filename = str(fld_value_parent / name) + # if keep_extension is False, the extensions are removed if keep_extension is False: ext = [] diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index ba95c6af81..b21b4c7c5b 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1162,8 +1162,56 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): assert shelly.output_dir == res.output.file_copy.parent -@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +@pytest.mark.parametrize("results_function", [result_no_submitter]) def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): + """ + providing output name using input_spec (output_file_template in metadata), + the template has a suffix, the extension of the file will be moved to the end + the change: input file has directory with a dot + """ + cmd = "cp" + file = tmpdir.mkdir("data.inp").join("file.txt") + file.write("content") + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file_orig", + attr.ib( + type=File, + metadata={"position": 2, "help_string": "new file", "argstr": ""}, + ), + ), + ( + "file_copy", + attr.ib( + type=str, + metadata={ + "output_file_template": "{file_orig}_copy", + "help_string": "output file", + "argstr": "", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.file_copy.exists() + assert res.output.file_copy.name == "file_copy.txt" + # checking if it's created in a good place + assert shelly.output_dir == res.output.file_copy.parent + + +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): """ providing output name using input_spec (output_file_template in metadata) and the keep_extension is set to False, so the extension is removed completely. @@ -1209,7 +1257,7 @@ def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): +def test_shell_cmd_inputspec_9c(tmpdir, plugin, results_function): """ providing output name using input_spec (output_file_template in metadata) and the keep_extension is set to False, so the extension is removed completely, From 09caefbf0554e8846cf1bacd3cff88a4f5d7f8ea Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 16 Nov 2020 23:24:32 -0500 Subject: [PATCH 180/271] adding MultiInputFile and MultiOutputFile; adding a tests for ShellCommandTest that uses MultiOutputFile with a template and is able to recreate a list --- pydra/engine/helpers.py | 12 +- pydra/engine/helpers_file.py | 100 +++++++++++------ pydra/engine/specs.py | 15 ++- pydra/engine/tests/test_shelltask.py | 160 ++++++++++++++++++++++++++- 4 files changed, 250 insertions(+), 37 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 975ef8e58d..72c1b3e512 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -27,6 +27,8 @@ LazyField, MultiOutputObj, MultiInputObj, + MultiInputFile, + MultiOutputFile, ) from .helpers_file import hash_file, hash_dir, copyfile, is_existing_file @@ -311,7 +313,15 @@ def custom_validator(instance, attribute, value): or value is None or attribute.name.startswith("_") # e.g. _func or isinstance(value, LazyField) - or tp_attr in [ty.Any, inspect._empty, MultiOutputObj, MultiInputObj] + or tp_attr + in [ + ty.Any, + inspect._empty, + MultiOutputObj, + MultiInputObj, + MultiOutputFile, + MultiInputFile, + ] ): check_type = False # no checking of the type elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index eff8e20033..f80509f0bd 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -541,7 +541,7 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input based on the value from inputs_dict (checking the types of the fields, that have "output_file_template)" """ - from .specs import File + from .specs import File, MultiOutputFile if spec_type == "input": if field.type not in [str, ty.Union[str, bool]]: @@ -559,7 +559,7 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input f"type of {field.name} is str, consider using Union[str, bool]" ) elif spec_type == "output": - if field.type is not File: + if field.type not in [File, MultiOutputFile]: raise Exception( f"output {field.name} should be a File, but {field.type} set as the type" ) @@ -572,26 +572,30 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input # if input fld is set to False, the fld shouldn't be used (setting NOTHING) return attr.NOTHING else: # inputs_dict[field.name] is True or spec_type is output - template = field.metadata["output_file_template"] - # as default, we assume that keep_extension is True - keep_extension = field.metadata.get("keep_extension", True) - value = _template_formatting( - template, inputs_dict, keep_extension=keep_extension - ) + value = _template_formatting(field, inputs_dict) # changing path so it is in the output_dir if output_dir and value is not attr.NOTHING: # should be converted to str, it is also used for input fields that should be str - return str(output_dir / Path(value).name) + if type(value) is list: + return [str(output_dir / Path(val).name) for val in value] + else: + return str(output_dir / Path(value).name) else: return value -def _template_formatting(template, inputs_dict, keep_extension=True): +def _template_formatting(field, inputs_dict): """Formatting a single template based on values from inputs_dict. - Taking into account that field values and template could have file extensions - (assuming that if template has extension, the field value extension is removed, - if field has extension, and no template extension, than it is moved to the end), + Taking into account that the field with a template can be a MultiOutputFile + and the field values needed in the template can be a list - + returning a list of formatted templates in that case. """ + from .specs import MultiOutputFile + + template = field.metadata["output_file_template"] + # as default, we assume that keep_extension is True + keep_extension = field.metadata.get("keep_extension", True) + inp_fields = re.findall("{\w+}", template) if len(inp_fields) == 0: return template @@ -600,31 +604,61 @@ def _template_formatting(template, inputs_dict, keep_extension=True): fld_value = inputs_dict[fld_name] if fld_value is attr.NOTHING: return attr.NOTHING - fld_value_parent = Path(fld_value).parent - fld_value_name = Path(fld_value).name - - name, *ext = fld_value_name.split(".", maxsplit=1) - filename = str(fld_value_parent / name) - - # if keep_extension is False, the extensions are removed - if keep_extension is False: - ext = [] - if template.endswith(inp_fields[0]): - # if no suffix added in template, the simplest formatting should work - # recreating fld_value with the updated extension - fld_value_upd = ".".join([filename] + ext) - formatted_value = template.format(**{fld_name: fld_value_upd}) - elif "." not in template: # the template doesn't have its own extension - # if the fld_value has extension, it will be moved to the end - formatted_value = ".".join([template.format(**{fld_name: filename})] + ext) - else: # template has its own extension - # removing fld_value extension if any - formatted_value = template.format(**{fld_name: filename}) + # if field is MultiOutputFile and the fld_value is a list, + # each element of the list should be used separately in the template + # and return a list with formatted values + if field.type is MultiOutputFile and type(fld_value) is list: + formatted_value = [] + for el in fld_value: + formatted_value.append( + _element_formatting( + template, + fld_name=fld_name, + fld_value=el, + keep_extension=keep_extension, + ) + ) + else: + formatted_value = _element_formatting( + template, + fld_name=fld_name, + fld_value=fld_value, + keep_extension=keep_extension, + ) return formatted_value else: raise NotImplementedError("should we allow for more args in the template?") +def _element_formatting(template, fld_name, fld_value, keep_extension): + """Formatting a single template for a single element of field value (if a list). + Taking into account that field values and template could have file extensions + (assuming that if template has extension, the field value extension is removed, + if field has extension, and no template extension, than it is moved to the end), + """ + fld_value_parent = Path(fld_value).parent + fld_value_name = Path(fld_value).name + + name, *ext = fld_value_name.split(".", maxsplit=1) + filename = str(fld_value_parent / name) + + # if keep_extension is False, the extensions are removed + if keep_extension is False: + ext = [] + if template.endswith(f"{{{fld_name}}}"): + # if no suffix added in template, the simplest formatting should work + # recreating fld_value with the updated extension + fld_value_upd = ".".join([filename] + ext) + formatted_value = template.format(**{fld_name: fld_value_upd}) + elif "." not in template: # the template doesn't have its own extension + # if the fld_value has extension, it will be moved to the end + formatted_value = ".".join([template.format(**{fld_name: filename})] + ext) + else: # template has its own extension + # removing fld_value extension if any + formatted_value = template.format(**{fld_name: filename}) + return formatted_value + + def is_local_file(f): from .specs import File diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 874a304e43..af8b43494f 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -41,6 +41,14 @@ def converter(cls, value): return value +class MultiInputFile(MultiInputObj): + """A ty.List[File] object, converter changes a single file path to a list""" + + +class MultiOutputFile(MultiOutputObj): + """A ty.List[File] object, converter changes an 1-el list to the single value""" + + @attr.s(auto_attribs=True, kw_only=True) class SpecInfo: """Base data structure for metadata of specifications.""" @@ -400,7 +408,7 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out = {} for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type is File: + if fld.type in [File, MultiOutputFile]: # assuming that field should have either default or metadata, but not both if ( fld.default is None or fld.default == attr.NOTHING @@ -493,7 +501,10 @@ def _field_metadata(self, fld, inputs, output_dir): value = template_update_single( fld, inputs_templ, output_dir=output_dir, spec_type="output" ) - return Path(value) + if fld.type is MultiOutputFile and type(value) is list: + return [Path(val) for val in value] + else: + return Path(value) elif "callable" in fld.metadata: return fld.metadata["callable"](fld.name, output_dir) else: diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 6ae2b512b7..fdb26cef05 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -8,7 +8,14 @@ from ..task import ShellCommandTask from ..submitter import Submitter from ..core import Workflow -from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File +from ..specs import ( + ShellOutSpec, + ShellSpec, + SpecInfo, + File, + MultiOutputFile, + MultiInputObj, +) from .utils import result_no_submitter, result_submitter, use_validator if sys.platform.startswith("win"): @@ -2629,6 +2636,157 @@ def test_shell_cmd_outputspec_5a(): assert res.output.out1.exists() +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_6(tmpdir, plugin, results_function): + """ + providing output with output_file_name and using MultiOutputFile as a type. + the input field used in the template is a MultiInputObj, so it can be and is a list + """ + file = tmpdir.join("script.sh") + file.write(f'for var in "$@"; do touch file"$var".txt; done') + + cmd = "bash" + new_files_id = ["1", "2", "3"] + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "script", + attr.ib( + type=File, + metadata={ + "help_string": "script file", + "mandatory": True, + "position": 1, + "argstr": "", + }, + ), + ), + ( + "files_id", + attr.ib( + type=MultiInputObj, + metadata={ + "position": 2, + "argstr": "...", + "sep": " ", + "help_string": "list of name indices", + "mandatory": True, + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "new_files", + attr.ib( + type=MultiOutputFile, + metadata={ + "output_file_template": "file{files_id}.txt", + "help_string": "output file", + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + script=file, + files_id=new_files_id, + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + for file in res.output.new_files: + assert file.exists() + + +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_6a(tmpdir, plugin, results_function): + """ + providing output with output_file_name and using MultiOutputFile as a type. + the input field used in the template is a MultiInputObj, but a single element is used + """ + file = tmpdir.join("script.sh") + file.write(f'for var in "$@"; do touch file"$var".txt; done') + + cmd = "bash" + new_files_id = "1" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "script", + attr.ib( + type=File, + metadata={ + "help_string": "script file", + "mandatory": True, + "position": 1, + "argstr": "", + }, + ), + ), + ( + "files_id", + attr.ib( + type=MultiInputObj, + metadata={ + "position": 2, + "argstr": "...", + "sep": " ", + "help_string": "list of name indices", + "mandatory": True, + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "new_files", + attr.ib( + type=MultiOutputFile, + metadata={ + "output_file_template": "file{files_id}.txt", + "help_string": "output file", + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + script=file, + files_id=new_files_id, + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.new_files.exists() + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): """ From 28ef46ad29e58895ff84836e62070b39a3531562 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 17 Nov 2020 11:37:01 -0500 Subject: [PATCH 181/271] fixing the way how the env var are set in github action --- .github/workflows/testpydra.yml | 2 +- .github/workflows/testsingularity.yml | 4 ++-- .github/workflows/testslurm.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index f859781253..196d20fb2b 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Disable etelemetry - run: echo ::set-env name=NO_ET::TRUE + run: echo "NO_ET=TRUE" >> $GITHUB_ENV - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} uses: actions/setup-python@v2 with: diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 3341c75265..68fbc5812c 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -14,8 +14,8 @@ jobs: steps: - name: Set env run: | - echo ::set-env name=RELEASE_VERSION::v3.5.0 - echo ::set-env name=NO_ET::TRUE + echo "RELEASE_VERSION=v3.5.0" >> $GITHUB_ENV + echo "NO_ET=TRUE" >> $GITHUB_ENV - name: Setup Singularity uses: actions/checkout@v2 with: diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 1497b5b545..250bbd2eca 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Disable etelemetry - run: echo ::set-env name=NO_ET::TRUE + run: echo "NO_ET=TRUE" >> $GITHUB_ENV - uses: actions/checkout@v2 - name: Pull docker image run: | From 66c99a5ff8ac2a45062f961647e1286e6f29b75e Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 20 Nov 2020 11:15:14 -0500 Subject: [PATCH 182/271] adding uid property for task, so it can be used in the slurm fscriptsfilenames etc., instead of calculating checksums --- pydra/engine/core.py | 30 ++++++++++++++++++++++++++++-- pydra/engine/workers.py | 28 ++++++++++++---------------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index f5490cef83..a3e2793336 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -7,6 +7,7 @@ from pathlib import Path import typing as ty from copy import deepcopy, copy +from uuid import uuid4 import cloudpickle as cp from filelock import SoftFileLock @@ -183,6 +184,8 @@ def __init__( self.cache_locations = cache_locations self.allow_cache_override = True self._checksum = None + self._uid = None + self._uid_states = None # if True the results are not checked (does not propagate to nodes) self.task_rerun = rerun @@ -249,6 +252,29 @@ def checksum(self): ) return self._checksum + @property + def uid(self): + """ setting the unique id number for the task + It will be used to create unique names for slurm scripts etc. + without a need to run checksum + """ + if not self._uid: + self._uid = str(uuid4()) + return self._uid + + def uid_states(self): + """ setting a list of the unique id numbers for the task with a splitter + It will be used to create unique names for slurm scripts etc. + without a need to run checksum + """ + if self._uid_states is None: + self._uid_states = [] + if not hasattr(self.state, "inputs_ind"): + self.state.prepare_states(self.inputs) + for ind in range(len(self.state.inputs_ind)): + self._uid_states.append(str(uuid4())) + return self._uid_states + def checksum_states(self, state_index=None): """ Calculate a checksum for the specific state or all of the states of the task. @@ -546,8 +572,8 @@ def pickle_task(self): """ Pickling the tasks with full inputs""" pkl_files = self.cache_dir / "pkl_files" pkl_files.mkdir(exist_ok=True, parents=True) - task_main_path = pkl_files / f"{self.name}_{self.checksum}_task.pklz" - save(task_path=pkl_files, task=self, name_prefix=f"{self.name}_{self.checksum}") + task_main_path = pkl_files / f"{self.name}_{self.uid}_task.pklz" + save(task_path=pkl_files, task=self, name_prefix=f"{self.name}_{self.uid}") return task_main_path @property diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 540739b4e7..ede7277213 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -69,17 +69,16 @@ def __init__(self, loop=None, max_jobs=None): self._jobs = 0 def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): - if isinstance(task, TaskBase): - checksum = task.checksum cache_dir = task.cache_dir ind = None + uid = task.uid else: ind = task[0] - checksum = task[-1].checksum_states()[ind] cache_dir = task[-1].cache_dir + uid = task[-1].uid_states()[ind] - script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / checksum + script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid script_dir.mkdir(parents=True, exist_ok=True) if ind is None: if not (script_dir / "_task.pkl").exists(): @@ -91,7 +90,7 @@ def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): if not task_pkl.exists() or not task_pkl.stat().st_size: raise Exception("Missing or empty task!") - batchscript = script_dir / f"batchscript_{checksum}.sh" + batchscript = script_dir / f"batchscript_{uid}.sh" python_string = f"""'from pydra.engine.helpers import load_and_run; load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) ' """ bcmd = "\n".join( @@ -246,27 +245,24 @@ def run_el(self, runnable, rerun=False): script_dir, batch_script = self._prepare_runscripts(runnable, rerun=rerun) if (script_dir / script_dir.parts[1]) == gettempdir(): logger.warning("Temporary directories may not be shared across computers") - if isinstance(runnable, TaskBase): - checksum = runnable.checksum cache_dir = runnable.cache_dir name = runnable.name + uid = runnable.uid else: - checksum = runnable[-1].checksum_states()[runnable[0]] cache_dir = runnable[-1].cache_dir name = runnable[-1].name + uid = runnable[-1].uid_states()[runnable[0]] - return self._submit_job( - batch_script, name=name, checksum=checksum, cache_dir=cache_dir - ) + return self._submit_job(batch_script, name=name, uid=uid, cache_dir=cache_dir) - async def _submit_job(self, batchscript, name, checksum, cache_dir): + async def _submit_job(self, batchscript, name, uid, cache_dir): """Coroutine that submits task runscript and polls job until completion or error.""" - script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / checksum + script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid sargs = self.sbatch_args.split() jobname = re.search(r"(?<=-J )\S+|(?<=--job-name=)\S+", self.sbatch_args) if not jobname: - jobname = ".".join((name, checksum)) + jobname = ".".join((name, uid)) sargs.append(f"--job-name={jobname}") output = re.search(r"(?<=-o )\S+|(?<=--output=)\S+", self.sbatch_args) if not output: @@ -304,9 +300,9 @@ async def _submit_job(self, batchscript, name, checksum, cache_dir): done in ["CANCELLED", "TIMEOUT", "PREEMPTED"] and "--no-requeue" not in self.sbatch_args ): - if (cache_dir / f"{checksum}.lock").exists(): + if (cache_dir / f"{uid}.lock").exists(): # for pyt3.8 we could you missing_ok=True - (cache_dir / f"{checksum}.lock").unlink() + (cache_dir / f"{uid}.lock").unlink() cmd_re = ("scontrol", "requeue", jobid) await read_and_display_async(*cmd_re, hide_display=True) else: From ce7d172ac7d044d0e452db069d5262fe9b1ac9d0 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 20 Nov 2020 11:30:14 -0500 Subject: [PATCH 183/271] moving _prepare_runscripts to the SlurmWorker class --- pydra/engine/workers.py | 108 ++++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index ede7277213..52fffbc3dd 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -68,42 +68,6 @@ def __init__(self, loop=None, max_jobs=None): """Maximum number of concurrently running jobs.""" self._jobs = 0 - def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): - if isinstance(task, TaskBase): - cache_dir = task.cache_dir - ind = None - uid = task.uid - else: - ind = task[0] - cache_dir = task[-1].cache_dir - uid = task[-1].uid_states()[ind] - - script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid - script_dir.mkdir(parents=True, exist_ok=True) - if ind is None: - if not (script_dir / "_task.pkl").exists(): - save(script_dir, task=task) - else: - copyfile(task[1], script_dir / "_task.pklz") - - task_pkl = script_dir / "_task.pklz" - if not task_pkl.exists() or not task_pkl.stat().st_size: - raise Exception("Missing or empty task!") - - batchscript = script_dir / f"batchscript_{uid}.sh" - python_string = f"""'from pydra.engine.helpers import load_and_run; load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) ' - """ - bcmd = "\n".join( - ( - f"#!{interpreter}", - f"#SBATCH --output={str(script_dir / 'slurm-%j.out')}", - f"{sys.executable} -c " + python_string, - ) - ) - with batchscript.open("wt") as fp: - fp.writelines(bcmd) - return script_dir, batchscript - async def fetch_finished(self, futures): """ Awaits asyncio's :class:`asyncio.Task` until one is finished. @@ -256,6 +220,42 @@ def run_el(self, runnable, rerun=False): return self._submit_job(batch_script, name=name, uid=uid, cache_dir=cache_dir) + def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): + if isinstance(task, TaskBase): + cache_dir = task.cache_dir + ind = None + uid = task.uid + else: + ind = task[0] + cache_dir = task[-1].cache_dir + uid = task[-1].uid_states()[ind] + + script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid + script_dir.mkdir(parents=True, exist_ok=True) + if ind is None: + if not (script_dir / "_task.pkl").exists(): + save(script_dir, task=task) + else: + copyfile(task[1], script_dir / "_task.pklz") + + task_pkl = script_dir / "_task.pklz" + if not task_pkl.exists() or not task_pkl.stat().st_size: + raise Exception("Missing or empty task!") + + batchscript = script_dir / f"batchscript_{uid}.sh" + python_string = f"""'from pydra.engine.helpers import load_and_run; load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) ' + """ + bcmd = "\n".join( + ( + f"#!{interpreter}", + f"#SBATCH --output={str(script_dir / 'slurm-%j.out')}", + f"{sys.executable} -c " + python_string, + ) + ) + with batchscript.open("wt") as fp: + fp.writelines(bcmd) + return script_dir, batchscript + async def _submit_job(self, batchscript, name, uid, cache_dir): """Coroutine that submits task runscript and polls job until completion or error.""" script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid @@ -309,6 +309,42 @@ async def _submit_job(self, batchscript, name, uid, cache_dir): return True await asyncio.sleep(self.poll_delay) + def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): + if isinstance(task, TaskBase): + cache_dir = task.cache_dir + ind = None + uid = task.uid + else: + ind = task[0] + cache_dir = task[-1].cache_dir + uid = task[-1].uid_states()[ind] + + script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid + script_dir.mkdir(parents=True, exist_ok=True) + if ind is None: + if not (script_dir / "_task.pkl").exists(): + save(script_dir, task=task) + else: + copyfile(task[1], script_dir / "_task.pklz") + + task_pkl = script_dir / "_task.pklz" + if not task_pkl.exists() or not task_pkl.stat().st_size: + raise Exception("Missing or empty task!") + + batchscript = script_dir / f"batchscript_{uid}.sh" + python_string = f"""'from pydra.engine.helpers import load_and_run; load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) ' + """ + bcmd = "\n".join( + ( + f"#!{interpreter}", + f"#SBATCH --output={str(script_dir / 'slurm-%j.out')}", + f"{sys.executable} -c " + python_string, + ) + ) + with batchscript.open("wt") as fp: + fp.writelines(bcmd) + return script_dir, batchscript + async def _poll_job(self, jobid): cmd = ("squeue", "-h", "-j", jobid) logger.debug(f"Polling job {jobid}") From c6e3193aa3823a9e3df31ac277fc56843742e9f4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 20 Nov 2020 22:11:29 -0500 Subject: [PATCH 184/271] removing _graph_checsum from the submitter (no checksum should be used by worker/submitter before running the task) --- pydra/engine/core.py | 49 +++++++++++++++++++++------------------ pydra/engine/submitter.py | 3 --- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index a3e2793336..89ccd8077f 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -252,29 +252,6 @@ def checksum(self): ) return self._checksum - @property - def uid(self): - """ setting the unique id number for the task - It will be used to create unique names for slurm scripts etc. - without a need to run checksum - """ - if not self._uid: - self._uid = str(uuid4()) - return self._uid - - def uid_states(self): - """ setting a list of the unique id numbers for the task with a splitter - It will be used to create unique names for slurm scripts etc. - without a need to run checksum - """ - if self._uid_states is None: - self._uid_states = [] - if not hasattr(self.state, "inputs_ind"): - self.state.prepare_states(self.inputs) - for ind in range(len(self.state.inputs_ind)): - self._uid_states.append(str(uuid4())) - return self._uid_states - def checksum_states(self, state_index=None): """ Calculate a checksum for the specific state or all of the states of the task. @@ -287,6 +264,9 @@ def checksum_states(self, state_index=None): TODO """ + if is_workflow(self) and self.inputs._graph_checksums is attr.NOTHING: + self.inputs._graph_checksums = [nd.checksum for nd in self.graph_sorted] + if state_index is not None: inputs_copy = deepcopy(self.inputs) for key, ind in self.state.inputs_ind[state_index].items(): @@ -314,6 +294,29 @@ def checksum_states(self, state_index=None): checksum_list.append(self.checksum_states(state_index=ind)) return checksum_list + @property + def uid(self): + """ setting the unique id number for the task + It will be used to create unique names for slurm scripts etc. + without a need to run checksum + """ + if not self._uid: + self._uid = str(uuid4()) + return self._uid + + def uid_states(self): + """ setting a list of the unique id numbers for the task with a splitter + It will be used to create unique names for slurm scripts etc. + without a need to run checksum + """ + if self._uid_states is None: + self._uid_states = [] + if not hasattr(self.state, "inputs_ind"): + self.state.prepare_states(self.inputs) + for ind in range(len(self.state.inputs_ind)): + self._uid_states.append(str(uuid4())) + return self._uid_states + def set_state(self, splitter, combiner=None): """ Set a particular state on this task. diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index 56d711de1c..fd76d5c626 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -51,9 +51,6 @@ def __call__(self, runnable, cache_locations=None, rerun=False): runnable.create_connections(nd) if nd.allow_cache_override: nd.cache_dir = runnable.cache_dir - runnable.inputs._graph_checksums = [ - nd.checksum for nd in runnable.graph_sorted - ] if is_workflow(runnable) and runnable.state is None: self.loop.run_until_complete(self.submit_workflow(runnable, rerun=rerun)) else: From 623132c0f678722ebc3ebee0c7c5d0c4917a923f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 21 Nov 2020 13:28:21 -0500 Subject: [PATCH 185/271] adding attributes to the BaseSpec so I can track if input has changed and if new hash value has to be calculated --- pydra/engine/specs.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index af8b43494f..c4d004b9e9 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -66,10 +66,16 @@ class SpecInfo: class BaseSpec: """The base dataclass specs for all inputs and outputs.""" + inp_hash = None + # a flag to check if anything has changed since the last hash was calculated + changed = None + def __setattr__(self, name, value): """changing settatr, so the converter and validator is run if input is set after __init__ """ + if name == "inp_hash": + breakpoint() if inspect.stack()[1][3] == "__init__": # or name.startswith("_"): super().__setattr__(name, value) else: @@ -80,6 +86,7 @@ def __setattr__(self, name, value): super().__setattr__(name, value) # validate all fields that have set a validator attr.validate(self) + super().__setattr__("changed", True) def collect_additional_outputs(self, inputs, output_dir): """Get additional outputs.""" @@ -90,6 +97,9 @@ def hash(self): """Compute a basic hash for any given set of fields.""" from .helpers import hash_value, hash_function + # if inp_hash already calculated and nothing has changed, the old value can be used + if self.changed is False and self.inp_hash: + return self.inp_hash inp_dict = {} for field in attr_fields(self): if field.name in ["_graph_checksums", "bindings"] or field.metadata.get( @@ -104,9 +114,11 @@ def hash(self): ) inp_hash = hash_function(inp_dict) if hasattr(self, "_graph_checksums"): - return hash_function((inp_hash, self._graph_checksums)) - else: - return inp_hash + inp_hash = hash_function((inp_hash, self._graph_checksums)) + # setting inp_hash and changing the flag to False + super().__setattr__("inp_hash", inp_hash) + super().__setattr__("changed", False) + return inp_hash def retrieve_values(self, wf, state_index=None): """Get values contained by this spec.""" From d3a87592da538eab1aa23df66ed4e80b0f364c68 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sun, 22 Nov 2020 23:03:05 -0500 Subject: [PATCH 186/271] removing repeated function --- pydra/engine/workers.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 52fffbc3dd..4af2780e02 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -309,42 +309,6 @@ async def _submit_job(self, batchscript, name, uid, cache_dir): return True await asyncio.sleep(self.poll_delay) - def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): - if isinstance(task, TaskBase): - cache_dir = task.cache_dir - ind = None - uid = task.uid - else: - ind = task[0] - cache_dir = task[-1].cache_dir - uid = task[-1].uid_states()[ind] - - script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid - script_dir.mkdir(parents=True, exist_ok=True) - if ind is None: - if not (script_dir / "_task.pkl").exists(): - save(script_dir, task=task) - else: - copyfile(task[1], script_dir / "_task.pklz") - - task_pkl = script_dir / "_task.pklz" - if not task_pkl.exists() or not task_pkl.stat().st_size: - raise Exception("Missing or empty task!") - - batchscript = script_dir / f"batchscript_{uid}.sh" - python_string = f"""'from pydra.engine.helpers import load_and_run; load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) ' - """ - bcmd = "\n".join( - ( - f"#!{interpreter}", - f"#SBATCH --output={str(script_dir / 'slurm-%j.out')}", - f"{sys.executable} -c " + python_string, - ) - ) - with batchscript.open("wt") as fp: - fp.writelines(bcmd) - return script_dir, batchscript - async def _poll_job(self, jobid): cmd = ("squeue", "-h", "-j", jobid) logger.debug(f"Polling job {jobid}") From f843e8aab5fb82c0f5d237f0f23a8c5ffb684404 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 23 Nov 2020 22:23:05 -0500 Subject: [PATCH 187/271] fixing changes in BaseSpec.hash --- pydra/engine/specs.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index c4d004b9e9..092570fc08 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -74,9 +74,7 @@ def __setattr__(self, name, value): """changing settatr, so the converter and validator is run if input is set after __init__ """ - if name == "inp_hash": - breakpoint() - if inspect.stack()[1][3] == "__init__": # or name.startswith("_"): + if inspect.stack()[1][3] == "__init__" or name in ["inp_hash", "changed"]: super().__setattr__(name, value) else: tp = attr.fields_dict(self.__class__)[name].type @@ -86,7 +84,7 @@ def __setattr__(self, name, value): super().__setattr__(name, value) # validate all fields that have set a validator attr.validate(self) - super().__setattr__("changed", True) + super().__setattr__("changed", True) def collect_additional_outputs(self, inputs, output_dir): """Get additional outputs.""" @@ -102,9 +100,12 @@ def hash(self): return self.inp_hash inp_dict = {} for field in attr_fields(self): - if field.name in ["_graph_checksums", "bindings"] or field.metadata.get( - "output_file_template" - ): + if field.name in [ + "_graph_checksums", + "bindings", + "inp_hash", + "changed", + ] or field.metadata.get("output_file_template"): continue # removing values that are notset from hash calculation if getattr(self, field.name) is attr.NOTHING: @@ -116,8 +117,8 @@ def hash(self): if hasattr(self, "_graph_checksums"): inp_hash = hash_function((inp_hash, self._graph_checksums)) # setting inp_hash and changing the flag to False - super().__setattr__("inp_hash", inp_hash) - super().__setattr__("changed", False) + self.inp_hash = inp_hash + self.changed = False return inp_hash def retrieve_values(self, wf, state_index=None): From cbff65e45f0a934a7a917a5d88be19ce4a5b79ef Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 23 Nov 2020 22:26:56 -0500 Subject: [PATCH 188/271] reverting name for the lockfiles that use checksum; saving checksum in an extra json file, so could be read if the job is cancelled and the locfile has to be removed --- pydra/engine/core.py | 9 ++++++++- pydra/engine/workers.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 89ccd8077f..b93991a530 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -438,7 +438,10 @@ def _run(self, rerun=False, **kwargs): lockfile = self.cache_dir / (checksum + ".lock") # Eagerly retrieve cached - see scenarios in __init__() self.hooks.pre_run(self) - # TODO add signal handler for processes killed after lock acquisition + # adding info file with the checksum in case the task was cancelled + # and the lockfile has to be removed + with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: + json.dump({"checksum": self.checksum}, jsonfile) with SoftFileLock(lockfile): if not (rerun or self.task_rerun): result = self.result() @@ -962,6 +965,10 @@ async def _run(self, submitter=None, rerun=False, **kwargs): "Workflow output cannot be None, use set_output to define output(s)" ) checksum = self.checksum + # adding info file with the checksum in case the task was cancelled + # and the lockfile has to be removed + with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: + json.dump({"checksum": checksum}, jsonfile) lockfile = self.cache_dir / (checksum + ".lock") # Eagerly retrieve cached if not (rerun or self.task_rerun): diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 4af2780e02..1ae47803cb 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -1,6 +1,6 @@ """Execution workers.""" import asyncio -import sys, os +import sys, os, json import re from tempfile import gettempdir from pathlib import Path @@ -300,9 +300,13 @@ async def _submit_job(self, batchscript, name, uid, cache_dir): done in ["CANCELLED", "TIMEOUT", "PREEMPTED"] and "--no-requeue" not in self.sbatch_args ): - if (cache_dir / f"{uid}.lock").exists(): - # for pyt3.8 we could you missing_ok=True - (cache_dir / f"{uid}.lock").unlink() + # loading info about task with a specific uid + info_file = cache_dir / f"{uid}_info.json" + if info_file.exists(): + checksum = json.loads(info_file.read_text())["checksum"] + if (cache_dir / f"{checksum}.lock").exists(): + # for pyt3.8 we could you missing_ok=True + (cache_dir / f"{checksum}.lock").unlink() cmd_re = ("scontrol", "requeue", jobid) await read_and_display_async(*cmd_re, hide_display=True) else: From c8dbddf7669469d80e92f7603698d5bdc97e5315 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 23 Nov 2020 22:28:41 -0500 Subject: [PATCH 189/271] small fix to test_submitter --- pydra/engine/submitter.py | 2 -- pydra/engine/tests/test_submitter.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index fd76d5c626..2cf15f3c9a 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -178,8 +178,6 @@ async def _run_workflow(self, wf, rerun=False): logger.debug(f"Retrieving inputs for {task}") # TODO: add state idx to retrieve values to reduce waiting task.inputs.retrieve_values(wf) - # checksum has to be updated, so resetting - task._checksum = None if is_workflow(task) and not task.state: await self.submit_workflow(task, rerun=rerun) else: diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index a2ddabd59c..12e5e06377 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -185,7 +185,8 @@ def test_slurm_wf_cf(tmpdir): # ensure only workflow was executed with slurm sdirs = [sd for sd in script_dir.listdir() if sd.isdir()] assert len(sdirs) == 1 - assert sdirs[0].basename == wf.checksum + # slurm scripts should be in the dirs that are using uid in the name + assert sdirs[0].basename == wf.uid @pytest.mark.skipif(not slurm_available, reason="slurm not installed") From f2a1dd82813a2fccd7485b97f7b7dd3dbc3a050f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 23 Nov 2020 23:58:00 -0500 Subject: [PATCH 190/271] saving hash values for files to prevent recomputing --- pydra/engine/specs.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 092570fc08..718fb957c8 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -5,7 +5,7 @@ import inspect import re -from .helpers_file import template_update_single +from .helpers_file import template_update_single, is_existing_file def attr_fields(x): @@ -69,12 +69,17 @@ class BaseSpec: inp_hash = None # a flag to check if anything has changed since the last hash was calculated changed = None + files_hash = None def __setattr__(self, name, value): """changing settatr, so the converter and validator is run if input is set after __init__ """ - if inspect.stack()[1][3] == "__init__" or name in ["inp_hash", "changed"]: + if inspect.stack()[1][3] == "__init__" or name in [ + "inp_hash", + "changed", + "files_hash", + ]: super().__setattr__(name, value) else: tp = attr.fields_dict(self.__class__)[name].type @@ -98,6 +103,8 @@ def hash(self): # if inp_hash already calculated and nothing has changed, the old value can be used if self.changed is False and self.inp_hash: return self.inp_hash + if self.files_hash is None: + self.files_hash = {} inp_dict = {} for field in attr_fields(self): if field.name in [ @@ -105,14 +112,28 @@ def hash(self): "bindings", "inp_hash", "changed", + "files_hash", ] or field.metadata.get("output_file_template"): continue # removing values that are notset from hash calculation if getattr(self, field.name) is attr.NOTHING: continue - inp_dict[field.name] = hash_value( - value=getattr(self, field.name), tp=field.type, metadata=field.metadata - ) + value = getattr(self, field.name) + # checking if the value is a file that already has calculated hash value + if str(value) in self.files_hash: + inp_dict[field.name] = self.files_hash[str(value)] + else: + inp_dict[field.name] = hash_value( + value=value, tp=field.type, metadata=field.metadata + ) + # if file/dir, the hash value will be saved to prevent recomputation + if ( + field.type in [File, Directory] + or "pydra.engine.specs.File" in str(field.type) + or "pydra.engine.specs.File" in str(field.type) + ) and is_existing_file(value): + self.files_hash[str(value)] = inp_dict[field.name] + inp_hash = hash_function(inp_dict) if hasattr(self, "_graph_checksums"): inp_hash = hash_function((inp_hash, self._graph_checksums)) From 311d59c499a4359ffa6be3925aeda50e2a0b7676 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 24 Nov 2020 16:18:15 -0500 Subject: [PATCH 191/271] removing an extra json file with checksums at the end of the run; reseting _uid after loading teh task from a pickle file --- pydra/engine/core.py | 14 +++++++++----- pydra/engine/helpers.py | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index b93991a530..5c612aae70 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -475,6 +475,8 @@ def _run(self, rerun=False, **kwargs): self.hooks.post_run_task(self, result) self.audit.finalize_audit(result) save(odir, result=result, task=self) + # removing the additional file with the chcksum + (self.cache_dir / f"{self.uid}_info.json").unlink() # # function etc. shouldn't change anyway, so removing orig_inputs = dict( (k, v) for (k, v) in orig_inputs.items() if not k.startswith("_") @@ -965,11 +967,6 @@ async def _run(self, submitter=None, rerun=False, **kwargs): "Workflow output cannot be None, use set_output to define output(s)" ) checksum = self.checksum - # adding info file with the checksum in case the task was cancelled - # and the lockfile has to be removed - with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: - json.dump({"checksum": checksum}, jsonfile) - lockfile = self.cache_dir / (checksum + ".lock") # Eagerly retrieve cached if not (rerun or self.task_rerun): result = self.result() @@ -987,6 +984,11 @@ async def _run(self, submitter=None, rerun=False, **kwargs): task.cache_locations = task._cache_locations + self.cache_locations self.create_connections(task) # TODO add signal handler for processes killed after lock acquisition + # adding info file with the checksum in case the task was cancelled + # and the lockfile has to be removed + with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: + json.dump({"checksum": checksum}, jsonfile) + lockfile = self.cache_dir / (checksum + ".lock") self.hooks.pre_run(self) with SoftFileLock(lockfile): # # Let only one equivalent process run @@ -1011,6 +1013,8 @@ async def _run(self, submitter=None, rerun=False, **kwargs): self.hooks.post_run_task(self, result) self.audit.finalize_audit(result=result) save(odir, result=result, task=self) + # removing the additional file with the chcksum + (self.cache_dir / f"{self.uid}_info.json").unlink() os.chdir(cwd) self.hooks.post_run(self, result) return result diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 72c1b3e512..85bf5ae645 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -803,6 +803,7 @@ def load_task(task_pkl, ind=None): _, inputs_dict = task.get_input_el(ind) task.inputs = attr.evolve(task.inputs, **inputs_dict) task.state = None + task._uid = None return task From c087ec7581d08f84d7ce6c6dc0121f2fa0b336f8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 24 Nov 2020 16:19:59 -0500 Subject: [PATCH 192/271] improving the way hash values is calculated - for files the hash values and the time of the last modification is saved, so the time could be compared every single time input has to be hashed --- pydra/engine/specs.py | 48 ++++++++++++++++++++-------- pydra/engine/tests/test_shelltask.py | 8 ++--- pydra/engine/tests/test_specs.py | 45 +++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 718fb957c8..b7af892ca5 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -89,6 +89,7 @@ def __setattr__(self, name, value): super().__setattr__(name, value) # validate all fields that have set a validator attr.validate(self) + # TODO: consider using on_setattr super().__setattr__("changed", True) def collect_additional_outputs(self, inputs, output_dir): @@ -102,7 +103,13 @@ def hash(self): # if inp_hash already calculated and nothing has changed, the old value can be used if self.changed is False and self.inp_hash: - return self.inp_hash + # if there are files in self.files_hash, checking if they didn't changed + check_files = [ + Path(val[0]).stat().st_mtime == val[1] + for val in self.files_hash.values() + ] + if all(check_files): + return self.inp_hash if self.files_hash is None: self.files_hash = {} inp_dict = {} @@ -119,20 +126,33 @@ def hash(self): if getattr(self, field.name) is attr.NOTHING: continue value = getattr(self, field.name) - # checking if the value is a file that already has calculated hash value - if str(value) in self.files_hash: - inp_dict[field.name] = self.files_hash[str(value)] - else: - inp_dict[field.name] = hash_value( - value=value, tp=field.type, metadata=field.metadata - ) - # if file/dir, the hash value will be saved to prevent recomputation + # checking if the field name is in the dictionary with pre-computed hash values for files + if field.name in self.files_hash: + filename, mtime, cont_hash = self.files_hash[field.name] + # if the name of the file and the time of last modification is the same, + # we are reusing the content hash value if ( - field.type in [File, Directory] - or "pydra.engine.specs.File" in str(field.type) - or "pydra.engine.specs.File" in str(field.type) - ) and is_existing_file(value): - self.files_hash[str(value)] = inp_dict[field.name] + str(Path(value)) == filename + and Path(value).stat().st_mtime == mtime + ): + inp_dict[field.name] = cont_hash + continue + + inp_dict[field.name] = hash_value( + value=value, tp=field.type, metadata=field.metadata + ) + # if file/dir, the hash value will be saved to prevent recomputation + if ( + field.type in [File, Directory] + or "pydra.engine.specs.File" in str(field.type) + or "pydra.engine.specs.File" in str(field.type) + ) and is_existing_file(value): + # saving tuple with full pathname, time modification and hash value + self.files_hash[field.name] = ( + str(Path(value)), + Path(value).stat().st_mtime, + inp_dict[field.name], + ) inp_hash = hash_function(inp_dict) if hasattr(self, "_graph_checksums"): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index fdb26cef05..4f54d40fc4 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1176,7 +1176,7 @@ def test_shell_cmd_inputspec_9(tmpdir, plugin, results_function): """ cmd = "cp" file = tmpdir.mkdir("data_inp").join("file.txt") - file.write("content") + file.write("content\n") my_input_spec = SpecInfo( name="Input", @@ -1228,7 +1228,7 @@ def test_shell_cmd_inputspec_9a(tmpdir, plugin, results_function): """ cmd = "cp" file = tmpdir.mkdir("data.inp").join("file.txt") - file.write("content") + file.write("content\n") my_input_spec = SpecInfo( name="Input", @@ -1275,7 +1275,7 @@ def test_shell_cmd_inputspec_9b(tmpdir, plugin, results_function): """ cmd = "cp" file = tmpdir.join("file.txt") - file.write("content") + file.write("content\n") my_input_spec = SpecInfo( name="Input", @@ -1326,7 +1326,7 @@ def test_shell_cmd_inputspec_9c(tmpdir, plugin, results_function): """ cmd = "cp" file = tmpdir.join("file.txt") - file.write("content") + file.write("content\n") my_input_spec = SpecInfo( name="Input", diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 605cbd33ff..3a415c3a36 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -1,5 +1,6 @@ from pathlib import Path import typing as ty +from copy import copy from ..specs import ( BaseSpec, @@ -241,6 +242,48 @@ def test_input_file_hash_2a(tmpdir): def test_input_file_hash_3(tmpdir): + """ input spec with File types, checking when the checksum changes""" + file = tmpdir.join("in_file_1.txt") + with open(file, "w") as f: + f.write("hello") + + input_spec = SpecInfo( + name="Inputs", fields=[("in_file", File), ("in_int", int)], bases=(BaseSpec,) + ) + inputs = make_klass(input_spec) + + my_inp = inputs(in_file=file, in_int=3) + # original hash and files_hash (dictionary contains info about files) + hash1 = my_inp.hash + files_hash1 = copy(my_inp.files_hash) + + # changing int input + my_inp.in_int = 5 + hash2 = my_inp.hash + files_hash2 = copy(my_inp.files_hash) + # hash should be different + assert hash1 != hash2 + # files_hash should be the same, and the tuple for in_file shouldn't be recomputed + assert files_hash1 == files_hash2 + assert id(files_hash1["in_file"]) == id(files_hash2["in_file"]) + + # recreating the file + with open(file, "w") as f: + f.write("hello") + + hash3 = my_inp.hash + files_hash3 = copy(my_inp.files_hash) + # hash should be the same, + # but the entry for in_file in files_hash should be different (modification time) + assert hash3 == hash2 + assert files_hash3["in_file"] != files_hash2["in_file"] + # different timestamp + assert files_hash3["in_file"][1] != files_hash2["in_file"][1] + # the same content hash + assert files_hash3["in_file"][2] == files_hash2["in_file"][2] + + +def test_input_file_hash_4(tmpdir): """ input spec with nested list, that contain ints and Files, checking changes in checksums """ @@ -278,7 +321,7 @@ def test_input_file_hash_3(tmpdir): assert hash1 != hash3 -def test_input_file_hash_4(tmpdir): +def test_input_file_hash_5(tmpdir): """ input spec with File in nested containers, checking changes in checksums""" file = tmpdir.join("in_file_1.txt") with open(file, "w") as f: From d9d12058fb41b17b08c1e8eb4abb179858cc1d48 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 24 Nov 2020 16:42:55 -0500 Subject: [PATCH 193/271] Update pydra/engine/specs.py Co-authored-by: Satrajit Ghosh --- pydra/engine/specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index b7af892ca5..0ea6c9cc16 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -68,7 +68,7 @@ class BaseSpec: inp_hash = None # a flag to check if anything has changed since the last hash was calculated - changed = None + changed = True files_hash = None def __setattr__(self, name, value): From 9bbdc53f52dabe9cb68109234deb1f028d84be76 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 24 Nov 2020 23:54:36 -0500 Subject: [PATCH 194/271] changing the place where content hash values for files are checked: creating an attribute in the BaseClass and passing it to hash_value and has_file --- pydra/engine/core.py | 2 +- pydra/engine/helpers.py | 12 +++--- pydra/engine/helpers_file.py | 21 +++++++++-- pydra/engine/specs.py | 63 +++++++------------------------- pydra/engine/tests/test_specs.py | 31 +++++++++++----- pydra/engine/workers.py | 2 +- 6 files changed, 62 insertions(+), 69 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 5c612aae70..c29c3ac95a 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -6,7 +6,7 @@ import os from pathlib import Path import typing as ty -from copy import deepcopy, copy +from copy import deepcopy from uuid import uuid4 import cloudpickle as cp diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 85bf5ae645..ab170a1abd 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -661,14 +661,16 @@ def hash_function(obj): return sha256(str(obj).encode()).hexdigest() -def hash_value(value, tp=None, metadata=None): +def hash_value(value, tp=None, metadata=None, precalculated=None): """calculating hash or returning values recursively""" if metadata is None: metadata = {} if isinstance(value, (tuple, list)): - return [hash_value(el, tp, metadata) for el in value] + return [hash_value(el, tp, metadata, precalculated) for el in value] elif isinstance(value, dict): - dict_hash = {k: hash_value(v, tp, metadata) for (k, v) in value.items()} + dict_hash = { + k: hash_value(v, tp, metadata, precalculated) for (k, v) in value.items() + } # returning a sorted object return [list(el) for el in sorted(dict_hash.items(), key=lambda x: x[0])] else: # not a container @@ -677,13 +679,13 @@ def hash_value(value, tp=None, metadata=None): and is_existing_file(value) and "container_path" not in metadata ): - return hash_file(value) + return hash_file(value, precalculated=precalculated) elif ( (tp is File or "pydra.engine.specs.Directory" in str(tp)) and is_existing_file(value) and "container_path" not in metadata ): - return hash_dir(value) + return hash_dir(value, precalculated=precalculated) else: return value diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index f80509f0bd..a1ab27f691 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -66,7 +66,9 @@ def split_filename(fname): return pth, fname, ext -def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True): +def hash_file( + afile, chunk_len=8192, crypto=sha256, raise_notfound=True, precalculated=None +): """Compute hash of a file using 'crypto' module.""" from .specs import LazyField @@ -77,6 +79,14 @@ def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True): raise RuntimeError('File "%s" not found.' % afile) return None + # if the path exists already in precalculated + # the time of the last modification will be compared + # and the precalculated hash value will be used if the file has not change + if precalculated and str(Path(afile)) in precalculated: + pre_mtime, pre_cont_hash = precalculated[str(Path(afile))] + if Path(afile).stat().st_mtime == pre_mtime: + return pre_cont_hash + crypto_obj = crypto() with open(afile, "rb") as fp: while True: @@ -84,7 +94,11 @@ def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True): if not data: break crypto_obj.update(data) - return crypto_obj.hexdigest() + + cont_hash = crypto_obj.hexdigest() + if precalculated is not None: + precalculated[str(Path(afile))] = (Path(afile).stat().st_mtime, cont_hash) + return cont_hash def hash_dir( @@ -93,6 +107,7 @@ def hash_dir( ignore_hidden_files=False, ignore_hidden_dirs=False, raise_notfound=True, + precalculated=None, ): """Compute hash of directory contents. @@ -142,7 +157,7 @@ def hash_dir( if not is_existing_file(dpath / filename): file_hashes.append(str(dpath / filename)) else: - this_hash = hash_file(dpath / filename) + this_hash = hash_file(dpath / filename, precalculated=precalculated) file_hashes.append(this_hash) crypto_obj = crypto() diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 0ea6c9cc16..d607fa837f 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -5,7 +5,7 @@ import inspect import re -from .helpers_file import template_update_single, is_existing_file +from .helpers_file import template_update_single def attr_fields(x): @@ -66,10 +66,14 @@ class SpecInfo: class BaseSpec: """The base dataclass specs for all inputs and outputs.""" - inp_hash = None - # a flag to check if anything has changed since the last hash was calculated - changed = True - files_hash = None + def __attrs_post_init__(self): + self.files_hash = {} + for field in attr_fields(self): + if ( + field.name not in ["_graph_checksums", "bindings", "files_hash"] + and field.metadata.get("output_file_template") is None + ): + self.files_hash[field.name] = {} def __setattr__(self, name, value): """changing settatr, so the converter and validator is run @@ -86,11 +90,10 @@ def __setattr__(self, name, value): # if the type has a converter, e.g., MultiInputObj if hasattr(tp, "converter"): value = tp.converter(value) + self.files_hash[name] = {} super().__setattr__(name, value) # validate all fields that have set a validator attr.validate(self) - # TODO: consider using on_setattr - super().__setattr__("changed", True) def collect_additional_outputs(self, inputs, output_dir): """Get additional outputs.""" @@ -101,24 +104,11 @@ def hash(self): """Compute a basic hash for any given set of fields.""" from .helpers import hash_value, hash_function - # if inp_hash already calculated and nothing has changed, the old value can be used - if self.changed is False and self.inp_hash: - # if there are files in self.files_hash, checking if they didn't changed - check_files = [ - Path(val[0]).stat().st_mtime == val[1] - for val in self.files_hash.values() - ] - if all(check_files): - return self.inp_hash - if self.files_hash is None: - self.files_hash = {} inp_dict = {} for field in attr_fields(self): if field.name in [ "_graph_checksums", "bindings", - "inp_hash", - "changed", "files_hash", ] or field.metadata.get("output_file_template"): continue @@ -126,40 +116,15 @@ def hash(self): if getattr(self, field.name) is attr.NOTHING: continue value = getattr(self, field.name) - # checking if the field name is in the dictionary with pre-computed hash values for files - if field.name in self.files_hash: - filename, mtime, cont_hash = self.files_hash[field.name] - # if the name of the file and the time of last modification is the same, - # we are reusing the content hash value - if ( - str(Path(value)) == filename - and Path(value).stat().st_mtime == mtime - ): - inp_dict[field.name] = cont_hash - continue - inp_dict[field.name] = hash_value( - value=value, tp=field.type, metadata=field.metadata + value=value, + tp=field.type, + metadata=field.metadata, + precalculated=self.files_hash[field.name], ) - # if file/dir, the hash value will be saved to prevent recomputation - if ( - field.type in [File, Directory] - or "pydra.engine.specs.File" in str(field.type) - or "pydra.engine.specs.File" in str(field.type) - ) and is_existing_file(value): - # saving tuple with full pathname, time modification and hash value - self.files_hash[field.name] = ( - str(Path(value)), - Path(value).stat().st_mtime, - inp_dict[field.name], - ) - inp_hash = hash_function(inp_dict) if hasattr(self, "_graph_checksums"): inp_hash = hash_function((inp_hash, self._graph_checksums)) - # setting inp_hash and changing the flag to False - self.inp_hash = inp_hash - self.changed = False return inp_hash def retrieve_values(self, wf, state_index=None): diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 3a415c3a36..8223b5bb38 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -1,6 +1,6 @@ from pathlib import Path import typing as ty -from copy import copy +from copy import deepcopy from ..specs import ( BaseSpec, @@ -242,7 +242,7 @@ def test_input_file_hash_2a(tmpdir): def test_input_file_hash_3(tmpdir): - """ input spec with File types, checking when the checksum changes""" + """ input spec with File types, checking when the hash and file_hash change""" file = tmpdir.join("in_file_1.txt") with open(file, "w") as f: f.write("hello") @@ -255,32 +255,43 @@ def test_input_file_hash_3(tmpdir): my_inp = inputs(in_file=file, in_int=3) # original hash and files_hash (dictionary contains info about files) hash1 = my_inp.hash - files_hash1 = copy(my_inp.files_hash) + files_hash1 = deepcopy(my_inp.files_hash) + # file name should be in files_hash1[in_file] + filename = str(Path(file)) + assert filename in files_hash1["in_file"] # changing int input my_inp.in_int = 5 hash2 = my_inp.hash - files_hash2 = copy(my_inp.files_hash) + files_hash2 = deepcopy(my_inp.files_hash) # hash should be different assert hash1 != hash2 - # files_hash should be the same, and the tuple for in_file shouldn't be recomputed + # files_hash should be the same, and the tuple for filename shouldn't be recomputed assert files_hash1 == files_hash2 - assert id(files_hash1["in_file"]) == id(files_hash2["in_file"]) + assert id(files_hash1["in_file"][filename]) == id(files_hash2["in_file"][filename]) # recreating the file with open(file, "w") as f: f.write("hello") hash3 = my_inp.hash - files_hash3 = copy(my_inp.files_hash) + files_hash3 = deepcopy(my_inp.files_hash) # hash should be the same, # but the entry for in_file in files_hash should be different (modification time) assert hash3 == hash2 - assert files_hash3["in_file"] != files_hash2["in_file"] + assert files_hash3["in_file"][filename] != files_hash2["in_file"][filename] # different timestamp - assert files_hash3["in_file"][1] != files_hash2["in_file"][1] + assert files_hash3["in_file"][filename][0] != files_hash2["in_file"][filename][0] # the same content hash - assert files_hash3["in_file"][2] == files_hash2["in_file"][2] + assert files_hash3["in_file"][filename][1] == files_hash2["in_file"][filename][1] + + # setting the in_file again + my_inp.in_file = file + # filename should be removed from files_hash + assert my_inp.files_hash["in_file"] == {} + # will be saved again when hash is calculated + assert my_inp.hash == hash3 + assert filename in my_inp.files_hash["in_file"] def test_input_file_hash_4(tmpdir): diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 1ae47803cb..b87c4ecc06 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -213,7 +213,7 @@ def run_el(self, runnable, rerun=False): cache_dir = runnable.cache_dir name = runnable.name uid = runnable.uid - else: + else: # runnable is a tuple (ind, pkl file, task) cache_dir = runnable[-1].cache_dir name = runnable[-1].name uid = runnable[-1].uid_states()[runnable[0]] From 255edb93d9eb0089629c199843a66b2c6abaac6b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 25 Nov 2020 00:15:49 -0500 Subject: [PATCH 195/271] removing uid_states - note really used --- pydra/engine/core.py | 14 -------------- pydra/engine/workers.py | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index c29c3ac95a..b51bc6ff49 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -185,7 +185,6 @@ def __init__( self.allow_cache_override = True self._checksum = None self._uid = None - self._uid_states = None # if True the results are not checked (does not propagate to nodes) self.task_rerun = rerun @@ -304,19 +303,6 @@ def uid(self): self._uid = str(uuid4()) return self._uid - def uid_states(self): - """ setting a list of the unique id numbers for the task with a splitter - It will be used to create unique names for slurm scripts etc. - without a need to run checksum - """ - if self._uid_states is None: - self._uid_states = [] - if not hasattr(self.state, "inputs_ind"): - self.state.prepare_states(self.inputs) - for ind in range(len(self.state.inputs_ind)): - self._uid_states.append(str(uuid4())) - return self._uid_states - def set_state(self, splitter, combiner=None): """ Set a particular state on this task. diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index b87c4ecc06..edd51e7e3b 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -216,7 +216,7 @@ def run_el(self, runnable, rerun=False): else: # runnable is a tuple (ind, pkl file, task) cache_dir = runnable[-1].cache_dir name = runnable[-1].name - uid = runnable[-1].uid_states()[runnable[0]] + uid = f"{runnable[-1].uid}_{runnable[0]}" return self._submit_job(batch_script, name=name, uid=uid, cache_dir=cache_dir) @@ -228,7 +228,7 @@ def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): else: ind = task[0] cache_dir = task[-1].cache_dir - uid = task[-1].uid_states()[ind] + uid = f"{task[-1].uid}_{ind}" script_dir = cache_dir / f"{self.__class__.__name__}_scripts" / uid script_dir.mkdir(parents=True, exist_ok=True) From c79fe93f2249a6f93198fca999ddeb7bd4160c3a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 25 Nov 2020 14:13:00 -0500 Subject: [PATCH 196/271] fixing tasks uid, resetting it after loading from file or copying the workflow graph to a new uid --- pydra/engine/core.py | 6 ++---- pydra/engine/helpers.py | 4 +++- pydra/engine/submitter.py | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index b51bc6ff49..66d983a81b 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -184,7 +184,7 @@ def __init__( self.cache_locations = cache_locations self.allow_cache_override = True self._checksum = None - self._uid = None + self._uid = str(uuid4()) # if True the results are not checked (does not propagate to nodes) self.task_rerun = rerun @@ -295,12 +295,10 @@ def checksum_states(self, state_index=None): @property def uid(self): - """ setting the unique id number for the task + """ the unique id number for the task It will be used to create unique names for slurm scripts etc. without a need to run checksum """ - if not self._uid: - self._uid = str(uuid4()) return self._uid def set_state(self, splitter, combiner=None): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index ab170a1abd..8a915a5da0 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -8,6 +8,7 @@ import os import sys from hashlib import sha256 +from uuid import uuid4 import subprocess as sp import getpass import re @@ -805,7 +806,8 @@ def load_task(task_pkl, ind=None): _, inputs_dict = task.get_input_el(ind) task.inputs = attr.evolve(task.inputs, **inputs_dict) task.state = None - task._uid = None + # resetting uid for task + task._uid = str(uuid4()) return task diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index 2cf15f3c9a..03728f16a6 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -1,6 +1,7 @@ """Handle execution backends.""" import asyncio import time +from uuid import uuid4 from .workers import SerialWorker, ConcurrentFuturesWorker, SlurmWorker, DaskWorker from .core import is_workflow from .helpers import get_open_loop, load_and_run_async @@ -155,6 +156,9 @@ async def _run_workflow(self, wf, rerun=False): # creating a copy of the graph that will be modified # the copy contains new lists with original runnable objects graph_copy = wf.graph.copy() + # resetting uid for nodes in the copied workflows + for nd in graph_copy.nodes: + nd._uid = str(uuid4()) # keep track of pending futures task_futures = set() tasks, tasks_follow_errored = get_runnable_tasks(graph_copy) From 136e7a48d2b60fdd2ed53a56fdca9d49e6b01edc Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 25 Nov 2020 14:19:47 -0500 Subject: [PATCH 197/271] Update pydra/engine/helpers.py Co-authored-by: Satrajit Ghosh --- pydra/engine/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 8a915a5da0..3a616acdc2 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -807,7 +807,7 @@ def load_task(task_pkl, ind=None): task.inputs = attr.evolve(task.inputs, **inputs_dict) task.state = None # resetting uid for task - task._uid = str(uuid4()) + task._uid = uuid4().hex return task From e521fad32ebddcb835ed6ace13ad7487ed29d3b8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 25 Nov 2020 14:19:55 -0500 Subject: [PATCH 198/271] Update pydra/engine/core.py Co-authored-by: Satrajit Ghosh --- pydra/engine/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 66d983a81b..4582f67ff1 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -184,7 +184,7 @@ def __init__( self.cache_locations = cache_locations self.allow_cache_override = True self._checksum = None - self._uid = str(uuid4()) + self._uid = uuid4().hex # if True the results are not checked (does not propagate to nodes) self.task_rerun = rerun From 9991f398a8a7f199383f74eedb140400815ea233 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 25 Nov 2020 14:25:14 -0500 Subject: [PATCH 199/271] Update pydra/engine/submitter.py Co-authored-by: Satrajit Ghosh --- pydra/engine/submitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index 03728f16a6..33ab9a5db3 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -158,7 +158,7 @@ async def _run_workflow(self, wf, rerun=False): graph_copy = wf.graph.copy() # resetting uid for nodes in the copied workflows for nd in graph_copy.nodes: - nd._uid = str(uuid4()) + nd._uid = uuid4().hex # keep track of pending futures task_futures = set() tasks, tasks_follow_errored = get_runnable_tasks(graph_copy) From 623069c1602a682bd6f918fa9d485cc0644e0601 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 26 Nov 2020 21:11:05 -0500 Subject: [PATCH 200/271] fixing singularity command and the working directory (should work for all chache_dir, not only for home, tmp) --- pydra/engine/task.py | 7 +++---- pydra/engine/tests/test_singularity.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 1d17b577f7..98a1b36503 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -828,9 +828,8 @@ def _container_args_single(self, image, ind=None): if self.inputs.container_xargs is not None: cargs.extend(self.inputs.container_xargs) - cargs.append(image) - # insert bindings before image - idx = len(cargs) - 1 - cargs[idx:-1] = self.binds("-B", ind) + cargs.extend(self.binds("-B", ind)) + cargs.extend(["--pwd", str(self.output_cpath)]) + cargs.append(image) return cargs diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 4dae2f36ef..1ebd7d7ceb 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -30,11 +30,11 @@ def test_singularity_1_nosubm(tmpdir): assert singu.inputs.container == "singularity" assert ( singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw {image} {cmd}" + == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {cmd}" ) res = singu() - assert "SingularityTask" in res.output.stdout + assert "output_pydra" in res.output.stdout assert res.output.return_code == 0 @@ -48,7 +48,7 @@ def test_singularity_2_nosubm(tmpdir): singu = SingularityTask(name="singu", executable=cmd, image=image, cache_dir=tmpdir) assert ( singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw {image} {' '.join(cmd)}" + == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" ) res = singu() @@ -66,7 +66,7 @@ def test_singularity_2(plugin, tmpdir): singu = SingularityTask(name="singu", executable=cmd, image=image, cache_dir=tmpdir) assert ( singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw {image} {' '.join(cmd)}" + == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" ) with Submitter(plugin=plugin) as sub: @@ -91,7 +91,7 @@ def test_singularity_2_singuflag(plugin, tmpdir): ) assert ( shingu.cmdline - == f"singularity exec -B {shingu.output_dir}:/output_pydra:rw {image} {' '.join(cmd)}" + == f"singularity exec -B {shingu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" ) with Submitter(plugin=plugin) as sub: @@ -115,7 +115,7 @@ def test_singularity_2a(plugin, tmpdir): ) assert ( singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw {image} {cmd_exec} {' '.join(cmd_args)}" + == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {cmd_exec} {' '.join(cmd_args)}" ) with Submitter(plugin=plugin) as sub: @@ -214,7 +214,7 @@ def test_singularity_st_1(plugin, tmpdir): assert singu.state.splitter == "singu.executable" res = singu(plugin=plugin) - assert "SingularityTask" in res[0].output.stdout + assert "/output_pydra" in res[0].output.stdout assert res[1].output.stdout == "" assert res[0].output.return_code == res[1].output.return_code == 0 @@ -249,9 +249,9 @@ def test_singularity_st_3(plugin, tmpdir): assert singu.state.splitter == ["singu.image", "singu.executable"] res = singu(plugin=plugin) - assert "SingularityTask" in res[0].output.stdout + assert "/output_pydra" in res[0].output.stdout assert "Alpine" in res[1].output.stdout - assert "SingularityTask" in res[2].output.stdout + assert "/output_pydra" in res[2].output.stdout assert "Ubuntu" in res[3].output.stdout From 18a07dc003ae6e43ae51a372df2710991af88d8b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 26 Nov 2020 21:39:08 -0500 Subject: [PATCH 201/271] fixing one tets --- pydra/engine/tests/test_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index e75857a489..d07fef765b 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1143,7 +1143,7 @@ def test_singularity_cmd(tmpdir): singu = SingularityTask(name="singi", executable="pwd", image=image) assert ( singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw {image} pwd" + == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} pwd" ) singu.inputs.bindings = [ ("/local/path", "/container/path", "ro"), @@ -1151,7 +1151,7 @@ def test_singularity_cmd(tmpdir): ] assert singu.cmdline == ( "singularity exec -B /local/path:/container/path:ro" - f" -B /local2:/container2:rw -B {singu.output_dir}:/output_pydra:rw {image} pwd" + f" -B /local2:/container2:rw -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} pwd" ) From 91aec909cd8fc4cd795c4ef5ae4048eaec845c73 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 2 Dec 2020 00:40:09 -0500 Subject: [PATCH 202/271] fixing input_spec for ShellCommandTask with the container_info; fixing automatic binding detections for ContainerTask that have files as input --- pydra/engine/helpers_file.py | 14 ++++++-- pydra/engine/specs.py | 32 +++++++++++++---- pydra/engine/task.py | 17 +++++---- pydra/engine/tests/test_dockertask.py | 50 ++++++++++++++++++++++++-- pydra/engine/tests/test_shelltask.py | 2 +- pydra/engine/tests/test_tasks_files.py | 8 +++-- 6 files changed, 102 insertions(+), 21 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index f80509f0bd..99a27bc272 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -660,9 +660,17 @@ def _element_formatting(template, fld_name, fld_value, keep_extension): def is_local_file(f): - from .specs import File - - return f.type is File and "container_path" not in f.metadata + # breakpoint() + from .specs import File, Directory, MultiInputFile + + if "container_path" not in f.metadata and ( + f.type in [File, Directory, MultiInputFile] + or "pydra.engine.specs.File" in str(f.type) + or "pydra.engine.specs.Directory" in str(f.type) + ): + return True + else: + return False def is_existing_file(value): diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index af8b43494f..2577094a71 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -153,7 +153,11 @@ def check_fields_input_spec(self): # will check after adding all fields to names require_to_check[fld.name] = mdata["requires"] - if fld.type is File: + if ( + fld.type in [File, Directory] + or "pydra.engine.specs.File" in str(fld.type) + or "pydra.engine.specs.Directory" in str(fld.type) + ): self._file_check_n_bindings(fld) for nm, required in require_to_check.items(): @@ -163,9 +167,22 @@ def check_fields_input_spec(self): def _file_check_n_bindings(self, field): """for tasks without container, this is simple check if the file exists""" - file = Path(getattr(self, field.name)) - if not file.exists(): - raise AttributeError(f"the file from the {field.name} input does not exist") + if isinstance(getattr(self, field.name), list): + # if value is a list and type is a list of Files/Directory, checking all elements + if field.type in [ty.List[File], ty.List[Directory]]: + for el in getattr(self, field.name): + file = Path(el) + if not file.exists() and field.type in [File, Directory]: + raise FileNotFoundError( + f"the file from the {field.name} input does not exist" + ) + else: + file = Path(getattr(self, field.name)) + # error should be raised only if the type is strictly File or Directory + if not file.exists() and field.type in [File, Directory]: + raise FileNotFoundError( + f"the file from the {field.name} input does not exist" + ) def check_metadata(self): """Check contained metadata.""" @@ -606,6 +623,8 @@ class ContainerSpec(ShellSpec): """Mount points to be bound into the container.""" def _file_check_n_bindings(self, field): + if field.name == "image": + return file = Path(getattr(self, field.name)) if field.metadata.get("container_path"): # if the path is in a container the input should be treated as a str (hash as a str) @@ -617,8 +636,9 @@ def _file_check_n_bindings(self, field): if self.bindings is None: self.bindings = [] self.bindings.append((file.parent, f"/pydra_inp_{field.name}", "ro")) - else: - raise Exception( + # error should be raised only if the type is strictly File or Directory + elif field.type in [File, Directory]: + raise FileNotFoundError( f"the file from {field.name} input does not exist, " f"if the file comes from the container, " f"use field.metadata['container_path']=True" diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 1d17b577f7..81a1876f6d 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -234,8 +234,14 @@ def __new__(cls, container_info=None, *args, **kwargs): ) if type_cont == "docker": + # changing base class of spec if user defined + if "input_spec" in kwargs: + kwargs["input_spec"].bases = (DockerSpec,) return DockerTask(image=image, bindings=bind, *args, **kwargs) elif type_cont == "singularity": + # changing base class of spec if user defined + if "input_spec" in kwargs: + kwargs["input_spec"].bases = (SingularitySpec,) return SingularityTask(image=image, bindings=bind, *args, **kwargs) else: raise Exception( @@ -376,14 +382,13 @@ def _field_value(self, field, state_ind, ind, check_file=False): if value is attr.NOTHING or value is None: return None if check_file: - if is_local_file(field): + if is_local_file(field) and getattr(self, "bind_paths", None): value = str(value) # changing path to the cpath (the directory should be mounted) - if getattr(self, "bind_paths", None): - lpath = Path(value) - cdir = self.bind_paths(ind=ind)[lpath.parent][0] - cpath = cdir.joinpath(lpath.name) - value = str(cpath) + lpath = Path(value) + cdir = self.bind_paths(ind=ind)[lpath.parent][0] + cpath = cdir.joinpath(lpath.name) + value = str(cpath) return value def _command_shelltask_executable(self, field, state_ind, ind): diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index e9e8e09237..5d314a8897 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -5,7 +5,7 @@ from ..task import DockerTask, ShellCommandTask from ..submitter import Submitter from ..core import Workflow -from ..specs import ShellOutSpec, SpecInfo, File, DockerSpec +from ..specs import ShellOutSpec, SpecInfo, File, DockerSpec, ShellSpec from .utils import no_win, need_docker @@ -657,7 +657,7 @@ def test_docker_outputspec_1(plugin, tmpdir): @no_win @need_docker -def test_docker_inputspec_1(plugin, tmpdir): +def test_docker_inputspec_1(tmpdir): """ a simple customized input spec for docker task """ filename = str(tmpdir.join("file_pydra.txt")) with open(filename, "w") as f: @@ -699,7 +699,7 @@ def test_docker_inputspec_1(plugin, tmpdir): @no_win @need_docker -def test_docker_inputspec_1a(plugin, tmpdir): +def test_docker_inputspec_1a(tmpdir): """ a simple customized input spec for docker task a default value is used """ @@ -736,6 +736,50 @@ def test_docker_inputspec_1a(plugin, tmpdir): assert res.output.stdout == "hello from pydra" +@no_win +@need_docker +def test_docker_inputspec_1_dockerflag(tmpdir): + """ a simple customized input spec for docker task + using ShellTask with container_info + """ + filename = str(tmpdir.join("file_pydra.txt")) + with open(filename, "w") as f: + f.write("hello from pydra") + + cmd = "cat" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file", + attr.ib( + type=File, + metadata={ + "mandatory": True, + "position": 1, + "argstr": "", + "help_string": "input file", + }, + ), + ) + ], + bases=(ShellSpec,), + ) + + docky = ShellCommandTask( + name="docky", + executable=cmd, + file=filename, + input_spec=my_input_spec, + container_info=("docker", "busybox"), + strip=True, + ) + + res = docky() + assert res.output.stdout == "hello from pydra" + + @no_win @need_docker def test_docker_inputspec_2(plugin, tmpdir): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index fdb26cef05..16cf01c9f9 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1456,7 +1456,7 @@ def test_shell_cmd_inputspec_10_err(tmpdir): ) shelly.cache_dir = tmpdir - with pytest.raises(AttributeError) as e: + with pytest.raises(FileNotFoundError): res = shelly() diff --git a/pydra/engine/tests/test_tasks_files.py b/pydra/engine/tests/test_tasks_files.py index 3b11445821..69e8fc3909 100644 --- a/pydra/engine/tests/test_tasks_files.py +++ b/pydra/engine/tests/test_tasks_files.py @@ -134,7 +134,9 @@ def test_broken_file(tmpdir): sub(nn) nn2 = file_add2_annot(name="add2_annot", file=file) - with pytest.raises(AttributeError, match="file from the file input does not exist"): + with pytest.raises( + FileNotFoundError, match="file from the file input does not exist" + ): with Submitter(plugin="cf") as sub: sub(nn2) @@ -161,7 +163,9 @@ def test_broken_file_link(tmpdir): # raises error before task is run nn2 = file_add2_annot(name="add2_annot", file=file_link) - with pytest.raises(AttributeError, match="file from the file input does not exist"): + with pytest.raises( + FileNotFoundError, match="file from the file input does not exist" + ): with Submitter(plugin="cf") as sub: sub(nn2) From b061375376761c4e03556be466f8b01363b00866 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 3 Dec 2020 11:37:50 -0600 Subject: [PATCH 203/271] Added quotes around .[dev] and .[dask] in the Developer installation instructions to make the command universal (these commands previously failed on mac zsh). --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index eadc677ffc..ccbc45a163 100644 --- a/README.rst +++ b/README.rst @@ -78,7 +78,7 @@ Pydra requires Python 3.7+. To install in developer mode: git clone git@github.com:nipype/pydra.git cd pydra - pip install -e .[dev] + pip install -e ".[dev]" In order to run pydra's test locally: @@ -94,7 +94,7 @@ If you want to test execution with Dask: git clone git@github.com:nipype/pydra.git cd pydra - pip install -e .[dask] + pip install -e ".[dask]" From 70b15bbd2652f2faa62dc8e1bb179efab63af840 Mon Sep 17 00:00:00 2001 From: chasejohnson3 Date: Thu, 3 Dec 2020 12:03:16 -0600 Subject: [PATCH 204/271] Added quotes around .[dev] and .[dask] in the Developer installation instructions to make the command universal (these commands previously failed on mac zsh). (#386) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index eadc677ffc..ccbc45a163 100644 --- a/README.rst +++ b/README.rst @@ -78,7 +78,7 @@ Pydra requires Python 3.7+. To install in developer mode: git clone git@github.com:nipype/pydra.git cd pydra - pip install -e .[dev] + pip install -e ".[dev]" In order to run pydra's test locally: @@ -94,7 +94,7 @@ If you want to test execution with Dask: git clone git@github.com:nipype/pydra.git cd pydra - pip install -e .[dask] + pip install -e ".[dask]" From 2e1ec36da038f6f1ab5a05a1ec6d1ad1a81a7dd0 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 3 Dec 2020 15:30:39 -0600 Subject: [PATCH 205/271] Allowing Directories to be passed in as output fields for a pydra task, but validation of the parameter needs to be implemented. --- pydra/engine/specs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 9974499d6e..6dfe7cb2c7 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -460,6 +460,8 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out[fld.name] = self._field_metadata( fld, inputs, output_dir ) + elif fld.type in [Directory]: + pass else: raise Exception("not implemented (collect_additional_output)") return additional_out From 082c44be9ca6ae283641c529b0c1184bd4b4ef7f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 3 Dec 2020 22:52:11 -0500 Subject: [PATCH 206/271] fixing __call__ function, so the submitter is not overwriiten when state --- pydra/engine/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 4582f67ff1..01782c5e9c 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -401,7 +401,7 @@ def __call__(self, submitter=None, plugin=None, rerun=False, **kwargs): plugin = plugin or self.plugin if plugin: submitter = Submitter(plugin=plugin) - elif self.state: + elif submitter is None and self.state: submitter = Submitter() if submitter: From 6a1f429a3a8feabdc743d6af5f9a82990942a882 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Fri, 4 Dec 2020 11:44:01 -0600 Subject: [PATCH 207/271] Added class placeholders for Int, Float, Bool, Str, and List. Added these classes to the types allowed in an output field. --- pydra/engine/specs.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 6dfe7cb2c7..66beb6c002 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -19,6 +19,20 @@ class File: class Directory: """An :obj:`os.pathlike` object, designating a folder.""" +class Int: + """An Int object, designating a whole number.""" + +class Float: + """""" + +class Bool: + """""" + +class Str: + """""" + +class List: + """""" class MultiInputObj: """A ty.List[ty.Any] object, converter changes a single values to a list""" @@ -444,7 +458,7 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out = {} for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type in [File, MultiOutputFile]: + if fld.type in [File, MultiOutputFile, Directory, Float, Bool, Str, List]: # assuming that field should have either default or metadata, but not both if ( fld.default is None or fld.default == attr.NOTHING @@ -460,8 +474,20 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out[fld.name] = self._field_metadata( fld, inputs, output_dir ) - elif fld.type in [Directory]: - pass +# elif fld.type in [Directory]: +# pass +# elif fld.type in [Int]: +# raise Exception("not implemented (collect_additional_output)") +# elif fld.type in [Float]: +# raise Exception("not implemented (collect_additional_output)") +# elif fld.type in [File]: +# raise Exception("not implemented (collect_additional_output)") +# elif fld.type in [Bool]: +# raise Exception("not implemented (collect_additional_output)") +# elif fld.type in [Str]: +# raise Exception("not implemented (collect_additional_output)") +# elif fld.type in [List]: +# raise Exception("not implemented (collect_additional_output)") else: raise Exception("not implemented (collect_additional_output)") return additional_out From f470554b6911fc2699f697a89f14b36a7e63b6ee Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 5 Dec 2020 00:32:52 -0500 Subject: [PATCH 208/271] refining TaskBase.__call__ function, adding plugin_kwargs --- pydra/engine/core.py | 24 ++--- pydra/engine/tests/test_submitter.py | 21 +++- pydra/engine/tests/test_workflow.py | 138 +++++++++++++-------------- pydra/engine/workers.py | 4 +- 4 files changed, 98 insertions(+), 89 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 01782c5e9c..4b64d29106 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -392,26 +392,28 @@ def output_dir(self): return [self._cache_dir / checksum for checksum in self.checksum_states()] return self._cache_dir / self.checksum - def __call__(self, submitter=None, plugin=None, rerun=False, **kwargs): + def __call__( + self, submitter=None, plugin=None, plugin_kwargs=None, rerun=False, **kwargs + ): """Make tasks callable themselves.""" from .submitter import Submitter if submitter and plugin: raise Exception("Specify submitter OR plugin, not both") - plugin = plugin or self.plugin - if plugin: - submitter = Submitter(plugin=plugin) - elif submitter is None and self.state: - submitter = Submitter() + elif submitter: + pass + # if there is plugin provided or the task is a Workflow or has a state, + # the submitter will be created using provided plugin, self.plugin or "cf" + elif plugin or self.state or is_workflow(self): + plugin = plugin or self.plugin or "cf" + if plugin_kwargs is None: + plugin_kwargs = {} + submitter = Submitter(plugin=plugin, **plugin_kwargs) if submitter: with submitter as sub: res = sub(self) - else: - if is_workflow(self): - raise NotImplementedError( - "TODO: linear workflow execution - assign submitter or plugin for now" - ) + else: # tasks without state could be run without a submitter res = self._run(rerun=rerun, **kwargs) return res diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 12e5e06377..1230cf6508 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -23,17 +23,30 @@ def sleep_add_one(x): def test_callable_wf(plugin, tmpdir): wf = gen_basic_wf() + res = wf() + assert res.output.out == 9 + del wf, res - with pytest.raises(NotImplementedError): - wf() - + # providing plugin + wf = gen_basic_wf() res = wf(plugin="cf") assert res.output.out == 9 del wf, res + # providing plugin_kwargs wf = gen_basic_wf() - wf.cache_dir = tmpdir + res = wf(plugin="cf", plugin_kwargs={"n_procs": 2}) + assert res.output.out == 9 + del wf, res + + # providing wrong plugin_kwargs + wf = gen_basic_wf() + with pytest.raises(TypeError, match="an unexpected keyword argument"): + wf(plugin="cf", plugin_kwargs={"sbatch_args": "-N2"}) + # providing submitter + wf = gen_basic_wf() + wf.cache_dir = tmpdir sub = Submitter(plugin) res = wf(submitter=sub) assert res.output.out == 9 diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index ecf84e5091..10439bdfe8 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -129,6 +129,20 @@ def test_wf_1_call_plug(plugin, tmpdir): assert wf.output_dir.exists() +def test_wf_1_call_noplug_nosubm(plugin, tmpdir): + """using wf.__call_ without plugin or submitter""" + wf = Workflow(name="wf_1", input_spec=["x"]) + wf.add(add2(name="add2", x=wf.lzin.x)) + wf.set_output([("out", wf.add2.lzout.out)]) + wf.inputs.x = 2 + wf.cache_dir = tmpdir + + wf() + results = wf.result() + assert 4 == results.output.out + assert wf.output_dir.exists() + + def test_wf_1_call_exception(plugin, tmpdir): """using wf.__call_ with plugin and submitter - should raise an exception""" wf = Workflow(name="wf_1", input_spec=["x"]) @@ -152,7 +166,6 @@ def test_wf_2(plugin, tmpdir): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -175,7 +188,6 @@ def test_wf_2a(plugin, tmpdir): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -199,7 +211,6 @@ def test_wf_2b(plugin, tmpdir): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -224,7 +235,6 @@ def test_wf_2c_multoutp(plugin, tmpdir): wf.set_output([("out_add2", wf.add2.lzout.out), ("out_mult", wf.mult.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -250,7 +260,6 @@ def test_wf_2d_outpasdict(plugin, tmpdir): wf.set_output({"out_add2": wf.add2.lzout.out, "out_mult": wf.mult.lzout.out}) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -309,7 +318,6 @@ def test_wf_4(plugin, tmpdir): wf.add(add2(name="add2", x=wf.addvar.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -330,7 +338,6 @@ def test_wf_4a(plugin, tmpdir): wf.add(add2(name="add2", x=wf.addvar.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -435,7 +442,6 @@ def test_wf_st_1(plugin, tmpdir): wf.split("x") wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir checksum_before = wf.checksum @@ -461,7 +467,6 @@ def test_wf_st_1_call_subm(plugin, tmpdir): wf.split("x") wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -478,14 +483,15 @@ def test_wf_st_1_call_subm(plugin, tmpdir): def test_wf_st_1_call_plug(plugin, tmpdir): - """ Workflow with one task, a splitter for the workflow""" + """ Workflow with one task, a splitter for the workflow + using Workflow.__call__(plugin) + """ wf = Workflow(name="wf_spl_1", input_spec=["x"]) wf.add(add2(name="add2", x=wf.lzin.x)) wf.split("x") wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir wf(plugin=plugin) @@ -500,6 +506,54 @@ def test_wf_st_1_call_plug(plugin, tmpdir): assert odir.exists() +def test_wf_st_1_call_selfplug(plugin, tmpdir): + """ Workflow with one task, a splitter for the workflow + using Workflow.__call__() and using self.plugin + """ + wf = Workflow(name="wf_spl_1", input_spec=["x"]) + wf.add(add2(name="add2", x=wf.lzin.x)) + + wf.split("x") + wf.inputs.x = [1, 2] + wf.set_output([("out", wf.add2.lzout.out)]) + wf.plugin = plugin + wf.cache_dir = tmpdir + + wf() + results = wf.result() + # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] + assert results[0].output.out == 3 + assert results[1].output.out == 4 + # checking all directories + assert wf.output_dir + for odir in wf.output_dir: + assert odir.exists() + + +def test_wf_st_1_call_noplug_nosubm(plugin, tmpdir): + """ Workflow with one task, a splitter for the workflow + using Workflow.__call__() without plugin and submitter + (a submitter should be created within the __call__ function) + """ + wf = Workflow(name="wf_spl_1", input_spec=["x"]) + wf.add(add2(name="add2", x=wf.lzin.x)) + + wf.split("x") + wf.inputs.x = [1, 2] + wf.set_output([("out", wf.add2.lzout.out)]) + wf.cache_dir = tmpdir + + wf() + results = wf.result() + # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] + assert results[0].output.out == 3 + assert results[1].output.out == 4 + # checking all directories + assert wf.output_dir + for odir in wf.output_dir: + assert odir.exists() + + def test_wf_st_noinput_1(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"]) @@ -528,7 +582,6 @@ def test_wf_ndst_1(plugin, tmpdir): wf.add(add2(name="add2", x=wf.lzin.x).split("x")) wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir checksum_before = wf.checksum @@ -550,7 +603,6 @@ def test_wf_ndst_updatespl_1(plugin, tmpdir): wf.add(add2(name="add2", x=wf.lzin.x)) wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir wf.add2.split("x") @@ -575,7 +627,6 @@ def test_wf_ndst_updatespl_1a(plugin, tmpdir): task_add2.split("x") wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -599,7 +650,6 @@ def test_wf_ndst_updateinp_1(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.add2.split("x") wf.add2.inputs.x = wf.lzin.y wf.cache_dir = tmpdir @@ -620,7 +670,6 @@ def test_wf_ndst_noinput_1(plugin, tmpdir): wf.add(add2(name="add2", x=wf.lzin.x).split("x")) wf.inputs.x = [] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir checksum_before = wf.checksum @@ -642,7 +691,6 @@ def test_wf_st_2(plugin, tmpdir): wf.split("x").combine(combiner="x") wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -664,7 +712,6 @@ def test_wf_ndst_2(plugin, tmpdir): wf.add(add2(name="add2", x=wf.lzin.x).split("x").combine(combiner="x")) wf.inputs.x = [1, 2] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -688,7 +735,6 @@ def test_wf_st_3(plugin, tmpdir): wf.inputs.y = [11, 12] wf.split(("x", "y")) wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -735,7 +781,6 @@ def test_wf_ndst_3(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -757,7 +802,6 @@ def test_wf_st_4(plugin, tmpdir): wf.split(("x", "y"), x=[1, 2], y=[11, 12]) wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -782,7 +826,6 @@ def test_wf_ndst_4(plugin, tmpdir): wf.add(add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir wf.inputs.a = [1, 2] wf.inputs.b = [11, 12] @@ -854,7 +897,6 @@ def test_wf_st_6(plugin, tmpdir): wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]) wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -881,7 +923,6 @@ def test_wf_ndst_6(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -903,7 +944,6 @@ def test_wf_ndst_7(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = 11 wf.set_output([("out", wf.iden.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -926,7 +966,6 @@ def test_wf_ndst_8(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.iden.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -952,7 +991,6 @@ def test_wf_ndst_9(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.iden.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -977,7 +1015,6 @@ def test_wf_3sernd_ndst_1(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.add2_2nd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1058,7 +1095,6 @@ def test_wf_3nd_st_2(plugin, tmpdir): wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("x") wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1211,7 +1247,6 @@ def test_wf_3nd_ndst_4(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1282,7 +1317,6 @@ def test_wf_3nd_ndst_5(plugin, tmpdir): wf.inputs.z = [10, 100] wf.set_output([("out", wf.addvar.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1314,7 +1348,6 @@ def test_wf_3nd_ndst_6(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1340,7 +1373,6 @@ def test_wf_ndstLR_1(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1371,7 +1403,6 @@ def test_wf_ndstLR_1a(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = [11, 12] wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1405,7 +1436,6 @@ def test_wf_ndstLR_2(plugin, tmpdir): wf.inputs.y = [10, 20] wf.inputs.z = [100, 200] wf.set_output([("out", wf.addvar.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1455,7 +1485,6 @@ def test_wf_ndstLR_2a(plugin, tmpdir): wf.inputs.y = [10, 20] wf.inputs.z = [100, 200] wf.set_output([("out", wf.addvar.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1501,7 +1530,6 @@ def test_wf_ndstinner_1(plugin, tmpdir): wf.add(add2(name="add2", x=wf.list.lzout.out).split("x")) wf.inputs.x = 1 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1527,7 +1555,6 @@ def test_wf_ndstinner_2(plugin, tmpdir): wf.inputs.x = 1 wf.inputs.y = 10 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1553,7 +1580,6 @@ def test_wf_ndstinner_3(plugin, tmpdir): wf.inputs.x = 1 wf.inputs.y = [10, 100] wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1581,7 +1607,6 @@ def test_wf_ndstinner_4(plugin, tmpdir): wf.inputs.x = 1 wf.inputs.y = 10 wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1611,7 +1636,6 @@ def test_wf_st_singl_1(plugin, tmpdir): wf.split("x", x=[1, 2], y=11) wf.combine("x") wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1636,7 +1660,6 @@ def test_wf_ndst_singl_1(plugin, tmpdir): wf.inputs.x = [1, 2] wf.inputs.y = 11 wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1660,7 +1683,6 @@ def test_wf_st_singl_2(plugin, tmpdir): wf.split("x", x=[1, 2, 3], y=11) wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1689,7 +1711,6 @@ def test_wf_ndst_singl_2(plugin, tmpdir): wf.inputs.x = [1, 2, 3] wf.inputs.y = 11 wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1717,7 +1738,6 @@ def test_wfasnd_1(plugin, tmpdir): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1742,7 +1762,6 @@ def test_wfasnd_wfinp_1(plugin, tmpdir): wf.add(wfnd) wf.inputs.x = 2 wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir checksum_before = wf.checksum @@ -1770,7 +1789,6 @@ def test_wfasnd_wfndupdate(plugin, tmpdir): wfnd.inputs.x = wf.lzin.x wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1802,7 +1820,6 @@ def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): # trying to set after add... wf.wfnd.inputs.x = wf.lzin.x wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1817,7 +1834,6 @@ def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): wf.inputs.x = wf_o.lzin.x wf_o.add(wf) wf_o.set_output([("out", wf_o.wf.lzout.out)]) - wf_o.plugin = plugin wf_o.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1842,7 +1858,6 @@ def test_wfasnd_st_1(plugin, tmpdir): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir checksum_before = wf.checksum @@ -1870,7 +1885,6 @@ def test_wfasnd_st_updatespl_1(plugin, tmpdir): wf.add(wfnd) wfnd.split("x") wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1897,7 +1911,6 @@ def test_wfasnd_ndst_1(plugin, tmpdir): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1918,14 +1931,12 @@ def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): wfnd.add(add2(name="add2", x=wfnd.lzin.x)) wfnd.set_output([("out", wfnd.add2.lzout.out)]) # TODO: without this the test is failing - wfnd.plugin = plugin wfnd.inputs.x = [2, 4] wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wfnd.add2.split("x") wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -1951,7 +1962,6 @@ def test_wfasnd_wfst_1(plugin, tmpdir): wf.split("x") wf.inputs.x = [2, 4] wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin with Submitter(plugin=plugin) as sub: sub(wf) @@ -1984,7 +1994,6 @@ def test_wfasnd_st_2(plugin, tmpdir): wf.add(wfnd) wf.add(add2(name="add2", x=wf.wfnd.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -2012,7 +2021,6 @@ def test_wfasnd_wfst_2(plugin, tmpdir): wf.inputs.x = [2, 4] wf.inputs.y = [1, 10] wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -2046,7 +2054,6 @@ def test_wfasnd_ndst_3(plugin, tmpdir): wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -2106,7 +2113,6 @@ def test_wfasnd_4(plugin, tmpdir): wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -2127,14 +2133,11 @@ def test_wfasnd_ndst_4(plugin, tmpdir): wfnd.add(add2(name="add2_1st", x=wfnd.lzin.x).split("x")) wfnd.add(add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) - # TODO: without this the test is failing - wfnd.plugin = plugin wfnd.inputs.x = [2, 4] wf = Workflow(name="wf", input_spec=["x"]) wf.add(wfnd) wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -2161,7 +2164,6 @@ def test_wfasnd_wfst_4(plugin, tmpdir): wf.split("x") wf.inputs.x = [2, 4] wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin with Submitter(plugin=plugin) as sub: sub(wf) @@ -2189,7 +2191,6 @@ def test_wf_nostate_cachedir(plugin, tmpdir): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin with Submitter(plugin=plugin) as sub: sub(wf) @@ -2214,7 +2215,6 @@ def test_wf_nostate_cachedir_relativepath(tmpdir, plugin): wf.set_output([("out", wf.add2.lzout.out)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin with Submitter(plugin=plugin) as sub: sub(wf) @@ -2241,7 +2241,6 @@ def test_wf_nostate_cachelocations(plugin, tmpdir): wf1.set_output([("out", wf1.add2.lzout.out)]) wf1.inputs.x = 2 wf1.inputs.y = 3 - wf1.plugin = plugin t0 = time.time() with Submitter(plugin=plugin) as sub: @@ -2262,7 +2261,6 @@ def test_wf_nostate_cachelocations(plugin, tmpdir): wf2.set_output([("out", wf2.add2.lzout.out)]) wf2.inputs.x = 2 wf2.inputs.y = 3 - wf2.plugin = plugin t0 = time.time() with Submitter(plugin=plugin) as sub: @@ -3637,7 +3635,7 @@ def test_workflow_combine1(tmpdir): } ) wf1.cache_dir = tmpdir - result = wf1(plugin="cf") + result = wf1() assert result.output.out_pow == [1, 1, 4, 8] assert result.output.out_iden1 == [[1, 4], [1, 8]] @@ -3652,7 +3650,7 @@ def test_workflow_combine2(tmpdir): wf1.add(identity(name="identity", x=wf1.power.lzout.out).combine("power.b")) wf1.set_output({"out_pow": wf1.power.lzout.out, "out_iden": wf1.identity.lzout.out}) wf1.cache_dir = tmpdir - result = wf1(plugin="cf") + result = wf1() assert result.output.out_pow == [[1, 4], [1, 8]] assert result.output.out_iden == [[1, 4], [1, 8]] @@ -3672,7 +3670,6 @@ def test_wf_lzoutall_1(plugin, tmpdir): wf.set_output([("out", wf.add_sub.lzout.out_add)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: @@ -3694,7 +3691,6 @@ def test_wf_lzoutall_1a(plugin, tmpdir): wf.set_output([("out_all", wf.add_sub.lzout.all_)]) wf.inputs.x = 2 wf.inputs.y = 3 - wf.plugin = plugin wf.cache_dir = tmpdir with Submitter(plugin=plugin) as sub: diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index edd51e7e3b..b7cae050b4 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -181,9 +181,7 @@ class SlurmWorker(DistributedWorker): "(?P\\d*) +(?P\\w*)\\+? +" "(?P\\d+):\\d+" ) - def __init__( - self, loop=None, max_jobs=None, poll_delay=1, sbatch_args=None, **kwargs - ): + def __init__(self, loop=None, max_jobs=None, poll_delay=1, sbatch_args=None): """ Initialize SLURM Worker. From 43084f4b97ef85bfd3ccd0ad12a1fdb1deaaf94c Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Mon, 7 Dec 2020 15:23:51 -0600 Subject: [PATCH 209/271] Added a placeholder for logic to add callable for Int, Bool, Float, Str, and List --- pydra/engine/specs.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 66beb6c002..554b755e89 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -458,7 +458,7 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out = {} for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type in [File, MultiOutputFile, Directory, Float, Bool, Str, List]: + if fld.type in [File, MultiOutputFile, Directory, Float, Bool, Str, List]: # assuming that field should have either default or metadata, but not both if ( fld.default is None or fld.default == attr.NOTHING @@ -474,20 +474,13 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out[fld.name] = self._field_metadata( fld, inputs, output_dir ) -# elif fld.type in [Directory]: -# pass -# elif fld.type in [Int]: -# raise Exception("not implemented (collect_additional_output)") -# elif fld.type in [Float]: -# raise Exception("not implemented (collect_additional_output)") -# elif fld.type in [File]: -# raise Exception("not implemented (collect_additional_output)") -# elif fld.type in [Bool]: -# raise Exception("not implemented (collect_additional_output)") -# elif fld.type in [Str]: -# raise Exception("not implemented (collect_additional_output)") -# elif fld.type in [List]: -# raise Exception("not implemented (collect_additional_output)") +# if fld.type in [Float, Bool, Str, List]: +# if not (#something): +# raise AttributeError( +# f"{fld.type} has to have a callable in metadata" +# ) +# else: +# additional_out["callable"] = # Get the callable else: raise Exception("not implemented (collect_additional_output)") return additional_out From e8129ff5fd64e0716329573b81278f0eff9c8ee0 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 09:47:59 -0600 Subject: [PATCH 210/271] Added pydra.specs.Int to the list of accepted output_spec fields. --- pydra/engine/specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 554b755e89..f5eff2712b 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -458,7 +458,7 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out = {} for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type in [File, MultiOutputFile, Directory, Float, Bool, Str, List]: + if fld.type in [File, MultiOutputFile, Directory, Int, Float, Bool, Str, List]: # assuming that field should have either default or metadata, but not both if ( fld.default is None or fld.default == attr.NOTHING From f419aabe00b1b72a2343083240bfc5e72610f0fd Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 11:45:11 -0600 Subject: [PATCH 211/271] If a output_spec field has type Int, Float, Bool, Str, or List, callable is required in the metadata. --- pydra/engine/specs.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index f5eff2712b..3cb4341b38 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -19,21 +19,27 @@ class File: class Directory: """An :obj:`os.pathlike` object, designating a folder.""" + class Int: """An Int object, designating a whole number.""" + class Float: """""" + class Bool: """""" + class Str: """""" + class List: """""" + class MultiInputObj: """A ty.List[ty.Any] object, converter changes a single values to a list""" @@ -458,7 +464,16 @@ def collect_additional_outputs(self, inputs, output_dir): additional_out = {} for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type in [File, MultiOutputFile, Directory, Int, Float, Bool, Str, List]: + if fld.type in [ + File, + MultiOutputFile, + Directory, + Int, + Float, + Bool, + Str, + List, + ]: # assuming that field should have either default or metadata, but not both if ( fld.default is None or fld.default == attr.NOTHING @@ -471,16 +486,17 @@ def collect_additional_outputs(self, inputs, output_dir): fld, output_dir ) elif fld.metadata: - additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir - ) -# if fld.type in [Float, Bool, Str, List]: -# if not (#something): -# raise AttributeError( -# f"{fld.type} has to have a callable in metadata" -# ) -# else: -# additional_out["callable"] = # Get the callable + if ( + fld.type in [Int, Float, Bool, Str, List] + and "callable" not in fld.metadata + ): + raise AttributeError( + f"{fld.type} has to have a callable in metadata" + ) + else: + additional_out[fld.name] = self._field_metadata( + fld, inputs, output_dir + ) else: raise Exception("not implemented (collect_additional_output)") return additional_out From d293fea6be74243b6ace4bc6a12a8c77fd4fd4e9 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 11:48:11 -0600 Subject: [PATCH 212/271] Added descriptions to classes used as spec field types. --- pydra/engine/specs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 3cb4341b38..2ef8de27b3 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -25,19 +25,19 @@ class Int: class Float: - """""" + """A Float object, designating a decimal number""" class Bool: - """""" + """A Boolean object, designating a True/False value""" class Str: - """""" + """A String object, designating a string""" class List: - """""" + """A List object, designating a list of objects""" class MultiInputObj: From 8977b69d6e947d537354857393bd7911e0cb70ee Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 12:37:23 -0600 Subject: [PATCH 213/271] Added a test for specifying an output_spec field type. --- .../engine/tests/test_shelltask_outputspec.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pydra/engine/tests/test_shelltask_outputspec.py diff --git a/pydra/engine/tests/test_shelltask_outputspec.py b/pydra/engine/tests/test_shelltask_outputspec.py new file mode 100644 index 0000000000..86ab365749 --- /dev/null +++ b/pydra/engine/tests/test_shelltask_outputspec.py @@ -0,0 +1,52 @@ +import nest_asyncio + +nest_asyncio.apply() + +import pydra +import attr +import pytest + + +def test_output_int(): + cmd = "echo" + args = ["newfile_1.txt", "newfile_2.txt"] + + my_output_spec = pydra.specs.SpecInfo( + name="Output", + fields=[ + ( + "out1", + attr.ib( + type=pydra.specs.File, + metadata={ + "output_file_template": "{args}", + "help_string": "output file", + }, + ), + ), + ( + "out_len", + attr.ib( + type=pydra.specs.Int, + metadata={"help_string": "output file", "value": "val"}, + ), + ), + ], + bases=(pydra.specs.ShellOutSpec,), + ) + + shelly = pydra.ShellCommandTask( + name="shelly", executable=cmd, args=args, output_spec=my_output_spec + ).split("args") + + print("cmndline = ", shelly.cmdline) + + # with pydra.Submitter(plugin="cf") as sub: + # sub(shelly) + # shelly() + # shelly.result() + + with pytest.raises(Exception) as e: + shelly() + # print(shelly.result()) + assert " has to have a callable" in str(e.value) From 9a293ac68849d4a55a9926efdb93aa1c8159b675 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 13:35:02 -0600 Subject: [PATCH 214/271] Added a test for requiring a callable in the metadata for a output_spec field with type Int. --- pydra/engine/tests/test_shelltask.py | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 1361f3f08b..4dbbf9ca0d 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -15,6 +15,7 @@ File, MultiOutputFile, MultiInputObj, + Int, ) from .utils import result_no_submitter, result_submitter, use_validator @@ -2618,6 +2619,34 @@ def gather_output(executable, output_dir, ble): shelly() +def test_shell_cmd_outputspec_4c_error(): + """ + customised output_spec, adding Int to the output, + requiring a function to collect output + """ + cmd = "echo" + args = ["newfile_1.txt", "newfile_2.txt"] + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out", + attr.ib( + type=Int, metadata={"help_string": "output file", "value": "val"} + ), + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", executable=cmd, args=args, output_spec=my_output_spec + ).split("args") + with pytest.raises(Exception) as e: + shelly() + assert "has to have a callable" in str(e.value) + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_5(plugin, results_function, tmpdir): """ From 81cdfaae0df8c158c1db123f8cccf0fd871edf61 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 8 Dec 2020 13:37:09 -0600 Subject: [PATCH 215/271] Removed an an outputspec test file after moving its test into pydra/engine/tests/test_shelltask.py --- .../engine/tests/test_shelltask_outputspec.py | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 pydra/engine/tests/test_shelltask_outputspec.py diff --git a/pydra/engine/tests/test_shelltask_outputspec.py b/pydra/engine/tests/test_shelltask_outputspec.py deleted file mode 100644 index 86ab365749..0000000000 --- a/pydra/engine/tests/test_shelltask_outputspec.py +++ /dev/null @@ -1,52 +0,0 @@ -import nest_asyncio - -nest_asyncio.apply() - -import pydra -import attr -import pytest - - -def test_output_int(): - cmd = "echo" - args = ["newfile_1.txt", "newfile_2.txt"] - - my_output_spec = pydra.specs.SpecInfo( - name="Output", - fields=[ - ( - "out1", - attr.ib( - type=pydra.specs.File, - metadata={ - "output_file_template": "{args}", - "help_string": "output file", - }, - ), - ), - ( - "out_len", - attr.ib( - type=pydra.specs.Int, - metadata={"help_string": "output file", "value": "val"}, - ), - ), - ], - bases=(pydra.specs.ShellOutSpec,), - ) - - shelly = pydra.ShellCommandTask( - name="shelly", executable=cmd, args=args, output_spec=my_output_spec - ).split("args") - - print("cmndline = ", shelly.cmdline) - - # with pydra.Submitter(plugin="cf") as sub: - # sub(shelly) - # shelly() - # shelly.result() - - with pytest.raises(Exception) as e: - shelly() - # print(shelly.result()) - assert " has to have a callable" in str(e.value) From bf381f36aedcd6cce5993bd38ed6dd8f8a83b8e9 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 8 Dec 2020 18:37:51 -0500 Subject: [PATCH 216/271] fixing numpy tests --- pydra/engine/tests/test_numpy_examples.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_numpy_examples.py b/pydra/engine/tests/test_numpy_examples.py index 572d8707a2..72895a64ae 100644 --- a/pydra/engine/tests/test_numpy_examples.py +++ b/pydra/engine/tests/test_numpy_examples.py @@ -17,7 +17,7 @@ def arrayout(val): return np.array([val, val]) -def test_multiout(plugin, tmpdir): +def test_multiout(tmpdir): """ testing a simple function that returns a numpy array""" wf = Workflow("wf", input_spec=["val"], val=2) wf.add(arrayout(name="mo", val=wf.lzin.val)) @@ -25,7 +25,7 @@ def test_multiout(plugin, tmpdir): wf.set_output([("array", wf.mo.lzout.b)]) wf.cache_dir = tmpdir - with Submitter(plugin=plugin, n_procs=2) as sub: + with Submitter(plugin="cf", n_procs=2) as sub: sub(runnable=wf) results = wf.result(return_inputs=True) @@ -34,7 +34,7 @@ def test_multiout(plugin, tmpdir): assert np.array_equal(results[1].output.array, np.array([2, 2])) -def test_multiout_st(plugin, tmpdir): +def test_multiout_st(tmpdir): """ testing a simple function that returns a numpy array, adding splitter""" wf = Workflow("wf", input_spec=["val"], val=[0, 1, 2]) wf.add(arrayout(name="mo", val=wf.lzin.val)) @@ -43,7 +43,7 @@ def test_multiout_st(plugin, tmpdir): wf.set_output([("array", wf.mo.lzout.b)]) wf.cache_dir = tmpdir - with Submitter(plugin=plugin, n_procs=2) as sub: + with Submitter(plugin="cf", n_procs=2) as sub: sub(runnable=wf) results = wf.result(return_inputs=True) From f74fc7f91d5117577fc93143b896e0c0cafc45d8 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 8 Dec 2020 18:57:12 -0500 Subject: [PATCH 217/271] changing checksum_states (used in result) so it keeps a track of hash values of all files and doesn't recalculate for each element of the state --- pydra/engine/core.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 4b64d29106..cfa4edf993 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -274,7 +274,16 @@ def checksum_states(self, state_index=None): key.split(".")[1], getattr(inputs_copy, key.split(".")[1])[ind], ) + # setting files_hash again in case it was cleaned by setting specific element + # that might be important for outer splitter of input variable with big files + # the file can be changed with every single index even if there are only two files + inputs_copy.files_hash = self.inputs.files_hash input_hash = inputs_copy.hash + # updating self.inputs.files_hash, so big files hashes + # doesn't have to be recompute for the next element + for key, val in inputs_copy.files_hash.items(): + if val: + self.inputs.files_hash[key].update(val) if is_workflow(self): con_hash = hash_function(self._connections) hash_list = [input_hash, con_hash] From a33c5c05ecc2f4036e42ad49608f2b25772ecbf1 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Wed, 9 Dec 2020 10:44:21 -0600 Subject: [PATCH 218/271] Used python types for the types output_spec fields [int, float, bool, str, list]. --- pydra/engine/specs.py | 32 ++++++---------------------- pydra/engine/tests/test_shelltask.py | 3 +-- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 2ef8de27b3..ffc4f0fc11 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -20,26 +20,6 @@ class Directory: """An :obj:`os.pathlike` object, designating a folder.""" -class Int: - """An Int object, designating a whole number.""" - - -class Float: - """A Float object, designating a decimal number""" - - -class Bool: - """A Boolean object, designating a True/False value""" - - -class Str: - """A String object, designating a string""" - - -class List: - """A List object, designating a list of objects""" - - class MultiInputObj: """A ty.List[ty.Any] object, converter changes a single values to a list""" @@ -468,11 +448,11 @@ def collect_additional_outputs(self, inputs, output_dir): File, MultiOutputFile, Directory, - Int, - Float, - Bool, - Str, - List, + int, + float, + bool, + str, + list, ]: # assuming that field should have either default or metadata, but not both if ( @@ -487,7 +467,7 @@ def collect_additional_outputs(self, inputs, output_dir): ) elif fld.metadata: if ( - fld.type in [Int, Float, Bool, Str, List] + fld.type in [int, float, bool, str, list] and "callable" not in fld.metadata ): raise AttributeError( diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 4dbbf9ca0d..bbea69bf9a 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -15,7 +15,6 @@ File, MultiOutputFile, MultiInputObj, - Int, ) from .utils import result_no_submitter, result_submitter, use_validator @@ -2633,7 +2632,7 @@ def test_shell_cmd_outputspec_4c_error(): ( "out", attr.ib( - type=Int, metadata={"help_string": "output file", "value": "val"} + type=int, metadata={"help_string": "output file", "value": "val"} ), ) ], From c9336f62c9632924636bc3bc1aeb4374511b0dd6 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 9 Dec 2020 20:14:39 -0500 Subject: [PATCH 219/271] adding a test with a bigger splitter over args for singularity --- pydra/engine/tests/test_singularity.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 1ebd7d7ceb..08c8d69c72 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -255,6 +255,25 @@ def test_singularity_st_3(plugin, tmpdir): assert "Ubuntu" in res[3].output.stdout +@need_singularity +@pytest.mark.xfail( + reason="slurm can complain if the number of submitted jobs exceeds the limit" +) +@pytest.mark.parametrize("n", [10, 50, 100]) +def test_singularity_st_1_pr(plugin, tmpdir, n): + """ splitter over args (checking bigger splitters)""" + args_n = list(range(n)) + image = "library://sylabsed/linux/alpine" + singu = SingularityTask( + name="singu", executable="echo", image=image, cache_dir=tmpdir, args=args_n + ).split("args") + assert singu.state.splitter == "singu.args" + res = singu(plugin=plugin) + assert "1" in res[1].output.stdout + assert str(n - 1) in res[-1].output.stdout + assert res[0].output.return_code == res[1].output.return_code == 0 + + @need_singularity def test_wf_singularity_1(plugin, tmpdir): """ a workflow with two connected task From 1460da93f0db236d2a51afe2f74061fdbb97d980 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 9 Dec 2020 20:15:55 -0500 Subject: [PATCH 220/271] adding a test with a bigger splitter over args for singularity --- pydra/engine/tests/test_singularity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 08c8d69c72..45721d84a1 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -260,7 +260,7 @@ def test_singularity_st_3(plugin, tmpdir): reason="slurm can complain if the number of submitted jobs exceeds the limit" ) @pytest.mark.parametrize("n", [10, 50, 100]) -def test_singularity_st_1_pr(plugin, tmpdir, n): +def test_singularity_st_4(plugin, tmpdir, n): """ splitter over args (checking bigger splitters)""" args_n = list(range(n)) image = "library://sylabsed/linux/alpine" From 3c6f8cb308f1424447b72188aa88f2c15dba20ea Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 9 Dec 2020 20:47:44 -0500 Subject: [PATCH 221/271] fixing the test with bigger splitter, so it runs only with slurm for now --- pydra/engine/tests/test_singularity.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 45721d84a1..4ad0b5db7c 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -17,6 +17,10 @@ shutil.which("singularity") is None, reason="no singularity available" ) +need_slurm = pytest.mark.skipif( + not bool(shutil.which("sbatch")), reason="no singularity available" +) + @need_singularity def test_singularity_1_nosubm(tmpdir): @@ -256,19 +260,20 @@ def test_singularity_st_3(plugin, tmpdir): @need_singularity +@need_slurm @pytest.mark.xfail( reason="slurm can complain if the number of submitted jobs exceeds the limit" ) @pytest.mark.parametrize("n", [10, 50, 100]) -def test_singularity_st_4(plugin, tmpdir, n): - """ splitter over args (checking bigger splitters)""" +def test_singularity_st_4(tmpdir, n): + """ splitter over args (checking bigger splitters if slurm available)""" args_n = list(range(n)) image = "library://sylabsed/linux/alpine" singu = SingularityTask( name="singu", executable="echo", image=image, cache_dir=tmpdir, args=args_n ).split("args") assert singu.state.splitter == "singu.args" - res = singu(plugin=plugin) + res = singu(plugin="slurm") assert "1" in res[1].output.stdout assert str(n - 1) in res[-1].output.stdout assert res[0].output.return_code == res[1].output.return_code == 0 From 52be443e2f7d39d62cbf3465455a2394b84de42f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 9 Dec 2020 20:58:37 -0500 Subject: [PATCH 222/271] fixing singularity GA workflow: adding singularity path to GITHUB_PATH --- .github/workflows/testsingularity.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 0d839478ff..2696faa890 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -40,8 +40,9 @@ jobs: cd .. - name: Echo singularity version run: | + echo "${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/" >> $GITHUB_PATH echo ${{ github.ref }} - ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/singularity --version + singularity --version - name: Set up Python ${{ matrix.python-version }} From 0337f9ab906138ed14df2d6e99039490b43e8868 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 9 Dec 2020 21:20:00 -0500 Subject: [PATCH 223/271] change prefix --- .github/workflows/testsingularity.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 2696faa890..afcec67e25 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -34,13 +34,12 @@ jobs: - name: Build run: | cd singularity - ./mconfig --without-suid -p ${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64 + ./mconfig -p /usr/local/ make -C ./builddir sudo make -C ./builddir install cd .. - name: Echo singularity version run: | - echo "${{ runner.tool_cache }}/singularity/${{ env.RELEASE_VERSION }}/x64/bin/" >> $GITHUB_PATH echo ${{ github.ref }} singularity --version From 2bf7d54e0ea9da77151ac7a528a387ff38bcf22a Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 9 Dec 2020 21:28:24 -0500 Subject: [PATCH 224/271] add back without suid --- .github/workflows/testsingularity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index afcec67e25..946390dc84 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -34,7 +34,7 @@ jobs: - name: Build run: | cd singularity - ./mconfig -p /usr/local/ + ./mconfig --without-suid -p /usr/local/ make -C ./builddir sudo make -C ./builddir install cd .. From a98b767e78aca3da8c0fcc14f5c7b910dd8d14da Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 11:45:06 -0600 Subject: [PATCH 225/271] Metadata can access stdout and stderr. Added task output as a parameter to collect_additional outputs to give metadata access to stdout and stderr. --- pydra/engine/core.py | 2 +- pydra/engine/specs.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 4582f67ff1..9ef6036867 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -474,7 +474,7 @@ def _collect_outputs(self, output_dir): run_output = self.output_ output_klass = make_klass(self.output_spec) output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) - other_output = output.collect_additional_outputs(self.inputs, output_dir) + other_output = output.collect_additional_outputs(self.inputs, output_dir, run_output) return attr.evolve(output, **run_output, **other_output) def split(self, splitter, overwrite=False, **kwargs): diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index ffc4f0fc11..d0788cd86b 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -439,9 +439,14 @@ class ShellOutSpec: stderr: ty.Union[File, str] """The process' standard input.""" - def collect_additional_outputs(self, inputs, output_dir): + def collect_additional_outputs(self, inputs, output_dir, outputs): """Collect additional outputs from shelltask output_spec.""" additional_out = {} +# for fld in attr_fields(self): +# if fld.name == "stdout": +# stdout = str(fld) +# if fld.name == "stderr": +# stderr = str(fld) for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: if fld.type in [ @@ -475,7 +480,7 @@ def collect_additional_outputs(self, inputs, output_dir): ) else: additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir + fld, inputs, output_dir,outputs ) else: raise Exception("not implemented (collect_additional_output)") @@ -503,7 +508,7 @@ def generated_output_names(self, inputs, output_dir): output_names.append(fld.name) elif ( fld.metadata - and self._field_metadata(fld, inputs, output_dir) + and self._field_metadata(fld, inputs, output_dir, outputs) != attr.NOTHING ): output_names.append(fld.name) @@ -539,7 +544,7 @@ def _field_defaultvalue(self, fld, output_dir): else: raise AttributeError(f"no file matches {default.name}") - def _field_metadata(self, fld, inputs, output_dir): + def _field_metadata(self, fld, inputs, output_dir, outputs=None): """Collect output file if metadata specified.""" if self._check_requires(fld, inputs) is False: return attr.NOTHING @@ -568,6 +573,10 @@ def _field_metadata(self, fld, inputs, output_dir): call_args_val[argnm] = output_dir elif argnm == "inputs": call_args_val[argnm] = inputs + elif argnm == "stdout": + call_args_val[argnm] = outputs["stdout"] + elif argnm == "stderr": + call_args_val[argnm] = outputs["stderr"] else: try: call_args_val[argnm] = getattr(inputs, argnm) From 3f0a2346e41f9166c22cae7f22479d1b66517979 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 12:01:52 -0600 Subject: [PATCH 226/271] Added a test for passing stdout into a callable in field with type int. --- pydra/engine/tests/test_shelltask.py | 48 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index bbea69bf9a..30e87d7b59 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -3,7 +3,7 @@ import os, sys import pytest from pathlib import Path - +import re from ..task import ShellCommandTask from ..submitter import Submitter @@ -2864,6 +2864,51 @@ def test_shell_cmd_outputspec_6a(tmpdir, plugin, results_function): assert res.output.stdout == "" assert res.output.new_files.exists() +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_7(tmpdir, plugin, results_function): + cmd = "echo" + args = ["newfile_1.txt", "newfile_2.txt"] + + def get_file_index(stdout): + stdout = re.sub(r'.*_', "", stdout) + stdout = re.sub(r'.txt', "", stdout) + print(stdout) + return int(stdout) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out1", + attr.ib( + type=File, + metadata={ + "output_file_template": "{args}", + "help_string": "output file", + }, + ), + ), + ( + "out_file_index", + attr.ib( + type=int, + metadata={ + "help_string": "output file", + "callable": get_file_index, + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + + shelly = ShellCommandTask(name="shelly", executable=cmd, args=args, output_spec=my_output_spec).split("args") + + results = results_function(shelly, plugin) + for index, res in enumerate(results): + assert res.output.out_file_index == index+1 + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): @@ -2904,7 +2949,6 @@ def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): assert res[i].output.stdout == "" assert res[i].output.out1.exists() - # customised output_spec for tasks in workflows From 1b674c1f7bd2ffc05d6f110247d6083b6ce7dafd Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 12:05:44 -0600 Subject: [PATCH 227/271] Moved all output_spec tests relating to int callables to the same section with names test_shell_cmd_outputspec_7... --- pydra/engine/tests/test_shelltask.py | 59 +++++++++++++++------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 30e87d7b59..1ff924cf36 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2618,33 +2618,6 @@ def gather_output(executable, output_dir, ble): shelly() -def test_shell_cmd_outputspec_4c_error(): - """ - customised output_spec, adding Int to the output, - requiring a function to collect output - """ - cmd = "echo" - args = ["newfile_1.txt", "newfile_2.txt"] - - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out", - attr.ib( - type=int, metadata={"help_string": "output file", "value": "val"} - ), - ) - ], - bases=(ShellOutSpec,), - ) - shelly = ShellCommandTask( - name="shelly", executable=cmd, args=args, output_spec=my_output_spec - ).split("args") - with pytest.raises(Exception) as e: - shelly() - assert "has to have a callable" in str(e.value) - @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_5(plugin, results_function, tmpdir): @@ -2866,6 +2839,10 @@ def test_shell_cmd_outputspec_6a(tmpdir, plugin, results_function): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_7(tmpdir, plugin, results_function): + """ + customised output_spec, adding Int to the output, + requiring a callable with parameter stdout + """ cmd = "echo" args = ["newfile_1.txt", "newfile_2.txt"] @@ -2909,6 +2886,34 @@ def get_file_index(stdout): for index, res in enumerate(results): assert res.output.out_file_index == index+1 +def test_shell_cmd_outputspec_7b_error(): + """ + customised output_spec, adding Int to the output, + requiring a function to collect output + """ + cmd = "echo" + args = ["newfile_1.txt", "newfile_2.txt"] + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "out", + attr.ib( + type=int, metadata={"help_string": "output file", "value": "val"} + ), + ) + ], + bases=(ShellOutSpec,), + ) + shelly = ShellCommandTask( + name="shelly", executable=cmd, args=args, output_spec=my_output_spec + ).split("args") + with pytest.raises(Exception) as e: + shelly() + assert "has to have a callable" in str(e.value) + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): From caaf9d1181e033b81973089b818801c628a5d7b8 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 12:13:13 -0600 Subject: [PATCH 228/271] Created the shell to test stderr in output_spec callables. --- pydra/engine/tests/test_shelltask.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 1ff924cf36..5e0b528091 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2914,6 +2914,43 @@ def test_shell_cmd_outputspec_7b_error(): assert "has to have a callable" in str(e.value) +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_7c_error(tmpdir, plugin, results_function): + """ + customised output_spec, adding Int to the output, + requiring a callable with parameter stdout + """ + cmd = "echo" + args = ["newfile_1.txt", "newfile_2.txt"] + + def get_file_index(stdout): + stdout = re.sub(r'.*_', "", stdout) + stdout = re.sub(r'.txt', "", stdout) + print(stdout) + return int(stdout) + + + my_input_spec = pydra.specs.SpecInfo( + name="Input", + fields=[ + ( + "text", + attr.ib( + type=File, + metadata={"position": 1, "argstr": "", "help_string": "text", "mandatory": True}, + ), + ) + ], + bases=(pydra.specs.ShellSpec,), + ) + + shelly = pydra.ShellCommandTask( + name="shelly", executable=cmd_exec, NOT_DEFINED=hello, input_spec=my_input_spec + ) + + results = results_function(shelly, plugin) + print(results) + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): From 0d9f21d47576225f5d467e4a2d222521d187535e Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 13:51:46 -0600 Subject: [PATCH 229/271] Added an output_spec test to demonstrate using stdout and stderr as callable parameters for an output_spec field. --- pydra/engine/tests/test_shelltask.py | 62 +++++++++------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 5e0b528091..d217bfddae 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2838,10 +2838,10 @@ def test_shell_cmd_outputspec_6a(tmpdir, plugin, results_function): assert res.output.new_files.exists() @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_7(tmpdir, plugin, results_function): +def test_shell_cmd_outputspec_7a(tmpdir, plugin, results_function): """ - customised output_spec, adding Int to the output, - requiring a callable with parameter stdout + customised output_spec, adding int and str to the output, + requiring two callables with parameters stdout and stderr """ cmd = "echo" args = ["newfile_1.txt", "newfile_2.txt"] @@ -2851,7 +2851,10 @@ def get_file_index(stdout): stdout = re.sub(r'.txt', "", stdout) print(stdout) return int(stdout) - + + def get_stderr(stderr): + return f"stderr: {stderr}" + my_output_spec = SpecInfo( name="Output", fields=[ @@ -2874,7 +2877,17 @@ def get_file_index(stdout): "callable": get_file_index, }, ), - ) + ), + ( + "stderr_field", + attr.ib( + type=int, + metadata={ + "help_string": "The standard error output", + "callable": get_stderr, + }, + ), + ), ], bases=(ShellOutSpec,), ) @@ -2885,6 +2898,8 @@ def get_file_index(stdout): results = results_function(shelly, plugin) for index, res in enumerate(results): assert res.output.out_file_index == index+1 + assert res.output.stderr_field == f"stderr: {res.output.stderr}" + def test_shell_cmd_outputspec_7b_error(): """ @@ -2914,43 +2929,6 @@ def test_shell_cmd_outputspec_7b_error(): assert "has to have a callable" in str(e.value) -@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_shell_cmd_outputspec_7c_error(tmpdir, plugin, results_function): - """ - customised output_spec, adding Int to the output, - requiring a callable with parameter stdout - """ - cmd = "echo" - args = ["newfile_1.txt", "newfile_2.txt"] - - def get_file_index(stdout): - stdout = re.sub(r'.*_', "", stdout) - stdout = re.sub(r'.txt', "", stdout) - print(stdout) - return int(stdout) - - - my_input_spec = pydra.specs.SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=File, - metadata={"position": 1, "argstr": "", "help_string": "text", "mandatory": True}, - ), - ) - ], - bases=(pydra.specs.ShellSpec,), - ) - - shelly = pydra.ShellCommandTask( - name="shelly", executable=cmd_exec, NOT_DEFINED=hello, input_spec=my_input_spec - ) - - results = results_function(shelly, plugin) - print(results) - @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): From c49d8a39ce9146b84cf77e1c894b068341e1bb7c Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 14:04:25 -0600 Subject: [PATCH 230/271] Fixed tests failing because of the undefined outputs variable in _field_metadata --- pydra/engine/specs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index d0788cd86b..f150e431c3 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -473,14 +473,18 @@ def collect_additional_outputs(self, inputs, output_dir, outputs): elif fld.metadata: if ( fld.type in [int, float, bool, str, list] - and "callable" not in fld.metadata ): - raise AttributeError( - f"{fld.type} has to have a callable in metadata" + if "callable" not in fld.metadata: + raise AttributeError( + f"{fld.type} has to have a callable in metadata" + ) + else: + additional_out[fld.name] = self._field_metadata( + fld, inputs, output_dir,outputs ) else: additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir,outputs + fld, inputs, output_dir ) else: raise Exception("not implemented (collect_additional_output)") @@ -508,7 +512,7 @@ def generated_output_names(self, inputs, output_dir): output_names.append(fld.name) elif ( fld.metadata - and self._field_metadata(fld, inputs, output_dir, outputs) + and self._field_metadata(fld, inputs, output_dir) != attr.NOTHING ): output_names.append(fld.name) From c36d6183935e60ede0ede090d858f8cacc558d12 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 14:10:06 -0600 Subject: [PATCH 231/271] Removed commented-out code. --- pydra/engine/specs.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index f150e431c3..0a80a1988f 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -442,11 +442,6 @@ class ShellOutSpec: def collect_additional_outputs(self, inputs, output_dir, outputs): """Collect additional outputs from shelltask output_spec.""" additional_out = {} -# for fld in attr_fields(self): -# if fld.name == "stdout": -# stdout = str(fld) -# if fld.name == "stderr": -# stderr = str(fld) for fld in attr_fields(self): if fld.name not in ["return_code", "stdout", "stderr"]: if fld.type in [ @@ -471,20 +466,18 @@ def collect_additional_outputs(self, inputs, output_dir, outputs): fld, output_dir ) elif fld.metadata: - if ( - fld.type in [int, float, bool, str, list] - ): + if fld.type in [int, float, bool, str, list]: if "callable" not in fld.metadata: raise AttributeError( f"{fld.type} has to have a callable in metadata" ) else: additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir,outputs - ) + fld, inputs, output_dir, outputs + ) else: additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir + fld, inputs, output_dir ) else: raise Exception("not implemented (collect_additional_output)") From fddc0cfc995584b8f94e332b79070f9f296b8c62 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 14:17:38 -0600 Subject: [PATCH 232/271] Converting to black formatting. --- pydra/engine/core.py | 4 +++- pydra/engine/tests/test_shelltask.py | 26 ++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 9ef6036867..0d5a1e0324 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -474,7 +474,9 @@ def _collect_outputs(self, output_dir): run_output = self.output_ output_klass = make_klass(self.output_spec) output = output_klass(**{f.name: None for f in attr.fields(output_klass)}) - other_output = output.collect_additional_outputs(self.inputs, output_dir, run_output) + other_output = output.collect_additional_outputs( + self.inputs, output_dir, run_output + ) return attr.evolve(output, **run_output, **other_output) def split(self, splitter, overwrite=False, **kwargs): diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index d217bfddae..7248d94f0f 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2618,7 +2618,6 @@ def gather_output(executable, output_dir, ble): shelly() - @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_5(plugin, results_function, tmpdir): """ @@ -2837,6 +2836,7 @@ def test_shell_cmd_outputspec_6a(tmpdir, plugin, results_function): assert res.output.stdout == "" assert res.output.new_files.exists() + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_7a(tmpdir, plugin, results_function): """ @@ -2845,10 +2845,10 @@ def test_shell_cmd_outputspec_7a(tmpdir, plugin, results_function): """ cmd = "echo" args = ["newfile_1.txt", "newfile_2.txt"] - + def get_file_index(stdout): - stdout = re.sub(r'.*_', "", stdout) - stdout = re.sub(r'.txt', "", stdout) + stdout = re.sub(r".*_", "", stdout) + stdout = re.sub(r".txt", "", stdout) print(stdout) return int(stdout) @@ -2872,10 +2872,7 @@ def get_stderr(stderr): "out_file_index", attr.ib( type=int, - metadata={ - "help_string": "output file", - "callable": get_file_index, - }, + metadata={"help_string": "output file", "callable": get_file_index}, ), ), ( @@ -2891,13 +2888,14 @@ def get_stderr(stderr): ], bases=(ShellOutSpec,), ) - - - shelly = ShellCommandTask(name="shelly", executable=cmd, args=args, output_spec=my_output_spec).split("args") - + + shelly = ShellCommandTask( + name="shelly", executable=cmd, args=args, output_spec=my_output_spec + ).split("args") + results = results_function(shelly, plugin) for index, res in enumerate(results): - assert res.output.out_file_index == index+1 + assert res.output.out_file_index == index + 1 assert res.output.stderr_field == f"stderr: {res.output.stderr}" @@ -2929,7 +2927,6 @@ def test_shell_cmd_outputspec_7b_error(): assert "has to have a callable" in str(e.value) - @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): """ @@ -2969,6 +2966,7 @@ def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): assert res[i].output.stdout == "" assert res[i].output.out1.exists() + # customised output_spec for tasks in workflows From 5d31c0127c4b0559904a4ab49aef09a436abb993 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 14:29:47 -0600 Subject: [PATCH 233/271] Changed the definition of _field_metadata to not have a default value for outputs so each use is uniform. --- pydra/engine/specs.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 0a80a1988f..b1d3a89891 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -466,19 +466,21 @@ def collect_additional_outputs(self, inputs, output_dir, outputs): fld, output_dir ) elif fld.metadata: - if fld.type in [int, float, bool, str, list]: - if "callable" not in fld.metadata: - raise AttributeError( - f"{fld.type} has to have a callable in metadata" - ) - else: - additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir, outputs - ) + if ( + fld.type in [int, float, bool, str, list] + and "callable" not in fld.metadata + ): + raise AttributeError( + f"{fld.type} has to have a callable in metadata" + ) else: additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir + fld, inputs, output_dir, outputs ) + # else: + # additional_out[fld.name] = self._field_metadata( + # fld, inputs, output_dir, outputs + # ) else: raise Exception("not implemented (collect_additional_output)") return additional_out @@ -505,7 +507,7 @@ def generated_output_names(self, inputs, output_dir): output_names.append(fld.name) elif ( fld.metadata - and self._field_metadata(fld, inputs, output_dir) + and self._field_metadata(fld, inputs, output_dir, outputs=None) != attr.NOTHING ): output_names.append(fld.name) @@ -541,7 +543,7 @@ def _field_defaultvalue(self, fld, output_dir): else: raise AttributeError(f"no file matches {default.name}") - def _field_metadata(self, fld, inputs, output_dir, outputs=None): + def _field_metadata(self, fld, inputs, output_dir, outputs): """Collect output file if metadata specified.""" if self._check_requires(fld, inputs) is False: return attr.NOTHING From dff1c419bec6e5fb7355c4df0d5ae602bc435158 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 15:05:26 -0600 Subject: [PATCH 234/271] Made the outputs parameter of _field_metadata None by default to avoid changing its signature --- pydra/engine/specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index b1d3a89891..240e92dfdf 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -543,7 +543,7 @@ def _field_defaultvalue(self, fld, output_dir): else: raise AttributeError(f"no file matches {default.name}") - def _field_metadata(self, fld, inputs, output_dir, outputs): + def _field_metadata(self, fld, inputs, output_dir, outputs=None): """Collect output file if metadata specified.""" if self._check_requires(fld, inputs) is False: return attr.NOTHING From 0efcc880dd7c3bd16090dc38262c3a0032bfecff Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Thu, 10 Dec 2020 15:53:20 -0600 Subject: [PATCH 235/271] Added outputs to the second collect_additional_outputs so they have the same signature. --- pydra/engine/specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 240e92dfdf..e7d3737a69 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -95,7 +95,7 @@ def __setattr__(self, name, value): # validate all fields that have set a validator attr.validate(self) - def collect_additional_outputs(self, inputs, output_dir): + def collect_additional_outputs(self, inputs, output_dir, outputs): """Get additional outputs.""" return {} From 64accf53669355dc1196f3c1ddde683f8cceac3c Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Fri, 11 Dec 2020 23:12:19 -0500 Subject: [PATCH 236/271] ref: remove output after task is run to save memory --- pydra/engine/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index cfa4edf993..609cac4ef3 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -470,6 +470,7 @@ def _run(self, rerun=False, **kwargs): self.hooks.post_run_task(self, result) self.audit.finalize_audit(result) save(odir, result=result, task=self) + self.output_ = None # removing the additional file with the chcksum (self.cache_dir / f"{self.uid}_info.json").unlink() # # function etc. shouldn't change anyway, so removing From 5147e05d6edc834198f5f7f6d7f791fb38b6c985 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Fri, 11 Dec 2020 23:13:55 -0500 Subject: [PATCH 237/271] Update pydra/engine/tests/test_shelltask.py --- pydra/engine/tests/test_shelltask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 7248d94f0f..a3a879f4c2 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2878,7 +2878,7 @@ def get_stderr(stderr): ( "stderr_field", attr.ib( - type=int, + type=str, metadata={ "help_string": "The standard error output", "callable": get_stderr, From ae5075c67d4ef994b99c283b00414a9d596740a1 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 18 Dec 2020 20:45:25 -0500 Subject: [PATCH 238/271] updating zenodo --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 749fddafb7..b17416c09b 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -50,6 +50,11 @@ "name": "Nijholt, Bas", "orcid": "0000-0003-0383-4986" }, + { + "affiliation": "University of Iowa", + "name": "Johnson, Charles E.", + "orcid": "0000-0001-7814-3501" + }, { "affiliation": "MIT, HMS", "name": "Ghosh, Satrajit", From 1cc671caa6702189fbd0d5da499f6b9e5288c8cf Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Mon, 21 Dec 2020 17:31:11 -0600 Subject: [PATCH 239/271] Added and passed test to make a directory and return it in the output_spec using output_file_template. --- pydra/engine/helpers_file.py | 9 ++++-- pydra/engine/tests/test_shelltask.py | 46 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index c75042e903..4b94eadf10 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -556,7 +556,8 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input based on the value from inputs_dict (checking the types of the fields, that have "output_file_template)" """ - from .specs import File, MultiOutputFile + + from .specs import File, MultiOutputFile, Directory if spec_type == "input": if field.type not in [str, ty.Union[str, bool]]: @@ -574,7 +575,11 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input f"type of {field.name} is str, consider using Union[str, bool]" ) elif spec_type == "output": - if field.type not in [File, MultiOutputFile]: + if field.type not in [ + File, + MultiOutputFile, + Directory, + ]: raise Exception( f"output {field.name} should be a File, but {field.type} set as the type" ) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index a3a879f4c2..4a1f6a8bed 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2927,6 +2927,52 @@ def test_shell_cmd_outputspec_7b_error(): assert "has to have a callable" in str(e.value) +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_7c(tmpdir, plugin, results_function): + """ + customised output_spec, adding Directory to the output, + """ + + def get_lowest_directory(directory_path): + return str(directory_path).replace(str(Path(directory_path).parents[0]), "") + + cmd = "mkdir" + args = [f"{tmpdir}/dir1", f"{tmpdir}/dir2"] + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "resultsDir", + attr.ib( + type=File, + metadata={ + "output_file_template": "{args}", + "help_string": "output file", + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + args=args, + output_spec=my_output_spec, + resultsDir="outdir", + ).split("args") + + with Submitter(plugin=plugin) as sub: + shelly(submitter=sub) + res = shelly.result() + + for index, arg_dir in enumerate(args): + assert Path(Path(tmpdir) / Path(arg_dir)).exists() == True + assert get_lowest_directory(arg_dir) == f"/dir{index+1}" + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): """ From eb67b4f42f33d498d6e07d1d2a2d8ae139993058 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Tue, 22 Dec 2020 11:39:38 -0600 Subject: [PATCH 240/271] Added a test to specify an output directory name from an input field. --- pydra/engine/helpers_file.py | 7 +-- pydra/engine/tests/test_shelltask.py | 71 +++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 4b94eadf10..65718cb05d 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -556,7 +556,6 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input based on the value from inputs_dict (checking the types of the fields, that have "output_file_template)" """ - from .specs import File, MultiOutputFile, Directory if spec_type == "input": @@ -575,11 +574,7 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input f"type of {field.name} is str, consider using Union[str, bool]" ) elif spec_type == "output": - if field.type not in [ - File, - MultiOutputFile, - Directory, - ]: + if field.type not in [File, MultiOutputFile, Directory]: raise Exception( f"output {field.name} should be a File, but {field.type} set as the type" ) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 4a1f6a8bed..7bfaa14c4c 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -13,6 +13,7 @@ ShellSpec, SpecInfo, File, + Directory, MultiOutputFile, MultiInputObj, ) @@ -2930,7 +2931,7 @@ def test_shell_cmd_outputspec_7b_error(): @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_outputspec_7c(tmpdir, plugin, results_function): """ - customised output_spec, adding Directory to the output, + customised output_spec, adding Directory to the output named by args """ def get_lowest_directory(directory_path): @@ -2945,7 +2946,7 @@ def get_lowest_directory(directory_path): ( "resultsDir", attr.ib( - type=File, + type=Directory, metadata={ "output_file_template": "{args}", "help_string": "output file", @@ -2964,15 +2965,73 @@ def get_lowest_directory(directory_path): resultsDir="outdir", ).split("args") - with Submitter(plugin=plugin) as sub: - shelly(submitter=sub) - res = shelly.result() - + res = results_function(shelly, plugin) for index, arg_dir in enumerate(args): assert Path(Path(tmpdir) / Path(arg_dir)).exists() == True assert get_lowest_directory(arg_dir) == f"/dir{index+1}" +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_outputspec_7d(tmpdir, plugin, results_function): + """ + customised output_spec, adding Directory to the output named by input spec + """ + + def get_lowest_directory(directory_path): + return str(directory_path).replace(str(Path(directory_path).parents[0]), "") + + cmd = "mkdir" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "resultsDir", + attr.ib( + type=str, + metadata={ + "position": 1, + "help_string": "new directory", + "argstr": "", + }, + ), + ) + ], + bases=(ShellSpec,), + ) + + my_output_spec = SpecInfo( + name="Output", + fields=[ + ( + "resultsDir", + attr.ib( + type=Directory, + metadata={ + "output_file_template": "{resultsDir}", + "help_string": "output file", + }, + ), + ) + ], + bases=(ShellOutSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + input_spec=my_input_spec, + output_spec=my_output_spec, + resultsDir=Path(tmpdir) / Path("test"), + ) + + res = results_function(shelly, plugin) + assert (Path(tmpdir) / Path("test")).exists() == True + assert get_lowest_directory(res.output.resultsDir) == get_lowest_directory( + Path(tmpdir) / Path("test") + ) + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmpdir): """ From 696ed54d57f01188637b5de954b67309ce870afe Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 11 Jan 2021 22:04:54 -0500 Subject: [PATCH 241/271] using asyncio.SelectorEventLoop() to get a loop --- pydra/engine/helpers.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 3a616acdc2..4b3e9a0f5a 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -648,12 +648,9 @@ def get_open_loop(): """ if os.name == "nt": loop = asyncio.ProactorEventLoop() # for subprocess' pipes on Windows - asyncio.set_event_loop(loop) else: - loop = asyncio.get_event_loop() - if loop.is_closed(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = asyncio.SelectorEventLoop() + asyncio.set_event_loop(loop) return loop From 3a8cb104b2587ac41ffc30ed16f848379311beac Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 12 Jan 2021 16:42:52 -0500 Subject: [PATCH 242/271] restoring get_event_loop for linux, but adding try/except block and creating a new loop with new_event_loop if needed --- pydra/engine/helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 4b3e9a0f5a..09ed6beeb3 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -649,8 +649,16 @@ def get_open_loop(): if os.name == "nt": loop = asyncio.ProactorEventLoop() # for subprocess' pipes on Windows else: - loop = asyncio.SelectorEventLoop() - asyncio.set_event_loop(loop) + try: + loop = asyncio.get_event_loop() + # in case RuntimeError: There is no current event loop in thread 'MainThread' + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + else: + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) return loop From e0fde5cdfcbb3f2b1b1a79604635562818a5b838 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 13 Jan 2021 11:04:48 -0500 Subject: [PATCH 243/271] adding tests that doesn't use automatic bindings, but user defined bindings. the path to input files is provided as a container_path --- pydra/engine/tests/test_dockertask.py | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 5d314a8897..84dc3fe596 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -736,6 +736,54 @@ def test_docker_inputspec_1a(tmpdir): assert res.output.stdout == "hello from pydra" +@no_win +@need_docker +def test_docker_inputspec_1b(tmpdir): + """ a simple customized input spec for docker task + instead of using automatic binding I provide the bindings + and name of the file inside the container + """ + filename = str(tmpdir.join("file_pydra.txt")) + with open(filename, "w") as f: + f.write("hello from pydra") + + cmd = "cat" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "file", + attr.ib( + type=File, + metadata={ + "mandatory": True, + "position": 1, + "argstr": "", + "help_string": "input file", + "container_path": True, + }, + ), + ) + ], + bases=(DockerSpec,), + ) + + docky = DockerTask( + name="docky", + image="busybox", + executable=cmd, + # container_path is set to True, so providing the filename inside the container + file="/in_container/file_pydra.txt", + bindings=[(str(tmpdir), "/in_container")], + input_spec=my_input_spec, + strip=True, + ) + + res = docky() + assert res.output.stdout == "hello from pydra" + + @no_win @need_docker def test_docker_inputspec_1_dockerflag(tmpdir): From 750f3f0cec85546cce41298b77168e934733b336 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Wed, 3 Feb 2021 14:02:35 -0600 Subject: [PATCH 244/271] If a ShellCommandTask contains a MultiInputObj in its input_spec but a value is not given, do not include that argument in the cmdline. --- pydra/engine/task.py | 2 +- .../engine/tests/test_shelltask_inputspec.py | 48 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index a689c0052c..4a7f739337 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -475,7 +475,7 @@ def _command_pos_args(self, field, state_ind, ind): if "{" in argstr and "}" in argstr: cmd_el_str = argstr_formatting(argstr, self.inputs) else: # argstr has a simple form, e.g. "-f", or "--f" - if value: + if value and value != "NOTHING": cmd_el_str = f"{argstr} {value}" else: cmd_el_str = "" diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index e43cb445cf..9e1543cd6a 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -3,7 +3,7 @@ import pytest from ..task import ShellCommandTask -from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File +from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File, MultiInputObj from .utils import use_validator @@ -533,6 +533,52 @@ def test_shell_cmd_inputs_mandatory_1(): assert "mandatory" in str(e.value) +def test_shell_cmd_inputs_not_given_1(): + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "arg1", + attr.ib( + type=MultiInputObj, + metadata={ + "argstr": "--arg1", + "help_string": "Command line argument 1", + }, + ), + ), + ( + "arg2", + attr.ib( + type=MultiInputObj, + metadata={ + "argstr": "--arg2", + "help_string": "Command line argument 2", + }, + ), + ), + ( + "arg3", + attr.ib( + type=File, + metadata={ + "argstr": "--arg3", + "help_string": "Command line argument 3", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + shelly = ShellCommandTask( + name="shelly", executable="executable", input_spec=my_input_spec + ) + + shelly.inputs.arg2 = "argument2" + + assert shelly.cmdline == f"executable --arg2 argument2" + + def test_shell_cmd_inputs_template_1(): """ additional inputs, one uses output_file_template (and argstr)""" my_input_spec = SpecInfo( From b1e45a320d5f0765e348a150d4cfd86a87310512 Mon Sep 17 00:00:00 2001 From: Chase Johnson Date: Mon, 8 Feb 2021 12:05:57 -0600 Subject: [PATCH 245/271] Adjusted the converter for MultiInputObj so a value ot attr.NOTHING is not converted to a list. --- pydra/engine/specs.py | 5 ++++- pydra/engine/task.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index e7d3737a69..6ec6ca8a19 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -27,7 +27,10 @@ class MultiInputObj: def converter(cls, value): from .helpers import ensure_list - return ensure_list(value) + if value == attr.NOTHING: + return value + else: + return ensure_list(value) class MultiOutputObj: diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 4a7f739337..a689c0052c 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -475,7 +475,7 @@ def _command_pos_args(self, field, state_ind, ind): if "{" in argstr and "}" in argstr: cmd_el_str = argstr_formatting(argstr, self.inputs) else: # argstr has a simple form, e.g. "-f", or "--f" - if value and value != "NOTHING": + if value: cmd_el_str = f"{argstr} {value}" else: cmd_el_str = "" From e5c13fbe0297c4b8e6b944f6c26a75764da0781c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 12 Feb 2021 21:10:23 -0500 Subject: [PATCH 246/271] updated zenodo id for stat task, but doesnt work (complains that docker doesnt exist...) - xfail for now --- pydra/engine/tests/test_boutiques.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_boutiques.py b/pydra/engine/tests/test_boutiques.py index 4f6665bac3..2979613d25 100644 --- a/pydra/engine/tests/test_boutiques.py +++ b/pydra/engine/tests/test_boutiques.py @@ -127,6 +127,7 @@ def test_boutiques_wf_1(maskfile, plugin, tmpdir): @no_win @need_bosh_docker @pytest.mark.flaky(reruns=3) +@pytest.mark.xfail(reason="issues with bosh for 4472771") @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) @@ -145,9 +146,10 @@ def test_boutiques_wf_2(maskfile, plugin, tmpdir): maskfile=wf.lzin.maskfile, ) ) + # used to be "3240521", but can't access anymore wf.add( BoshTask( - name="stat", zenodo_id="3240521", input_file=wf.bet.lzout.outfile, v=True + name="stat", zenodo_id="4472771", input_file=wf.bet.lzout.outfile, v=True ) ) wf.add(ShellCommandTask(name="cat", executable="cat", args=wf.stat.lzout.output)) From 480fd5890d7bccdf9513d54b40a600cb5b1a2eee Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 12 Feb 2021 22:44:28 -0500 Subject: [PATCH 247/271] some changes in the run methods: moving cache reading after creating lockfiles, the same with saving info files with checksums --- pydra/engine/core.py | 30 +++++++++++++++--------------- pydra/engine/helpers.py | 2 ++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 09240cd3c4..a757a9f369 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -433,15 +433,15 @@ def _run(self, rerun=False, **kwargs): lockfile = self.cache_dir / (checksum + ".lock") # Eagerly retrieve cached - see scenarios in __init__() self.hooks.pre_run(self) - # adding info file with the checksum in case the task was cancelled - # and the lockfile has to be removed - with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: - json.dump({"checksum": self.checksum}, jsonfile) with SoftFileLock(lockfile): if not (rerun or self.task_rerun): result = self.result() if result is not None: return result + # adding info file with the checksum in case the task was cancelled + # and the lockfile has to be removed + with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: + json.dump({"checksum": self.checksum}, jsonfile) # Let only one equivalent process run odir = self.output_dir if not self.can_resume and odir.exists(): @@ -965,11 +965,6 @@ async def _run(self, submitter=None, rerun=False, **kwargs): "Workflow output cannot be None, use set_output to define output(s)" ) checksum = self.checksum - # Eagerly retrieve cached - if not (rerun or self.task_rerun): - result = self.result() - if result is not None: - return result # creating connections that were defined after adding tasks to the wf for task in self.graph.nodes: # if workflow has task_rerun=True and propagate_rerun=True, @@ -981,15 +976,18 @@ async def _run(self, submitter=None, rerun=False, **kwargs): task.propagate_rerun = self.propagate_rerun task.cache_locations = task._cache_locations + self.cache_locations self.create_connections(task) - # TODO add signal handler for processes killed after lock acquisition - # adding info file with the checksum in case the task was cancelled - # and the lockfile has to be removed - with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: - json.dump({"checksum": checksum}, jsonfile) lockfile = self.cache_dir / (checksum + ".lock") self.hooks.pre_run(self) with SoftFileLock(lockfile): - # # Let only one equivalent process run + # retrieve cached results + if not (rerun or self.task_rerun): + result = self.result() + if result is not None: + return result + # adding info file with the checksum in case the task was cancelled + # and the lockfile has to be removed + with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile: + json.dump({"checksum": checksum}, jsonfile) odir = self.output_dir if not self.can_resume and odir.exists(): shutil.rmtree(odir) @@ -1015,6 +1013,8 @@ async def _run(self, submitter=None, rerun=False, **kwargs): (self.cache_dir / f"{self.uid}_info.json").unlink() os.chdir(cwd) self.hooks.post_run(self, result) + if result is None: + raise Exception("This should never happen, please open new issue") return result async def _run_task(self, submitter, rerun=False): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 09ed6beeb3..1ebd2c4e7b 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -109,6 +109,8 @@ def load_result(checksum, cache_locations): """ if not cache_locations: return None + # TODO: if there are issues with loading, we might need to + # TODO: sleep and repeat loads (after checkin that there are no lock files!) for location in cache_locations: if (location / checksum).exists(): result_file = location / checksum / "_result.pklz" From c0e0987f85e97ebf0f95b04be0841c7690c61cfd Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Sat, 13 Feb 2021 00:26:23 -0500 Subject: [PATCH 248/271] fixing hashing for numpy objects str from numpy array doesnt work well (doesnt return all of the elements); adding cp.dumps for other complex objects --- pydra/engine/helpers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 1ebd2c4e7b..632a1fbb6b 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -673,7 +673,7 @@ def hash_value(value, tp=None, metadata=None, precalculated=None): """calculating hash or returning values recursively""" if metadata is None: metadata = {} - if isinstance(value, (tuple, list)): + if isinstance(value, (tuple, list, set)): return [hash_value(el, tp, metadata, precalculated) for el in value] elif isinstance(value, dict): dict_hash = { @@ -694,8 +694,21 @@ def hash_value(value, tp=None, metadata=None, precalculated=None): and "container_path" not in metadata ): return hash_dir(value, precalculated=precalculated) - else: + elif type(value).__module__ == "numpy": # numpy objects + return sha256(value.tostring()).hexdigest() + elif ( + isinstance( + value, (int, float, complex, bool, str, bytes, LazyField, os.PathLike) + ) + or value is None + ): return value + else: + warnings.warn( + f"pydra doesn't fully support hashing for {type(value)}, " + f"cp.dumps is used in hash functions, so it could depend on the system" + ) + return sha256(cp.dumps(value)).hexdigest() def output_from_inputfields(output_spec, input_spec): From 60b354a350c9ccf33e98bf9a5d80afca4f8f4d7a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 15 Feb 2021 00:40:50 -0500 Subject: [PATCH 249/271] adding callable to the user guide --- docs/output_spec.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/output_spec.rst b/docs/output_spec.rst index af0eacc366..52062d863f 100644 --- a/docs/output_spec.rst +++ b/docs/output_spec.rst @@ -62,7 +62,7 @@ The metadata dictionary for `output_spec` can include: A short description of the input field. The same as in `input_spec`. `output_file_template` (`str`): - If provided, the field is treated also as an output field and it is added to the output spec. + If provided the output file name (or list of file names) is created using the template. The template can use other fields, e.g. `{file1}`. The same as in `input_spec`. `output_field_name` (`str`, used together with `output_file_template`) @@ -79,3 +79,10 @@ The metadata dictionary for `output_spec` can include: The fields do not have to be a part of the `output_file_template` and if any field from the list is not provided in the input, a `NOTHING` is returned for the specific output. This has a different meaning than the `requires` form the `input_spec`. + +`callable` (`function`): + If provided the output file name (or list of file names) is created using the function. + The function can take `field` (the specific output field will be passed to the function), + `output_dir` (task `output_dir` wil be used), `stdout`, `stderr` (`stdout` and `stderr` of + the task will be sent) `inputs` (entire `inputs` will be passed) or any input field name + (a specific input field will be sent). From 6bd33d0b10806e21152cdd193ceeec8fc11801f7 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 15 Feb 2021 23:55:21 -0500 Subject: [PATCH 250/271] reverting some changes to hash_value: approach of adding cp.dumps(ob) to hash_value has to be better tested --- pydra/engine/helpers.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 632a1fbb6b..5ef127f203 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -696,19 +696,8 @@ def hash_value(value, tp=None, metadata=None, precalculated=None): return hash_dir(value, precalculated=precalculated) elif type(value).__module__ == "numpy": # numpy objects return sha256(value.tostring()).hexdigest() - elif ( - isinstance( - value, (int, float, complex, bool, str, bytes, LazyField, os.PathLike) - ) - or value is None - ): - return value else: - warnings.warn( - f"pydra doesn't fully support hashing for {type(value)}, " - f"cp.dumps is used in hash functions, so it could depend on the system" - ) - return sha256(cp.dumps(value)).hexdigest() + return value def output_from_inputfields(output_spec, input_spec): From 6ad8b991bbc22fb14226b3e6625b4a7238a2a09c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 17 Feb 2021 08:50:16 -0500 Subject: [PATCH 251/271] small fix copyfile_input --- pydra/engine/helpers_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 65718cb05d..60ed1a7566 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -499,16 +499,17 @@ def ensure_list(filename): # not sure if this might be useful for Function Task def copyfile_input(inputs, output_dir): """Implement the base class method.""" - from .specs import attr_fields, File + from .specs import attr_fields, File, MultiInputFile map_copyfiles = {} for fld in attr_fields(inputs): copy = fld.metadata.get("copyfile") - if copy is not None and fld.type is not File: + if copy is not None and fld.type not in [File, MultiInputFile]: raise Exception( f"if copyfile set, field has to be a File " f"but {fld.type} provided" ) - if copy in [True, False]: + file = getattr(inputs, fld.name) + if copy in [True, False] and file != attr.NOTHING: file = getattr(inputs, fld.name) newfile = output_dir.joinpath(Path(getattr(inputs, fld.name)).name) copyfile(file, newfile, copy=copy) From 6e664844fed86f6c29458bd4b86c5d864580982f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 17 Feb 2021 09:11:25 -0500 Subject: [PATCH 252/271] adding option for list in copyfile_input --- pydra/engine/helpers_file.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 60ed1a7566..7b1836f68b 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -510,10 +510,16 @@ def copyfile_input(inputs, output_dir): ) file = getattr(inputs, fld.name) if copy in [True, False] and file != attr.NOTHING: - file = getattr(inputs, fld.name) - newfile = output_dir.joinpath(Path(getattr(inputs, fld.name)).name) - copyfile(file, newfile, copy=copy) - map_copyfiles[fld.name] = str(newfile) + if isinstance(file, list): + map_copyfiles[fld.name] = [] + for el in file: + newfile = output_dir.joinpath(Path(el).name) + copyfile(el, newfile, copy=copy) + map_copyfiles[fld.name].append(str(newfile)) + else: + newfile = output_dir.joinpath(Path(file).name) + copyfile(file, newfile, copy=copy) + map_copyfiles[fld.name] = str(newfile) return map_copyfiles or None From d3643753efeed16806cc960dc7069944aa77136b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 18 Feb 2021 23:47:32 -0500 Subject: [PATCH 253/271] allowing for any position of the shelltask input field (not excluding 0); allowing for position duplication in the input spec as long as one value only is provided --- pydra/engine/task.py | 32 +++++++++++------- .../engine/tests/test_shelltask_inputspec.py | 33 ++++++++++++++++++- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index a689c0052c..e1340c08c2 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -346,6 +346,10 @@ def _command_args_single(self, state_ind, ind=None): "bindings", ]: continue + elif getattr(self.inputs, f.name) is attr.NOTHING and not f.metadata.get( + "readonly" + ): + continue elif f.name == "executable": pos_args.append( self._command_shelltask_executable( @@ -424,18 +428,22 @@ def _command_pos_args(self, field, state_ind, ind): if pos is None: # position will be set at the end pass - elif not isinstance(pos, int): - raise Exception(f"position should be an integer, but {pos} given") - elif pos == 0: - raise Exception(f"position can't be 0") - elif pos < 0: # position -1 is for args - pos = pos - 1 - # checking if the position is not already used - elif pos in self._positions_provided: - raise Exception( - f"{field.name} can't have provided position, {pos} is already used" - ) - self._positions_provided.append(pos) + else: + if not isinstance(pos, int): + raise Exception(f"position should be an integer, but {pos} given") + # checking if the position is not already used + if pos in self._positions_provided: + raise Exception( + f"{field.name} can't have provided position, {pos} is already used" + ) + else: + self._positions_provided.append(pos) + + if pos >= 0: + pos = pos + 1 # position 0 is for executable + else: # pos < 0: + pos = pos - 1 # position -1 is for args + value = self._field_value( field=field, state_ind=state_ind, ind=ind, check_file=True ) diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 9e1543cd6a..c19671eae3 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -158,13 +158,44 @@ def test_shell_cmd_inputs_2_err(): ) shelly = ShellCommandTask( - executable="executable", inpA="inp1", inpB="inp1", input_spec=my_input_spec + executable="executable", inpA="inp1", inpB="inp2", input_spec=my_input_spec ) with pytest.raises(Exception) as e: shelly.cmdline assert "1 is already used" in str(e.value) +def test_shell_cmd_inputs_2_noerr(): + """ additional inputs with provided positions + (duplication of teh position doesn't lead to error, since only one field has value) + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=str, + metadata={"position": 1, "help_string": "inpA", "argstr": ""}, + ), + ), + ( + "inpB", + attr.ib( + type=str, + metadata={"position": 1, "help_string": "inpB", "argstr": ""}, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + executable="executable", inpA="inp1", input_spec=my_input_spec + ) + shelly.cmdline + + def test_shell_cmd_inputs_3(): """ additional inputs: positive pos, negative pos and no pos """ my_input_spec = SpecInfo( From 30c114d32deb379d7f2e1bc383ab96ec92016ffe Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 19 Feb 2021 20:40:01 -0500 Subject: [PATCH 254/271] RF: Rewrite position_adjustment as position_sort --- pydra/engine/helpers.py | 67 ++++++++++++++---------------- pydra/engine/task.py | 10 +++-- pydra/engine/tests/test_helpers.py | 6 +-- 3 files changed, 41 insertions(+), 42 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 5ef127f203..ad96792755 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -820,42 +820,39 @@ def load_task(task_pkl, ind=None): return task -def position_adjustment(pos_args): +def position_sort(args): """ - sorting elements with the first element - position, - the negative positions should go to the end of the list - everything that has no position (i.e. it's None), - should go between elements with positive positions an with negative pos. - Returns a list of sorted args. - """ - # sorting all elements of the command - try: - pos_args.sort() - except TypeError: # if some positions are None - pos_args_none = [] - pos_args_int = [] - for el in pos_args: - if el[0] is None: - pos_args_none.append(el) - else: - pos_args_int.append(el) - pos_args_int.sort() - last_el = pos_args_int[-1][0] - for el_none in pos_args_none: - last_el += 1 - pos_args_int.append((last_el, el_none[1])) - pos_args = pos_args_int - - # if args available, they should be moved at the of the list - while pos_args[0][0] < 0: - pos_args.append(pos_args.pop(0)) - - # dropping the position index - cmd_args = [] - for el in pos_args: - cmd_args += el[1] - - return cmd_args + Sort objects by position, following Python indexing conventions. + + Ordering is postive positions, lowest to highest, followed by unspecified + positions (``None``) and negative positions, lowest to highest. + + >>> position_sort([(None, "d"), (-3, "e"), (2, "b"), (-2, "f"), (5, "c"), (1, "a")]) + ['a', 'b', 'c', 'd', 'e', 'f'] + + Parameters + ---------- + args : list of (int/None, object) tuples + + Returns + ------- + list of objects + """ + import bisect + pos, none, neg = [], [], [] + for entry in args: + position = entry[0] + if position is None: + # Take existing order + none.append(entry[1]) + elif position < 0: + # Sort negatives while collecting + bisect.insort(neg, entry) + else: + # Sort positives while collecting + bisect.insort(pos, entry) + + return [arg for _, arg in pos] + none + [arg for _, arg in neg] def argstr_formatting(argstr, inputs, value_updates=None): diff --git a/pydra/engine/task.py b/pydra/engine/task.py index e1340c08c2..b232fa8ac1 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -61,7 +61,7 @@ from .helpers import ( ensure_list, execute, - position_adjustment, + position_sort, argstr_formatting, output_from_inputfields, ) @@ -367,9 +367,11 @@ def _command_args_single(self, state_ind, ind=None): if pos_val: pos_args.append(pos_val) - # sorted elements of the command - cmd_args = position_adjustment(pos_args) - return cmd_args + # Sort command and arguments by position + cmd_args = position_sort(pos_args) + # pos_args values are each a list of arguments, so concatenate lists after sorting + return sum(cmd_args, []) + def _field_value(self, field, state_ind, ind, check_file=False): """ diff --git a/pydra/engine/tests/test_helpers.py b/pydra/engine/tests/test_helpers.py index d7eae8cd44..759dc9bb18 100644 --- a/pydra/engine/tests/test_helpers.py +++ b/pydra/engine/tests/test_helpers.py @@ -14,7 +14,7 @@ get_available_cpus, save, load_and_run, - position_adjustment, + position_sort, ) from .. import helpers_file from ..specs import File, Directory @@ -303,6 +303,6 @@ def test_load_and_run_wf(tmpdir): [(None, "b"), (1, "a"), (None, "c")], ], ) -def test_position_adjustment(pos_args): - final_args = position_adjustment(pos_args) +def test_position_sort(pos_args): + final_args = position_sort(pos_args) assert final_args == ["a", "b", "c"] From b30da771a0d103128ae44859d861815df505a289 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 19 Feb 2021 20:41:32 -0500 Subject: [PATCH 255/271] RF: Simplify branching on position shift --- pydra/engine/task.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index b232fa8ac1..8f6aeb2d4e 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -427,10 +427,7 @@ def _command_pos_args(self, field, state_ind, ind): # assuming that input that has no arstr is not used in the command return None pos = field.metadata.get("position", None) - if pos is None: - # position will be set at the end - pass - else: + if pos is not None: if not isinstance(pos, int): raise Exception(f"position should be an integer, but {pos} given") # checking if the position is not already used @@ -438,13 +435,12 @@ def _command_pos_args(self, field, state_ind, ind): raise Exception( f"{field.name} can't have provided position, {pos} is already used" ) - else: - self._positions_provided.append(pos) - if pos >= 0: - pos = pos + 1 # position 0 is for executable - else: # pos < 0: - pos = pos - 1 # position -1 is for args + self._positions_provided.append(pos) + + # Shift non-negatives up to allow executable to be 0 + # Shift negatives down to allow args to be -1 + pos += 1 if pos >= 0 else -1 value = self._field_value( field=field, state_ind=state_ind, ind=ind, check_file=True From bfdf54616cddeb61b1754f1c57bdfcd8bd0e5d11 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 19 Feb 2021 21:03:01 -0500 Subject: [PATCH 256/271] DOC: positive -> nonnegative --- docs/input_spec.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/input_spec.rst b/docs/input_spec.rst index a8454b3df4..9d2750ca2a 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -129,8 +129,8 @@ In the example we used multiple keys in the metadata dictionary including `help_ If no `argstr` is used the field is not part of the command. `position` (`int`): - Position of the field in the command, could be positive or negative integer. - If nothing is provided the field will be inserted between all fields with positive positions + Position of the field in the command, could be nonnegative or negative integer. + If nothing is provided the field will be inserted between all fields with nonnegative positions and fields with negative positions. `allowed_values` (`list`): From 86d0b9cf759516d5db6504d1c32475728a53ac03 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 19 Feb 2021 23:01:18 -0500 Subject: [PATCH 257/271] STY: Black --- pydra/engine/helpers.py | 1 + pydra/engine/task.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index ad96792755..9b9c0f0022 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -839,6 +839,7 @@ def position_sort(args): list of objects """ import bisect + pos, none, neg = [], [], [] for entry in args: position = entry[0] diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 8f6aeb2d4e..4aa99b3bd5 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -372,7 +372,6 @@ def _command_args_single(self, state_ind, ind=None): # pos_args values are each a list of arguments, so concatenate lists after sorting return sum(cmd_args, []) - def _field_value(self, field, state_ind, ind, check_file=False): """ Checking value of the specific field, if value is not set, None is returned. From 53d4d24475f5941535306373bd16be159b299277 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sat, 20 Feb 2021 10:24:10 -0500 Subject: [PATCH 258/271] RF: Simplify output_spec definition and population (#424) --- pydra/engine/task.py | 56 +++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index e1340c08c2..f9c8fb9f19 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -149,47 +149,31 @@ def __init__( rerun=rerun, ) if output_spec is None: - if "return" not in func.__annotations__: - output_spec = SpecInfo( - name="Output", fields=[("out", ty.Any)], bases=(BaseSpec,) - ) - else: + name = "Output" + fields = [("out", ty.Any)] + if "return" in func.__annotations__: return_info = func.__annotations__["return"] # e.g. python annotation: fun() -> ty.NamedTuple("Output", [("out", float)]) # or pydra decorator: @pydra.mark.annotate({"return": ty.NamedTuple(...)}) if hasattr(return_info, "__name__") and hasattr( return_info, "__annotations__" ): - output_spec = SpecInfo( - name=return_info.__name__, - fields=list(return_info.__annotations__.items()), - bases=(BaseSpec,), - ) + name = return_info.__name__ + fields = list(return_info.__annotations__.items()) # e.g. python annotation: fun() -> {"out": int} # or pydra decorator: @pydra.mark.annotate({"return": {"out": int}}) elif isinstance(return_info, dict): - output_spec = SpecInfo( - name="Output", - fields=list(return_info.items()), - bases=(BaseSpec,), - ) + fields = list(return_info.items()) # e.g. python annotation: fun() -> (int, int) # or pydra decorator: @pydra.mark.annotate({"return": (int, int)}) elif isinstance(return_info, tuple): - output_spec = SpecInfo( - name="Output", - fields=[ - ("out{}".format(n + 1), t) - for n, t in enumerate(return_info) - ], - bases=(BaseSpec,), - ) + fields = [(f"out{i}", t) for i, t in enumerate(return_info, 1)] # e.g. python annotation: fun() -> int # or pydra decorator: @pydra.mark.annotate({"return": int}) else: - output_spec = SpecInfo( - name="Output", fields=[("out", return_info)], bases=(BaseSpec,) - ) + fields = [("out", return_info)] + output_spec = SpecInfo(name=name, fields=fields, bases=(BaseSpec,)) + self.output_spec = output_spec def _run_task(self): @@ -200,18 +184,16 @@ def _run_task(self): output_names = [el[0] for el in self.output_spec.fields] if output is None: self.output_ = {nm: None for nm in output_names} + elif len(output_names) == 1: + # if only one element in the fields, everything should be returned together + self.output_ = {output_names[0]: output} + elif isinstance(output, tuple) and len(output_names) == len(output): + self.output_ = dict(zip(output_names, output)) else: - if len(output_names) == 1: - # if only one element in the fields, everything should be returned together - self.output_ = {output_names[0]: output} - else: - if isinstance(output, tuple) and len(output_names) == len(output): - self.output_ = dict(zip(output_names, output)) - else: - raise Exception( - f"expected {len(self.output_spec.fields)} elements, " - f"but {output} were returned" - ) + raise RuntimeError( + f"expected {len(self.output_spec.fields)} elements, " + f"but {output} were returned" + ) class ShellCommandTask(TaskBase): From 6a4af70ebf68ec512b28c2266a8fa83a7b720ec3 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sat, 20 Feb 2021 00:01:14 -0500 Subject: [PATCH 259/271] RF: Various adjustments to ShellCommandTask * Rename ind to index * Factor out check_file aspect of _field_value to ContainerTask * Document some parameters and return types --- pydra/engine/task.py | 136 +++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index f9c8fb9f19..951af992bf 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -310,42 +310,45 @@ def command_args(self): command_args_list = [] self.state.prepare_states(self.inputs) for ii, el in enumerate(self.state.states_ind): - command_args_list.append(self._command_args_single(el, ind=ii)) + command_args_list.append(self._command_args_single(el, index=ii)) return command_args_list else: - return self._command_args_single(self.inputs) + return self._command_args_single() - def _command_args_single(self, state_ind, ind=None): - """Get command line arguments for a single state""" + def _command_args_single(self, state_ind=None, index=None): + """Get command line arguments for a single state + + Parameters + ---------- + state_ind : dict[str, int] + Keys are inputs being mapped over, values are indices within that input + index : int + Index in flattened list of states + """ pos_args = [] # list for (position, command arg) self._positions_provided = [] - for f in attr_fields(self.inputs): - # these inputs will eb used in container_args - if isinstance(self, ContainerTask) and f.name in [ + for field in attr_fields(self.inputs): + name, meta = field.name, field.metadata + # these inputs will be used in container_args + if isinstance(self, ContainerTask) and name in [ "container", "image", "container_xargs", "bindings", ]: continue - elif getattr(self.inputs, f.name) is attr.NOTHING and not f.metadata.get( - "readonly" - ): + if getattr(self.inputs, name) is attr.NOTHING and not meta.get("readonly"): continue - elif f.name == "executable": + if name == "executable": pos_args.append( - self._command_shelltask_executable( - field=f, state_ind=state_ind, ind=ind - ) - ) - elif f.name == "args": - pos_val = self._command_shelltask_args( - field=f, state_ind=state_ind, ind=ind + self._command_shelltask_executable(field, state_ind, index) ) + elif name == "args": + pos_val = self._command_shelltask_args(field, state_ind, index) if pos_val: pos_args.append(pos_val) else: - pos_val = self._command_pos_args(field=f, state_ind=state_ind, ind=ind) + pos_val = self._command_pos_args(field, state_ind, index) if pos_val: pos_args.append(pos_val) @@ -353,50 +356,39 @@ def _command_args_single(self, state_ind, ind=None): cmd_args = position_adjustment(pos_args) return cmd_args - def _field_value(self, field, state_ind, ind, check_file=False): + def _field_value(self, field, state_ind, index, check_file=False): """ Checking value of the specific field, if value is not set, None is returned. If state_ind and ind, taking a specific element of the field. - If check_file is True, checking if field is a a local file - and settings bindings if needed. + check_file has no effect, but subclasses can use it to validate or modify + filenames. """ name = f"{self.name}.{field.name}" + value = getattr(self.inputs, field.name) if self.state and name in state_ind: - value = getattr(self.inputs, field.name)[state_ind[name]] - else: - value = getattr(self.inputs, field.name) - if value is attr.NOTHING or value is None: - return None - if check_file: - if is_local_file(field) and getattr(self, "bind_paths", None): - value = str(value) - # changing path to the cpath (the directory should be mounted) - lpath = Path(value) - cdir = self.bind_paths(ind=ind)[lpath.parent][0] - cpath = cdir.joinpath(lpath.name) - value = str(cpath) + value = value[state_ind[name]] + if value == attr.NOTHING: + value = None return value - def _command_shelltask_executable(self, field, state_ind, ind): + def _command_shelltask_executable(self, field, state_ind, index): """Returining position and value for executable ShellTask input""" pos = 0 # executable should be the first el. of the command - value = self._field_value(field=field, state_ind=state_ind, ind=ind) + value = self._field_value(field, state_ind, index) if value is None: - raise Exception("executable has to be set") + raise ValueError("executable has to be set") return pos, ensure_list(value, tuple2list=True) - def _command_shelltask_args(self, field, state_ind, ind): + def _command_shelltask_args(self, field, state_ind, index): """Returining position and value for args ShellTask input""" pos = -1 # assuming that args is the last el. of the command - value = self._field_value( - field=field, state_ind=state_ind, ind=ind, check_file=True - ) + value = self._field_value(field, state_ind, index, check_file=True) if value is None: return None else: return pos, ensure_list(value, tuple2list=True) - def _command_pos_args(self, field, state_ind, ind): + def _command_pos_args(self, field, state_ind, index): """ Checking all additional input fields, setting pos to None, if position not set. Creating a list with additional parts of the command that comes from @@ -426,9 +418,7 @@ def _command_pos_args(self, field, state_ind, ind): else: # pos < 0: pos = pos - 1 # position -1 is for args - value = self._field_value( - field=field, state_ind=state_ind, ind=ind, check_file=True - ) + value = self._field_value(field, state_ind, index, check_file=True) if field.metadata.get("readonly", False) and value is not None: raise Exception(f"{field.name} is read only, the value can't be provided") elif value is None and not field.metadata.get("readonly", False): @@ -583,6 +573,22 @@ def __init__( **kwargs, ) + def _field_value(self, field, state_ind, index, check_file=False): + """ + Checking value of the specific field, if value is not set, None is returned. + If state_ind and ind, taking a specific element of the field. + If check_file is True, checking if field is a a local file + and settings bindings if needed. + """ + value = super()._field_value(field, state_ind, index) + if value and check_file and is_local_file(field): + # changing path to the cpath (the directory should be mounted) + lpath = Path(str(value)) + cdir = self.bind_paths(index)[lpath.parent][0] + cpath = cdir.joinpath(lpath.name) + value = str(cpath) + return value + def container_check(self, container_type): """Get container-specific CLI arguments.""" if self.inputs.container is None: @@ -594,16 +600,22 @@ def container_check(self, container_type): if self.inputs.image is attr.NOTHING: raise AttributeError("Container image is not specified") - def bind_paths(self, ind=None): - """Return bound mount points: ``dict(lpath: (cpath, mode))``.""" + def bind_paths(self, index=None): + """Get bound mount points + + Returns + ------- + mount points: dict + mapping from local path to tuple of container path + mode + """ bind_paths = {} output_dir_cpath = None if self.inputs.bindings is None: self.inputs.bindings = [] - if ind is None: + if index is None: output_dir = self.output_dir else: - output_dir = self.output_dir[ind] + output_dir = self.output_dir[index] for binding in self.inputs.bindings: if len(binding) == 3: lpath, cpath, mode = binding @@ -623,16 +635,16 @@ def bind_paths(self, ind=None): bind_paths[output_dir] = (self.output_cpath, "rw") return bind_paths - def binds(self, opt, ind=None): + def binds(self, opt, index=None): """ Specify mounts to bind from local filesystems to container and working directory. - Uses py:meth:`binds_paths` + Uses py:meth:`bind_paths` """ bargs = [] - for (key, val) in self.bind_paths(ind).items(): - bargs.extend([opt, f"{key}:{val[0]}:{val[1]}"]) + for lpath, (cpath, mode) in self.bind_paths(index).items(): + bargs.extend([opt, f"{lpath}:{cpath}:{mode}"]) return bargs @@ -713,23 +725,23 @@ def container_args(self): if f"{self.name}.image" in el: cargs_list.append( self._container_args_single( - self.inputs.image[el[f"{self.name}.image"]], ind=ii + self.inputs.image[el[f"{self.name}.image"]], index=ii ) ) else: cargs_list.append( - self._container_args_single(self.inputs.image, ind=ii) + self._container_args_single(self.inputs.image, index=ii) ) return cargs_list else: return self._container_args_single(self.inputs.image) - def _container_args_single(self, image, ind=None): + def _container_args_single(self, image, index=None): cargs = ["docker", "run"] if self.inputs.container_xargs is not None: cargs.extend(self.inputs.container_xargs) - cargs.extend(self.binds("-v", ind)) + cargs.extend(self.binds("-v", index)) cargs.extend(["-w", str(self.output_cpath)]) cargs.append(image) @@ -807,24 +819,24 @@ def container_args(self): if f"{self.name}.image" in el: cargs_list.append( self._container_args_single( - self.inputs.image[el[f"{self.name}.image"]], ind=ii + self.inputs.image[el[f"{self.name}.image"]], index=ii ) ) else: cargs_list.append( - self._container_args_single(self.inputs.image, ind=ii) + self._container_args_single(self.inputs.image, index=ii) ) return cargs_list else: return self._container_args_single(self.inputs.image) - def _container_args_single(self, image, ind=None): + def _container_args_single(self, image, index=None): cargs = ["singularity", "exec"] if self.inputs.container_xargs is not None: cargs.extend(self.inputs.container_xargs) - cargs.extend(self.binds("-B", ind)) + cargs.extend(self.binds("-B", index)) cargs.extend(["--pwd", str(self.output_cpath)]) cargs.append(image) return cargs From 5179b7880e86a8091aceb05338ca389e2522f447 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sat, 20 Feb 2021 00:16:51 -0500 Subject: [PATCH 260/271] TEST: Fix FSL BET test --- pydra/engine/tests/test_shelltask.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 7bfaa14c4c..0d60e1ad07 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -4040,12 +4040,13 @@ def change_name(file): ) # TODO: not sure why this has to be string - in_file = Path(os.path.dirname(os.path.abspath(__file__))) / "data" / "foo.nii" + in_file = Path(__file__).parent / "data_tests" / "test.nii.gz" # separate command into exec + args shelly = ShellCommandTask( name="bet_task", executable="bet", in_file=in_file, input_spec=bet_input_spec ) + out_file = shelly.output_dir / "test_brain.nii.gz" assert shelly.inputs.executable == "bet" - assert shelly.cmdline == f"bet {in_file} {in_file}_brain" + assert shelly.cmdline == f"bet {in_file} {out_file}" # res = shelly(plugin="cf") From dd97142ed456638820a71887be34daad0393bc6e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sat, 20 Feb 2021 10:29:06 -0500 Subject: [PATCH 261/271] RF: Make state_ind optional for BoshTask._command_args_single Also rename ind to index --- pydra/engine/boutiques.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pydra/engine/boutiques.py b/pydra/engine/boutiques.py index 9e5678f761..83f80b7f96 100644 --- a/pydra/engine/boutiques.py +++ b/pydra/engine/boutiques.py @@ -178,9 +178,9 @@ def _prepare_output_spec(self, names_subset=None): spec = SpecInfo(name="Outputs", fields=fields, bases=(ShellOutSpec,)) return spec - def _command_args_single(self, state_ind, ind=None): + def _command_args_single(self, state_ind=None, index=None): """Get command line arguments for a single state""" - input_filepath = self._bosh_invocation_file(state_ind=state_ind, ind=ind) + input_filepath = self._bosh_invocation_file(state_ind=state_ind, index=index) cmd_list = ( self.inputs.executable + [str(self.bosh_file), input_filepath] @@ -189,7 +189,7 @@ def _command_args_single(self, state_ind, ind=None): ) return cmd_list - def _bosh_invocation_file(self, state_ind, ind=None): + def _bosh_invocation_file(self, state_ind=None, index=None): """creating bosh invocation file - json file with inputs values""" input_json = {} for f in attr_fields(self.inputs): @@ -208,7 +208,7 @@ def _bosh_invocation_file(self, state_ind, ind=None): input_json[f.name] = value - filename = self.cache_dir / f"{self.name}-{ind}.json" + filename = self.cache_dir / f"{self.name}-{index}.json" with open(filename, "w") as jsonfile: json.dump(input_json, jsonfile) From d730edf8e172929d3cebfc1b037a53be61360b2d Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 22 Feb 2021 09:00:19 -0500 Subject: [PATCH 262/271] ENH: Allow attr_fields to exclude fields (#426) * ENH: Allow attr_fields to exclude fields * RF: Use exclude parameter to avoid common branching patterns --- pydra/engine/boutiques.py | 4 +- pydra/engine/specs.py | 149 ++++++++++++++++---------------------- pydra/engine/task.py | 13 +--- 3 files changed, 67 insertions(+), 99 deletions(-) diff --git a/pydra/engine/boutiques.py b/pydra/engine/boutiques.py index 83f80b7f96..560a30b1dd 100644 --- a/pydra/engine/boutiques.py +++ b/pydra/engine/boutiques.py @@ -192,9 +192,7 @@ def _command_args_single(self, state_ind=None, index=None): def _bosh_invocation_file(self, state_ind=None, index=None): """creating bosh invocation file - json file with inputs values""" input_json = {} - for f in attr_fields(self.inputs): - if f.name in ["executable", "args"]: - continue + for f in attr_fields(self.inputs, exclude_names=("executable", "args")): if self.state and f"{self.name}.{f.name}" in state_ind: value = getattr(self.inputs, f.name)[state_ind[f"{self.name}.{f.name}"]] else: diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 6ec6ca8a19..4bc8e52546 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -8,8 +8,8 @@ from .helpers_file import template_update_single -def attr_fields(x): - return x.__attrs_attrs__ +def attr_fields(spec, exclude_names=()): + return [field for field in spec.__attrs_attrs__ if field.name not in exclude_names] class File: @@ -70,13 +70,13 @@ class BaseSpec: """The base dataclass specs for all inputs and outputs.""" def __attrs_post_init__(self): - self.files_hash = {} - for field in attr_fields(self): - if ( - field.name not in ["_graph_checksums", "bindings", "files_hash"] - and field.metadata.get("output_file_template") is None - ): - self.files_hash[field.name] = {} + self.files_hash = { + field.name: {} + for field in attr_fields( + self, exclude_names=("_graph_checksums", "bindings", "files_hash") + ) + if field.metadata.get("output_file_template") is None + } def __setattr__(self, name, value): """changing settatr, so the converter and validator is run @@ -108,14 +108,12 @@ def hash(self): from .helpers import hash_value, hash_function inp_dict = {} - for field in attr_fields(self): - if field.name in [ - "_graph_checksums", - "bindings", - "files_hash", - ] or field.metadata.get("output_file_template"): + for field in attr_fields( + self, exclude_names=("_graph_checksums", "bindings", "files_hash") + ): + if field.metadata.get("output_file_template"): continue - # removing values that are notset from hash calculation + # removing values that are not set from hash calculation if getattr(self, field.name) is attr.NOTHING: continue value = getattr(self, field.name) @@ -317,11 +315,7 @@ def check_metadata(self): "xor", "sep", } - # special inputs, don't have to follow rules for standard inputs - special_input = ["_func", "_graph_checksums"] - - fields = [fld for fld in attr_fields(self) if fld.name not in special_input] - for fld in fields: + for fld in attr_fields(self, exclude_names=("_func", "_graph_checksums")): mdata = fld.metadata # checking keys from metadata if set(mdata.keys()) - supported_keys: @@ -398,11 +392,7 @@ def check_metadata(self): "xor", "sep", } - # special inputs, don't have to follow rules for standard inputs - special_input = ["_func", "_graph_checksums"] - - fields = [fld for fld in attr_fields(self) if fld.name not in special_input] - for fld in fields: + for fld in attr_fields(self, exclude_names=("_func", "_graph_checksums")): mdata = fld.metadata # checking keys from metadata if set(mdata.keys()) - supported_keys: @@ -445,47 +435,36 @@ class ShellOutSpec: def collect_additional_outputs(self, inputs, output_dir, outputs): """Collect additional outputs from shelltask output_spec.""" additional_out = {} - for fld in attr_fields(self): - if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type in [ - File, - MultiOutputFile, - Directory, - int, - float, - bool, - str, - list, - ]: - # assuming that field should have either default or metadata, but not both - if ( - fld.default is None or fld.default == attr.NOTHING - ) and not fld.metadata: # TODO: is it right? - raise AttributeError( - "File has to have default value or metadata" - ) - elif fld.default != attr.NOTHING: - additional_out[fld.name] = self._field_defaultvalue( - fld, output_dir - ) - elif fld.metadata: - if ( - fld.type in [int, float, bool, str, list] - and "callable" not in fld.metadata - ): - raise AttributeError( - f"{fld.type} has to have a callable in metadata" - ) - else: - additional_out[fld.name] = self._field_metadata( - fld, inputs, output_dir, outputs - ) - # else: - # additional_out[fld.name] = self._field_metadata( - # fld, inputs, output_dir, outputs - # ) - else: - raise Exception("not implemented (collect_additional_output)") + for fld in attr_fields(self, exclude_names=("return_code", "stdout", "stderr")): + if fld.type not in [ + File, + MultiOutputFile, + Directory, + int, + float, + bool, + str, + list, + ]: + raise Exception("not implemented (collect_additional_output)") + # assuming that field should have either default or metadata, but not both + if ( + fld.default is None or fld.default == attr.NOTHING + ) and not fld.metadata: # TODO: is it right? + raise AttributeError("File has to have default value or metadata") + elif fld.default != attr.NOTHING: + additional_out[fld.name] = self._field_defaultvalue(fld, output_dir) + elif fld.metadata: + if ( + fld.type in [int, float, bool, str, list] + and "callable" not in fld.metadata + ): + raise AttributeError( + f"{fld.type} has to have a callable in metadata" + ) + additional_out[fld.name] = self._field_metadata( + fld, inputs, output_dir, outputs + ) return additional_out def generated_output_names(self, inputs, output_dir): @@ -496,26 +475,22 @@ def generated_output_names(self, inputs, output_dir): # checking the input (if all mandatory fields are provided, etc.) inputs.check_fields_input_spec() output_names = ["return_code", "stdout", "stderr"] - for fld in attr_fields(self): - if fld.name not in ["return_code", "stdout", "stderr"]: - if fld.type is File: - # assuming that field should have either default or metadata, but not both - if ( - fld.default is None or fld.default == attr.NOTHING - ) and not fld.metadata: # TODO: is it right? - raise AttributeError( - "File has to have default value or metadata" - ) - elif fld.default != attr.NOTHING: - output_names.append(fld.name) - elif ( - fld.metadata - and self._field_metadata(fld, inputs, output_dir, outputs=None) - != attr.NOTHING - ): - output_names.append(fld.name) - else: - raise Exception("not implemented (collect_additional_output)") + for fld in attr_fields(self, exclude_names=("return_code", "stdout", "stderr")): + if fld.type is not File: + raise Exception("not implemented (collect_additional_output)") + # assuming that field should have either default or metadata, but not both + if ( + fld.default in (None, attr.NOTHING) and not fld.metadata + ): # TODO: is it right? + raise AttributeError("File has to have default value or metadata") + elif fld.default != attr.NOTHING: + output_names.append(fld.name) + elif ( + fld.metadata + and self._field_metadata(fld, inputs, output_dir, outputs=None) + != attr.NOTHING + ): + output_names.append(fld.name) return output_names def _field_defaultvalue(self, fld, output_dir): diff --git a/pydra/engine/task.py b/pydra/engine/task.py index d692a7ed3b..fd9996d8ce 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -327,16 +327,11 @@ def _command_args_single(self, state_ind=None, index=None): """ pos_args = [] # list for (position, command arg) self._positions_provided = [] - for field in attr_fields(self.inputs): + for field in attr_fields( + self.inputs, + exclude_names=("container", "image", "container_xargs", "bindings"), + ): name, meta = field.name, field.metadata - # these inputs will be used in container_args - if isinstance(self, ContainerTask) and name in [ - "container", - "image", - "container_xargs", - "bindings", - ]: - continue if getattr(self.inputs, name) is attr.NOTHING and not meta.get("readonly"): continue if name == "executable": From ec46cc53390efccbb1f78100eb18e2015f5bcd33 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 22 Feb 2021 11:45:41 -0500 Subject: [PATCH 263/271] CI: Squash gh-pages history on each push (#427) Option available in gh-pages v2.2.0+ --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e679bfd49..dbdd6b0603 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ docs_deploy: &docs - run: name: Install gh-pages tool command: | - npm install -g --silent gh-pages@2.0.1 + npm install -g --silent gh-pages@3.1.0 - checkout - run: name: Set git settings @@ -30,7 +30,7 @@ docs_deploy: &docs command: touch docs/_build/html/.nojekyll - run: name: Deploy docs to gh-pages branch - command: gh-pages --dotfiles --message "doc(update) [skip ci]" --dist docs/_build/html + command: gh-pages --no-history --dotfiles --message "doc(update) [skip ci]" --dist docs/_build/html version: 2 jobs: From 7bf22ff0f6aeaea80b9561df284ca6ad5641cbfa Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 22 Feb 2021 12:07:02 -0500 Subject: [PATCH 264/271] CI: Use gh-pages@3.0.0 due to tschaub/gh-pages#354 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbdd6b0603..e025c480f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ docs_deploy: &docs - run: name: Install gh-pages tool command: | - npm install -g --silent gh-pages@3.1.0 + npm install -g --silent gh-pages@3.0.0 - checkout - run: name: Set git settings From b5cce7155a4ed022cee8c4fba7150092d80288cc Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 24 Feb 2021 22:58:06 -0500 Subject: [PATCH 265/271] adding full traceback to the error files; adding tests --- pydra/engine/core.py | 16 ++++++--- pydra/engine/helpers.py | 3 +- pydra/engine/tests/test_task.py | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index a757a9f369..317c45ac1f 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -3,7 +3,7 @@ import attr import json import logging -import os +import os, sys from pathlib import Path import typing as ty from copy import deepcopy @@ -13,6 +13,7 @@ from filelock import SoftFileLock import shutil from tempfile import mkdtemp +from traceback import format_exception from . import state from . import helpers_state as hlpst @@ -463,7 +464,9 @@ def _run(self, rerun=False, **kwargs): self._run_task() result.output = self._collect_outputs(output_dir=odir) except Exception as e: - record_error(self.output_dir, e) + etype, eval, etr = sys.exc_info() + traceback = format_exception(etype, eval, etr) + record_error(self.output_dir, error=traceback) result.errored = True raise finally: @@ -1001,7 +1004,9 @@ async def _run(self, submitter=None, rerun=False, **kwargs): await self._run_task(submitter, rerun=rerun) result.output = self._collect_outputs() except Exception as e: - record_error(self.output_dir, e) + etype, eval, etr = sys.exc_info() + traceback = format_exception(etype, eval, etr) + record_error(self.output_dir, error=traceback) result.errored = True self._errored = True raise @@ -1095,7 +1100,10 @@ def _collect_outputs(self): f"Tasks {getattr(self, val.name)._errored} raised an error" ) else: - raise ValueError(f"Task {val.name} raised an error") + raise ValueError( + f"Task {val.name} raised an error, " + f"full crash report is here: {getattr(self, val.name).output_dir / '_error.pklz'}" + ) return attr.evolve(output, **output_wf) def create_dotfile(self, type="simple", export=None, name=None): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 9b9c0f0022..7deaeb77e5 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -784,10 +784,11 @@ def load_and_run( except Exception as excinfo: # creating result and error files if missing errorfile = task.output_dir / "_error.pklz" - if not resultfile.exists(): + if not errorfile.exists(): # not sure if this is needed etype, eval, etr = sys.exc_info() traceback = format_exception(etype, eval, etr) errorfile = record_error(task.output_dir, error=traceback) + if not resultfile.exists(): # not sure if this is needed result = Result(output=None, runtime=None, errored=True) save(task.output_dir, result=result) raise type(excinfo)( diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index d07fef765b..09a577ea48 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -2,6 +2,9 @@ import os, sys import attr import pytest +import cloudpickle as cp +from pathlib import Path +import re from ... import mark from ..core import Workflow @@ -1302,3 +1305,57 @@ def myhook_postrun(task, result, *args): # only post run task hook should be called assert len(hook_messages) == 1 assert "postrun task hook was called" in hook_messages[0] + + +def test_traceback(): + """ checking if the error raised in a function is properly returned; + checking if there is an error filename in the error message that contains + full traceback including the line in the python function + """ + + @mark.task + def fun_error(x): + raise Exception("Error from the function") + + task = fun_error(name="error", x=[3, 4]).split("x") + + with pytest.raises(Exception, match="from the function") as exinfo: + res = task() + + # getting error file from the error message + error_file_match = re.findall("[a-zA-Z0-9/_]+error.pklz", str(exinfo.value)) + assert len(error_file_match) # should be one error file in the error message + error_file = Path(error_file_match[0]) + assert error_file.name == "_error.pklz" + error_tb = cp.loads(error_file.read_bytes())["error message"] + # the error traceback should be a list and should point to a specific line in the function + assert isinstance(error_tb, list) + assert "in fun_error" in error_tb[-2] + + +def test_traceback_wf(): + """ checking if the error raised in a function is properly returned by a workflow; + checking if there is an error filename in the error message that contains + full traceback including the line in the python function + """ + + @mark.task + def fun_error(x): + raise Exception("Error from the function") + + wf = Workflow(name="wf", input_spec=["x"], x=[3, 4]).split("x") + wf.add(fun_error(name="error", x=wf.lzin.x)) + wf.set_output([("out", wf.error.lzout.out)]) + + with pytest.raises(Exception, match="Task error raised an error") as exinfo: + res = wf() + + # getting error file from the error message + error_file_match = re.findall("[a-zA-Z0-9/_]+error.pklz", str(exinfo.value)) + assert len(error_file_match) # should be one error file in the error message + error_file = Path(error_file_match[0]) + assert error_file.name == "_error.pklz" + error_tb = cp.loads(error_file.read_bytes())["error message"] + # the error traceback should be a list and should point to a specific line in the function + assert isinstance(error_tb, list) + assert "in fun_error" in error_tb[-2] From fe99b9061ff51c03a2c29daa7733fe3324f39b0a Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 25 Feb 2021 12:04:06 -0500 Subject: [PATCH 266/271] fixing the test so it works for windows --- pydra/engine/tests/test_task.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 09a577ea48..fc3b2ac91f 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1307,7 +1307,7 @@ def myhook_postrun(task, result, *args): assert "postrun task hook was called" in hook_messages[0] -def test_traceback(): +def test_traceback(tmpdir): """ checking if the error raised in a function is properly returned; checking if there is an error filename in the error message that contains full traceback including the line in the python function @@ -1317,23 +1317,24 @@ def test_traceback(): def fun_error(x): raise Exception("Error from the function") - task = fun_error(name="error", x=[3, 4]).split("x") + task = fun_error(name="error", x=[3, 4], cache_dir=tmpdir).split("x") with pytest.raises(Exception, match="from the function") as exinfo: res = task() # getting error file from the error message - error_file_match = re.findall("[a-zA-Z0-9/_]+error.pklz", str(exinfo.value)) - assert len(error_file_match) # should be one error file in the error message - error_file = Path(error_file_match[0]) - assert error_file.name == "_error.pklz" + error_file_match = str(exinfo.value).split("here: ")[-1].split("_error.pklz")[0] + error_file = Path(error_file_match) / "_error.pklz" + # checking if the file exists + assert error_file.exists() + # reading error message from the pickle file error_tb = cp.loads(error_file.read_bytes())["error message"] # the error traceback should be a list and should point to a specific line in the function assert isinstance(error_tb, list) assert "in fun_error" in error_tb[-2] -def test_traceback_wf(): +def test_traceback_wf(tmpdir): """ checking if the error raised in a function is properly returned by a workflow; checking if there is an error filename in the error message that contains full traceback including the line in the python function @@ -1343,7 +1344,7 @@ def test_traceback_wf(): def fun_error(x): raise Exception("Error from the function") - wf = Workflow(name="wf", input_spec=["x"], x=[3, 4]).split("x") + wf = Workflow(name="wf", input_spec=["x"], x=[3, 4], cache_dir=tmpdir).split("x") wf.add(fun_error(name="error", x=wf.lzin.x)) wf.set_output([("out", wf.error.lzout.out)]) @@ -1351,10 +1352,11 @@ def fun_error(x): res = wf() # getting error file from the error message - error_file_match = re.findall("[a-zA-Z0-9/_]+error.pklz", str(exinfo.value)) - assert len(error_file_match) # should be one error file in the error message - error_file = Path(error_file_match[0]) - assert error_file.name == "_error.pklz" + error_file_match = str(exinfo.value).split("here: ")[-1].split("_error.pklz")[0] + error_file = Path(error_file_match) / "_error.pklz" + # checking if the file exists + assert error_file.exists() + # reading error message from the pickle file error_tb = cp.loads(error_file.read_bytes())["error message"] # the error traceback should be a list and should point to a specific line in the function assert isinstance(error_tb, list) From 40099c71af62df696bc26ea5121032322ae6c450 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 1 Mar 2021 20:19:36 -0500 Subject: [PATCH 267/271] CI: Install libssl1.1 (update for Ubuntu 20.04) (#435) * CI: Install libssl1.1 (update for Ubuntu 20.04) * CI: Test with Singularity 3.7.1 --- .github/workflows/testsingularity.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index 946390dc84..b926ee1cfc 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -14,13 +14,13 @@ jobs: steps: - name: Set env run: | - echo "RELEASE_VERSION=v3.5.0" >> $GITHUB_ENV + echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV echo "NO_ET=TRUE" >> $GITHUB_ENV - name: Setup Singularity uses: actions/checkout@v2 with: repository: hpcng/singularity - ref: 'v3.5.0' + ref: 'v3.7.1' path: 'singularity' - name: Setup GO uses: actions/setup-go@v2 @@ -29,7 +29,7 @@ jobs: - name: Install OS deps run: | sudo apt-get update - sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \ + sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \ libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y - name: Build run: | From 334028c9d4797db5052f35cfbfcb3b6289146cae Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 8 Mar 2021 00:08:22 -0500 Subject: [PATCH 268/271] allowing for float formatting in argstr and output_file_template; changing template_formatting so it allows for multiple input fields in the templates --- pydra/engine/helpers.py | 2 + pydra/engine/helpers_file.py | 155 +++++--- pydra/engine/specs.py | 11 +- pydra/engine/tests/test_shelltask.py | 42 +++ .../engine/tests/test_shelltask_inputspec.py | 349 ++++++++++++++++++ 5 files changed, 502 insertions(+), 57 deletions(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 9b9c0f0022..7e40db1b70 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -866,6 +866,8 @@ def argstr_formatting(argstr, inputs, value_updates=None): inputs_dict.update(value_updates) # getting all fields that should be formatted, i.e. {field_name}, ... inp_fields = re.findall("{\w+}", argstr) + inp_fields_float = re.findall("{\w+:[0-9.]+f}", argstr) + inp_fields += [re.sub(":[0-9.]+f", "", el) for el in inp_fields_float] val_dict = {} for fld in inp_fields: fld_name = fld[1:-1] # extracting the name form {field_name} diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 7b1836f68b..50b0234f36 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -10,6 +10,7 @@ import logging from pathlib import Path import typing as ty +from copy import copy related_filetype_sets = [(".hdr", ".img", ".mat"), (".nii", ".mat"), (".BRIK", ".HEAD")] """List of neuroimaging file types that are to be interpreted together.""" @@ -547,7 +548,7 @@ def template_update(inputs, output_dir, map_copyfiles=None): " has to be a string or Union[str, bool]" ) dict_[fld.name] = template_update_single( - field=fld, inputs_dict=dict_, output_dir=output_dir + field=fld, inputs=inputs, output_dir=output_dir ) # using is and == so it covers list and numpy arrays updated_templ_dict = { @@ -558,7 +559,7 @@ def template_update(inputs, output_dir, map_copyfiles=None): return updated_templ_dict -def template_update_single(field, inputs_dict, output_dir=None, spec_type="input"): +def template_update_single(field, inputs, output_dir=None, spec_type="input"): """Update a single template from the input_spec or output_spec based on the value from inputs_dict (checking the types of the fields, that have "output_file_template)" @@ -571,7 +572,7 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input f"fields with output_file_template" "has to be a string or Union[str, bool]" ) - inp_val_set = inputs_dict[field.name] + inp_val_set = getattr(inputs, field.name) if inp_val_set is not attr.NOTHING and not isinstance(inp_val_set, (str, bool)): raise Exception( f"{field.name} has to be str or bool, but {inp_val_set} set" @@ -588,13 +589,13 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input else: raise Exception(f"spec_type can be input or output, but {spec_type} provided") # for inputs that the value is set (so the template is ignored) - if spec_type == "input" and isinstance(inputs_dict[field.name], str): - return inputs_dict[field.name] - elif spec_type == "input" and inputs_dict[field.name] is False: + if spec_type == "input" and isinstance(getattr(inputs, field.name), str): + return getattr(inputs, field.name) + elif spec_type == "input" and getattr(inputs, field.name) is False: # if input fld is set to False, the fld shouldn't be used (setting NOTHING) return attr.NOTHING else: # inputs_dict[field.name] is True or spec_type is output - value = _template_formatting(field, inputs_dict) + value = _template_formatting(field, inputs) # changing path so it is in the output_dir if output_dir and value is not attr.NOTHING: # should be converted to str, it is also used for input fields that should be str @@ -606,11 +607,13 @@ def template_update_single(field, inputs_dict, output_dir=None, spec_type="input return value -def _template_formatting(field, inputs_dict): - """Formatting a single template based on values from inputs_dict. +def _template_formatting(field, inputs): + """Formatting the field template based on the values from inputs. Taking into account that the field with a template can be a MultiOutputFile and the field values needed in the template can be a list - returning a list of formatted templates in that case. + Allowing for multiple input values used in teh template as longs as + there is no more than one file (i.e. File, PathLike or string with extensions) """ from .specs import MultiOutputFile @@ -619,70 +622,112 @@ def _template_formatting(field, inputs_dict): keep_extension = field.metadata.get("keep_extension", True) inp_fields = re.findall("{\w+}", template) + inp_fields_fl = re.findall("{\w+:[0-9.]+f}", template) + inp_fields += [re.sub(":[0-9.]+f", "", el) for el in inp_fields_fl] if len(inp_fields) == 0: return template - elif len(inp_fields) == 1: - fld_name = inp_fields[0][1:-1] - fld_value = inputs_dict[fld_name] + + val_dict = {} + file_template = None + from .specs import attr_fields_dict, File + + for fld in inp_fields: + fld_name = fld[1:-1] # extracting the name form {field_name} + fld_value = getattr(inputs, fld_name) if fld_value is attr.NOTHING: + # if value is NOTHING, nothing should be added to the command return attr.NOTHING - # if field is MultiOutputFile and the fld_value is a list, - # each element of the list should be used separately in the template - # and return a list with formatted values - if field.type is MultiOutputFile and type(fld_value) is list: - formatted_value = [] - for el in fld_value: - formatted_value.append( - _element_formatting( - template, - fld_name=fld_name, - fld_value=el, - keep_extension=keep_extension, + else: + # checking for fields that can be treated as a file: + # have type File, or value that is path like (including str with extensions) + if ( + attr_fields_dict(inputs)[fld_name].type is File + or isinstance(fld_value, os.PathLike) + or (isinstance(fld_value, str) and "." in fld_value) + ): + if file_template: + raise Exception( + f"can't have multiple paths in {field.name} template," + f" but {template} provided" ) + else: + file_template = (fld_name, fld_value) + else: + val_dict[fld_name] = fld_value + + # if field is MultiOutputFile and some elements from val_dict are lists, + # each element of the list should be used separately in the template + # and return a list with formatted values + if field.type is MultiOutputFile and any( + [isinstance(el, list) for el in val_dict.values()] + ): + # all fields that are lists + keys_list = [k for k, el in val_dict.items() if isinstance(el, list)] + if any( + [len(val_dict[key]) != len(val_dict[keys_list[0]]) for key in keys_list[1:]] + ): + raise Exception( + f"all fields used in {field.name} template have to have the same length" + f" or be a single value" + ) + formatted_value = [] + for ii in range(len(val_dict[keys_list[0]])): + val_dict_el = copy(val_dict) + # updating values to a single element from the list + for key in keys_list: + val_dict_el[key] = val_dict[key][ii] + + formatted_value.append( + _element_formatting( + template, val_dict_el, file_template, keep_extension=keep_extension ) - else: - formatted_value = _element_formatting( - template, - fld_name=fld_name, - fld_value=fld_value, - keep_extension=keep_extension, ) - return formatted_value else: - raise NotImplementedError("should we allow for more args in the template?") + formatted_value = _element_formatting( + template, val_dict, file_template, keep_extension=keep_extension + ) + return formatted_value -def _element_formatting(template, fld_name, fld_value, keep_extension): - """Formatting a single template for a single element of field value (if a list). - Taking into account that field values and template could have file extensions +def _element_formatting(template, values_template_dict, file_template, keep_extension): + """Formatting a single template for a single element (if a list). + Taking into account that a file used in the template (file_template) + and the template itself could have file extensions (assuming that if template has extension, the field value extension is removed, - if field has extension, and no template extension, than it is moved to the end), + if field has extension, and no template extension, than it is moved to the end). + For values_template_dict the simple formatting can be used (no file values inside) """ - fld_value_parent = Path(fld_value).parent - fld_value_name = Path(fld_value).name - - name, *ext = fld_value_name.split(".", maxsplit=1) - filename = str(fld_value_parent / name) - - # if keep_extension is False, the extensions are removed - if keep_extension is False: + if file_template: + fld_name_file, fld_value_file = file_template + # splitting the filename for name and extension, + # the final value used for formatting depends on the template and keep_extension flag + name, *ext = Path(fld_value_file).name.split(".", maxsplit=1) + filename = str(Path(fld_value_file).parent / name) + # updating values_template_dic with the name of file + values_template_dict[fld_name_file] = filename + # if keep_extension is False, the extensions are removed + if keep_extension is False: + ext = [] + else: ext = [] - if template.endswith(f"{{{fld_name}}}"): - # if no suffix added in template, the simplest formatting should work + + # if file_template is at the end of the template, the simplest formatting should work + if file_template and template.endswith(f"{{{fld_name_file}}}"): # recreating fld_value with the updated extension - fld_value_upd = ".".join([filename] + ext) - formatted_value = template.format(**{fld_name: fld_value_upd}) - elif "." not in template: # the template doesn't have its own extension - # if the fld_value has extension, it will be moved to the end - formatted_value = ".".join([template.format(**{fld_name: filename})] + ext) - else: # template has its own extension - # removing fld_value extension if any - formatted_value = template.format(**{fld_name: filename}) + values_template_dict[fld_name_file] = ".".join([filename] + ext) + formatted_value = template.format(**values_template_dict) + # file_template provided, but the template doesn't have its own extension + elif file_template and "." not in template: + # if the fld_value_file has extension, it will be moved to the end + formatted_value = ".".join([template.format(**values_template_dict)] + ext) + # template has its own extension or no file_template provided + # the simplest formatting, if file_template is provided it's used without the extension + else: + formatted_value = template.format(**values_template_dict) return formatted_value def is_local_file(f): - # breakpoint() from .specs import File, Directory, MultiInputFile if "container_path" not in f.metadata and ( diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 4bc8e52546..aedd08ac59 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -12,6 +12,14 @@ def attr_fields(spec, exclude_names=()): return [field for field in spec.__attrs_attrs__ if field.name not in exclude_names] +def attr_fields_dict(spec, exclude_names=()): + return { + field.name: field + for field in spec.__attrs_attrs__ + if field.name not in exclude_names + } + + class File: """An :obj:`os.pathlike` object, designating a file.""" @@ -532,9 +540,8 @@ def _field_metadata(self, fld, inputs, output_dir, outputs=None): # if the field is set in input_spec with output_file_template, # than the field already should have value elif "output_file_template" in fld.metadata: - inputs_templ = attr.asdict(inputs) value = template_update_single( - fld, inputs_templ, output_dir=output_dir, spec_type="output" + fld, inputs=inputs, output_dir=output_dir, spec_type="output" ) if fld.type is MultiOutputFile and type(value) is list: return [Path(val) for val in value] diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 0d60e1ad07..e6af16a278 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -1055,6 +1055,48 @@ def test_shell_cmd_inputspec_7b(plugin, results_function, tmpdir): assert res.output.out1.exists() +@pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) +def test_shell_cmd_inputspec_7c(plugin, results_function, tmpdir): + """ + providing output name using input_spec, + using name_tamplate with txt extension (extension from args should be removed + """ + cmd = "touch" + args = "newfile_tmp.txt" + + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "out1", + attr.ib( + type=str, + metadata={ + "output_file_template": "{args}.txt", + "help_string": "output file", + }, + ), + ) + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + args=args, + input_spec=my_input_spec, + cache_dir=tmpdir, + ) + + res = results_function(shelly, plugin) + assert res.output.stdout == "" + assert res.output.out1.exists() + # checking if the file is created in a good place + assert shelly.output_dir == res.output.out1.parent + assert res.output.out1.name == "newfile_tmp.txt" + + @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) def test_shell_cmd_inputspec_8(plugin, results_function, tmpdir): """ diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index c19671eae3..73a25be62d 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -82,6 +82,34 @@ def test_shell_cmd_inputs_1b(): assert shelly.cmdline == "executable inp-1 arg" +def test_shell_cmd_inputs_1_st(): + """ additional input with provided position, checking cmdline when splitter """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=str, + metadata={"position": 1, "help_string": "inp1", "argstr": ""}, + ), + ) + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + name="shelly", + executable="executable", + args="arg", + inpA=["inp1", "inp2"], + input_spec=my_input_spec, + ).split("inpA") + # cmdline should be a list + assert shelly.cmdline[0] == "executable inp1 arg" + assert shelly.cmdline[1] == "executable inp2 arg" + + def test_shell_cmd_inputs_2(): """ additional inputs with provided positions """ my_input_spec = SpecInfo( @@ -537,6 +565,32 @@ def test_shell_cmd_inputs_format_2(): assert shelly.cmdline == "executable -v el_1 -v el_2" +def test_shell_cmd_inputs_format_3(): + """ adding float formatting for argstr with input field""" + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=float, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "-v {inpA:.5f}", + }, + ), + ) + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + executable="executable", inpA=0.007, input_spec=my_input_spec + ) + assert shelly.cmdline == "executable -v 0.00700" + + def test_shell_cmd_inputs_mandatory_1(): """ additional inputs with mandatory=True""" my_input_spec = SpecInfo( @@ -1304,6 +1358,301 @@ def test_shell_cmd_inputs_template_8(tmpdir): ) +def test_shell_cmd_inputs_template_9(tmpdir): + """ additional inputs, one uses output_file_template with two fields: + one File and one ints - the output should be recreated from the template + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "inpInt", + attr.ib( + type=int, + metadata={ + "position": 2, + "help_string": "inp int", + "argstr": "-i", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 3, + "help_string": "outA", + "argstr": "-o", + "output_file_template": "{inpA}_{inpInt}_out.txt", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("inpA.t") + inpA_file.write("content") + + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file, inpInt=3 + ) + + assert ( + shelly.cmdline + == f"executable {tmpdir.join('inpA.t')} -i 3 -o {str(shelly.output_dir / 'inpA_3_out.txt')}" + ) + # checking if outA in the output fields + assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + + +def test_shell_cmd_inputs_template_9a(tmpdir): + """ additional inputs, one uses output_file_template with two fields: + one file and one string without extension - should be fine + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "inpStr", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "inp str", + "argstr": "-i", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 3, + "help_string": "outA", + "argstr": "-o", + "output_file_template": "{inpA}_{inpStr}_out.txt", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("inpA.t") + inpA_file.write("content") + + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=inpA_file, inpStr="hola" + ) + + assert ( + shelly.cmdline + == f"executable {tmpdir.join('inpA.t')} -i hola -o {str(shelly.output_dir / 'inpA_hola_out.txt')}" + ) + # checking if outA in the output fields + assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + + +def test_shell_cmd_inputs_template_9b_err(tmpdir): + """ output_file_template with two fields that are both Files, + an exception should be raised + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "inpFile", + attr.ib( + type=File, + metadata={ + "position": 2, + "help_string": "inp file", + "argstr": "-i", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 3, + "help_string": "outA", + "argstr": "-o", + "output_file_template": "{inpA}_{inpFile}_out.txt", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("inpA.t") + inpA_file.write("content") + + inpFile_file = tmpdir.join("inpFile.t") + inpFile_file.write("content") + + shelly = ShellCommandTask( + executable="executable", + input_spec=my_input_spec, + inpA=inpA_file, + inpFile=inpFile_file, + ) + # the template has two files so the exception should be raised + with pytest.raises(Exception, match="can't have multiple paths"): + shelly.cmdline + + +def test_shell_cmd_inputs_template_9c_err(tmpdir): + """ output_file_template with two fields: a file and a string with extension, + that should be used as an additional file and the exception should be raised + """ + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=File, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "", + "mandatory": True, + }, + ), + ), + ( + "inpStr", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "inp str with extension", + "argstr": "-i", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 3, + "help_string": "outA", + "argstr": "-o", + "output_file_template": "{inpA}_{inpStr}_out.txt", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + inpA_file = tmpdir.join("inpA.t") + inpA_file.write("content") + + shelly = ShellCommandTask( + executable="executable", + input_spec=my_input_spec, + inpA=inpA_file, + inpStr="hola.txt", + ) + # inptStr has an extension so should be treated as a second file in the template formatting + # and teh exception should be raised + with pytest.raises(Exception, match="can't have multiple paths"): + shelly.cmdline + + +def test_shell_cmd_inputs_template_10(): + """ output_file_template uses a float field with formatting""" + my_input_spec = SpecInfo( + name="Input", + fields=[ + ( + "inpA", + attr.ib( + type=float, + metadata={ + "position": 1, + "help_string": "inpA", + "argstr": "{inpA:.1f}", + "mandatory": True, + }, + ), + ), + ( + "outA", + attr.ib( + type=str, + metadata={ + "position": 2, + "help_string": "outA", + "argstr": "-o", + "output_file_template": "file_{inpA:.1f}_out", + }, + ), + ), + ], + bases=(ShellSpec,), + ) + + shelly = ShellCommandTask( + executable="executable", input_spec=my_input_spec, inpA=3.3456 + ) + # outA has argstr in the metadata fields, so it's a part of the command line + # the full path will be use din the command line + assert ( + shelly.cmdline == f"executable 3.3 -o {str(shelly.output_dir / 'file_3.3_out')}" + ) + # checking if outA in the output fields + assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + + # TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_di(tmpdir, use_validator): """ example from #279 """ From 911918751c14d4d4b7f56c98b4d2ce1324378e6f Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 19 Mar 2021 12:50:24 -0400 Subject: [PATCH 269/271] changing hashing function for numpy objects: tostring/tobytes was not working for dtype=object, so using tolist to create hash values for elements --- pydra/engine/helpers.py | 5 ++- pydra/engine/tests/test_numpy_examples.py | 43 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index a0a5114b75..be16e00886 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -695,7 +695,10 @@ def hash_value(value, tp=None, metadata=None, precalculated=None): ): return hash_dir(value, precalculated=precalculated) elif type(value).__module__ == "numpy": # numpy objects - return sha256(value.tostring()).hexdigest() + return [ + hash_value(el, tp, metadata, precalculated) + for el in ensure_list(value.tolist()) + ] else: return value diff --git a/pydra/engine/tests/test_numpy_examples.py b/pydra/engine/tests/test_numpy_examples.py index 72895a64ae..b3709c467d 100644 --- a/pydra/engine/tests/test_numpy_examples.py +++ b/pydra/engine/tests/test_numpy_examples.py @@ -2,10 +2,13 @@ import typing as ty import importlib import pytest +import pickle as pk from ..submitter import Submitter from ..core import Workflow from ...mark import task, annotate +from .utils import identity +from ..helpers import hash_value if importlib.util.find_spec("numpy") is None: pytest.skip("can't find numpy library", allow_module_level=True) @@ -51,3 +54,43 @@ def test_multiout_st(tmpdir): assert results[0] == {"wf.val": [0, 1, 2]} for el in range(3): assert np.array_equal(results[1].output.array[el], np.array([el, el])) + + +def test_numpy_hash_1(): + """hashing check for numeric numpy array""" + A = np.array([1, 2]) + A_pk = pk.loads(pk.dumps(A)) + assert (A == A_pk).all() + assert hash_value(A) == hash_value(A_pk) + + +def test_numpy_hash_2(): + """hashing check for numpy array of type object""" + A = np.array([["NDAR"]], dtype=object) + A_pk = pk.loads(pk.dumps(A)) + assert (A == A_pk).all() + assert hash_value(A) == hash_value(A_pk) + + +def test_task_numpyinput_1(tmpdir): + """ task with numeric numpy array as an input""" + nn = identity(name="NA", x=[np.array([1, 2]), np.array([3, 4])]) + nn.cache_dir = tmpdir + nn.split("x") + # checking the results + results = nn() + assert (results[0].output.out == np.array([1, 2])).all() + assert (results[1].output.out == np.array([3, 4])).all() + + +def test_task_numpyinput_2(tmpdir): + """ task with numpy array of type object as an input""" + nn = identity( + name="NA", + x=[np.array(["VAL1"], dtype=object), np.array(["VAL2"], dtype=object)], + ) + nn.cache_dir = tmpdir + nn.split("x") + # checking the results + results = nn() + assert (results[0].output.out == np.array(["VAL1"], dtype=object)).all() From a0937e4c4d52038b17327101294593263a7c390c Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Thu, 15 Apr 2021 12:52:42 -0400 Subject: [PATCH 270/271] make token clear in publish (#449) --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6f80ac01f5..84f3ac6b2e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,8 +24,8 @@ jobs: - name: Build and publish env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python setup.py bdist_wheel twine upload dist/* From d721046d54ac7b641486050f4d757fc59007cdca Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 15 Apr 2021 21:48:32 -0400 Subject: [PATCH 271/271] fixing __call__ function, so it updates inputs when kwargs provided --- pydra/engine/core.py | 2 +- pydra/engine/tests/test_workflow.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 317c45ac1f..6f5fded68f 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -422,6 +422,7 @@ def __call__( if submitter: with submitter as sub: + self.inputs = attr.evolve(self.inputs, **kwargs) res = sub(self) else: # tasks without state could be run without a submitter res = self._run(rerun=rerun, **kwargs) @@ -961,7 +962,6 @@ def create_connections(self, task, detailed=False): ) async def _run(self, submitter=None, rerun=False, **kwargs): - # self.inputs = dc.replace(self.inputs, **kwargs) don't need it? # output_spec needs to be set using set_output or at workflow initialization if self.output_spec is None: raise ValueError( diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 10439bdfe8..c0a30ea46f 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -158,6 +158,26 @@ def test_wf_1_call_exception(plugin, tmpdir): assert "Specify submitter OR plugin" in str(e.value) +def test_wf_1_inp_in_call(tmpdir): + """ Defining input in __call__""" + wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) + wf.add(add2(name="add2", x=wf.lzin.x)) + wf.set_output([("out", wf.add2.lzout.out)]) + wf.inputs.x = 1 + results = wf(x=2) + assert 4 == results.output.out + + +def test_wf_1_upd_in_run(tmpdir): + """ Updating input in __call__ """ + wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) + wf.add(add2(name="add2", x=wf.lzin.x)) + wf.set_output([("out", wf.add2.lzout.out)]) + wf.inputs.x = 1 + results = wf(x=2) + assert 4 == results.output.out + + def test_wf_2(plugin, tmpdir): """ workflow with 2 tasks, no splitter""" wf = Workflow(name="wf_2", input_spec=["x", "y"]) @@ -554,6 +574,27 @@ def test_wf_st_1_call_noplug_nosubm(plugin, tmpdir): assert odir.exists() +def test_wf_st_1_inp_in_call(tmpdir): + """ Defining input in __call__""" + wf = Workflow(name="wf_spl_1", input_spec=["x"], cache_dir=tmpdir).split("x") + wf.add(add2(name="add2", x=wf.lzin.x)) + wf.set_output([("out", wf.add2.lzout.out)]) + results = wf(x=[1, 2]) + assert results[0].output.out == 3 + assert results[1].output.out == 4 + + +def test_wf_st_1_upd_inp_call(tmpdir): + """ Updating input in __call___""" + wf = Workflow(name="wf_spl_1", input_spec=["x"], cache_dir=tmpdir).split("x") + wf.add(add2(name="add2", x=wf.lzin.x)) + wf.inputs.x = [11, 22] + wf.set_output([("out", wf.add2.lzout.out)]) + results = wf(x=[1, 2]) + assert results[0].output.out == 3 + assert results[1].output.out == 4 + + def test_wf_st_noinput_1(plugin, tmpdir): """ Workflow with one task, a splitter for the workflow""" wf = Workflow(name="wf_spl_1", input_spec=["x"])