Bump luigi-tools

pyproject.toml

@@ -6,7 +6,6 @@ target-version = ["py38"]
 # PYLINT
 [tool.pylint.messages-control]
 disable=[
-	"bad-continuation",
 	"duplicate-code",
 	"empty-docstring",
 	"fixme",

requirements/base.pip

@@ -7,7 +7,7 @@ diameter_synthesis>=0.4.1,<1
 gitpython
 jinja2
 joblib
-luigi
+luigi>=3.1
 luigi-tools>=0.0.15
 matplotlib
 morph_tool>=2.9.0,<3

src/synthesis_workflow/tasks/circuit.py

@@ -5,6 +5,12 @@
 import yaml
 from atlas_analysis.planes.planes import load_planes_centerline
 from atlas_analysis.planes.planes import save_planes_centerline
+from luigi.parameter import OptionalPathParameter
+from luigi.parameter import PathParameter
+from luigi_tools.parameter import RatioParameter
+from luigi_tools.task import ParamRef
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import copy_params
 from voxcell import VoxelData
 
 from synthesis_workflow.circuit import build_circuit
@@ -20,12 +26,6 @@
 from synthesis_workflow.tasks.config import GetCellComposition
 from synthesis_workflow.tasks.config import GetSynthesisInputs
 from synthesis_workflow.tasks.config import SynthesisConfig
-from synthesis_workflow.tasks.luigi_tools import OptionalPathParameter
-from synthesis_workflow.tasks.luigi_tools import ParamRef
-from synthesis_workflow.tasks.luigi_tools import PathParameter
-from synthesis_workflow.tasks.luigi_tools import RatioParameter
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import copy_params
 
 
 class CreateAtlasLayerAnnotations(WorkflowTask):

src/synthesis_workflow/tasks/cli.py

@@ -52,6 +52,37 @@ def _process_param(param):
     return param_doc, param_type, choices, interval, optional
 
 
+def format_description(
+    param,
+    default_str="{doc} Default value: {default}.",
+    optional_str="(optional) {doc}",
+    type_str="({type}) {doc}",
+    choices_str="{doc} Choices: {choices}.",
+    interval_str="{doc} Permitted values: {interval}.",
+    param_no_value=None,
+):
+    """Format the description of a parameter."""
+    if param_no_value is None:
+        param_no_value = _PARAM_NO_VALUE
+
+    try:
+        param_doc, param_type, choices, interval, optional = _process_param(param)
+        if optional:
+            param_doc = optional_str + param_doc
+        if param_type is not None:
+            param_doc = type_str.format(doc=param_doc, type=param_type.replace(":", ""))
+        if choices is not None:
+            param_doc = choices_str.format(doc=param_doc, choices=choices)
+        if interval is not None:
+            param_doc = interval_str.format(doc=param_doc, interval=interval)
+        # pylint: disable=protected-access
+        if hasattr(param, "_default") and param._default not in param_no_value:
+            param_doc = default_str.format(doc=param_doc, default=param._default)
+    except AttributeError:
+        param_doc = param.description
+    return param_doc
+
+
 class ArgParser:
     """Class to build parser and parse arguments."""
 
@@ -102,8 +133,8 @@ def _get_parsers(self):
             "--create-dependency-graph",
             help=(
                 "Create the dependency graph of a workflow instead of running it. "
-                "Pass either 'ascii' to print the graph to screen or a path to render "
-                "it as an image (depending on the extension of the given path)."
+                "Pass a path to render it as an image (depending on the extension of the given "
+                "path)."
             ),
         )
 
@@ -126,37 +157,28 @@ def _get_workflow_parsers(parser=None):
 
         workflow_parser = parser.add_subparsers(help="Possible workflows", dest="workflow")
 
-        def format_description(param):
-            try:
-                param_doc, param_type, choices, interval, optional = _process_param(param)
-                if optional:
-                    param_doc = "(optional) " + param_doc
-                if param_type is not None:
-                    param_type = f"({param_type.replace(':', '')})"
-                    param_doc = f"{param_type} {param_doc}"
-                if choices is not None:
-                    param_doc = f"{param_doc} Choices: {choices}."
-                if interval is not None:
-                    param_doc = f"{param_doc} Permitted values: {interval}."
-                # pylint: disable=protected-access
-                if hasattr(param, "_default") and param._default not in _PARAM_NO_VALUE:
-                    param_doc = f"{param_doc} Default value: {param._default}."
-            except AttributeError:
-                param_doc = param.description
-            return param_doc
-
         for workflow_name, task in WORKFLOW_TASKS.items():
             try:
                 task_name = task.__name__
                 doc = task.__doc__
+                if ".. graphviz::" in doc:
+                    doc = re.sub(
+                        (
+                            "The complete phase has the following dependency graph:"
+                            r"\n\n    .. graphviz:: .*\.dot"
+                        ),
+                        "",
+                        doc,
+                        flags=re.DOTALL,
+                    ).strip()
                 subparser = workflow_parser.add_parser(workflow_name, help=doc)
                 for param, param_obj in task.get_params():
                     param_name = "--" + param.replace("_", "-")
                     subparser.add_argument(
                         param_name,
                         help=format_description(param_obj),
                         # pylint: disable=protected-access
-                        **param_obj._parser_kwargs(param_name, task_name),
+                        **param_obj._parser_kwargs(param, task_name),
                     )
                 parsers[workflow_name] = subparser
             except (AttributeError, TypeError):
@@ -176,6 +198,12 @@ def _setup_logging(log_level, log_file=None, log_file_level=None):
     setup_logging(log_level, log_file, log_file_level)
 
 
+def _build_parser():
+    """Build the parser."""
+    tmp = ArgParser().parser
+    return tmp
+
+
 def main(arguments=None):
     """Main function."""
     if arguments is None:
@@ -201,30 +229,36 @@ def main(arguments=None):
     luigi_config = {k: v for k, v in vars(args).items() if k in LUIGI_PARAMETERS}
 
     # Prepare workflow task and aguments
-    task = WORKFLOW_TASKS[args.workflow]
-    args_dict = {k: v for k, v in vars(args).items() if k in task.get_param_names()}
+    task_cls = WORKFLOW_TASKS[args.workflow]
+    args_dict = {k.split(task_cls.get_task_family() + "_")[-1]: v for k, v in vars(args).items()}
+    args_dict = {
+        k: v for k, v in args_dict.items() if v is not None and k in task_cls.get_param_names()
+    }
+    task = WORKFLOW_TASKS[args.workflow](**args_dict)
 
     # Export the dependency graph of the workflow instead of running it
     if args.create_dependency_graph is not None:
-        task = WORKFLOW_TASKS[args.workflow](**args_dict)
-        g = get_dependency_graph(task)
+        g = get_dependency_graph(task, allow_orphans=True)
 
         # Create URLs
         base_f = Path(inspect.getfile(synthesis_workflow)).parent
         node_kwargs = {}
         for _, child in g:
+            if child is None:
+                continue
             url = (
                 Path(inspect.getfile(child.__class__)).relative_to(base_f).with_suffix("")
                 / "index.html"
             )
             anchor = "#" + ".".join(child.__module__.split(".")[1:] + [child.__class__.__name__])
             node_kwargs[child] = {"URL": "../../" + url.as_posix() + anchor}
+
         dot = graphviz_dependency_graph(g, node_kwargs=node_kwargs)
         render_dependency_graph(dot, args.create_dependency_graph)
-        sys.exit()
+        return
 
     # Run the luigi task
-    luigi.build([WORKFLOW_TASKS[args.workflow](**args_dict)], **luigi_config)
+    luigi.build([task], **luigi_config)
 
 
 if __name__ == "__main__":

src/synthesis_workflow/tasks/config.py

@@ -8,12 +8,11 @@
 import luigi
 import yaml
 from git import Repo
-
-from synthesis_workflow.tasks.luigi_tools import ExtParameter
-from synthesis_workflow.tasks.luigi_tools import OptionalChoiceParameter
-from synthesis_workflow.tasks.luigi_tools import OptionalIntParameter
-from synthesis_workflow.tasks.luigi_tools import OutputLocalTarget
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
+from luigi.parameter import OptionalChoiceParameter
+from luigi.parameter import OptionalIntParameter
+from luigi_tools.parameter import ExtParameter
+from luigi_tools.target import OutputLocalTarget
+from luigi_tools.task import WorkflowTask
 
 # Add some warning filters
 warnings.filterwarnings("ignore", module="diameter_synthesis.build_diameters")

src/synthesis_workflow/tasks/diametrizer.py

@@ -15,17 +15,17 @@
 from diameter_synthesis.plotting import plot_distribution_fit
 from joblib import Parallel
 from joblib import delayed
+from luigi_tools.parameter import BoolParameter
+from luigi_tools.task import ParamRef
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import copy_params
 from morphio.mut import Morphology
 from neurom import load_morphologies
 from tqdm import tqdm
 
 from synthesis_workflow.tasks.config import DiametrizerConfig
 from synthesis_workflow.tasks.config import OutputLocalTarget
 from synthesis_workflow.tasks.config import RunnerConfig
-from synthesis_workflow.tasks.luigi_tools import BoolParameter
-from synthesis_workflow.tasks.luigi_tools import ParamRef
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import copy_params
 from synthesis_workflow.tools import update_morphs_df
 
 matplotlib.use("Agg")

src/synthesis_workflow/tasks/synthesis.py

@@ -9,6 +9,14 @@
 import pandas as pd
 import yaml
 from diameter_synthesis.build_models import build as build_diameter_models
+from luigi.parameter import OptionalPathParameter
+from luigi.parameter import PathParameter
+from luigi_tools.parameter import BoolParameter
+from luigi_tools.parameter import RatioParameter
+from luigi_tools.target import OutputLocalTarget
+from luigi_tools.task import ParamRef
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import copy_params
 from neurots import extract_input
 from placement_algorithm.app.compact_annotations import _collect_annotations
 from region_grower.synthesize_morphologies import SynthesizeMorphologies
@@ -31,14 +39,6 @@
 from synthesis_workflow.tasks.config import RunnerConfig
 from synthesis_workflow.tasks.config import SynthesisConfig
 from synthesis_workflow.tasks.config import SynthesisLocalTarget
-from synthesis_workflow.tasks.luigi_tools import BoolParameter
-from synthesis_workflow.tasks.luigi_tools import OptionalPathParameter
-from synthesis_workflow.tasks.luigi_tools import OutputLocalTarget
-from synthesis_workflow.tasks.luigi_tools import ParamRef
-from synthesis_workflow.tasks.luigi_tools import PathParameter
-from synthesis_workflow.tasks.luigi_tools import RatioParameter
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import copy_params
 from synthesis_workflow.tools import find_case_insensitive_file
 from synthesis_workflow.tools import load_neurondb_to_dataframe
 

src/synthesis_workflow/tasks/vacuum_synthesis.py

@@ -5,16 +5,16 @@
 import luigi
 import morphio
 import pandas as pd
+from luigi.parameter import PathParameter
+from luigi_tools.task import ParamRef
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import copy_params
 
 from synthesis_workflow.tasks.config import MorphsDfLocalTarget
 from synthesis_workflow.tasks.config import RunnerConfig
 from synthesis_workflow.tasks.config import SynthesisConfig
 from synthesis_workflow.tasks.config import SynthesisLocalTarget
 from synthesis_workflow.tasks.config import ValidationLocalTarget
-from synthesis_workflow.tasks.luigi_tools import ParamRef
-from synthesis_workflow.tasks.luigi_tools import PathParameter
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import copy_params
 from synthesis_workflow.tasks.synthesis import BuildSynthesisDistributions
 from synthesis_workflow.tasks.synthesis import BuildSynthesisParameters
 from synthesis_workflow.vacuum_synthesis import VACUUM_SYNTH_MORPHOLOGY_PATH

src/synthesis_workflow/tasks/validation.py

@@ -8,6 +8,14 @@
 import pkg_resources
 import yaml
 from atlas_analysis.planes.planes import load_planes_centerline
+from luigi.parameter import OptionalNumericalParameter
+from luigi.parameter import PathParameter
+from luigi_tools.parameter import BoolParameter
+from luigi_tools.task import ParamRef
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import WorkflowWrapperTask
+from luigi_tools.task import copy_params
+from luigi_tools.util import WorkflowError
 from neurom.view import matplotlib_impl
 from voxcell import CellCollection
 from voxcell import VoxelData
@@ -24,14 +32,6 @@
 from synthesis_workflow.tasks.config import SynthesisConfig
 from synthesis_workflow.tasks.config import ValidationConfig
 from synthesis_workflow.tasks.config import ValidationLocalTarget
-from synthesis_workflow.tasks.luigi_tools import BoolParameter
-from synthesis_workflow.tasks.luigi_tools import OptionalNumericalParameter
-from synthesis_workflow.tasks.luigi_tools import ParamRef
-from synthesis_workflow.tasks.luigi_tools import PathParameter
-from synthesis_workflow.tasks.luigi_tools import WorkflowError
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import WorkflowWrapperTask
-from synthesis_workflow.tasks.luigi_tools import copy_params
 from synthesis_workflow.tasks.synthesis import AddScalingRulesToParameters
 from synthesis_workflow.tasks.synthesis import ApplySubstitutionRules
 from synthesis_workflow.tasks.synthesis import BuildMorphsDF

src/synthesis_workflow/tasks/workflows.py

@@ -1,12 +1,12 @@
 """Luigi tasks for validation workflows."""
 import luigi
 import pandas as pd
+from luigi_tools.parameter import BoolParameter
+from luigi_tools.task import WorkflowTask
+from luigi_tools.task import WorkflowWrapperTask
 
 from synthesis_workflow.tasks.config import GetSynthesisInputs
 from synthesis_workflow.tasks.config import ValidationLocalTarget
-from synthesis_workflow.tasks.luigi_tools import BoolParameter
-from synthesis_workflow.tasks.luigi_tools import WorkflowTask
-from synthesis_workflow.tasks.luigi_tools import WorkflowWrapperTask
 from synthesis_workflow.tasks.synthesis import ApplySubstitutionRules
 from synthesis_workflow.tasks.vacuum_synthesis import PlotVacuumMorphologies
 from synthesis_workflow.tasks.validation import MorphologyValidationReports
@@ -137,7 +137,6 @@ class ValidateRescaling(WorkflowTask):
 
     def requires(self):
         """ """
-        # pylint: disable=no-self-use
         return ApplySubstitutionRules()
 
     def run(self):