Feat: Add -dgdpi for workflow diagram and --version and fix logger

.gitignore

@@ -1,6 +1,8 @@
 # data
 .eggs
 .idea
+*.log
+*.png
 *.pyc
 *.swp
 *py~

MANIFEST.in

@@ -4,3 +4,5 @@ include src/version.py
 include src/morphval/templates/*
 include src/synthesis_workflow/defaults/*
 include requirements/*.pip
+
+recursive-include src/synthesis_workflow/_templates/ *.cfg *.conf *.json *.py

examples/rat_vaccum/logging.conf

@@ -8,7 +8,7 @@ keys=consoleHandler,fileHandler
 keys=PrettyFormatter
 
 [logger_root]
-level=DEBUG
+level=INFO
 handlers=consoleHandler,fileHandler
 
 [logger_luigi]
@@ -18,7 +18,7 @@ qualname=luigi
 propagate=0
 
 [logger_luigi_interface]
-level=DEBUG
+level=INFO
 handlers=consoleHandler,fileHandler
 qualname=luigi-interface
 propagate=0

requirements/test.pip

@@ -1,7 +1,9 @@
 dir-content-diff>=1
 dir-content-diff-plugins>=0.0.3
+graphviz>=0.14
 pandas>=1.5
 pytest>=6.2
+pytest-console-scripts>=1.3
 pytest-cov>=3
 pytest-html>=3
 pytest-xdist>=3.0.2

setup.py

@@ -49,8 +49,8 @@
     },
     entry_points={
         "console_scripts": [
-            "synthesis_workflow=synthesis_workflow.tasks.cli:main",
-            "morph_validation=morphval.cli:main",
+            "synthesis-workflow=synthesis_workflow.tasks.cli:main",
+            "morph-validation=morphval.cli:main",
         ],
     },
     include_package_data=True,

src/synthesis_workflow/_templates/logging.conf

@@ -0,0 +1,38 @@
+[loggers]
+keys=root,luigi,luigi_interface
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=PrettyFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler,fileHandler
+
+[logger_luigi]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=luigi
+propagate=0
+
+[logger_luigi_interface]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=luigi-interface
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+formatter=PrettyFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+formatter=PrettyFormatter
+args=('logs.log',)
+
+[formatter_PrettyFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

src/synthesis_workflow/tasks/cli.py

@@ -14,7 +14,7 @@
 
 import synthesis_workflow
 from synthesis_workflow.tasks import workflows
-from synthesis_workflow.utils import setup_logging
+from synthesis_workflow.utils import _TEMPLATES
 
 L = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@
 
 LOGGING_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 
-LUIGI_PARAMETERS = ["workers", "local_scheduler", "log_level"]
+LUIGI_PARAMETERS = ["workers", "log_level"]
 
 
 _PARAM_NO_VALUE = [luigi.parameter._no_value, None]  # pylint: disable=protected-access
@@ -97,17 +97,22 @@ def parser(self):
     def _get_parsers(self):
         """Return the main argument parser."""
         parser = argparse.ArgumentParser(
-            description="Run the synthesis workflow",
+            description="Run the workflow",
+        )
+        parser.add_argument(
+            "--version",
+            action="version",
+            version=f"%(prog)s, version {synthesis_workflow.__version__}",
         )
 
-        parser.add_argument("-c", "--config-path", help="Path to the Luigi config file")
+        parser.add_argument("-c", "--config-path", help="Path to the Luigi config file.")
 
         parser.add_argument(
-            "-l",
-            "--local-scheduler",
+            "-m",
+            "--master-scheduler",
             default=False,
             action="store_true",
-            help="Use Luigi's local scheduler instead of master scheduler.",
+            help="Use Luigi's master scheduler instead of local scheduler.",
         )
 
         parser.add_argument(
@@ -138,6 +143,12 @@ def _get_parsers(self):
             ),
         )
 
+        parser.add_argument(
+            "-dgdpi",
+            "--dependency-graph-dpi",
+            help="The DPI used for the dependency graph export.",
+        )
+
         return self._get_workflow_parsers(parser)
 
     @staticmethod
@@ -171,7 +182,7 @@ def _get_workflow_parsers(parser=None):
                         doc,
                         flags=re.DOTALL,
                     ).strip()
-                subparser = workflow_parser.add_parser(workflow_name, help=doc)
+                subparser = workflow_parser.add_parser(workflow_name, help=doc, description=doc)
                 for param, param_obj in task.get_params():
                     param_name = "--" + param.replace("_", "-")
                     subparser.add_argument(
@@ -193,23 +204,57 @@ def parse_args(self, argv):
         return args
 
 
-def _setup_logging(log_level, log_file=None, log_file_level=None):
-    """Setup logging."""
-    setup_logging(log_level, log_file, log_file_level)
-
-
 def _build_parser():
     """Build the parser."""
     tmp = ArgParser().parser
     return tmp
 
 
+def export_dependency_graph(task, output_file, dpi=None):
+    """Export the dependency graph of the given task."""
+    g = get_dependency_graph(task, allow_orphans=True)
+
+    # Create URLs
+    base_f = Path(inspect.getfile(synthesis_workflow)).parent
+    node_kwargs = {}
+    for _, child in g:
+        if child is None:
+            continue
+        url = (
+            Path(inspect.getfile(child.__class__)).relative_to(base_f).with_suffix("")
+            / "index.html"
+        )
+        anchor = "#" + ".".join(child.__module__.split(".")[1:] + [child.__class__.__name__])
+        node_kwargs[child] = {"URL": "../../" + url.as_posix() + anchor}
+
+    graph_attrs = {}
+    if dpi is not None:
+        graph_attrs["dpi"] = dpi
+    dot = graphviz_dependency_graph(g, node_kwargs=node_kwargs, graph_attrs=graph_attrs)
+    render_dependency_graph(dot, output_file)
+
+
 def main(arguments=None):
     """Main function."""
-    if arguments is None:
-        arguments = sys.argv[1:]
+    # Setup logging
+    logging.getLogger("luigi").propagate = False
+    logging.getLogger("luigi-interface").propagate = False
+    luigi_config = luigi.configuration.get_config()
+    logging_conf = luigi_config.get("core", "logging_conf_file", None)
+    if logging_conf is not None and not Path(logging_conf).exists():
+        L.warning(
+            "The core->logging_conf_file entry is not a valid path so the default logging "
+            "configuration is taken."
+        )
+        logging_conf = None
+    if logging_conf is None:
+        logging_conf = str(_TEMPLATES / "logging.conf")
+        luigi_config.set("core", "logging_conf_file", logging_conf)
+    logging.config.fileConfig(str(logging_conf), disable_existing_loggers=False)
 
     # Parse arguments
+    if arguments is None:
+        arguments = sys.argv[1:]
     parser = ArgParser()
     args = parser.parse_args(arguments)
 
@@ -219,42 +264,26 @@ def main(arguments=None):
     if args is None or args.workflow is None:
         L.critical("Arguments must contain one workflow. Check help with -h/--help argument.")
         parser.parser.print_help()
-        sys.exit()
+        return
 
     # Set luigi.cfg path
     if args.config_path is not None:
         os.environ["LUIGI_CONFIG_PATH"] = args.config_path
 
     # Get arguments to configure luigi
     luigi_config = {k: v for k, v in vars(args).items() if k in LUIGI_PARAMETERS}
+    luigi_config["local_scheduler"] = not args.master_scheduler
 
     # Prepare workflow task and arguments
     task_cls = WORKFLOW_TASKS[args.workflow]
     args_dict = {k.split(task_cls.get_task_family() + "_")[-1]: v for k, v in vars(args).items()}
-    args_dict = {
-        k: v for k, v in args_dict.items() if v is not None and k in task_cls.get_param_names()
-    }
+    task_params = [i for i, j in task_cls.get_params()]
+    args_dict = {k: v for k, v in args_dict.items() if v is not None and k in task_params}
     task = WORKFLOW_TASKS[args.workflow](**args_dict)
 
     # Export the dependency graph of the workflow instead of running it
     if args.create_dependency_graph is not None:
-        g = get_dependency_graph(task, allow_orphans=True)
-
-        # Create URLs
-        base_f = Path(inspect.getfile(synthesis_workflow)).parent
-        node_kwargs = {}
-        for _, child in g:
-            if child is None:
-                continue
-            url = (
-                Path(inspect.getfile(child.__class__)).relative_to(base_f).with_suffix("")
-                / "index.html"
-            )
-            anchor = "#" + ".".join(child.__module__.split(".")[1:] + [child.__class__.__name__])
-            node_kwargs[child] = {"URL": "../../" + url.as_posix() + anchor}
-
-        dot = graphviz_dependency_graph(g, node_kwargs=node_kwargs)
-        render_dependency_graph(dot, args.create_dependency_graph)
+        export_dependency_graph(task, args.create_dependency_graph, dpi=args.dependency_graph_dpi)
         return
 
     # Run the luigi task

src/synthesis_workflow/utils.py

@@ -1,14 +1,23 @@
 """Utils functions."""
 import json
 import logging
+from pathlib import Path
 
 import dictdiffer
 import numpy as np
 import pandas as pd
 from jsonpath_ng import parse
+from pkg_resources import resource_filename
 
 # pylint:disable=too-many-nested-blocks
 
+_TEMPLATES = Path(
+    resource_filename(
+        "synthesis_workflow",
+        "_templates",
+    )
+)
+
 
 class DisableLogger:
     """Context manager to disable logging."""
@@ -38,8 +47,8 @@ def setup_logging(
 ):
     """Setup logging."""
     if logger is None:
-        root = logging.getLogger()
-        root.setLevel(logging.DEBUG)
+        logger = logging.getLogger()
+        logger.setLevel(logging.DEBUG)
 
     # Setup logging formatter
     if log_format is None:
@@ -52,7 +61,7 @@ def setup_logging(
     console = logging.StreamHandler()
     console.setFormatter(formatter)
     console.setLevel(log_level)
-    root.addHandler(console)
+    logger.addHandler(console)
 
     # Setup file logging handler
     if log_file is not None:
@@ -61,7 +70,7 @@ def setup_logging(
         fh = logging.FileHandler(log_file, mode="w")
         fh.setLevel(log_file_level)
         fh.setFormatter(formatter)
-        root.addHandler(fh)
+        logger.addHandler(fh)
 
 
 def create_parameter_diff(param, param_spec):

tests/test_cli.py

@@ -0,0 +1,47 @@
+"""Tests for the synthesis_workflow.cli module."""
+import re
+
+from synthesis_workflow.tasks import cli
+
+
+class TestCLI:
+    """Test the CLI of the synthesis-workflow package."""
+
+    def test_help(self, capsys):
+        """Test the --help argument."""
+        try:
+            cli.main(arguments=["--help"])
+        except SystemExit:
+            pass
+        captured = capsys.readouterr()
+        assert (
+            re.match(
+                r"usage: \S+ .*Run the workflow\n\npositional arguments:\s*"
+                r"{ValidateSynthesis,ValidateVacuumSynthesis,ValidateRescaling}\s*"
+                r"Possible workflows.*",
+                captured.out,
+                flags=re.DOTALL,
+            )
+            is not None
+        )
+
+    def test_dependency_graph(self, vacuum_working_directory):
+        """Test the --create-dependency-graph argument."""
+        root_dir = vacuum_working_directory[0].parent
+        cli.main(
+            arguments=[
+                "--create-dependency-graph",
+                str(root_dir / "dependency_graph.png"),
+                "ValidateVacuumSynthesis",
+            ]
+        )
+
+        assert (root_dir / "dependency_graph.png").exists()
+
+
+def test_entry_point(script_runner):
+    """Test the entry point."""
+    ret = script_runner.run("synthesis-workflow", "--version")
+    assert ret.success
+    assert ret.stdout.startswith("synthesis-workflow, version ")
+    assert ret.stderr == ""