Simplify doc and improve its generation

Change-Id: Ib52e7e36b7304eeb6384452438308e9c015e32d2

doc/source/conf.py

@@ -95,25 +95,78 @@
 
 # autodoc settings
 autodoc_typehints = "signature"
-autodoc_default_options = {"members": True}
+autodoc_default_options = {
+    "members": True,
+    "show-inheritance": True,
+}
 autoclass_content = "both"
 
+add_module_names = False
+
+
+import importlib
+import luigi
 import re
 
+import morphval
+import synthesis_workflow
+import synthesis_workflow.tasks
+from synthesis_workflow.tasks.cli import _PARAM_NO_VALUE
+from synthesis_workflow.tasks.cli import _process_param
+
+
 SKIP = [
     r".*\.L",
     r".*tasks\..*\.requires",
     r".*tasks\..*\.run",
     r".*tasks\..*\.output",
 ]
 
+IMPORT_MAPPING = {
+    "morphval": morphval,
+    "synthesis_workflow": synthesis_workflow,
+    "tasks": synthesis_workflow.tasks,
+}
+
 
 def maybe_skip_member(app, what, name, obj, skip, options):
     skip = None
     for pattern in SKIP:
         if re.match(pattern, name) is not None:
             skip = True
             break
+
+    if not skip:
+        try:
+            package, module, *path = name.split(".")
+            root_package = IMPORT_MAPPING[package]
+            actual_module = importlib.import_module(
+                root_package.__name__ + "." + module
+            )
+            task = getattr(actual_module, path[-2])
+            actual_obj = getattr(task, path[-1])
+            if isinstance(actual_obj, luigi.Parameter):
+                if hasattr(actual_obj, "description") and actual_obj.description:
+                    help_str, param_type, choices, interval, optional = _process_param(
+                        actual_obj
+                    )
+                    if optional:
+                        help_str = "(optional) " + help_str
+                    if param_type is not None:
+                        help_str += f"\n\n:type: {param_type}"
+                    if choices is not None:
+                        help_str += f"\n\n:choices: {choices}"
+                    if interval is not None:
+                        help_str += f"\n\n:permitted values: {interval}"
+                    if (
+                        hasattr(actual_obj, "_default")
+                        and actual_obj._default not in _PARAM_NO_VALUE
+                    ):
+                        help_str += f"\n\n:default value: {actual_obj._default}"
+                    obj.docstring = help_str
+        except:
+            pass
+
     return skip
 
 

requirements.pip

@@ -21,4 +21,4 @@ seaborn
 tns>=2.2.7
 tmd
 tqdm
-voxcell>=2.7.3
+voxcell>=2.7.3,<3

setup.py

@@ -38,7 +38,7 @@
     "tns>=2.2.7",
     "tmd",
     "tqdm",
-    "voxcell>=2.7.3",
+    "voxcell>=2.7.3,<3",
 ]
 
 doc_reqs = [

src/synthesis_workflow/tasks/circuit.py

@@ -34,17 +34,22 @@
 class CreateAtlasLayerAnnotations(WorkflowTask):
     """Create the annotation file for layers from an atlas."""
 
-    layer_annotations_path = luigi.Parameter(default="layer_annotation.nrrd")
-    """str: Path to save layer annotations constructed from atlas."""
-
-    use_half = BoolParameter(default=False)
-    """bool: Set to True to use half of the atlas (left or right hemisphere)."""
-
-    half_axis = luigi.IntParameter(default=0)
-    """int: Direction to select half of the atlas (can be 0, 1 or 2)."""
-
-    half_side = luigi.IntParameter(default=0)
-    """int: Side to choose to halve the atlas (0=left, 1=right)."""
+    layer_annotations_path = luigi.Parameter(
+        default="layer_annotation.nrrd",
+        description=":str: Path to save layer annotations constructed from atlas.",
+    )
+    use_half = BoolParameter(
+        default=False,
+        description=":bool: Set to True to use half of the atlas (left or right hemisphere).",
+    )
+    half_axis = luigi.IntParameter(
+        default=0,
+        description=":int: Direction to select half of the atlas (can be 0, 1 or 2).",
+    )
+    half_side = luigi.IntParameter(
+        default=0,
+        description=":int: Side to choose to halve the atlas (0=left, 1=right).",
+    )
 
     def run(self):
         """"""
@@ -88,55 +93,40 @@ class CreateAtlasPlanes(WorkflowTask):
     plane_type = luigi.ChoiceParameter(
         default="centerline",
         choices=["aligned", "centerline"],
-        description="Type of planes creation algorithm.",
+        description=(
+            ":str: Type of planes creation algorithm. It can take the value 'centerline', "
+            "so the center line is computed between first_bound and last_bound with internal "
+            "algorithm (from atlas-analysis package), or the value 'aligned' (warning: "
+            "experimental) so center line is a straight line, along the centerline_axis."
+        ),
     )
-    """str: Type of planes creation algorithm, two choices:
-
-        * centerline: centerline is computed between first_bound and last_bound with
-          internal algorithm (from atlas-analysis package), (warning: experimental)
-        * aligned: centerline is a straight line, along the centerline_axis
-    """
-
     plane_count = luigi.IntParameter(
-        default=10, description="Number of planes to create slices of atlas."
+        default=10, description=":int: Number of planes to create slices of atlas."
     )
-    """int: Number of planes to create slices of atlas."""
-
     slice_thickness = luigi.FloatParameter(
-        default=100, description="Thickness of slices (in micrometer)."
+        default=100, description=":float: Thickness of slices (in micrometer)."
     )
-    """float: Thickness of slices (in micrometer)."""
-
     centerline_first_bound = luigi.ListParameter(
         default=[126, 181, 220],
         description=(
-            "(only for plane_type == centerline) Location of first bound for centerline "
-            "(in voxcell index)."
+            ":list(int): (only for plane_type == centerline) Location of first bound for "
+            "centerline (in voxcell index)."
         ),
     )
-    """list: (only for plane_type == centerline) Location of first bound for centerline
-    (in voxcell index)."""
-
     centerline_last_bound = luigi.ListParameter(
         default=[407, 110, 66],
         description=(
-            "(only for plane_type == centerline) Location of last bound for centerline "
-            "(in voxcell index)."
+            ":list(int): (only for plane_type == centerline) Location of last bound for "
+            "centerline (in voxcell index)."
         ),
     )
-    """list: (only for plane_type == centerline) Location of last bound for centerline
-    (in voxcell index)."""
-
     centerline_axis = luigi.IntParameter(
         default=0,
-        description="(only for plane_type = aligned) Axis along which to create planes.",
+        description=":str: (only for plane_type = aligned) Axis along which to create planes.",
     )
-    """str: (only for plane_type = aligned) Axis along which to create planes."""
-
     atlas_planes_path = luigi.Parameter(
-        default="atlas_planes", description="Path to save atlas planes."
+        default="atlas_planes", description=":str: Path to save atlas planes."
     )
-    """str: Path to save atlas planes."""
 
     def requires(self):
         """"""
@@ -169,29 +159,24 @@ class BuildCircuit(WorkflowTask):
     """Generate cell positions and me-types from atlas, compositions and taxonomy.
 
     Attributes:
-        mtype_taxonomy_path (str): path to the taxonomy file (TSV)
+        mtype_taxonomy_path (str): Path to the taxonomy file (TSV).
     """
 
     cell_composition_path = luigi.Parameter(
         default="cell_composition.yaml",
-        description="Path to the cell composition file (YAML).",
+        description=":str: Path to the cell composition file (YAML).",
     )
-    """str: Path to the cell composition file (YAML)."""
-
     density_factor = RatioParameter(
         default=0.01,
         left_op=luigi.parameter.operator.lt,
-        description="The density of positions generated from the atlas.",
+        description=":float: The density of positions generated from the atlas.",
     )
-    """float: The density of positions generated from the atlas."""
-
     mask_path = luigi.Parameter(
-        default=None, description="Path to save thickness mask (NCx only)."
+        default=None, description=":str: Path to save thickness mask (NCx only)."
+    )
+    seed = luigi.IntParameter(
+        default=None, description=":int: Pseudo-random generator seed."
     )
-    """str: Path to save thickness mask (NCx only)."""
-
-    seed = luigi.IntParameter(default=None, description="Pseudo-random generator seed.")
-    """int: Pseudo-random generator seed."""
 
     def requires(self):
         """"""
@@ -234,16 +219,18 @@ class SliceCircuit(WorkflowTask):
         mtypes (list): List of mtypes to consider.
     """
 
-    sliced_circuit_path = luigi.Parameter(default="sliced_circuit_somata.mvd3")
-    """str: Path to save sliced circuit somata mvd3."""
-
-    n_cells = luigi.IntParameter(default=10)
-    """int: Number of cells per mtype to consider."""
-
+    sliced_circuit_path = luigi.Parameter(
+        default="sliced_circuit_somata.mvd3",
+        description=":str: Path to save sliced circuit somata mvd3.",
+    )
+    n_cells = luigi.IntParameter(
+        default=10, description=":int: Number of cells per mtype to consider."
+    )
     hemisphere = OptionalChoiceParameter(
-        default=None, choices=["left", "right"], description="The hemisphere side."
+        default=None,
+        choices=["left", "right"],
+        description=":str: The hemisphere side.",
     )
-    """str: (optional) The hemisphere side."""
 
     def requires(self):
         """"""

src/synthesis_workflow/tasks/cli.py

@@ -29,6 +29,28 @@
 LUIGI_PARAMETERS = ["workers", "local_scheduler", "log_level"]
 
 
+_PARAM_NO_VALUE = [luigi.parameter._no_value, None]  # pylint: disable=protected-access
+
+
+def _process_param(param):
+    desc = param.description
+    choices = None
+    interval = None
+    optional = False
+    if isinstance(param, luigi.OptionalParameter):
+        optional = True
+    if isinstance(param, luigi.ChoiceParameter):
+        desc, choices = desc.rsplit("Choices: ", 1)
+    if isinstance(param, luigi.NumericalParameter):
+        desc, interval = desc.rsplit("permitted values: ", 1)
+    try:
+        param_type, param_doc = re.match("(:.*?:)? *(.*)", desc).groups()
+    except AttributeError:
+        param_type = None
+        param_doc = desc
+    return param_doc, param_type, choices, interval, optional
+
+
 class ArgParser:
     """Class to build parser and parse arguments."""
 
@@ -105,6 +127,27 @@ def _get_workflow_parsers(parser=None):
             help="Possible workflows", dest="workflow"
         )
 
+        def format_description(param):
+            try:
+                param_doc, param_type, choices, interval, optional = _process_param(
+                    param
+                )
+                if optional:
+                    param_doc = "(optional) " + param_doc
+                if param_type is not None:
+                    param_type = f"({param_type.replace(':', '')})"
+                    param_doc = f"{param_type} {param_doc}"
+                if choices is not None:
+                    param_doc = f"{param_doc} Choices: {choices}."
+                if interval is not None:
+                    param_doc = f"{param_doc} Permitted values: {interval}."
+                # pylint: disable=protected-access
+                if hasattr(param, "_default") and param._default not in _PARAM_NO_VALUE:
+                    param_doc = f"{param_doc} Default value: {param._default}."
+            except AttributeError:
+                param_doc = param.description
+            return param_doc
+
         for workflow_name, task in WORKFLOW_TASKS.items():
             try:
                 task_name = task.__name__
@@ -114,9 +157,9 @@ def _get_workflow_parsers(parser=None):
                     param_name = "--" + param.replace("_", "-")
                     subparser.add_argument(
                         param_name,
-                        help=param_obj.description,
+                        help=format_description(param_obj),
                         # pylint: disable=protected-access
-                        **param_obj._parser_kwargs(param_name, task_name)
+                        **param_obj._parser_kwargs(param_name, task_name),
                     )
                 parsers[workflow_name] = subparser
             except (AttributeError, TypeError):
@@ -215,7 +258,7 @@ def main(arguments=None):
             dot.edge(
                 parent.__class__.__name__,
                 child.__class__.__name__,
-                **default_edge_attrs
+                **default_edge_attrs,
             )
         filepath = Path(args.create_dependency_graph)
         filename = filepath.with_suffix("")