Improve doc and tests

Change-Id: I07a21b25697ed4e0a13f0f2e9aa28f2e305ec79a

.gitignore

@@ -10,3 +10,5 @@ dist
 .eggs
 
 reports
+doc/source/autoapi/
+doc/_images

README.rst

@@ -1,17 +1,82 @@
-synthesis-workflow
+Synthesis Workflow
 ==================
 
-Workflow used for synthesis and its validation.
+This project contains several workflows used for neuron synthesis and the validation of this process.
+It is devided into two packages:
+
+	* **synthesis-workflow**, which contains the workflow tasks and tools.
+	* **MorphVal**, which is a library used for morphology validation and can be used as a standalone.
 
 
 Installation
 ------------
-In a fresh virtualenv:
 
-.. code:: bash
+This package rely on several internal requirements which can be found in the BBP devpi index.
+This index must be specified to pip:
+
+.. code::
 
     pip install --index-url https://bbpteam.epfl.ch/repository/devpi/simple synthesis-workflow
 
 
 Usage
 -----
+
+Synthesis workflow
+~~~~~~~~~~~~~~~~~~
+
+The usual command is the following:
+
+.. code::
+
+    synthesis_workflow <workflow>
+
+You can get help and complete parameter description with the following commands:
+
+.. code::
+
+    synthesis_workflow --help
+    synthesis_workflow <workflow> --help
+
+You can also run a complete ``luigi`` command in order to fine-control task parameters:
+
+.. code::
+
+    luigi --module synthesis_workflow.tasks.workflows --help
+    luigi --module synthesis_workflow.tasks.workflows <workflow> --help
+    luigi --module synthesis_workflow.tasks.workflows <workflow> [specific arguments]
+
+.. note::
+
+	The ``synthesis_workflow`` command (or the complete ``luigi`` command) must be
+	executed from a directory containing a ``luigi.cfg`` file.
+	A simple example of such file is given in the ``examples`` directory.
+	Complete examples for BBP usecases are provided here: `<https://bbpcode.epfl.ch/browse/code/project/proj82/tree/entities/synthesis_workflow?h=refs/heads/master>`_
+
+Morphology validation
+~~~~~~~~~~~~~~~~~~~~~
+
+The usual command is the following:
+
+.. code::
+
+    morph_validation -t <path to reference data> -r <path to test data> -o <output path> -c <YAML config file> --bio-compare
+
+You can get help and complete parameter description with the following command:
+
+.. code::
+
+    morph_validation --help
+
+Workflows
+---------
+
+The package ``synthesis-workflow`` contains many tasks that are organized in workflows.
+The two main workflows are the following:
+
+    * ``synthesis-workflow.tasks.ValidateVacuumSynthesis``: this workflow generates cells
+      in vacuum (e.g. with no atlas information) and compute some simple validation features.
+    * ``synthesis-workflow.tasks.ValidateSynthesis``: this workflow generates cells in a
+      given atlas and compute many validation features.
+
+More details on these workflows and their tasks can be found in the API Reference.

doc/Makefile

@@ -17,4 +17,18 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
+	# Generate images
+	mkdir -p source/autoapi/tasks/validation/ source/autoapi/tasks/vacuum_synthesis/
+	pdftocairo -png -r 300 -f 5 -l 5 ../tests/data/in_small_O1/out/validation/scales/statistics.pdf source/autoapi/tasks/validation/scale_statistics
+	pdftocairo -png -r 300 -f 1 -l 1 ../tests/data/in_small_O1/out/validation/collages/L3_TPC:A.pdf source/autoapi/tasks/validation/collages
+	pdftocairo -png -r 300 -f 1 -l 1 ../tests/data/in_small_O1/out/validation/path_distance_fit.pdf source/autoapi/tasks/validation/path_distance_fit
+	pdftocairo -png -r 300 -f 1 -l 1 ../tests/data/in_vacuum/out/validation/morphometrics/morphometrics_basal_dendrite.pdf source/autoapi/tasks/validation/morphometrics
+	pdftocairo -png -r 300 -f 1 -l 1 ../tests/data/in_vacuum/out/validation/vacuum_morphologies.pdf source/autoapi/tasks/vacuum_synthesis/vacuum_morphologies
+
+	# Generate dependency graphs
+	cd ../tests/data/in_small_O1; synthesis_workflow -dg ../../../doc/source/autoapi/tasks/workflows/ValidateSynthesis.dot ValidateSynthesis; cd ../../../doc
+	cd ../tests/data/in_small_O1; synthesis_workflow -dg ../../../doc/source/autoapi/tasks/workflows/ValidateRescaling.dot ValidateRescaling; cd ../../../doc
+	cd ../tests/data/in_vacuum; synthesis_workflow -dg ../../../doc/source/autoapi/tasks/workflows/ValidateVacuumSynthesis.dot ValidateVacuumSynthesis; cd ../../../doc
+
+	# Generate the documentation
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/source/api_ref.rst

@@ -0,0 +1,11 @@
+API Reference
+=============
+
+This page contains auto-generated API reference documentation.
+
+.. toctree::
+   :titlesonly:
+
+   autoapi/morphval/index
+   autoapi/synthesis_workflow/index
+   autoapi/tasks/index

doc/source/conf.py

@@ -19,10 +19,10 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'synthesis-workflow'
+project = "synthesis-workflow"
 
 # The short X.Y version
-version = get_distribution('synthesis_workflow').version
+version = get_distribution("synthesis_workflow").version
 
 # The full version, including alpha/beta/rc tags
 release = version
@@ -34,8 +34,9 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.napoleon',
-    'autoapi.extension',
+    "autoapi.extension",
+    "sphinx.ext.graphviz",
+    "sphinx.ext.napoleon",
 ]
 
 autoapi_dirs = [
@@ -48,6 +49,7 @@
 ]
 autoapi_python_use_implicit_namespaces = True
 autoapi_keep_files = False
+autoapi_add_toctree_entry = False
 autoapi_options = [
     "imported-members",
     "members",
@@ -72,26 +74,48 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx-bluebrain-theme'
+html_theme = "sphinx-bluebrain-theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 # html_static_path = ['_static']
 
 html_theme_options = {
-    'metadata_distribution': 'synthesis-workflow',
+    "metadata_distribution": "synthesis-workflow",
 }
 
-html_title = 'Synthesis Workflow'
+html_title = "Synthesis Workflow"
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
 
 # autosummary settings
-autosummary_generate=True
+autosummary_generate = True
 
 # autodoc settings
-autodoc_typehints='signature'
-autodoc_default_options = {'members': True}
+autodoc_typehints = "signature"
+autodoc_default_options = {"members": True}
 autoclass_content = "both"
+
+import re
+
+SKIP = [
+    r".*\.L",
+    r".*tasks\..*\.requires",
+    r".*tasks\..*\.run",
+    r".*tasks\..*\.output",
+]
+
+
+def maybe_skip_member(app, what, name, obj, skip, options):
+    skip = None
+    for pattern in SKIP:
+        if re.match(pattern, name) is not None:
+            skip = True
+            break
+    return skip
+
+
+def setup(app):
+    app.connect("autoapi-skip-member", maybe_skip_member)

doc/source/index.rst

@@ -1,10 +1,21 @@
 .. include:: ../../README.rst
 
 
+API Reference
+-------------
+
+The :doc:`./api_ref` page contains detailed documentation of:
+
+	* :py:mod:`main workflows<tasks.workflows>`
+	* :py:mod:`all tasks<tasks>`
+	* :py:mod:`all synthesis_workflow functions<synthesis_workflow>`
+	* :py:mod:`all morphval functions<morphval>`
+
+
 .. toctree::
    :hidden:
    :maxdepth: 2
 
    Home <self>
    changelog
-   autoapi/index
+   api_ref

requirements-doc.pip

@@ -1,3 +1,4 @@
+graphviz
 sphinx
 sphinx-autoapi
 sphinx-bluebrain-theme

requirements-test.pip

@@ -1,4 +1,4 @@
-diff_pdf_visually
+diff_pdf_visually>=1.5.1
 pytest
 pytest-cov
 pytest-html

setup.py

@@ -42,13 +42,14 @@
 ]
 
 doc_reqs = [
+    "graphviz",
     "sphinx",
     "sphinx-autoapi",
     "sphinx-bluebrain-theme",
 ]
 
 test_reqs = [
-    "diff_pdf_visually",
+    "diff_pdf_visually>=1.5.1",
     "pytest",
     "pytest-cov",
     "pytest-html",
@@ -59,8 +60,8 @@
 
 setup(
     name="synthesis-workflow",
-    author="bbp-ou-cell",
-    author_email="bbp-ou-cell@groupes.epfl.ch",
+    author="bbp-ou-cells",
+    author_email="bbp-ou-cells@groupes.epfl.ch",
     version=VERSION,
     description="Workflow used for synthesis and its validation.",
     long_description=README,

src/morphval/__init__.py

@@ -1,4 +1,4 @@
-"""Morphology Statistical Validation Package"""
+"""Morphology Statistical Validation Package."""
 import pkg_resources
 
 __version__ = pkg_resources.get_distribution("synthesis_workflow").version

src/morphval/cli.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-"""CLI for MorphVal package"""
+"""CLI for MorphVal package."""
 import argparse
 import pkg_resources
 
@@ -8,7 +8,7 @@
 
 
 def get_parser():
-    """return the argument parser"""
+    """Return the argument parser."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "-t", "--test-dir", required=True, help="full path to directory with test data"
@@ -52,7 +52,7 @@ def get_parser():
 
 
 def main():
-    """Main function of MorphVal package"""
+    """Main function of MorphVal package."""
     args = get_parser().parse_args()
 
     if args.example_config: