Add tests for the workflows and reorganize the code

Change-Id: I69e1677f6ef41a92a4caecc691f6c6dc5f600a5f

MANIFEST.in

@@ -1,6 +1,5 @@
-graft tests
-
 global-exclude *.py[co] .DS_Store
 
-include morphval/templates/*
-include synthesis_workflow/defaults/morphval_default_config.yaml
+include src/version.py
+include src/morphval/templates/*
+include src/synthesis_workflow/defaults/morphval_default_config.yaml

requirements-test.pip

@@ -1,4 +1,4 @@
-brainbuilder
+diff_pdf_visually
 pytest
 pytest-cov
 pytest-html

requirements.pip

@@ -1,7 +1,10 @@
-atlas_analysis
+atlas_analysis>0.0.1
+brainbuilder>=0.14
 bluepyefe
 bluepyopt
-diameter_synthesis
+bluepymm
+diameter_synthesis>=0.1.7
+gitpython
 h5py
 joblib
 luigi
@@ -11,11 +14,11 @@ morphio
 neuroc
 neurom
 pandas
-placement_algorithm
-region_grower
+placement_algorithm>=2.1.1
+region_grower>=0.1.10
 scipy
 seaborn
-tns
+tns>=2.2.7
 tmd
 tqdm
-voxcell
+voxcell>=2.7.3

setup.py

@@ -13,30 +13,30 @@
     README = f.read()
 
 reqs = [
-    "atlas_analysis",
-    "brainbuilder",
+    "atlas_analysis>0.0.1",
+    "brainbuilder>=0.14",
     "bluepyefe",
     "bluepyopt",
     "bluepymm",
-    "diameter_synthesis",
+    "diameter_synthesis>=0.1.7",
     "gitpython",
-    "h5py",
+    "h5py<3",
     "joblib",
     "luigi",
     "matplotlib",
     "morph_validator",
     "morphio",
     "neuroc",
-    "neurom",
+    "neurom!=2.0.1.dev4",
     "pandas",
-    "placement_algorithm",
-    "region_grower",
+    "placement_algorithm>=2.1.1",
+    "region_grower>=0.1.10",
     "scipy",
     "seaborn",
-    "tns",
+    "tns>=2.2.7",
     "tmd",
     "tqdm",
-    "voxcell",
+    "voxcell>=2.7.3",
 ]
 
 doc_reqs = [
@@ -45,13 +45,14 @@
 ]
 
 test_reqs = [
+    "diff_pdf_visually",
     "pytest",
     "pytest-cov",
     "pytest-html",
     "pytest-mpl",
 ]
 
-VERSION = imp.load_source("", "synthesis_workflow/version.py").__version__
+VERSION = imp.load_source("", "src/version.py").VERSION
 
 setup(
     name="synthesis-workflow",
@@ -67,7 +68,8 @@
         "Source": "ssh://bbpcode.epfl.ch/cells/synthesis-workflow",
     },
     license="BBP-internal-confidential",
-    packages=find_packages(),
+    packages=find_packages("src", exclude=["tests"]),
+    package_dir={"": "src"},
     python_requires=">=3.6",
     install_requires=reqs,
     extras_require={"docs": doc_reqs, "test": test_reqs},

morphval/validation_main.py → src/morphval/validation_main.py

@@ -8,6 +8,7 @@
 import neurom
 import numpy as np
 import pandas as pd
+import pkg_resources
 from joblib import delayed
 from joblib import Parallel
 from joblib import cpu_count
@@ -16,11 +17,12 @@
 from morphval import validation
 
 
-BASE_PATH = os.path.dirname(os.path.abspath(__file__))
-TEMPLATE_FILE = os.path.join(BASE_PATH, "templates/report_template.jinja2")
-SUMMARY_TEMPLATE_FILE = os.path.join(
-    BASE_PATH, "templates/report_summary_template.jinja2"
+_distribution = pkg_resources.get_distribution("synthesis-workflow")
+TEMPLATES = (
+    Path(_distribution.get_resource_filename("morphval", "morphval")) / "templates"
 )
+TEMPLATE_FILE = (TEMPLATES / "report_template.jinja2").as_posix()
+SUMMARY_TEMPLATE_FILE = (TEMPLATES / "report_summary_template.jinja2").as_posix()
 
 
 def save_csv(dir_name, feature, data):

synthesis_workflow/__init__.py → src/synthesis_workflow/__init__.py

@@ -1,7 +1,10 @@
 """Workflow for neuronal synthesis validation."""
+import pkg_resources
 from morphio import SectionType
 
 
+__version__ = pkg_resources.get_distribution("synthesis_workflow").version
+
 STR_TO_TYPES = {
     "basal": SectionType.basal_dendrite,
     "apical": SectionType.apical_dendrite,

synthesis_workflow/tasks/circuit.py → src/synthesis_workflow/tasks/circuit.py

@@ -4,10 +4,8 @@
 import yaml
 
 import luigi
-import numpy as np
 
 from voxcell import VoxelData
-from voxcell.nexus.voxelbrain import LocalAtlas
 from atlas_analysis.planes.planes import load_planes_centerline
 from atlas_analysis.planes.planes import save_planes_centerline
 
@@ -17,6 +15,7 @@
 from ..circuit import halve_atlas
 from ..circuit import slice_circuit
 from ..tools import ensure_dir
+from ..tools import get_layer_tags
 from .config import AtlasLocalTarget
 from .config import CircuitConfig
 from .config import CircuitLocalTarget
@@ -47,17 +46,7 @@ class CreateAtlasLayerAnnotations(WorkflowTask):
 
     def run(self):
         """"""
-        atlas = LocalAtlas(CircuitConfig().atlas_path)
-        names, ids = atlas.get_layers()  # pylint: disable=no-member
-        annotation = VoxelData.load_nrrd(
-            Path(CircuitConfig().atlas_path) / "brain_regions.nrrd"
-        )
-        layers = np.zeros_like(annotation.raw, dtype="uint8")
-        layer_mapping = {}
-        for layer_id, (ids_set, layer) in enumerate(zip(ids, names)):
-            layer_mapping[layer_id] = layer
-            layers[np.isin(annotation.raw, list(ids_set))] = layer_id + 1
-        annotation.raw = layers
+        annotation, layer_mapping = get_layer_tags(CircuitConfig().atlas_path)
 
         if self.use_half:
             annotation.raw = halve_atlas(

synthesis_workflow/tasks/config.py → src/synthesis_workflow/tasks/config.py

@@ -157,30 +157,34 @@ class ValidationLocalTarget(OutputLocalTarget):
     """Specific target for validation targets"""
 
 
-# Set default output paths
-OutputLocalTarget.set_default_prefix(PathConfig().result_path)
-AtlasLocalTarget.set_default_prefix(
-    # pylint: disable=protected-access
-    OutputLocalTarget._prefix
-    / PathConfig().atlas_subpath
-)
-CircuitLocalTarget.set_default_prefix(
-    # pylint: disable=protected-access
-    OutputLocalTarget._prefix
-    / PathConfig().circuit_subpath
-)
-MorphsDfLocalTarget.set_default_prefix(
-    # pylint: disable=protected-access
-    OutputLocalTarget._prefix
-    / PathConfig().morphs_df_subpath
-)
-SynthesisLocalTarget.set_default_prefix(
-    # pylint: disable=protected-access
-    OutputLocalTarget._prefix
-    / PathConfig().synthesis_subpath
-)
-ValidationLocalTarget.set_default_prefix(
-    # pylint: disable=protected-access
-    OutputLocalTarget._prefix
-    / PathConfig().validation_subpath
-)
+def reset_default_prefixes():
+    """Set default output paths for targets"""
+    OutputLocalTarget.set_default_prefix(PathConfig().result_path)
+    AtlasLocalTarget.set_default_prefix(
+        # pylint: disable=protected-access
+        OutputLocalTarget._prefix
+        / PathConfig().atlas_subpath
+    )
+    CircuitLocalTarget.set_default_prefix(
+        # pylint: disable=protected-access
+        OutputLocalTarget._prefix
+        / PathConfig().circuit_subpath
+    )
+    MorphsDfLocalTarget.set_default_prefix(
+        # pylint: disable=protected-access
+        OutputLocalTarget._prefix
+        / PathConfig().morphs_df_subpath
+    )
+    SynthesisLocalTarget.set_default_prefix(
+        # pylint: disable=protected-access
+        OutputLocalTarget._prefix
+        / PathConfig().synthesis_subpath
+    )
+    ValidationLocalTarget.set_default_prefix(
+        # pylint: disable=protected-access
+        OutputLocalTarget._prefix
+        / PathConfig().validation_subpath
+    )
+
+
+reset_default_prefixes()

synthesis_workflow/tasks/synthesis.py → src/synthesis_workflow/tasks/synthesis.py

@@ -210,9 +210,14 @@ def requires(self):
 
     def run(self):
         """"""
+        L.debug("reading morphs df from: %s", self.input().path)
         morphs_df = pd.read_csv(self.input().path)
-        mtypes = morphs_df.mtype.unique()
+
+        mtypes = sorted(morphs_df.mtype.unique())
+        L.debug("mtypes found: %s", mtypes)
+
         neurite_types = get_neurite_types(morphs_df, mtypes)
+        L.debug("neurite_types found: %s", neurite_types)
 
         diameter_model_function = partial(
             build_diameter_models, config=DiametrizerConfig().config_model

synthesis_workflow/tools.py → src/synthesis_workflow/tools.py

@@ -17,6 +17,8 @@
 from bluepy.v2 import Circuit
 from placement_algorithm.exceptions import SkipSynthesisError
 from morph_tool.utils import neurondb_dataframe, find_morph
+from voxcell import VoxelData
+from voxcell.nexus.voxelbrain import LocalAtlas
 
 
 def add_mtype_taxonomy(morphs_df, mtype_taxonomy):
@@ -120,6 +122,21 @@ def update_morphs_df(morphs_df_path, new_morphs_df):
     return pd.read_csv(morphs_df_path).merge(new_morphs_df, how="left")
 
 
+def get_layer_tags(atlas_dir):
+    """Create a VoxelData with layer tags"""
+    atlas = LocalAtlas(atlas_dir)
+
+    names, ids = atlas.get_layers()  # pylint: disable=no-member
+    br = VoxelData.load_nrrd(Path(atlas_dir) / "brain_regions.nrrd")
+    layers = np.zeros_like(br.raw, dtype="uint8")
+    layer_mapping = {}
+    for layer_id, (ids_set, layer) in enumerate(zip(ids, names)):
+        layer_mapping[layer_id] = layer
+        layers[np.isin(br.raw, list(ids_set))] = layer_id + 1
+    br.raw = layers
+    return br, layer_mapping
+
+
 class IdProcessingFormatter(logging.Formatter):
     """Logging formatter class"""
 

src/version.py

@@ -0,0 +1,2 @@
+""" Package version. """
+VERSION = "0.0.1.dev2"