Make plots more robust ; Fix collage tasks

synthesis_workflow/synthesis.py

@@ -466,17 +466,17 @@ def _get_target_layer(target_layer_str):
             tmd_parameters[neurite_type]["hard_limits"] = defaultdict(dict)
         if "min" in limits:
             lim = limits["min"]
-            tmd_parameters[neurite_type]["hard_limits"]["min"]["layer"] = int(
-                lim.get("layer")[1:]
-            )
-            tmd_parameters[neurite_type]["hard_limits"]["min"]["fraction"] = lim.get(
-                "fraction", 0.0
-            )
+            min_layer = int(lim.get("layer")[1:])
+            min_fraction = lim.get("fraction", 0.0)
+            tmd_parameters[neurite_type]["hard_limits"]["min"]["layer"] = min_layer
+            tmd_parameters[neurite_type]["hard_limits"]["min"]["fraction"] = min_fraction
+            L.debug("Add min hard limit to {}: {} in {} layer".format(
+                neurite_type, min_layer, min_fraction))
         if "max" in limits:
             lim = limits["max"]
-            tmd_parameters[neurite_type]["hard_limits"]["max"]["layer"] = int(
-                lim.get("layer")[1:]
-            )
-            tmd_parameters[neurite_type]["hard_limits"]["max"]["fraction"] = lim.get(
-                "fraction", 1.0
-            )
+            max_layer = int(lim.get("layer")[1:])
+            max_fraction = lim.get("fraction", 1.0)
+            tmd_parameters[neurite_type]["hard_limits"]["max"]["layer"] = max_layer
+            tmd_parameters[neurite_type]["hard_limits"]["max"]["fraction"] = max_fraction
+            L.debug("Add max hard limit to {}: {} in {} layer".format(
+                neurite_type, max_layer, max_fraction))

synthesis_workflow/utils_tasks.py

@@ -8,6 +8,8 @@
 
 
 logger = logging.getLogger("luigi-interface")
+warnings.filterwarnings("ignore", module="diameter_synthesis.build_diameters")
+warnings.filterwarnings("ignore", module="joblib")
 warnings.filterwarnings("ignore", module="luigi.parameter")
 warnings.filterwarnings("ignore", module="neurom.io")
 warnings.filterwarnings("ignore", module="neurom.features")

synthesis_workflow/validation.py

@@ -1,5 +1,6 @@
 """Functions for validation of synthesis to be used by luigi tasks."""
 import os
+import warnings
 from collections import defaultdict
 from functools import partial
 from pathlib import Path
@@ -157,12 +158,17 @@ def _plot_density_profile(
     mtype, circuit=None, x_pos=None, sample=None, voxeldata=None, sample_distance=None
 ):
     """Plot density profile of an mtype."""
-    plot_df = _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance)
     fig = plt.figure()
     ax = plt.gca()
     _plot_layers(x_pos, circuit.atlas, ax)
-    sns.violinplot(x="neurite_type", y="y", data=plot_df, ax=ax, bw=0.1)
-    ax.legend(loc="best")
+    try:
+        plot_df = _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance)
+        sns.violinplot(x="neurite_type", y="y", data=plot_df, ax=ax, bw=0.1)
+        ax.legend(loc="best")
+    except:
+        ax.text(
+            0.5, 0.5, 'ERROR WHEN GETTING POINT DEPTHS', horizontalalignment='center',
+            verticalalignment='center', transform=ax.transAxes)
     fig.suptitle(mtype)
     return fig
 
@@ -189,14 +195,20 @@ def plot_density_profiles(circuit, sample, region, sample_distance, output_path,
             delayed(f)(
                 mtype
             )
-            for mtype in sorted(circuit.cells.mtypes)
+            for mtype in tqdm(sorted(circuit.cells.mtypes))
         ):
             pdf.savefig(fig, bbox_inches="tight")
             plt.close(fig)
 
 
 def _plot_cells(circuit, mtype, sample, ax):
     """Plot cells for collage."""
+    max_sample = (circuit.cells.get(properties="mtype") == mtype).sum()
+    if sample > max_sample:
+        warnings.warn(
+            ("The sample value is set to '{}' for the type {} because there are no more "
+            "cells available of that type").format(max_sample, mtypes))
+        sample = max_sample
     gids = circuit.cells.ids(group={"mtype": mtype}, sample=sample)
 
     for gid in gids:
@@ -212,8 +224,14 @@ def _plot_collage_O1(
     """Plot collage for a given mtype (for multiprocessing)."""
     fig = plt.figure(mtype, figsize=figsize)
     ax = plt.gca()
-    _plot_layers(x_pos, circuit.atlas, ax)
-    _plot_cells(circuit, mtype, sample, ax)
+    try:
+        _plot_layers(x_pos, circuit.atlas, ax)
+    except:
+        warnings.warn("Unable to plot the layers for the type '{}'".format(mtype))
+    try:
+        _plot_cells(circuit, mtype, sample, ax)
+    except:
+        warnings.warn("Unable to plot the cells for the type '{}'".format(mtype))
     plt.axis(ax_limit)
 
     ax.set_rasterized(True)
@@ -265,7 +283,7 @@ def plot_collage_O1(circuit, sample, output_path, mtypes=None, nb_jobs=-1):
             delayed(f)(
                 mtype
             )
-            for mtype in sorted(circuit.cells.mtypes)
+            for mtype in tqdm(sorted(circuit.cells.mtypes))
         ):
             pdf.savefig(fig, bbox_inches="tight", dpi=100)
             plt.close(fig)
@@ -407,7 +425,7 @@ def plot_collage(
             delayed(f)(
                 plane_id
             )
-            for plane_id in plane_ids
+            for plane_id in tqdm(plane_ids)
         ):
             pdf.savefig(fig, bbox_inches="tight", dpi=100)
             plt.close(fig)

synthesis_workflow/validation_tasks.py

@@ -3,16 +3,20 @@
 import yaml
 
 import luigi
+import numpy as np
 import pandas as pd
+from voxcell import VoxelData
 
 from .synthesis_tasks import CreateAtlasPlanes
 from .synthesis_tasks import Synthesize
 from .synthesis_tasks import VacuumSynthesize
 from .synthesis_tasks import RescaleMorphologies
 from .synthesis_tasks import GetSynthetisedNeuriteLengths
+from .utils_tasks import BaseTask
 from .utils_tasks import BaseWrapperTask
 from .utils_tasks import circuitconfigs
 from .utils_tasks import load_circuit
+from .utils_tasks import logger as L
 from .utils_tasks import pathconfigs
 from .validation import convert_mvd3_to_morphs_df
 from .validation import plot_collage
@@ -116,6 +120,7 @@ class PlotDensityProfiles(luigi.Task):
     sample_distance = luigi.FloatParameter(default=10)
     sample = luigi.IntParameter(default=None)
     region = luigi.Parameter(default="O1")
+    nb_jobs = luigi.IntParameter(default=-1)
 
     def requires(self):
         """"""
@@ -131,7 +136,7 @@ def run(self):
         )
 
         plot_density_profiles(
-            circuit, self.sample, self.region, self.sample_distance, self.output().path
+            circuit, self.sample, self.region, self.sample_distance, self.output().path, self.nb_jobs
         )
 
     def output(self):
@@ -153,24 +158,33 @@ class PlotCollage(BaseWrapperTask):
     mtypes = luigi.ListParameter(default=None)
 
     def requires(self):
+        """"""
+        return ConvertMvd3()
+
+    def run(self):
         """"""
         if self.mtypes[0] == "all":
-            self.mtypes = pd.read_csv(
+            mtypes = pd.read_csv(
                 pathconfigs().synth_morphs_df_path
             ).mtype.unique()
+        else:
+            mtypes = self.mtypes
 
-        tasks = []
-        for mtype in self.mtypes:
-            collage_path = (Path(self.collage_base_path) / mtype).with_suffix(".pdf")
-            tasks.append(
-                PlotSingleCollage(
-                    mtype=mtype,
-                    collage_path=collage_path,
-                    sample=self.sample,
+        if self.collage_type == "O1":
+            yield PlotSingleCollage(
+                collage_base_path=self.collage_base_path,
+                collage_type=self.collage_type,
+                sample=self.sample,
+                mtype=mtypes,
+            )
+        elif self.collage_type == "Isocortex":
+            for mtype in mtypes:
+                yield PlotSingleCollage(
+                    collage_base_path=self.collage_base_path,
                     collage_type=self.collage_type,
+                    sample=self.sample,
+                    mtype=mtype,
                 )
-            )
-        return tasks
 
 
 class PlotSingleCollage(luigi.Task):
@@ -181,31 +195,33 @@ class PlotSingleCollage(luigi.Task):
         sample (float): number of cells to use, if None, all available
     """
 
-    collage_path = luigi.Parameter(default="collage.pdf")
-    sample = luigi.IntParameter(default=20)
+    collage_base_path = luigi.Parameter(default="collages")
+    # collage_path = luigi.Parameter(default="collage.pdf")
     collage_type = luigi.Parameter(default="O1")
+    sample = luigi.IntParameter(default=20)
     mtype = luigi.Parameter()
 
     def requires(self):
         """"""
-        return {"synthesis": Synthesize(), "planes": CreateAtlasPlanes()}
+        return {"synthesis": Synthesize()}
 
     def run(self):
         """"""
 
+        L.debug("collage_path = {}".format(self.output().path))
+
         circuit = load_circuit(
             path_to_mvd3=self.input()["synthesis"].path,
             path_to_morphologies=pathconfigs().synth_output_path,
             path_to_atlas=circuitconfigs().atlas_path,
         )
-        import numpy as np
-        from voxcell import VoxelData
 
         if self.collage_type == "O1":
             plot_collage_O1(circuit, self.sample, self.output().path)
 
-        if self.collage_type == "Isocortex":
-            planes = np.load(self.input()["planes"].path)["planes"]
+        elif self.collage_type == "Isocortex":
+            planes_task = yield CreateAtlasPlanes()
+            planes = np.load(planes_task.path)["planes"]
             annotation_path = Path(circuitconfigs().atlas_path) / "layers.nrrd"
             layer_annotation = VoxelData.load_nrrd(annotation_path)
             plot_collage(
@@ -219,7 +235,11 @@ def run(self):
 
     def output(self):
         """"""
-        return luigi.LocalTarget(self.collage_path)
+        if self.collage_type == "O1":
+            collage_path = (Path(self.collage_base_path) / "collages").with_suffix(".pdf")
+        else:
+            collage_path = (Path(self.collage_base_path) / self.mtype).with_suffix(".pdf")
+        return luigi.LocalTarget(collage_path)
 
 
 class ValidateSynthesis(luigi.WrapperTask):