Trunk validation

examples/trunk_validation/logging.conf

@@ -0,0 +1,38 @@
+[loggers]
+keys=root,luigi,luigi_interface
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=PrettyFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+
+[logger_luigi]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=luigi
+propagate=0
+
+[logger_luigi_interface]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=luigi-interface
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+formatter=PrettyFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+formatter=PrettyFormatter
+args=('synthesis_workflow.log',)
+
+[formatter_PrettyFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

examples/trunk_validation/luigi.cfg

@@ -0,0 +1,66 @@
+# TODO: Update this file according to new target classes
+
+# luigi parameters
+# [core]
+# logging_conf_file = logging.conf
+
+# global parameters
+[SynthesisConfig]
+tmd_parameters_path = out_vacuum/tmd_parameters.json
+tmd_distributions_path = out_vacuum/tmd_distributions.json
+mtypes = ["L5_TPC:A"]
+
+[PathConfig]
+morphs_df_path = morphs_df.csv
+synth_morphs_df_path = out_vacuum/synth_morphs_df.csv
+synth_output_path = out_vacuum/synthesized_morphologies
+substituted_morphs_df_path = out_vacuum/substituted_morphs_df.csv
+morphology_path = repaired_morphology_path
+
+# prepare step
+[GetSynthesisInputs]
+url = git@bbpgitlab.epfl.ch:circuits/thncx.git
+git_synthesis_input_path = entities/synthesis_workflow/rat/vacuum/synthesis_input
+local_synthesis_input_path = synthesis_input
+
+[BuildMorphsDF]
+neurondb_path = /gpfs/bbp.cscs.ch/project/proj83/home/gevaert/morph-release/morph_release_old_code-2020-07-27/output/06_RepairUnravel-asc/neuronDB.xml
+morphology_dirs = {"repaired_morphology_path": "/gpfs/bbp.cscs.ch/project/proj83/home/gevaert/morph-release/morph_release_old_code-2020-07-27/output/06_RepairUnravel-asc", "repaired_morphology_path_h5": "/gpfs/bbp.cscs.ch/project/proj83/home/gevaert/morph-release/morph_release_old_code-2020-07-27/output/06_RepairUnravel-h5"}
+
+
+# synthesis setup
+[ApplySubstitutionRules]
+substitution_rules_path = substitution_rules.yaml
+
+[BuildSynthesisParameters]
+tmd_parameters_path = out_vacuum/tmd_parameters_no_scaling.json
+#input_tmd_parameters_path = tmd_specific_parameters.json
+morphology_path = repaired_morphology_path_h5
+
+[BuildSynthesisDistributions]
+morphology_path = repaired_morphology_path_h5
+
+# synthesize in vacuum
+[VacuumSynthesize]
+vacuum_synth_morphology_path = vacuum_synth_morphologies
+vacuum_synth_morphs_df_path = out_vacuum/vacuum_synth_morphs_df.csv
+n_cells = 100
+
+# validation plots
+[ValidateVacuumSynthesis]
+with_morphometrics = False 
+with_density_profiles = False
+with_vacuum_morphologies= False 
+with_score_matrix_reports=False
+with_trunk_validation=True
+with_morphology_validation_reports=False
+with_scale_statistics=False
+
+[PlotVacuumMorphologies]
+pdf_filename = figures_vacuum/vacuum_morphologies.pdf
+
+[PlotMorphometrics]
+morphometrics_path = figures_vacuum/morphometrics
+
+[PlotDensityProfiles]
+density_profiles_path = figures_vacuum/density_profiles.pdf

examples/trunk_validation/run.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/bash -l
+
+export OMP_NUM_THREADS=1
+
+python -m luigi --module synthesis_workflow.tasks.workflows ValidateVacuumSynthesis \
+    --local-scheduler \
+    --log-level INFO \

src/synthesis_workflow/synthesis.py

@@ -69,18 +69,19 @@ def _build_distributions_single_mtype(
     neurite_types=None,
     diameter_model_function=None,
     morphology_path=None,
+    trunk_method="simple",
 ):
     """Internal function for multiprocessing of tmd_distribution building."""
     morphology_paths = morphs_df.loc[morphs_df.mtype == mtype, morphology_path].to_list()
-    return (
-        mtype,
-        extract_input.distributions(
-            morphology_paths,
-            neurite_types=neurite_types[mtype],
-            diameter_input_morph=morphology_paths,
-            diameter_model=diameter_model_function,
-        ),
-    )
+    kwargs = {
+        "neurite_types": neurite_types[mtype],
+        "diameter_input_morph": morphology_paths,
+        "diameter_model": diameter_model_function,
+    }
+    if trunk_method != "simple":
+        kwargs["trunk_method"] = trunk_method
+
+    return mtype, extract_input.distributions(morphology_paths, **kwargs)
 
 
 def build_distributions(
@@ -92,6 +93,7 @@ def build_distributions(
     cortical_thickness,
     nb_jobs=-1,
     joblib_verbose=10,
+    trunk_method="simple",
 ):
     """Build tmd_distribution dictionary for synthesis.
 
@@ -111,6 +113,7 @@ def build_distributions(
         neurite_types=neurite_types,
         diameter_model_function=diameter_model_function,
         morphology_path=morphology_path,
+        trunk_method=trunk_method,
     )
 
     tmd_distributions = {

src/synthesis_workflow/tasks/synthesis.py

@@ -133,9 +133,7 @@ def output(self):
         return MorphsDfLocalTarget(PathConfig().substituted_morphs_df_path)
 
 
-@copy_params(
-    tmd_parameters_path=ParamRef(SynthesisConfig),
-)
+@copy_params(tmd_parameters_path=ParamRef(SynthesisConfig))
 class BuildSynthesisParameters(WorkflowTask):
     """Build the tmd_parameter.json for synthesis.
 
@@ -150,6 +148,7 @@ class BuildSynthesisParameters(WorkflowTask):
             "parameters from TNS."
         ),
     )
+    trunk_method = luigi.ChoiceParameter(default="simple", choices=["simple", "3d_angle"])
 
     def requires(self):
         """ """
@@ -175,10 +174,13 @@ def run(self):
         tmd_parameters = {}
         for mtype in mtypes:
             if self.input_tmd_parameters_path is None:
-                tmd_parameters[mtype] = extract_input.parameters(
-                    neurite_types=neurite_types[mtype],
-                    diameter_parameters=DiametrizerConfig().config_diametrizer,
-                )
+                kwargs = {
+                    "neurite_types": neurite_types[mtype],
+                    "diameter_parameters": DiametrizerConfig().config_diametrizer,
+                }
+                if self.trunk_method != "simple":
+                    kwargs["trunk_method"] = self.trunk_method
+                tmd_parameters[mtype] = extract_input.parameters(**kwargs)
             else:
                 try:
                     tmd_parameters[mtype] = custom_tmd_parameters[mtype]
@@ -208,6 +210,8 @@ class BuildSynthesisDistributions(WorkflowTask):
         nb_jobs (int): Number of workers.
     """
 
+    trunk_method = luigi.ChoiceParameter(default="simple", choices=["simple", "3d_angle"])
+
     def requires(self):
         """ """
         return ApplySubstitutionRules()
@@ -235,6 +239,7 @@ def run(self):
             self.morphology_path,
             SynthesisConfig().cortical_thickness,
             nb_jobs=self.nb_jobs,
+            trunk_method=self.trunk_method,
         )
 
         with self.output().open("w") as f:

src/synthesis_workflow/tasks/validation.py

@@ -48,6 +48,7 @@
 from synthesis_workflow.validation import plot_path_distance_fits
 from synthesis_workflow.validation import plot_scale_statistics
 from synthesis_workflow.validation import plot_score_matrix
+from synthesis_workflow.validation import trunk_validation
 
 L = logging.getLogger(__name__)
 
@@ -817,3 +818,58 @@ def output(self):
         return ValidationLocalTarget(
             (Path(self.collage_base_path) / self.mtype).with_suffix(".pdf")
         )
+
+
+class TrunkValidation(WorkflowTask):
+    """Trunk angle validation plots."""
+
+    in_atlas = BoolParameter(
+        default=False, description=":bool: Set to True to consider cells in an atlas."
+    )
+    validation_path = luigi.Parameter(
+        default="trunk_validation",
+        description=":str: Path to output directory (relative from ``PathConfig.result_path``).",
+    )
+    base_key = luigi.Parameter(
+        default="repaired_morphology_path",
+        description=":str: Base key to use in the morphology DataFrame.",
+    )
+    comp_key = luigi.Parameter(
+        default=SYNTH_MORPHOLOGY_PATH,
+        description=":str: Compared key to use in the morphology DataFrame.",
+    )
+    base_label = luigi.Parameter(default="bio", description=":str: Label for base morphologies.")
+    comp_label = luigi.Parameter(
+        default="synth", description=":str: Label for compared morphologies."
+    )
+
+    def requires(self):
+        """ """
+        if self.in_atlas:
+            return {"morphs": BuildMorphsDF(), "mvd3": ConvertMvd3()}
+        else:
+            return {"vacuum": VacuumSynthesize(), "morphs": ApplySubstitutionRules()}
+
+    def run(self):
+        """ """
+        morphs_df = pd.read_csv(self.input()["morphs"].path)
+        if self.in_atlas:
+            synth_morphs_df = pd.read_csv(self.input()["mvd3"].path)
+            comp_key = self.comp_key
+        else:
+            synth_morphs_df = pd.read_csv(self.input()["vacuum"]["out_morphs_df"].path)
+            comp_key = self.requires()["vacuum"].vacuum_synth_morphology_path
+
+        trunk_validation(
+            morphs_df,
+            synth_morphs_df,
+            self.output().pathlib_path,
+            base_key=self.base_key,
+            comp_key=comp_key,
+            base_label=self.base_label,
+            comp_label=self.comp_label,
+        )
+
+    def output(self):
+        """ """
+        return ValidationLocalTarget(self.validation_path)

src/synthesis_workflow/tasks/workflows.py

@@ -16,6 +16,7 @@
 from synthesis_workflow.tasks.validation import PlotPathDistanceFits
 from synthesis_workflow.tasks.validation import PlotScales
 from synthesis_workflow.tasks.validation import PlotScoreMatrix
+from synthesis_workflow.tasks.validation import TrunkValidation
 from synthesis_workflow.validation import plot_morphometrics
 
 
@@ -44,6 +45,9 @@ class ValidateSynthesis(WorkflowWrapperTask):
     with_score_matrix_reports = BoolParameter(
         default=True, description=":bool: Trigger score matrix reports."
     )
+    with_trunk_validation = BoolParameter(
+        default=False, description=":bool: Trigger trunk validation."
+    )
 
     def requires(self):
         """ """
@@ -62,6 +66,8 @@ def requires(self):
             tasks.append(MorphologyValidationReports())
         if self.with_score_matrix_reports:
             tasks.append(PlotScoreMatrix(in_atlas=True))
+        if self.with_trunk_validation:
+            tasks.append(TrunkValidation(in_atlas=True))
         return tasks
 
 
@@ -83,6 +89,9 @@ class ValidateVacuumSynthesis(WorkflowWrapperTask):
     with_score_matrix_reports = BoolParameter(
         default=True, description=":bool: Trigger score matrix reports."
     )
+    with_trunk_validation = BoolParameter(
+        default=False, description=":bool: Trigger trunk validation."
+    )
 
     def requires(self):
         """ """
@@ -95,6 +104,8 @@ def requires(self):
             tasks.append(PlotDensityProfiles(in_atlas=False))
         if self.with_score_matrix_reports:
             tasks.append(PlotScoreMatrix(in_atlas=False))
+        if self.with_trunk_validation:
+            tasks.append(TrunkValidation(in_atlas=False))
         return tasks