Fix axon_morphs_base_dir extraction

Change-Id: Id6f3137282d3e05f12b623786454f44c7d47a5fc

synthesis_workflow/synthesis.py

@@ -135,6 +135,30 @@ def build_distributions(
     return tmd_distributions
 
 
+def get_axon_base_dir(morphs_df, col_name="morphology_path"):
+    """Get the common base directory of morphologies"""
+    if morphs_df.empty:
+        raise RuntimeError("Can not get axon base dir from an empty DataFrame")
+
+    col = morphs_df[col_name]
+
+    # Get all parent directories
+    parents = col.apply(lambda x: str(Path(x).parent)).unique()
+
+    # Check they are all equal
+    if len(parents) != 1:
+        raise Exception("Base dirs are different for axon grafting.")
+
+    # Pick the first one
+    axon_morphs_base_dir = parents[0]
+
+    # Remove '-asc' suffix if present
+    if axon_morphs_base_dir.split("-")[-1] == "asc":
+        axon_morphs_base_dir = "-".join(axon_morphs_base_dir.split("-")[:-1])
+
+    return axon_morphs_base_dir
+
+
 def create_axon_morphologies_tsv(
     circuit_path,
     morphs_df_path,
@@ -211,7 +235,6 @@ def create_axon_morphologies_tsv(
     else:
         cells_df = CellCollection.load_mvd3(circuit_path).as_dataframe()
 
-    axon_morphs_base_dir = None
     axon_morphs = pd.DataFrame()
     for gid in cells_df.index:
         all_cells = morphs_df[
@@ -224,22 +247,11 @@ def create_axon_morphologies_tsv(
                 cell = all_cells.sample().iloc[0]
 
             axon_morphs.loc[gid, "morphology"] = cell["name"]
-
-            if axon_morphs_base_dir is None:
-                axon_morphs_base_dir = str(Path(cell["morphology_path"]).parent)
-            elif axon_morphs_base_dir != str(Path(cell["morphology_path"]).parent):
-                raise Exception("Base dirs are different for axon grafting.")
         else:
             L.info("Axon grafting: no cells for %s", cells_df.loc[gid, "mtype"])
 
     axon_morphs.index -= 1
     axon_morphs.to_csv(axon_morphs_path, sep="\t", index=True)
-    if (
-        axon_morphs_base_dir is not None
-        and axon_morphs_base_dir.split("-")[-1] == "asc"
-    ):
-        axon_morphs_base_dir = "-".join(axon_morphs_base_dir.split("-")[:-1])
-    return axon_morphs_base_dir
 
 
 def run_synthesize_morphologies(kwargs, nb_jobs=-1):

synthesis_workflow/tasks/config.py

@@ -78,7 +78,7 @@ class pathconfigs(luigi.Config):
     """Morphology path configuration."""
 
     morphs_df_path = luigi.Parameter(default="morphs_df.csv")
-    morphology_path = luigi.Parameter(default="morphology_path")
+    morphology_path = luigi.Parameter(default="repaired_morphology_path")
     synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv")
     synth_output_path = luigi.Parameter(default="synthesized_morphologies")
     substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv")

synthesis_workflow/tasks/synthesis.py

@@ -2,7 +2,6 @@
 import json
 import re
 from functools import partial
-from pathlib import Path
 
 import luigi
 import pandas as pd
@@ -17,6 +16,7 @@
 from ..synthesis import apply_substitutions
 from ..synthesis import build_distributions
 from ..synthesis import create_axon_morphologies_tsv
+from ..synthesis import get_axon_base_dir
 from ..synthesis import get_mean_neurite_lengths
 from ..synthesis import get_neurite_types
 from ..synthesis import rescale_morphologies
@@ -241,15 +241,18 @@ class Synthesize(BaseTask):
     Args:
         out_circuit_path (str): path to circuit mvd3 with morphology data
         ext (str): extension for morphology files
-        axon_morphs_path (str): path to .tsv file for axon grafting
         axon_morphs_base_dir (str): base dir for morphology used for axon (.h5 files)
         apical_points_path (str): path to .yaml file for recording apical points
+        morphology_path (str): column name to use in the DF to compute
+            axon_morphs_base_dir if it is not provided
+        nb_jobs (int): number of threads used for synthesis
     """
 
     out_circuit_path = luigi.Parameter(default="sliced_circuit_morphologies.mvd3")
     ext = ExtParameter(default="asc")
     axon_morphs_base_dir = luigi.OptionalParameter(default=None)
     apical_points_path = luigi.Parameter(default="apical_points.yaml")
+    morphology_path = luigi.Parameter(default=None)
     nb_jobs = luigi.IntParameter(default=-1)
 
     def requires(self):
@@ -275,12 +278,10 @@ def run(self):
 
         # Get base-morph-dir argument value
         if self.axon_morphs_base_dir is None:
-            morphs_df = pd.read_csv(self.input()["substituted_cells"].path)
-            axon_morphs_base_dir = str(
-                Path(morphs_df.iloc[0]["morphology_path"]).parent
+            axon_morphs_base_dir = get_axon_base_dir(
+                pd.read_csv(self.input()["substituted_cells"].path),
+                self.morphology_path,
             )
-            if not morphs_df["morphology_path"].str.match(axon_morphs_base_dir).all():
-                raise Exception("Base dirs are different for axon grafting.")
         else:
             axon_morphs_base_dir = self.axon_morphs_base_dir
 

synthesis_workflow/tasks/validation.py

@@ -55,7 +55,7 @@ class PlotMorphometrics(luigi.Task):
     morph_type = luigi.Parameter(default="in_circuit")
     config_features = luigi.DictParameter(default=None)
     morphometrics_path = luigi.Parameter(default="morphometrics")
-    base_key = luigi.Parameter(default="morphology_path")
+    base_key = luigi.Parameter(default="repaired_morphology_path")
     comp_key = luigi.Parameter(default="synth_morphology_path")
     base_label = luigi.Parameter(default="bio")
     comp_label = luigi.Parameter(default="synth")