Add task to build a morphs_df file

Change-Id: I45c84eb43974860d69802f8f5744ad30822eb3ad

synthesis_workflow/tasks/config.py

@@ -100,10 +100,12 @@ class PathConfig(luigi.Config):
     """Morphology path configuration."""
 
     ext = ExtParameter(default="asc")
-    # TODO: use result_path as suffix for all output paths
+    # TODO: use result_path as prefix for all output paths
     result_path = luigi.Parameter(default="out")
     morphs_df_path = luigi.Parameter(default="morphs_df.csv")
     morphology_path = luigi.Parameter(default="repaired_morphology_path")
+    mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (TSV)")
+    pc_in_types_path = luigi.Parameter(default="pc_in_types.yaml")
     synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv")
     synth_output_path = luigi.Parameter(default="synthesized_morphologies")
     substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv")

synthesis_workflow/tasks/synthesis.py

@@ -22,6 +22,7 @@
 from ..synthesis import rescale_morphologies
 from ..synthesis import run_synthesize_morphologies
 from ..tools import ensure_dir
+from ..tools import load_neurondb_to_dataframe
 from .circuit import SliceCircuit
 from .config import CircuitConfig
 from .config import DiametrizerConfig
@@ -38,6 +39,46 @@
 L = logging.getLogger(__name__)
 
 
+@copy_params(
+    pc_in_types_path=ParamLink(PathConfig),
+)
+class BuildMorphsDF(WorkflowTask):
+    """Generate the list of morphologies with their mtypes and paths.
+
+    Args:
+        neurondb_path (str): path to the neuronDB file (XML)
+        pc_in_types_path (str): path to the pc_in_types file (TSV)
+        morphology_dirs (str): dict (JSON format) in which keys are column names and values
+            are the paths to each morphology file
+        apical_points_path (str): path to the apical points file (JSON)
+    """
+
+    neurondb_path = luigi.Parameter(description="path to the neuronDB file (XML)")
+    morphology_dirs = luigi.DictParameter(
+        description=(
+            "dict (JSON format) in which keys are column names and values are the paths to each "
+            "morphology file"
+        )
+    )
+    apical_points_path = luigi.OptionalParameter(
+        default=None, description="path to the apical points file (JSON)"
+    )
+
+    def run(self):
+        """"""
+        morphs_df = load_neurondb_to_dataframe(
+            self.neurondb_path,
+            self.morphology_dirs,
+            self.pc_in_types_path,
+            self.apical_points_path,
+        )
+        morphs_df.to_csv(self.output().path)
+
+    def output(self):
+        """"""
+        return luigi.LocalTarget(PathConfig().morphs_df_path)
+
+
 class ApplySubstitutionRules(WorkflowTask):
     """Apply substitution rules to the morphology dataframe.
 
@@ -401,20 +442,22 @@ def output(self):
         return luigi.LocalTarget(self.rescaled_morphs_df_path)
 
 
+@copy_params(
+    mtype_taxonomy_path=ParamLink(PathConfig),
+)
 class BuildCircuit(WorkflowTask):
     """Generate cell positions and me-types from atlas, compositions and taxonomy.
 
     Args:
         cell_composition_path (str): path to the cell composition file (YAML)
-        mtype_taxonomy_path (str): path to the taxonomy file (tsv)
+        mtype_taxonomy_path (str): path to the taxonomy file (TSV)
         density_factor (float): density factor
         seed (int): pseudo-random generator seed
     """
 
     cell_composition_path = luigi.Parameter(
         description="path to the cell composition file (YAML)"
     )
-    mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (tsv)")
     density_factor = luigi.NumericalParameter(
         default=0.01,
         var_type=float,

synthesis_workflow/tasks/workflows.py

@@ -6,6 +6,8 @@
 from .luigi_tools import WorkflowTask
 from .luigi_tools import WorkflowWrapperTask
 from .synthesis import ApplySubstitutionRules
+from .synthesis import BuildCircuit
+from .synthesis import BuildMorphsDF
 from .validation import PlotCollage
 from .validation import PlotDensityProfiles
 from .validation import PlotMorphometrics
@@ -17,6 +19,8 @@
 class ValidateSynthesis(WorkflowWrapperTask):
     """Workflow to validate synthesis"""
 
+    build_circuit = luigi.BoolParameter(default=False)
+    build_morphs_df = luigi.BoolParameter(default=False)
     with_collage = luigi.BoolParameter(default=True)
     with_morphometrics = luigi.BoolParameter(default=True)
     with_density_profiles = luigi.BoolParameter(default=True)
@@ -26,6 +30,10 @@ class ValidateSynthesis(WorkflowWrapperTask):
     def requires(self):
         """"""
         tasks = []
+        if self.build_circuit:
+            tasks.append(BuildCircuit())
+        if self.build_morphs_df:
+            tasks.append(BuildMorphsDF())
         if self.with_collage:
             tasks.append(PlotCollage())
         if self.with_morphometrics:

synthesis_workflow/tools.py

@@ -6,6 +6,7 @@
 import traceback
 import warnings
 from collections import namedtuple
+from functools import partial
 from pathlib import Path
 
 import pandas as pd
@@ -28,21 +29,17 @@ def add_taxonomy(morphs_df, pc_in_types):
     return morphs_df
 
 
-def add_morphology_paths(morph_df, morphology_dirs):
+def add_morphology_paths(morphs_df, morphology_dirs):
     """Same as the path loader of morph_tool.utils.neurondb_dataframe, but add multiple columns.
 
     Args:
         morphology_dirs: (dict) If passed, a column with the path to each morphology file
             will be added for each entry of the dict, where the column name is the dict key
     """
-    for name, morphology_dir in morphology_dirs.items():
-        morph_df[name] = morph_df.apply(
-            lambda row, morphology_dir=morphology_dir: find_morph(
-                morphology_dir, row["name"]
-            ),
-            axis=1,
-        )
-    return morph_df
+    for col_name, morphology_dir in morphology_dirs.items():
+        f = partial(find_morph, Path(morphology_dir))
+        morphs_df[col_name] = morphs_df["name"].apply(f)
+    return morphs_df
 
 
 def add_apical_points(morphs_df, apical_points):
@@ -52,6 +49,7 @@ def add_apical_points(morphs_df, apical_points):
     """
 
     morphs_df["apical_point_isec"] = -1
+    # morphs_df["apical_point_isec_test"] = morphs_df["name"].map(apical_points)
     for name, apical_point in apical_points.items():
         morphs_df.loc[morphs_df.name == name, "apical_point_isec"] = apical_point
     morphs_df["apical_point_isec"] = morphs_df["apical_point_isec"].astype(int)
@@ -70,7 +68,7 @@ def load_neurondb_to_dataframe(
         pc_in_types (dict): dict of the form  [mtype]: [IN|PC]
         apical_points (dict): name of cell as key and apical point isec as value
     """
-    morphs_df = neurondb_dataframe(neurondb_path)
+    morphs_df = neurondb_dataframe(Path(neurondb_path))
 
     if morphology_dirs is not None:
         morphs_df = add_morphology_paths(morphs_df, morphology_dirs)