diff --git a/synthesis_workflow/tasks/config.py b/synthesis_workflow/tasks/config.py index 1457a1a..1d28b5b 100644 --- a/synthesis_workflow/tasks/config.py +++ b/synthesis_workflow/tasks/config.py @@ -100,10 +100,12 @@ class PathConfig(luigi.Config): """Morphology path configuration.""" ext = ExtParameter(default="asc") - # TODO: use result_path as suffix for all output paths + # TODO: use result_path as prefix for all output paths result_path = luigi.Parameter(default="out") morphs_df_path = luigi.Parameter(default="morphs_df.csv") morphology_path = luigi.Parameter(default="repaired_morphology_path") + mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (TSV)") + pc_in_types_path = luigi.Parameter(default="pc_in_types.yaml") synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv") synth_output_path = luigi.Parameter(default="synthesized_morphologies") substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv") diff --git a/synthesis_workflow/tasks/synthesis.py b/synthesis_workflow/tasks/synthesis.py index 6edd82c..ad82423 100644 --- a/synthesis_workflow/tasks/synthesis.py +++ b/synthesis_workflow/tasks/synthesis.py @@ -22,6 +22,7 @@ from ..synthesis import rescale_morphologies from ..synthesis import run_synthesize_morphologies from ..tools import ensure_dir +from ..tools import load_neurondb_to_dataframe from .circuit import SliceCircuit from .config import CircuitConfig from .config import DiametrizerConfig @@ -38,6 +39,46 @@ L = logging.getLogger(__name__) +@copy_params( + pc_in_types_path=ParamLink(PathConfig), +) +class BuildMorphsDF(WorkflowTask): + """Generate the list of morphologies with their mtypes and paths. + + Args: + neurondb_path (str): path to the neuronDB file (XML) + pc_in_types_path (str): path to the pc_in_types file (TSV) + morphology_dirs (str): dict (JSON format) in which keys are column names and values + are the paths to each morphology file + apical_points_path (str): path to the apical points file (JSON) + """ + + neurondb_path = luigi.Parameter(description="path to the neuronDB file (XML)") + morphology_dirs = luigi.DictParameter( + description=( + "dict (JSON format) in which keys are column names and values are the paths to each " + "morphology file" + ) + ) + apical_points_path = luigi.OptionalParameter( + default=None, description="path to the apical points file (JSON)" + ) + + def run(self): + """""" + morphs_df = load_neurondb_to_dataframe( + self.neurondb_path, + self.morphology_dirs, + self.pc_in_types_path, + self.apical_points_path, + ) + morphs_df.to_csv(self.output().path) + + def output(self): + """""" + return luigi.LocalTarget(PathConfig().morphs_df_path) + + class ApplySubstitutionRules(WorkflowTask): """Apply substitution rules to the morphology dataframe. @@ -401,12 +442,15 @@ def output(self): return luigi.LocalTarget(self.rescaled_morphs_df_path) +@copy_params( + mtype_taxonomy_path=ParamLink(PathConfig), +) class BuildCircuit(WorkflowTask): """Generate cell positions and me-types from atlas, compositions and taxonomy. Args: cell_composition_path (str): path to the cell composition file (YAML) - mtype_taxonomy_path (str): path to the taxonomy file (tsv) + mtype_taxonomy_path (str): path to the taxonomy file (TSV) density_factor (float): density factor seed (int): pseudo-random generator seed """ @@ -414,7 +458,6 @@ class BuildCircuit(WorkflowTask): cell_composition_path = luigi.Parameter( description="path to the cell composition file (YAML)" ) - mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (tsv)") density_factor = luigi.NumericalParameter( default=0.01, var_type=float, diff --git a/synthesis_workflow/tasks/workflows.py b/synthesis_workflow/tasks/workflows.py index 6fd1441..38513b3 100644 --- a/synthesis_workflow/tasks/workflows.py +++ b/synthesis_workflow/tasks/workflows.py @@ -6,6 +6,8 @@ from .luigi_tools import WorkflowTask from .luigi_tools import WorkflowWrapperTask from .synthesis import ApplySubstitutionRules +from .synthesis import BuildCircuit +from .synthesis import BuildMorphsDF from .validation import PlotCollage from .validation import PlotDensityProfiles from .validation import PlotMorphometrics @@ -17,6 +19,8 @@ class ValidateSynthesis(WorkflowWrapperTask): """Workflow to validate synthesis""" + build_circuit = luigi.BoolParameter(default=False) + build_morphs_df = luigi.BoolParameter(default=False) with_collage = luigi.BoolParameter(default=True) with_morphometrics = luigi.BoolParameter(default=True) with_density_profiles = luigi.BoolParameter(default=True) @@ -26,6 +30,10 @@ class ValidateSynthesis(WorkflowWrapperTask): def requires(self): """""" tasks = [] + if self.build_circuit: + tasks.append(BuildCircuit()) + if self.build_morphs_df: + tasks.append(BuildMorphsDF()) if self.with_collage: tasks.append(PlotCollage()) if self.with_morphometrics: diff --git a/synthesis_workflow/tools.py b/synthesis_workflow/tools.py index 1963c2e..90c4394 100644 --- a/synthesis_workflow/tools.py +++ b/synthesis_workflow/tools.py @@ -6,6 +6,7 @@ import traceback import warnings from collections import namedtuple +from functools import partial from pathlib import Path import pandas as pd @@ -28,21 +29,17 @@ def add_taxonomy(morphs_df, pc_in_types): return morphs_df -def add_morphology_paths(morph_df, morphology_dirs): +def add_morphology_paths(morphs_df, morphology_dirs): """Same as the path loader of morph_tool.utils.neurondb_dataframe, but add multiple columns. Args: morphology_dirs: (dict) If passed, a column with the path to each morphology file will be added for each entry of the dict, where the column name is the dict key """ - for name, morphology_dir in morphology_dirs.items(): - morph_df[name] = morph_df.apply( - lambda row, morphology_dir=morphology_dir: find_morph( - morphology_dir, row["name"] - ), - axis=1, - ) - return morph_df + for col_name, morphology_dir in morphology_dirs.items(): + f = partial(find_morph, Path(morphology_dir)) + morphs_df[col_name] = morphs_df["name"].apply(f) + return morphs_df def add_apical_points(morphs_df, apical_points): @@ -52,6 +49,7 @@ def add_apical_points(morphs_df, apical_points): """ morphs_df["apical_point_isec"] = -1 + # morphs_df["apical_point_isec_test"] = morphs_df["name"].map(apical_points) for name, apical_point in apical_points.items(): morphs_df.loc[morphs_df.name == name, "apical_point_isec"] = apical_point morphs_df["apical_point_isec"] = morphs_df["apical_point_isec"].astype(int) @@ -70,7 +68,7 @@ def load_neurondb_to_dataframe( pc_in_types (dict): dict of the form [mtype]: [IN|PC] apical_points (dict): name of cell as key and apical point isec as value """ - morphs_df = neurondb_dataframe(neurondb_path) + morphs_df = neurondb_dataframe(Path(neurondb_path)) if morphology_dirs is not None: morphs_df = add_morphology_paths(morphs_df, morphology_dirs)