diff --git a/synthesis_workflow/circuit_slicing.py b/synthesis_workflow/circuit_slicing.py index 934795d..85fbe93 100644 --- a/synthesis_workflow/circuit_slicing.py +++ b/synthesis_workflow/circuit_slicing.py @@ -84,6 +84,15 @@ def slice_atlas_bbox(cells, bbox): return slice_z_slice(cells, bbox[2]) +def generic_slicer_old(cells, n_cells, mtypes=None, bbox=None): + """Select n_cells mtype in mtypes and within bbox.""" + if mtypes is not None: + cells = slice_per_mtype(cells, mtypes) + if bbox is not None: + cells = slice_atlas_bbox(cells, bbox) + return slice_n_cells(cells, n_cells) + + def generic_slicer(cells, n_cells, mtypes=None, planes=None, hemisphere=None): """Select n_cells mtype in mtypes and within bbox.""" if mtypes is not None: diff --git a/synthesis_workflow/diametrizer_tasks.py b/synthesis_workflow/diametrizer_tasks.py index c749c56..60420ab 100644 --- a/synthesis_workflow/diametrizer_tasks.py +++ b/synthesis_workflow/diametrizer_tasks.py @@ -1,5 +1,4 @@ """Luigi tasks to diametrize cells.""" -import logging import multiprocessing import os import sys @@ -20,12 +19,11 @@ from .utils_tasks import diametrizerconfigs from .utils_tasks import get_morphs_df +from .utils_tasks import logger as L from .utils_tasks import update_morphs_df warnings.filterwarnings("ignore") matplotlib.use("Agg") -L = logging.getLogger(__name__) -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) def _build_diameter_model( diff --git a/synthesis_workflow/synthesis_tasks.py b/synthesis_workflow/synthesis_tasks.py index 57bd0b0..dab5fde 100644 --- a/synthesis_workflow/synthesis_tasks.py +++ b/synthesis_workflow/synthesis_tasks.py @@ -12,38 +12,36 @@ import numpy as np import pandas as pd import yaml -from voxcell import VoxelData - -from atlas_analysis.planes.planes import (_create_planes, - create_centerline_planes, - save_planes_centerline) +from atlas_analysis.planes.planes import _create_planes +from atlas_analysis.planes.planes import create_centerline_planes +from atlas_analysis.planes.planes import save_planes_centerline from diameter_synthesis.build_models import build as build_diameter_models from region_grower.utils import NumpyEncoder from tns import extract_input +from voxcell import VoxelData + +from .circuit_slicing import halve_atlas +from .circuit_slicing import generic_slicer +from .circuit_slicing import slice_circuit +from .synthesis import add_scaling_rules_to_parameters +from .synthesis import apply_substitutions +from .synthesis import build_distributions +from .synthesis import create_axon_morphologies_tsv +from .synthesis import get_mean_neurite_lengths +from .synthesis import get_neurite_types +from .synthesis import grow_vacuum_morphologies +from .synthesis import plot_vacuum_morphologies +from .synthesis import rescale_morphologies +from .synthesis import run_placement_algorithm +from .utils_tasks import BaseTask +from .utils_tasks import circuitconfigs +from .utils_tasks import diametrizerconfigs +from .utils_tasks import ensure_dir +from .utils_tasks import get_morphs_df +from .utils_tasks import logger as L +from .utils_tasks import pathconfigs +from .utils_tasks import synthesisconfigs -from .utils_tasks import ( - diametrizerconfigs, - circuitconfigs, - synthesisconfigs, - pathconfigs, -) -from .utils_tasks import get_morphs_df, ensure_dir -from .circuit_slicing import generic_slicer, slice_circuit -from .synthesis import ( - get_neurite_types, - apply_substitutions, - build_distributions, - create_axon_morphologies_tsv, - run_placement_algorithm, - get_mean_neurite_lengths, - grow_vacuum_morphologies, - plot_vacuum_morphologies, - rescale_morphologies, - add_scaling_rules_to_parameters, -) - -L = logging.getLogger(__name__) -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) warnings.filterwarnings("ignore") morphio.set_maximum_warnings(0) @@ -73,15 +71,18 @@ def output(self): return luigi.LocalTarget(pathconfigs().substituted_morphs_df_path) -class BuildSynthesisParameters(luigi.Task): +class BuildSynthesisParameters(BaseTask): """Build the tmd_parameter.json for synthesis. Args: to_use_flag (str): column name in morphology dataframe to select subset of cells + custom_tmd_parameters_path (str): custom path to input tmd_parameters. If not given, + the one from """ - tmd_parameters_path = luigi.Parameter(default="tmd_parameters.json") - to_use_flag = luigi.Parameter(default="all") + to_use_flag = luigi.Parameter(default=None) + input_tmd_parameters_path = luigi.Parameter(default=None) + tmd_parameters_path = luigi.Parameter(default=None) def requires(self): """""" @@ -94,14 +95,14 @@ def run(self): ) neurite_types = get_neurite_types(synthesisconfigs().pc_in_types_path, mtypes) - if synthesisconfigs().custom_tmd_parameters_path is not None: + if self.input_tmd_parameters_path is not None: L.info("Using custom tmd parameters") - with open(synthesisconfigs().custom_tmd_parameters_path, "r") as f: + with open(self.input_tmd_parameters_path, "r") as f: custom_tmd_parameters = json.load(f) tmd_parameters = {} for mtype in mtypes: - if synthesisconfigs().custom_tmd_parameters_path is None: + if self.input_tmd_parameters_path is None: tmd_parameters[mtype] = extract_input.parameters( neurite_types=neurite_types[mtype], diameter_parameters=diametrizerconfigs().config_diametrizer, @@ -125,7 +126,7 @@ def output(self): return luigi.LocalTarget(self.tmd_parameters_path) -class BuildSynthesisDistributions(luigi.Task): +class BuildSynthesisDistributions(BaseTask): """Build the tmd_distribution.json for synthesis. Args: @@ -134,8 +135,8 @@ class BuildSynthesisDistributions(luigi.Task): h5_path (str): base path to h5 morphologies for TNS distribution extraction """ - to_use_flag = luigi.Parameter(default="all") - morphology_path = luigi.Parameter(default="morphology_path") + to_use_flag = luigi.Parameter(default=None) + morphology_path = luigi.Parameter(default=None) h5_path = luigi.Parameter(default=None) def requires(self): @@ -144,7 +145,6 @@ def requires(self): def run(self): """""" - morphs_df = get_morphs_df( self.input().path, to_use_flag=self.to_use_flag, @@ -183,7 +183,7 @@ def requires(self): return [BuildSynthesisParameters(), BuildSynthesisDistributions()] -class CreateAtlasPlanes(luigi.Task): +class CreateAtlasPlanes(BaseTask): """Crerate plane cuts of an atlas.""" plane_type = luigi.Parameter(default="centerline") @@ -251,34 +251,37 @@ def output(self): return luigi.LocalTarget(self.atlas_planes_path + ".npz") -class SliceCircuit(luigi.Task): +class SliceCircuit(BaseTask): """Create a smaller circuit .mvd3 file for subsampling. Args: sliced_circuit_path (str): path to save sliced circuit somata mvd3 mtypes (list): list of mtypes to consider n_cells (int): number of cells per mtype to consider + hemisphere (str): 'left' or 'right' """ sliced_circuit_path = luigi.Parameter(default="sliced_circuit_somata.mvd3") - mtypes = luigi.ListParameter(default=["all"]) + mtypes = luigi.ListParameter(default=None) n_cells = luigi.IntParameter(default=10) - hemisphere = luigi.Parameter(default="left") - - def requires(self): - """""" - return CreateAtlasPlanes() + hemisphere = luigi.Parameter(default=None) def run(self): """""" if "all" in self.mtypes: # pylint: disable=unsupported-membership-test self.mtypes = None + if self.hemisphere is not None: + atlas_planes = yield CreateAtlasPlanes() + planes = np.load(atlas_planes.path)["planes"] + else: + planes = None + slicer = partial( generic_slicer, n_cells=self.n_cells, mtypes=self.mtypes, - planes=np.load(self.input().path)["planes"], + planes=planes, hemisphere=self.hemisphere, ) @@ -295,7 +298,7 @@ def output(self): return luigi.LocalTarget(self.sliced_circuit_path) -class Synthesize(luigi.Task): +class Synthesize(BaseTask): """Run placement-algorithm to synthesize morphologies. Args: @@ -363,12 +366,12 @@ def output(self): return luigi.LocalTarget(self.out_circuit_path) -class GetMeanNeuriteLengths(luigi.Task): +class GetMeanNeuriteLengths(BaseTask): """Get the mean neurite lenghts of a neuron population, per mtype and neurite type.""" - neurite_types = luigi.ListParameter(default=["apical"]) - mtypes = luigi.ListParameter(default=["all"]) - morphology_path = luigi.Parameter(default="morphology_path") + neurite_types = luigi.ListParameter(default=None) + mtypes = luigi.ListParameter(default=None) + morphology_path = luigi.Parameter(default=None) mean_lengths_path = luigi.Parameter(default="mean_neurite_lengths.yaml") def requires(self): @@ -397,7 +400,7 @@ def output(self): return luigi.LocalTarget(self.mean_lengths_path) -class GetSynthetisedNeuriteLengths(luigi.Task): +class GetSynthetisedNeuriteLengths(BaseTask): """Get the mean neurite lenghts of a neuron population, per mtype and neurite type.""" neurite_types = luigi.ListParameter(default=["apical"]) @@ -435,14 +438,12 @@ def output(self): return luigi.LocalTarget(self.mean_lengths_path) -class AddScalingRulesToParameters(luigi.Task): +class AddScalingRulesToParameters(BaseTask): """Add scaling rules to tmd_parameter.json.""" scaling_rules_path = luigi.Parameter(default="scaling_rules.yaml") hard_limits_path = luigi.Parameter(default="hard_limits.yaml") - tmd_parameters_with_scaling_path = luigi.Parameter( - default="tmd_parameters_with_scaling.json" - ) + tmd_parameters_path = luigi.Parameter(default=None) def requires(self): """""" @@ -493,13 +494,13 @@ def _get_target_layer(target_layer_str): def output(self): """""" - return luigi.LocalTarget(self.tmd_parameters_with_scaling_path) + return luigi.LocalTarget(self.tmd_parameters_path) -class VacuumSynthesize(luigi.Task): +class VacuumSynthesize(BaseTask): """Grow cells in vacuum, for annotation tasks.""" - mtypes = luigi.ListParameter(default=["all"]) + mtypes = luigi.ListParameter(default=None) vacuum_synth_morphology_path = luigi.Parameter(default="vacuum_synth_morphologies") vacuum_synth_morphs_df_path = luigi.Parameter(default="vacuum_synth_morphs_df.csv") n_cells = luigi.IntParameter(default=10) @@ -538,7 +539,7 @@ def output(self): return luigi.LocalTarget(self.vacuum_synth_morphs_df_path) -class PlotVacuumMorphologies(luigi.Task): +class PlotVacuumMorphologies(BaseTask): """Plot morphologies to obtain annotations.""" pdf_filename = luigi.Parameter(default="vacuum_morphologies.pdf") @@ -565,10 +566,10 @@ def output(self): return luigi.LocalTarget(self.pdf_filename) -class RescaleMorphologies(luigi.Task): +class RescaleMorphologies(BaseTask): """Rescale morphologies for synthesis input.""" - morphology_path = luigi.Parameter(default="morphology_path") + morphology_path = luigi.Parameter(default=None) rescaled_morphology_path = luigi.Parameter(default="rescaled_morphology_path") rescaled_morphology_base_path = luigi.Parameter(default="rescaled_morphologies") scaling_rules_path = luigi.Parameter(default="scaling_rules.yaml") diff --git a/synthesis_workflow/utils_tasks.py b/synthesis_workflow/utils_tasks.py index 7e1dd12..660e814 100644 --- a/synthesis_workflow/utils_tasks.py +++ b/synthesis_workflow/utils_tasks.py @@ -1,10 +1,14 @@ """utils functions for luigi tasks.""" +import logging from pathlib import Path import luigi import pandas as pd +logger = logging.getLogger("luigi-interface") + + def load_circuit( path_to_mvd3=None, path_to_morphologies=None, @@ -36,7 +40,7 @@ def get_morphs_df( """Get valid morphs_df for diametrizer (select using flag + remove duplicates).""" morphs_df = pd.read_csv(morphs_df_path) if to_use_flag != "all": - morphs_df = morphs_df[morphs_df[to_use_flag]] + morphs_df = morphs_df.loc[morphs_df[to_use_flag]] if h5_path is not None: morphs_df[morphology_path] = morphs_df[ "_".join(morphology_path.split("_")[:-1]) @@ -52,38 +56,16 @@ def update_morphs_df(morphs_df_path, new_morphs_df): class diametrizerconfigs(luigi.Config): """Diametrizer configuration.""" - model = luigi.Parameter( - config_path={"section": "DIAMETRIZER", "name": "model"}, default="generic", - ) - terminal_threshold = luigi.FloatParameter( - config_path={"section": "DIAMETRIZER", "name": "terminal_threshold"}, - default=2.0, - ) - taper_min = luigi.FloatParameter( - config_path={"section": "DIAMETRIZER", "name": "taper_min"}, default=-0.01, - ) - taper_max = luigi.FloatParameter( - config_path={"section": "DIAMETRIZER", "name": "taper_max"}, default=1e-6, - ) - asymmetry_threshold_basal = luigi.FloatParameter( - config_path={"section": "DIAMETRIZER", "name": "asymmetry_threshold_basal"}, - default=1.0, - ) - asymmetry_threshold_apical = luigi.FloatParameter( - config_path={"section": "DIAMETRIZER", "name": "asymmetry_threshold_apical"}, - default=0.2, - ) - neurite_types = luigi.ListParameter( - config_path={"section": "DIAMETRIZER", "name": "neurite_types"}, - default=["basal", "apical"], - ) + model = luigi.Parameter(default="generic") + terminal_threshold = luigi.FloatParameter(default=2.0) + taper_min = luigi.FloatParameter(default=-0.01) + taper_max = luigi.FloatParameter(default=1e-6) + asymmetry_threshold_basal = luigi.FloatParameter(default=1.0) + asymmetry_threshold_apical = luigi.FloatParameter(default=0.2) + neurite_types = luigi.ListParameter(default=["basal", "apical"]) - trunk_max_tries = luigi.IntParameter( - config_path={"section": "DIAMETRIZER", "name": "trunk_max_tries"}, default=100, - ) - n_samples = luigi.IntParameter( - config_path={"section": "DIAMETRIZER", "name": "n_samples"}, default=2, - ) + trunk_max_tries = luigi.IntParameter(default=100) + n_samples = luigi.IntParameter(default=2) def __init__(self, *args, **kwargs): """Init.""" @@ -114,64 +96,70 @@ def __init__(self, *args, **kwargs): class synthesisconfigs(luigi.Config): """Circuit configuration.""" - tmd_parameters_path = luigi.Parameter( - config_path={"section": "SYNTHESIS", "name": "tmd_parameters_path"}, - default="tmd_parameters.json", - ) - - custom_tmd_parameters_path = luigi.Parameter( - config_path={"section": "SYNTHESIS", "name": "custom_tmd_parameters_path"}, - default=None, - ) - - tmd_distributions_path = luigi.Parameter( - config_path={"section": "SYNTHESIS", "name": "tmd_distributions_path"}, - default="tmd_distributions.json", - ) - - pc_in_types_path = luigi.Parameter( - config_path={"section": "SYNTHESIS", "name": "pc_in_types_path"}, - default="pc_in_types.yaml", - ) - - cortical_thickness = luigi.Parameter( - config_path={"section": "SYNTHESIS", "name": "cortical_thickness"}, - default="[165, 149, 353, 190, 525, 700]", - ) + tmd_parameters_path = luigi.Parameter(default="tmd_parameters.json") + tmd_distributions_path = luigi.Parameter(default="tmd_distributions.json") + pc_in_types_path = luigi.Parameter(default="pc_in_types.yaml") + cortical_thickness = luigi.Parameter(default="[165, 149, 353, 190, 525, 700]") + to_use_flag = luigi.Parameter(default="all") + mtypes = luigi.ListParameter(default=["all"]) class circuitconfigs(luigi.Config): """Circuit configuration.""" - circuit_somata_path = luigi.Parameter( - config_path={"section": "CIRCUIT", "name": "circuit_somata_path"}, - default="circuit_somata.mvd3", - ) - - atlas_path = luigi.Parameter( - config_path={"section": "CIRCUIT", "name": "atlas_path"}, default=None - ) + circuit_somata_path = luigi.Parameter(default="circuit_somata.mvd3") + atlas_path = luigi.Parameter(default=None) class pathconfigs(luigi.Config): """Morphology path configuration.""" - morphs_df_path = luigi.Parameter( - config_path={"section": "PATHS", "name": "morphs_df_path"}, - default="morphs_df.csv", - ) + morphs_df_path = luigi.Parameter(default="morphs_df.csv") + morphology_path = luigi.Parameter(default="morphology_path") + synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv") + synth_output_path = luigi.Parameter(default="synthesized_morphologies") + substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv") - synth_morphs_df_path = luigi.Parameter( - config_path={"section": "PATHS", "name": "synth_morphs_df_path"}, - default="synth_morphs_df.csv", - ) - synth_output_path = luigi.Parameter( - config_path={"section": "PATHS", "name": "synth_output_path"}, - default="synthesized_morphologies", - ) +class BaseTask(luigi.Task): + """Base class used to add customisable global parameters""" + _global_configs = [diametrizerconfigs, synthesisconfigs, circuitconfigs, pathconfigs] - substituted_morphs_df_path = luigi.Parameter( - config_path={"section": "PATHS", "name": "substituted_morphs_df_path"}, - default="substituted_morphs_df.csv", - ) + def __init__(self, *args, **kwargs): + super(BaseTask, self).__init__(*args, **kwargs) + + # Replace run() method by a new one which adds logging before the actual run() method + if not hasattr(self, "_actual_run"): + # Rename actual run() method + self._actual_run = super(BaseTask, self).__getattribute__("run") + + # Replace by _run_debug() method + self.run = super(BaseTask, self).__getattribute__("_run_debug") + + def _run_debug(self): + class_name = self.__class__.__name__ + logger.debug("Attributes of {} class after global processing:".format(class_name)) + for name, attr in self.get_params(): + try: + logger.debug("Atribute: {} == {}".format(name, getattr(self, name))) + except: + logger.debug("Can't print '{}' attribute for unknown reason".format(name)) + + logger.debug("Running {} task".format(class_name)) + gen = self._actual_run() + logger.debug("{} task ended properly".format(class_name)) + return gen + + def __getattribute__(self, name): + tmp = super(BaseTask, self).__getattribute__(name) + if tmp is not None: + return tmp + for conf in self._global_configs: + tmp_conf = conf() + if hasattr(tmp_conf, name): + return getattr(tmp_conf, name) + return tmp + + +class BaseWrapperTask(BaseTask, luigi.WrapperTask): + pass diff --git a/synthesis_workflow/validation_tasks.py b/synthesis_workflow/validation_tasks.py index b35764d..a2c7cb0 100644 --- a/synthesis_workflow/validation_tasks.py +++ b/synthesis_workflow/validation_tasks.py @@ -1,5 +1,4 @@ """Luigi tasks for validation of synthesis.""" -import logging from pathlib import Path import yaml @@ -11,6 +10,7 @@ from .synthesis_tasks import VacuumSynthesize from .synthesis_tasks import RescaleMorphologies from .synthesis_tasks import GetSynthetisedNeuriteLengths +from .utils_tasks import BaseWrapperTask from .utils_tasks import circuitconfigs from .utils_tasks import load_circuit from .utils_tasks import pathconfigs @@ -21,9 +21,6 @@ from .validation import plot_morphometrics -L = logging.getLogger(__name__) - - class ConvertMvd3(luigi.Task): """Convert synthesize mvd3 file to morphs_df.csv file. @@ -142,7 +139,7 @@ def output(self): return luigi.LocalTarget(self.density_profiles_path) -class PlotCollage(luigi.WrapperTask): +class PlotCollage(BaseWrapperTask): """Plot collage. Args: @@ -153,13 +150,13 @@ class PlotCollage(luigi.WrapperTask): collage_base_path = luigi.Parameter(default="collages") sample = luigi.IntParameter(default=20) collage_type = luigi.Parameter(default="O1") - mtypes = luigi.ListParameter(["all"]) + mtypes = luigi.ListParameter(default=None) def requires(self): """""" if self.mtypes[0] == "all": self.mtypes = pd.read_csv( - pathconfigs().morphs_combos_df_path + pathconfigs().synth_morphs_df_path ).mtype.unique() tasks = [] @@ -231,5 +228,6 @@ class ValidateSynthesis(luigi.WrapperTask): def requires(self): """""" # tasks = [PlotMorphometrics(), PlotDensityProfiles(), PlotCollage()] - tasks = [PlotCollage()] + tasks = [PlotMorphometrics(), PlotDensityProfiles()] + # tasks = [PlotCollage()] return tasks