Skip to content

Commit

Permalink
Clean vacuum workflow
Browse files Browse the repository at this point in the history
Change-Id: I9d40c6f78ffba13f2428c805779c6386f1f1a981
  • Loading branch information
adrien-berchet committed Oct 23, 2020
1 parent 10d930b commit c7736c7
Show file tree
Hide file tree
Showing 7 changed files with 148 additions and 45 deletions.
38 changes: 38 additions & 0 deletions examples/luigi_cfg/logging.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
[loggers]
keys=root,luigi,luigi_interface

[handlers]
keys=consoleHandler,fileHandler

[formatters]
keys=PrettyFormatter

[logger_root]
level=DEBUG
handlers=consoleHandler,fileHandler

[logger_luigi]
level=INFO
handlers=consoleHandler,fileHandler
qualname=luigi
propagate=0

[logger_luigi_interface]
level=DEBUG
handlers=consoleHandler,fileHandler
qualname=luigi-interface
propagate=0

[handler_consoleHandler]
class=StreamHandler
formatter=PrettyFormatter
args=(sys.stdout,)

[handler_fileHandler]
class=FileHandler
formatter=PrettyFormatter
args=('synthesis_workflow.log',)

[formatter_PrettyFormatter]
format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
datefmt=%Y-%m-%d %H:%M:%S
33 changes: 18 additions & 15 deletions examples/luigi_cfg/luigi_vacuum.cfg
Original file line number Diff line number Diff line change
@@ -1,21 +1,25 @@
# global parameters
# luigi parameters
[core]
logging_conf_file = logging.conf

# global parameters
[SynthesisConfig]
tmd_parameters_path = out/tmd_parameters.json
tmd_distributions_path = out/tmd_distributions.json
tmd_parameters_path = out_vacuum/tmd_parameters.json
tmd_distributions_path = out_vacuum/tmd_distributions.json

[PathConfig]
morphs_df_path = synthesis_configs/morphs_df.csv
synth_morphs_df_path = out/synth_morphs_df.csv
synth_output_path = out/synthesized_morphologies
substituted_morphs_df_path = out/substituted_morphs_df.csv
synth_morphs_df_path = out_vacuum/synth_morphs_df.csv
synth_output_path = out_vacuum/synthesized_morphologies
substituted_morphs_df_path = out_vacuum/substituted_morphs_df.csv
morphology_path = vacuum_morphology_path

# synthesis setup
[ApplySubstitutionRules]
substitution_rules_path = synthesis_configs/substitution_rules.yaml

[BuildSynthesisParameters]
tmd_parameters_path = out/tmd_parameters_no_scaling.json
tmd_parameters_path = out_vacuum/tmd_parameters_no_scaling.json
input_tmd_parameters_path = synthesis_configs/tmd_specific_parameters.json
morphology_path = repaired_morphology_path_h5

Expand All @@ -24,23 +28,22 @@ morphology_path = repaired_morphology_path_h5

# synthesize in vacuum
[VacuumSynthesize]
vacuum_synth_morphology_path = out/vacuum_synth_morphologies
vacuum_synth_morphs_df_path = out/vacuum_synth_morphs_df.csv
vacuum_synth_morphology_path = out_vacuum/vacuum_synth_morphologies
vacuum_synth_morphs_df_path = out_vacuum/vacuum_synth_morphs_df.csv
n_cells = 10
mtypes = ["all"]

# validation plots
[ValidateVacuumSynthesis]
with_morphometrics = False
with_density_profiles = False
with_morphometrics = True
with_density_profiles = True
with_vacuum_morphologies= True

[PlotVacuumMorphologies]
pdf_filename = figures/vacuum_morphologies.pdf
pdf_filename = figures_vacuum/vacuum_morphologies.pdf

[PlotMorphometrics]
morphometrics_path = figures/morphometrics
morphometrics_path = figures_vacuum/morphometrics

[PlotDensityProfiles]
density_profiles_path = figures/density_profiles.pdf

density_profiles_path = figures_vacuum/density_profiles.pdf
22 changes: 16 additions & 6 deletions synthesis_workflow/tasks/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,20 @@ class RunnerConfig(luigi.Config):
class SynthesisConfig(luigi.Config):
"""Synthesis global configuration."""

tmd_parameters_path = luigi.Parameter(default="tmd_parameters.json")
tmd_distributions_path = luigi.Parameter(default="tmd_distributions.json")
cortical_thickness = luigi.ListParameter(default=[165, 149, 353, 190, 525, 700])
mtypes = luigi.ListParameter(default=None)
tmd_parameters_path = luigi.Parameter(
default="tmd_parameters.json", description="The path to the TMD parameters"
)
tmd_distributions_path = luigi.Parameter(
default="tmd_distributions.json",
description="The path to the TMD distributions",
)
cortical_thickness = luigi.ListParameter(
default=[165, 149, 353, 190, 525, 700],
description="The list of cortical thicknesses",
)
mtypes = luigi.ListParameter(
default=None, description="The list of mtypes to process"
)


class CircuitConfig(luigi.Config):
Expand All @@ -90,8 +100,8 @@ class PathConfig(luigi.Config):
"""Morphology path configuration."""

ext = ExtParameter(default="asc")
# TODO: use out_path as suffix for all output paths
out_path = luigi.Parameter(default="out")
# TODO: use result_path as suffix for all output paths
result_path = luigi.Parameter(default="out")
morphs_df_path = luigi.Parameter(default="morphs_df.csv")
morphology_path = luigi.Parameter(default="repaired_morphology_path")
synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv")
Expand Down
2 changes: 1 addition & 1 deletion synthesis_workflow/tasks/vacuum_synthesis.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def output(self):


@copy_params(
morphology_path=ParamLink(PathConfig, default="vacuum_morphology_path"),
morphology_path=ParamLink(PathConfig),
)
class PlotVacuumMorphologies(WorkflowTask):
"""Plot morphologies to obtain annotations."""
Expand Down
26 changes: 19 additions & 7 deletions synthesis_workflow/tasks/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from ..validation import plot_morphometrics
from ..validation import plot_path_distance_fits
from ..validation import plot_scale_statistics
from ..validation import VacuumCircuit
from .circuit import CreateAtlasPlanes
from .circuit import CreateAtlasLayerAnnotations
from .config import CircuitConfig
Expand Down Expand Up @@ -95,7 +96,7 @@ def run(self):
""""""
if self.morph_type == "in_vacuum":
synthesize_task = self.input()[0]
synth_morphs_df = pd.read_csv(synthesize_task.path)
synth_morphs_df = pd.read_csv(synthesize_task["out_morphs_df"].path)

rescalemorphologies_task = self.input()[1]
morphs_df = pd.read_csv(rescalemorphologies_task.path)
Expand Down Expand Up @@ -146,16 +147,27 @@ class PlotDensityProfiles(WorkflowTask):

def requires(self):
""""""
return Synthesize()
if self.region == "in_vacuum":
return VacuumSynthesize()
else:
return Synthesize()

def run(self):
""""""

circuit = load_circuit(
path_to_mvd3=self.input()["out_mvd3"].path,
path_to_morphologies=PathConfig().synth_output_path,
path_to_atlas=CircuitConfig().atlas_path,
)
if self.region == "in_vacuum":
df = pd.read_csv(self.input()["out_morphs_df"].path)
circuit = VacuumCircuit(
morphs_df=df,
cells=pd.DataFrame(df["mtype"].unique(), columns=["mtypes"]),
morphology_path=PathConfig().morphology_path,
)
else:
circuit = load_circuit(
path_to_mvd3=self.input()["out_mvd3"].path,
path_to_morphologies=PathConfig().synth_output_path,
path_to_atlas=CircuitConfig().atlas_path,
)

plot_density_profiles(
circuit,
Expand Down
2 changes: 1 addition & 1 deletion synthesis_workflow/tasks/workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def requires(self):
if self.with_vacuum_morphologies:
tasks.append(PlotVacuumMorphologies())
if self.with_density_profiles:
tasks.append(PlotDensityProfiles())
tasks.append(PlotDensityProfiles(region="in_vacuum"))
return tasks


Expand Down
70 changes: 55 additions & 15 deletions synthesis_workflow/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import re
import warnings
from collections import defaultdict
from collections import namedtuple
from functools import partial
from pathlib import Path
from tempfile import TemporaryDirectory
Expand All @@ -20,15 +21,17 @@
from matplotlib.backends.backend_pdf import PdfPages
from pyquaternion import Quaternion
from scipy.optimize import fmin
from tqdm import tqdm

from atlas_analysis.constants import CANONICAL
from bluepy.v2 import Circuit
from morph_validator.feature_configs import get_feature_configs
from morph_validator.plotting import get_features_df
from morph_validator.plotting import plot_violin_features
from morph_validator.spatial import relative_depth_volume
from morph_validator.spatial import sample_morph_voxel_values
from morphio.mut import Morphology
from neurom import viewer
from neurom.core.dataformat import COLS
from region_grower.atlas_helper import AtlasHelper
from region_grower.modify import scale_target_barcode
from tmd.io.io import load_population
Expand All @@ -49,6 +52,9 @@
matplotlib.use("Agg")


VacuumCircuit = namedtuple("VacuumCircuit", ["cells", "morphs_df", "morphology_path"])


def convert_mvd3_to_morphs_df(mvd3_path, synth_output_path, ext="asc"):
"""Convert the list of morphologies from mvd3 to morphology dataframe.
Expand Down Expand Up @@ -127,13 +133,14 @@ def plot_morphometrics(

all_features_df = pd.concat([base_features_df, comp_features_df])
ensure_dir(output_path)
plot_violin_features(
all_features_df,
["basal_dendrite", "apical_dendrite"],
output_dir=Path(output_path),
bw=0.1,
normalize=normalize,
)
with DisableLogger():
plot_violin_features(
all_features_df,
["basal_dendrite", "apical_dendrite"],
output_dir=Path(output_path),
bw=0.1,
normalize=normalize,
)


def _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance):
Expand All @@ -151,7 +158,30 @@ def _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance):
point_depths[neurite_type.name] += data.tolist()

df = pd.DataFrame.from_dict(point_depths, orient="index").T
return df.melt(var_name="neurite_type", value_name="y").dropna()
return (
df.melt(var_name="neurite_type", value_name="y")
.dropna()
.sort_values("neurite_type")
)


def _get_vacuum_depths_df(circuit, mtype):
"""Create dataframe with depths data for violin plots."""
morphs_df = circuit.morphs_df
path = circuit.morphology_path
cells = morphs_df.loc[morphs_df["mtype"] == mtype, path]
point_depths = defaultdict(list)
for cell_path in cells:
morph = Morphology(cell_path)
for i in morph.iter():
point_depths[i.type.name] += i.points[COLS.Y].tolist()

df = pd.DataFrame.from_dict(point_depths, orient="index").T
return (
df.melt(var_name="neurite_type", value_name="y")
.dropna()
.sort_values("neurite_type")
)


def _plot_layers(x_pos, atlas, ax):
Expand Down Expand Up @@ -187,11 +217,16 @@ def _plot_density_profile(
"""Plot density profile of an mtype."""
fig = plt.figure()
ax = plt.gca()
_plot_layers(x_pos, circuit.atlas, ax)
try:
plot_df = _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance)
sns.violinplot(x="neurite_type", y="y", data=plot_df, ax=ax, bw=0.1)
ax.legend(loc="best")
if isinstance(circuit, Circuit):
_plot_layers(x_pos, circuit.atlas, ax)
plot_df = _get_depths_df(circuit, mtype, sample, voxeldata, sample_distance)
ax.legend(loc="best")
elif isinstance(circuit, VacuumCircuit):
plot_df = _get_vacuum_depths_df(circuit, mtype)

with DisableLogger():
sns.violinplot(x="neurite_type", y="y", data=plot_df, ax=ax, bw=0.1)
except Exception: # pylint: disable=broad-except
ax.text(
0.5,
Expand All @@ -212,7 +247,12 @@ def plot_density_profiles(
WIP function, waiting on complete atlas to update.
"""
voxeldata = relative_depth_volume(circuit.atlas, in_region=region, relative=False)
if region == "in_vacuum":
voxeldata = None
else:
voxeldata = relative_depth_volume(
circuit.atlas, in_region=region, relative=False
)
x_pos = 0

ensure_dir(output_path)
Expand All @@ -226,7 +266,7 @@ def plot_density_profiles(
sample_distance=sample_distance,
)
for fig in Parallel(nb_jobs)(
delayed(f)(mtype) for mtype in tqdm(sorted(circuit.cells.mtypes))
delayed(f)(mtype) for mtype in sorted(circuit.cells.mtypes)
):
with DisableLogger():
pdf.savefig(fig, bbox_inches="tight")
Expand Down

0 comments on commit c7736c7

Please sign in to comment.