Use luigi-tools>=0.0.5 to automatically create parent directories of …

…task targets

Change-Id: Icda71c6ddab031bfab9797f12a9f8188ba1d5251

CHANGELOG.rst

@@ -1,6 +1,26 @@
 Changelog
 =========
 
+Version 0.0.11
+--------------
+
+Improvements
+~~~~~~~~~~~~
+- Use luigi-tools>=0.0.5
+
+Version 0.0.10
+--------------
+
+Improvements
+~~~~~~~~~~~~
+- Add methodology in the doc
+- Use luigi-tools package
+
+Bug Fixes
+~~~~~~~~~
+- Fix BuildAxonMorphologies to use worker from placement_algorithm
+- Fix PlotPathDistanceFits for mtypes with no fit
+
 Version 0.0.9
 -------------
 

requirements.pip

@@ -5,16 +5,20 @@ bluepyopt
 bluepymm
 diameter_synthesis>=0.1.7
 gitpython
-h5py
+h5py<3
+jinja2
 joblib
 luigi
+luigi-tools>=0.0.5
 matplotlib
+morph_tool>=2.2.21
 morph_validator
 morphio
 neuroc
-neurom
+neurom!=2.0.1.dev4
 pandas
 placement_algorithm>=2.1.1
+PyYAML
 region_grower>=0.1.10
 scipy
 seaborn

setup.py

@@ -24,7 +24,7 @@
     "jinja2",
     "joblib",
     "luigi",
-    "luigi-tools>=0.0.3",
+    "luigi-tools>=0.0.5",
     "matplotlib",
     "morph_tool>=2.2.21",
     "morph_validator",

src/synthesis_workflow/tasks/circuit.py

@@ -27,7 +27,6 @@
 from synthesis_workflow.tasks.luigi_tools import RatioParameter
 from synthesis_workflow.tasks.luigi_tools import WorkflowTask
 from synthesis_workflow.tasks.utils import GetSynthesisInputs
-from synthesis_workflow.tools import ensure_dir
 from synthesis_workflow.tools import get_layer_tags
 
 
@@ -61,11 +60,9 @@ def run(self):
             )
 
         annotation_path = self.output()["annotations"].path
-        ensure_dir(annotation_path)
         annotation.save_nrrd(annotation_path)
 
         layer_mapping_path = self.output()["layer_mapping"].path
-        ensure_dir(layer_mapping_path)
         yaml.dump(
             layer_mapping,
             open(
@@ -201,7 +198,6 @@ def run(self):
             mask=thickness_mask_path,
             seed=self.seed,
         )
-        ensure_dir(self.output().path)
         cells.save(self.output().path)
 
     def output(self):
@@ -251,7 +247,6 @@ def run(self):
             hemisphere=self.hemisphere,
         )
 
-        ensure_dir(self.output().path)
         cells = slice_circuit(self.input()["circuit"].path, self.output().path, _slicer)
 
         if len(cells.index) == 0:

src/synthesis_workflow/tasks/synthesis.py

@@ -36,7 +36,6 @@
 from synthesis_workflow.tasks.luigi_tools import WorkflowTask
 from synthesis_workflow.tasks.utils import CreateAnnotationsFile
 from synthesis_workflow.tasks.utils import GetSynthesisInputs
-from synthesis_workflow.tools import ensure_dir
 from synthesis_workflow.tools import find_case_insensitive_file
 from synthesis_workflow.tools import load_neurondb_to_dataframe
 
@@ -91,7 +90,6 @@ def run(self):
         # Remove duplicated morphologies in L23
         morphs_df.drop_duplicates(subset=["name"], inplace=True)
 
-        ensure_dir(self.output().path)
         morphs_df.to_csv(self.output().path)
 
     def output(self):
@@ -128,7 +126,6 @@ def run(self):
         substituted_morphs_df = apply_substitutions(
             pd.read_csv(self.input()["morphs_df"].path), substitution_rules
         )
-        ensure_dir(self.output().path)
         substituted_morphs_df.to_csv(self.output().path, index=False)
 
     def output(self):
@@ -325,7 +322,6 @@ def requires(self):
 
     def run(self):
         """"""
-        ensure_dir(self.output().path)
         if self.annotations_path is None:
             annotations_file = None
             neurondb_path = None
@@ -435,16 +431,12 @@ def run(self):
         out_morphologies = self.output()["out_morphologies"]
         out_apical_points = self.output()["apical_points"]
         debug_scales = self.output().get("debug_scales")
+
         if debug_scales is not None:
             debug_scales_path = debug_scales.path
         else:
             debug_scales_path = None
 
-        ensure_dir(axon_morphs_path)
-        ensure_dir(out_mvd3.path)
-        ensure_dir(out_apical_points.path)
-        ensure_dir(out_morphologies.path)
-
         # Get base-morph-dir argument value
         if self.axon_morphs_base_dir is None:
             axon_morphs_base_dir = get_axon_base_dir(

src/synthesis_workflow/tasks/vacuum_synthesis.py

@@ -1,7 +1,6 @@
 """Luigi tasks for morphology synthesis in vacuum."""
 import json
 import logging
-from pathlib import Path
 
 import luigi
 import morphio
@@ -17,7 +16,6 @@
 from synthesis_workflow.tasks.luigi_tools import WorkflowTask
 from synthesis_workflow.tasks.synthesis import BuildSynthesisDistributions
 from synthesis_workflow.tasks.synthesis import BuildSynthesisParameters
-from synthesis_workflow.tools import ensure_dir
 from synthesis_workflow.vacuum_synthesis import grow_vacuum_morphologies
 from synthesis_workflow.vacuum_synthesis import plot_vacuum_morphologies
 from synthesis_workflow.vacuum_synthesis import VACUUM_SYNTH_MORPHOLOGY_PATH
@@ -76,14 +74,14 @@ def run(self):
         else:
             mtypes = self.mtypes
 
-        Path(self.output()["out_morphologies"].path).mkdir(parents=True, exist_ok=True)
-        morphology_base_path = Path(self.output()["out_morphologies"].path).absolute()
+        morphology_base_path = self.output()["out_morphologies"].pathlib_path
+        morphology_base_path.mkdir(parents=True, exist_ok=True)
         vacuum_synth_morphs_df = grow_vacuum_morphologies(
             mtypes,
             self.n_cells,
             tmd_parameters,
             tmd_distributions,
-            morphology_base_path,
+            morphology_base_path.absolute(),
             vacuum_morphology_path=self.vacuum_synth_morphology_path,
             diametrizer=self.diametrizer,
             joblib_verbose=self.joblib_verbose,
@@ -124,7 +122,6 @@ def requires(self):
     def run(self):
         """"""
         vacuum_synth_morphs_df = pd.read_csv(self.input()["out_morphs_df"].path)
-        ensure_dir(self.output().path)
         plot_vacuum_morphologies(
             vacuum_synth_morphs_df,
             self.output().path,

src/version.py

@@ -1,2 +1,2 @@
 """Package version."""
-VERSION = "0.0.10"
+VERSION = "0.0.11.dev0"