Use specific targets to improve output directory tree

Change-Id: I0c175bc6fe574fc4970fcf98713c0faf38d182a8

examples/luigi_cfg/luigi_vacuum.cfg

@@ -1,3 +1,5 @@
+# TODO: Update this file according to new target classes
+
 # luigi parameters
 # [core]
 # logging_conf_file = logging.conf

synthesis_workflow/tasks/circuit.py

@@ -17,14 +17,16 @@
 from ..circuit import halve_atlas
 from ..circuit import slice_circuit
 from ..tools import ensure_dir
+from .config import AtlasLocalTarget
 from .config import CircuitConfig
+from .config import CircuitLocalTarget
 from .config import PathConfig
 from .config import SynthesisConfig
 from .luigi_tools import BoolParameter
 from .luigi_tools import copy_params
-from .luigi_tools import OutputLocalTarget
 from .luigi_tools import ParamLink
 from .luigi_tools import WorkflowTask
+from .utils import GetSynthesisInputs
 
 
 class CreateAtlasLayerAnnotations(WorkflowTask):
@@ -83,8 +85,8 @@ def output(self):
             annotation_base_name + "_layer_mapping"
         ).with_suffix(".yaml")
         return {
-            "annotations": OutputLocalTarget(annotation_path),
-            "layer_mapping": OutputLocalTarget(layer_mapping_path),
+            "annotations": AtlasLocalTarget(annotation_path),
+            "layer_mapping": AtlasLocalTarget(layer_mapping_path),
         }
 
 
@@ -139,7 +141,7 @@ def run(self):
 
     def output(self):
         """"""
-        return OutputLocalTarget(self.atlas_planes_path + ".npz")
+        return AtlasLocalTarget(self.atlas_planes_path + ".npz")
 
 
 @copy_params(
@@ -156,7 +158,8 @@ class BuildCircuit(WorkflowTask):
     """
 
     cell_composition_path = luigi.Parameter(
-        description="path to the cell composition file (YAML)"
+        default="cell_composition.yaml",
+        description="path to the cell composition file (YAML)",
     )
     density_factor = luigi.NumericalParameter(
         default=0.01,
@@ -169,11 +172,17 @@ class BuildCircuit(WorkflowTask):
     )
     seed = luigi.IntParameter(default=None, description="pseudo-random generator seed")
 
+    def requires(self):
+        """"""
+        return GetSynthesisInputs()
+
     def run(self):
         """"""
+        cell_composition_path = self.input().ppath / self.cell_composition_path
+        mtype_taxonomy_path = self.input().ppath / self.mtype_taxonomy_path
         cells = build_circuit(
-            self.cell_composition_path,
-            self.mtype_taxonomy_path,
+            cell_composition_path,
+            mtype_taxonomy_path,
             CircuitConfig().atlas_path,
             self.density_factor,
             self.seed,
@@ -183,7 +192,7 @@ def run(self):
 
     def output(self):
         """"""
-        return OutputLocalTarget(CircuitConfig().circuit_somata_path)
+        return CircuitLocalTarget(CircuitConfig().circuit_somata_path)
 
 
 @copy_params(
@@ -238,4 +247,4 @@ def run(self):
 
     def output(self):
         """"""
-        return OutputLocalTarget(self.sliced_circuit_path)
+        return CircuitLocalTarget(self.sliced_circuit_path)

synthesis_workflow/tasks/config.py

@@ -76,10 +76,11 @@ class SynthesisConfig(luigi.Config):
     """Synthesis global configuration."""
 
     tmd_parameters_path = luigi.Parameter(
-        default="tmd_parameters.json", description="The path to the TMD parameters"
+        default="tns_input/tmd_parameters.json",
+        description="The path to the TMD parameters",
     )
     tmd_distributions_path = luigi.Parameter(
-        default="tmd_distributions.json",
+        default="tns_input/tmd_distributions.json",
         description="The path to the TMD distributions",
     )
     cortical_thickness = luigi.ListParameter(
@@ -101,14 +102,25 @@ class CircuitConfig(luigi.Config):
 class PathConfig(luigi.Config):
     """Morphology path configuration."""
 
-    ext = ExtParameter(default="asc")
+    mtype_taxonomy_path = luigi.Parameter(
+        default="mtype_taxonomy.tsv", description="path to the taxonomy file (TSV)"
+    )
+    local_synthesis_input_path = luigi.Parameter(default="synthesis_input")
+
     result_path = luigi.Parameter(default="out")
-    morphs_df_path = luigi.Parameter(default="morphs_df.csv")
+    atlas_subpath = luigi.Parameter(default="atlas")
+    circuit_subpath = luigi.Parameter(default="circuit")
+    morphs_df_subpath = luigi.Parameter(default="morphs_df")
+    synthesis_subpath = luigi.Parameter(default="synthesis")
+    validation_subpath = luigi.Parameter(default="validation")
+
+    ext = ExtParameter(default="asc")
     morphology_path = luigi.Parameter(default="repaired_morphology_path")
-    mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (TSV)")
+    morphs_df_path = luigi.Parameter(default="morphs_df.csv")
+    substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv")
     synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv")
     synth_output_path = luigi.Parameter(default="synthesized_morphologies")
-    substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv")
+
     debug_region_grower_scales_path = luigi.Parameter(
         default="region_grower_scales_logs"
     )
@@ -117,17 +129,53 @@ class PathConfig(luigi.Config):
 class ValidationConfig(luigi.Config):
     """Validation configuration."""
 
-    validation_subpath = luigi.Parameter(default="validation")
     sample = OptionalIntParameter(default=None)
 
 
+class AtlasLocalTarget(OutputLocalTarget):
+    """Specific target for atlas targets"""
+
+
+class CircuitLocalTarget(OutputLocalTarget):
+    """Specific target for circuit targets"""
+
+
+class MorphsDfLocalTarget(OutputLocalTarget):
+    """Specific target for morphology dataframe targets"""
+
+
+class SynthesisLocalTarget(OutputLocalTarget):
+    """Specific target for synthesis targets"""
+
+
 class ValidationLocalTarget(OutputLocalTarget):
-    """Specific target for validation tasks"""
+    """Specific target for validation targets"""
 
 
+# Set default output paths
 OutputLocalTarget.set_default_prefix(PathConfig().result_path)
+AtlasLocalTarget.set_default_prefix(
+    # pylint: disable=protected-access
+    OutputLocalTarget._prefix
+    / PathConfig().atlas_subpath
+)
+CircuitLocalTarget.set_default_prefix(
+    # pylint: disable=protected-access
+    OutputLocalTarget._prefix
+    / PathConfig().circuit_subpath
+)
+MorphsDfLocalTarget.set_default_prefix(
+    # pylint: disable=protected-access
+    OutputLocalTarget._prefix
+    / PathConfig().morphs_df_subpath
+)
+SynthesisLocalTarget.set_default_prefix(
+    # pylint: disable=protected-access
+    OutputLocalTarget._prefix
+    / PathConfig().synthesis_subpath
+)
 ValidationLocalTarget.set_default_prefix(
     # pylint: disable=protected-access
     OutputLocalTarget._prefix
-    / ValidationConfig().validation_subpath
+    / PathConfig().validation_subpath
 )

synthesis_workflow/tasks/diametrizer.py

@@ -79,7 +79,9 @@ class BuildDiameterModels(WorkflowTask):
     plot_models = BoolParameter()
 
     def run(self):
-        """Run."""
+        """Run"""
+        raise DeprecationWarning("This task must be updated to be used")
+        # pylint: disable=unreachable
 
         config_model = DiametrizerConfig().config_model
         morphs_df = pd.read_csv(self.morphs_df_path)
@@ -156,11 +158,13 @@ class Diametrize(WorkflowTask):
     new_morphs_df_path = luigi.Parameter(default="diametrized_morphs_df.csv")
 
     def requires(self):
-        """Requres."""
+        """Requires"""
         return BuildDiameterModels()
 
     def run(self):
-        """Run."""
+        """Run"""
+        raise DeprecationWarning("This task must be updated to be used")
+        # pylint: disable=unreachable
 
         config = DiametrizerConfig().config_diametrizer