generalise git clone task

so we specify a folder location on a repo to get synthesis input files
this also makes by default the use of that task and CreateMorphsDF task
also fixes a small bug in plotvacuummorphologies

Change-Id: Iabbf19af9d7d0695e22c2ee9afc7bfec60bcf413

examples/luigi_cfg/luigi_vacuum.cfg

@@ -14,6 +14,17 @@ synth_output_path = out_vacuum/synthesized_morphologies
 substituted_morphs_df_path = out_vacuum/substituted_morphs_df.csv
 morphology_path = vacuum_morphology_path
 
+# prepare step
+[GetSynthesisInputs]
+url = ssh://<username>@bbpcode.epfl.ch/project/proj82
+git_synthesis_input_path = entities/bionames/rat/synthesis_input
+local_synthesis_input_path = synthesis_input
+
+[BuildMorphsDF]
+neurondb_path = /gpfs/bbp.cscs.ch/data/project_no_backup/proj82_no_backup/home/gevaert/morph_release_old_code-2020-07-27/output/06_RepairUnravel-asc/neuronDB.xml
+morphology_dirs = {"repaired_morphology_path": "/gpfs/bbp.cscs.ch/data/project_no_backup/proj82_no_backup/home/gevaert/morph_release_old_code-2020-07-27/output/06_RepairUnravel-asc", "repaired_morphology_path_h5": "/gpfs/bbp.cscs.ch/data/project_no_backup/proj82_no_backup/home/gevaert/morph_release_old_code-2020-07-27/output/06_RepairUnravel-h5"}
+
+
 # synthesis setup
 [ApplySubstitutionRules]
 substitution_rules_path = synthesis_configs/substitution_rules.yaml

synthesis_workflow/circuit.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from atlas_analysis.planes.planes import _create_planes, _create_centerline, _smoothing
+from brainbuilder.app.cells import _place as place
 
 LEFT = 0
 RIGHT = 1
@@ -48,6 +49,37 @@ def halve_atlas(annotated_volume, axis=0, side=LEFT):
     return annotated_volume
 
 
+def build_circuit(
+    cell_composition_path,
+    mtype_taxonomy_path,
+    atlas_path,
+    density_factor=0.01,
+    seed=None,
+):
+    """Based on YAML cell composition recipe, build a circuit as MVD3 file with:
+    - cell positions
+    - required cell properties: 'layer', 'mtype', 'etype'
+    - additional cell properties prescribed by the recipe and / or atlas
+    """
+    if seed is not None:
+        np.random.seed(seed)
+    return place(
+        composition_path=cell_composition_path,
+        mtype_taxonomy_path=mtype_taxonomy_path,
+        atlas_url=atlas_path,
+        mini_frequencies_path=None,
+        atlas_cache=None,
+        region=None,
+        mask_dset=None,
+        density_factor=density_factor,
+        soma_placement="basic",
+        atlas_properties=None,
+        sort_by=None,
+        append_hemisphere=False,
+        input_path=None,
+    )
+
+
 def slice_per_mtype(cells, mtypes):
     """Select cells of given mtype."""
     return cells[cells["mtype"].isin(mtypes)]

synthesis_workflow/synthesis.py

@@ -12,7 +12,6 @@
 from joblib import Parallel
 from tqdm import tqdm
 
-from brainbuilder.app.cells import _place as place
 from morphio.mut import Morphology
 from neuroc.scale import scale_section
 from neuroc.scale import ScaleParameters
@@ -543,34 +542,3 @@ def _process_scaling_rule(
             slope,
             intercept,
         )
-
-
-def build_circuit(
-    cell_composition_path,
-    mtype_taxonomy_path,
-    atlas_path,
-    density_factor=0.01,
-    seed=None,
-):
-    """Based on YAML cell composition recipe, build a circuit as MVD3 file with:
-    - cell positions
-    - required cell properties: 'layer', 'mtype', 'etype'
-    - additional cell properties prescribed by the recipe and / or atlas
-    """
-    if seed is not None:
-        np.random.seed(seed)
-    return place(
-        composition_path=cell_composition_path,
-        mtype_taxonomy_path=mtype_taxonomy_path,
-        atlas_url=atlas_path,
-        mini_frequencies_path=None,
-        atlas_cache=None,
-        region=None,
-        mask_dset=None,
-        density_factor=density_factor,
-        soma_placement="basic",
-        atlas_properties=None,
-        sort_by=None,
-        append_hemisphere=False,
-        input_path=None,
-    )

synthesis_workflow/tasks/circuit.py

@@ -11,12 +11,14 @@
 from atlas_analysis.planes.planes import load_planes_centerline
 from atlas_analysis.planes.planes import save_planes_centerline
 
+from ..circuit import build_circuit
 from ..circuit import circuit_slicer
+from ..circuit import create_planes
 from ..circuit import halve_atlas
 from ..circuit import slice_circuit
-from ..circuit import create_planes
 from ..tools import ensure_dir
 from .config import CircuitConfig
+from .config import PathConfig
 from .config import SynthesisConfig
 from .luigi_tools import copy_params
 from .luigi_tools import OutputLocalTarget
@@ -127,6 +129,48 @@ def output(self):
         return OutputLocalTarget(self.atlas_planes_path + ".npz")
 
 
+@copy_params(
+    mtype_taxonomy_path=ParamLink(PathConfig),
+)
+class BuildCircuit(WorkflowTask):
+    """Generate cell positions and me-types from atlas, compositions and taxonomy.
+
+    Args:
+        cell_composition_path (str): path to the cell composition file (YAML)
+        mtype_taxonomy_path (str): path to the taxonomy file (TSV)
+        density_factor (float): density factor
+        seed (int): pseudo-random generator seed
+    """
+
+    cell_composition_path = luigi.Parameter(
+        description="path to the cell composition file (YAML)"
+    )
+    density_factor = luigi.NumericalParameter(
+        default=0.01,
+        var_type=float,
+        min_value=0,
+        max_value=1,
+        description="The density of positions generated from the atlas",
+    )
+    seed = luigi.IntParameter(default=None, description="pseudo-random generator seed")
+
+    def run(self):
+        """"""
+        cells = build_circuit(
+            self.cell_composition_path,
+            self.mtype_taxonomy_path,
+            CircuitConfig().atlas_path,
+            self.density_factor,
+            self.seed,
+        )
+        ensure_dir(self.output().path)
+        cells.save(self.output().path)
+
+    def output(self):
+        """"""
+        return OutputLocalTarget(CircuitConfig().circuit_somata_path)
+
+
 @copy_params(
     mtypes=ParamLink(SynthesisConfig),
 )
@@ -146,7 +190,10 @@ class SliceCircuit(WorkflowTask):
 
     def requires(self):
         """"""
-        return CreateAtlasPlanes()
+        return {
+            "atlas_planes": CreateAtlasPlanes(),
+            "circuit": BuildCircuit(),
+        }
 
     def run(self):
         """"""
@@ -158,7 +205,7 @@ def run(self):
         ):
             mtypes = None
 
-        planes = load_planes_centerline(self.input().path)["planes"]
+        planes = load_planes_centerline(self.input()["atlas_planes"].path)["planes"]
 
         _slicer = partial(
             circuit_slicer,
@@ -169,9 +216,7 @@ def run(self):
         )
 
         ensure_dir(self.output().path)
-        cells = slice_circuit(
-            CircuitConfig().circuit_somata_path, self.output().path, _slicer
-        )
+        cells = slice_circuit(self.input()["circuit"].path, self.output().path, _slicer)
 
         if len(cells.index) == 0:
             raise Exception("No cells will be synthtesized, better stop here.")

synthesis_workflow/tasks/config.py

@@ -105,7 +105,6 @@ class PathConfig(luigi.Config):
     morphs_df_path = luigi.Parameter(default="morphs_df.csv")
     morphology_path = luigi.Parameter(default="repaired_morphology_path")
     mtype_taxonomy_path = luigi.Parameter(description="path to the taxonomy file (TSV)")
-    pc_in_types_path = luigi.Parameter(default="pc_in_types.yaml")
     synth_morphs_df_path = luigi.Parameter(default="synth_morphs_df.csv")
     synth_output_path = luigi.Parameter(default="synthesized_morphologies")
     substituted_morphs_df_path = luigi.Parameter(default="substituted_morphs_df.csv")

synthesis_workflow/tasks/synthesis.py

@@ -14,7 +14,6 @@
 
 from ..synthesis import add_scaling_rules_to_parameters
 from ..synthesis import apply_substitutions
-from ..synthesis import build_circuit
 from ..synthesis import build_distributions
 from ..synthesis import create_axon_morphologies_tsv
 from ..synthesis import get_axon_base_dir
@@ -41,14 +40,14 @@
 
 
 @copy_params(
-    pc_in_types_path=ParamLink(PathConfig),
+    mtype_taxonomy_path=ParamLink(PathConfig),
 )
 class BuildMorphsDF(WorkflowTask):
     """Generate the list of morphologies with their mtypes and paths.
 
     Args:
         neurondb_path (str): path to the neuronDB file (XML)
-        pc_in_types_path (str): path to the pc_in_types file (TSV)
+        mtype_taxonomy_path (str): path to the mtype_taxonomy.tsv file
         morphology_dirs (str): dict (JSON format) in which keys are column names and values
             are the paths to each morphology file
         apical_points_path (str): path to the apical points file (JSON)
@@ -70,9 +69,10 @@ def run(self):
         morphs_df = load_neurondb_to_dataframe(
             self.neurondb_path,
             self.morphology_dirs,
-            self.pc_in_types_path,
+            self.mtype_taxonomy_path,
             self.apical_points_path,
         )
+        ensure_dir(self.output().path)
         morphs_df.to_csv(self.output().path)
 
     def output(self):
@@ -89,13 +89,17 @@ class ApplySubstitutionRules(WorkflowTask):
 
     substitution_rules_path = luigi.Parameter(default="substitution_rules.yaml")
 
+    def requires(self):
+        """"""
+        return BuildMorphsDF()
+
     def run(self):
         """"""
         with open(self.substitution_rules_path, "rb") as sub_file:
             substitution_rules = yaml.full_load(sub_file)
 
         substituted_morphs_df = apply_substitutions(
-            pd.read_csv(PathConfig().morphs_df_path), substitution_rules
+            pd.read_csv(self.input().path), substitution_rules
         )
         ensure_dir(self.output().path)
         substituted_morphs_df.to_csv(self.output().path, index=False)
@@ -443,44 +447,3 @@ def run(self):
     def output(self):
         """"""
         return OutputLocalTarget(self.rescaled_morphs_df_path)
-
-
-@copy_params(
-    mtype_taxonomy_path=ParamLink(PathConfig),
-)
-class BuildCircuit(WorkflowTask):
-    """Generate cell positions and me-types from atlas, compositions and taxonomy.
-
-    Args:
-        cell_composition_path (str): path to the cell composition file (YAML)
-        mtype_taxonomy_path (str): path to the taxonomy file (TSV)
-        density_factor (float): density factor
-        seed (int): pseudo-random generator seed
-    """
-
-    cell_composition_path = luigi.Parameter(
-        description="path to the cell composition file (YAML)"
-    )
-    density_factor = luigi.NumericalParameter(
-        default=0.01,
-        var_type=float,
-        min_value=0,
-        max_value=1,
-        description="The density of positions generated from the atlas",
-    )
-    seed = luigi.IntParameter(default=None, description="pseudo-random generator seed")
-
-    def run(self):
-        """"""
-        cells = build_circuit(
-            self.cell_composition_path,
-            self.mtype_taxonomy_path,
-            CircuitConfig().atlas_path,
-            self.density_factor,
-            self.seed,
-        )
-        cells.save(self.output().path)
-
-    def output(self):
-        """"""
-        return OutputLocalTarget(CircuitConfig().circuit_somata_path)

synthesis_workflow/tasks/utils.py

@@ -6,11 +6,8 @@
 import luigi
 from git import Repo
 
-from .config import SynthesisConfig
-from .luigi_tools import copy_params
-from .luigi_tools import OutputLocalTarget
-from .luigi_tools import ParamLink
 from .luigi_tools import WorkflowTask
+from .luigi_tools import OutputLocalTarget
 
 
 class GitClone(WorkflowTask):
@@ -27,37 +24,38 @@ def output(self):
         return OutputLocalTarget(self.dest)
 
 
-@copy_params(
-    tmd_parameters_path=ParamLink(SynthesisConfig),
-    tmd_distributions_path=ParamLink(SynthesisConfig),
-)
-class GetOfficialConfiguration(WorkflowTask):
-    """Task to get official parameters from the git repository"""
+class GetSynthesisInputs(WorkflowTask):
+    """Task to get synthesis input files from a folder on git it repository.
 
-    url = luigi.Parameter()
-    specie = luigi.ChoiceParameter(choices=["rat", "mouse", "human"])
+    If no url is provided, this task will copy an existing folder to the target location.
+
+    Args:
+        url (str): url of repository, if None, git_synthesis_input_path should be an existing folder
+        version (str): version of repo to checkout (optional)
+        git_synthesis_input_path (str): path to folder in git repo with synthesis files
+        local_synthesis_input_path (str): path to local folder to copy these files
+    """
+
+    url = luigi.Parameter(default=None)
     version = luigi.OptionalParameter(default=None)
+    git_synthesis_input_path = luigi.Parameter(default="synthesis_input")
+    local_synthesis_input_path = luigi.Parameter(default="synthesis_input")
 
     def run(self):
         """"""
-        with TemporaryDirectory() as tmpdir:
-            dest = Path(tmpdir) / "tmp_repo"
-            # Note: can not be called with yield here because of the TemporaryDirectory
-            GitClone(url=self.url, dest=dest).run()
-            if self.version is not None:
-                r = Repo(dest)
-                r.git.checkout(self.version)
-            shutil.copy(
-                dest / "entities" / "bionames" / self.specie / "tmd_parameters.json",
-                self.output()["tmd_parameters"].path,
-            )
-            shutil.copy(
-                dest / "entities" / "bionames" / self.specie / "tmd_distributions.json",
-                self.output()["tmd_distributions"].path,
-            )
+        if self.url is None:
+            shutil.copytree(self.git_synthesis_input_path, self.output().path)
+        else:
+            with TemporaryDirectory() as tmpdir:
+                dest = Path(tmpdir) / "tmp_repo"
+                # Note: can not be called with yield here because of the TemporaryDirectory
+                GitClone(url=self.url, dest=dest).run()
+                if self.version is not None:
+                    r = Repo(dest)
+                    r.git.checkout(self.version)
+                shutil.copytree(
+                    dest / self.git_synthesis_input_path, self.output().path
+                )
 
     def output(self):
-        return {
-            "tmd_parameters": OutputLocalTarget(self.tmd_parameters_path),
-            "tmd_distributions": OutputLocalTarget(self.tmd_distributions_path),
-        }
+        return luigi.LocalTarget(self.local_synthesis_input_path)