Improve Synthesize dependency of PlotScale

Change-Id: Ic7e632c8d7de29df6c55c5208ba1fbfbe5dd76cc

synthesis_workflow/tasks/luigi_tools.py

@@ -9,6 +9,10 @@
 L = logging.getLogger(__name__)
 
 
+class WorkflowError(Exception):
+    """Exception raised when the workflow is not consistent"""
+
+
 def recursive_check(task, attr="rerun"):
     """Check if a task or any of its recursive dependencies has a given attribute set to True"""
     val = getattr(task, attr, False)

synthesis_workflow/tasks/synthesis.py

@@ -311,7 +311,9 @@ def output(self):
         return {
             "out_mvd3": luigi.LocalTarget(self.out_circuit_path),
             "out_morphologies": luigi.LocalTarget(PathConfig().synth_output_path),
-            "debug_scales": luigi.LocalTarget(self.debug_region_grower_scales),
+            "debug_scales": luigi.LocalTarget(self.debug_region_grower_scales)
+            if self.debug_region_grower_scales
+            else None,
         }
 
 

synthesis_workflow/tasks/validation.py

@@ -24,6 +24,7 @@
 from .luigi_tools import copy_params
 from .luigi_tools import ParamLink
 from .luigi_tools import WorkflowTask
+from .luigi_tools import WorkflowError
 from .synthesis import AddScalingRulesToParameters
 from .synthesis import BuildSynthesisDistributions
 from .synthesis import ApplySubstitutionRules
@@ -296,7 +297,7 @@ class PlotScales(WorkflowTask):
     """
 
     scales_base_path = luigi.Parameter(default="scales")
-    log_file = luigi.Parameter(default="synthesis_workflow.log")
+    log_file = luigi.OptionalParameter(default=None)
     neuron_type_position_regex = luigi.Parameter(
         default=r".*\[WORKER TASK ID=([0-9]*)\] Neurite type and position: (.*)"
     )
@@ -324,6 +325,17 @@ def run(self):
         else:
             mtypes = self.mtypes
 
+        if self.log_file is None:
+            debug_scales = self.requires().input()["debug_scales"]
+            if debug_scales is not None:
+                self.log_file = debug_scales.path
+            else:
+                raise WorkflowError(
+                    "%s task: either a 'log_file' argument must be provided, either the "
+                    "'Synthesize' task must be run with 'debug_region_grower_scales' set "
+                    "to a valid directory path" % self.__class__.__name__
+                )
+
         # Plot statistics
         scale_data = parse_log(
             self.log_file,

synthesis_workflow/tasks/workflows.py

@@ -10,6 +10,7 @@
 from .validation import PlotDensityProfiles
 from .validation import PlotMorphometrics
 from .validation import PlotPathDistanceFits
+from .validation import PlotScales
 from .vacuum_synthesis import PlotVacuumMorphologies
 
 
@@ -20,6 +21,7 @@ class ValidateSynthesis(WorkflowWrapperTask):
     with_morphometrics = luigi.BoolParameter(default=True)
     with_density_profiles = luigi.BoolParameter(default=True)
     with_path_distance_fits = luigi.BoolParameter(default=True)
+    with_scale_statistics = luigi.BoolParameter(default=True)
 
     def requires(self):
         """"""
@@ -32,6 +34,8 @@ def requires(self):
             tasks.append(PlotDensityProfiles())
         if self.with_path_distance_fits:
             tasks.append(PlotPathDistanceFits())
+        if self.with_scale_statistics:
+            tasks.append(PlotScales())
         return tasks
 
 

synthesis_workflow/validation.py

@@ -415,10 +415,6 @@ def _plot_collage(
     X, Y, layers = get_layer_info(
         layer_annotation, left_plane.point, rotation_matrix, n_pixels
     )
-    if with_y_field:
-        X_y, Y_y, orientation_u, orientation_v = get_y_info(
-            atlas, left_plane.point, rotation_matrix, n_pixels_y
-        )
 
     fig = plt.figure()
     plt.imshow(
@@ -443,6 +439,9 @@ def _plot_collage(
         )
     if with_y_field:
         # note: some of these parameters are harcoded for NCx plot, adjust as needed
+        X_y, Y_y, orientation_u, orientation_v = get_y_info(
+            atlas, left_plane.point, rotation_matrix, n_pixels_y
+        )
         plt.quiver(
             X_y,
             Y_y,
@@ -696,7 +695,7 @@ def _search(pattern, line, data):
             data.append(new_data)
 
     # List log files
-    log_files = glob.glob(log_file + "*")
+    log_files = glob.glob(log_file + "/*")
 
     # Read log file
     all_lines = []

tests/test_luigi_tools.py

@@ -249,7 +249,7 @@ def output(self):
     for task in [TaskA(), TaskB(), TaskC(), TaskD(), TaskE()]:
         all_targets[task.__class__.__name__] = task.output()
 
-    # Test that everything is runned when all rerun are False and targets are missing
+    # Test that everything is run when all rerun are False and targets are missing
     print("=================== FIRST BUILD ====================")
     for task_class in [TaskA, TaskB, TaskC, TaskD, TaskE]:
         set_new_state(task_class)
@@ -258,7 +258,7 @@ def output(self):
 
     assert all([check_not_empty_file(i.path) for i in luigi.task.flatten(all_targets)])
 
-    # Test that nothing is runned when all rerun are False and targets are present
+    # Test that nothing is run when all rerun are False and targets are present
     for i in luigi.task.flatten(all_targets):
         create_empty_file(i.path)
 
@@ -269,7 +269,7 @@ def output(self):
 
     assert all([check_empty_file(i.path) for i in luigi.task.flatten(all_targets)])
 
-    # Test that everything is runned when rerun = True for the root task and targets are present
+    # Test that everything is run when rerun = True for the root task and targets are present
     for i in luigi.task.flatten(all_targets):
         create_empty_file(i.path)
 
@@ -282,7 +282,7 @@ def output(self):
 
     assert all([check_not_empty_file(i.path) for i in luigi.task.flatten(all_targets)])
 
-    # Test that only the parents of the task with rerun = True are runned
+    # Test that only the parents of the task with rerun = True are run
     for i in luigi.task.flatten(all_targets):
         create_empty_file(i.path)