Fix: Raise an exception when the morphologies extracted from the Neur…

…onDB file can not be found

src/synthesis_workflow/tasks/config.py

@@ -51,10 +51,8 @@ def output(self):
 class GetSynthesisInputs(WorkflowTask):
     """Task to get synthesis input files from a folder on git repository.
 
-    If no url is provided, this task will copy an existing folder to the target location.
-
-    Attributes:
-        local_synthesis_input_path (str): Path to local folder to copy these files.
+    If no url is provided, this task will copy an existing folder to the target location
+    given in the 'local_synthesis_input_path' parameter of the 'PathConfig' task.
     """
 
     url = luigi.OptionalParameter(

src/synthesis_workflow/tasks/synthesis.py

@@ -81,6 +81,13 @@ def run(self):
         # Remove possibly duplicated morphologies
         morphs_df = morphs_df.drop_duplicates(subset=["name"])
 
+        missing_files = morphs_df.loc[morphs_df["path"].isnull()]
+        if not missing_files.empty:
+            raise RuntimeError(
+                "The following morphologies extracted from the MorphDB file do not exist: "
+                f"{missing_files['name'].tolist()}"
+            )
+
         morphs_df.to_csv(self.output().path)
 
     def output(self):
@@ -129,11 +136,7 @@ def output(self):
 
 
 class GetDefaultParameters(WorkflowTask):
-    """Build the tmd_parameter.json for synthesis.
-
-    Attributes:
-        tmd_parameters_path (str): The path to the TMD parameters.
-    """
+    """Build the tmd_parameter.json for synthesis."""
 
     default_tmd_parameters_path = luigi.PathParameter(
         default="neurots_input/tmd_parameters_default.json",
@@ -173,7 +176,11 @@ def output(self):
 
 @copy_params(tmd_parameters_path=ParamRef(SynthesisConfig))
 class BuildSynthesisParameters(WorkflowTask):
-    """Build the tmd_parameters.json for synthesis."""
+    """Build the tmd_parameters.json for synthesis.
+
+    Attributes:
+        tmd_parameters_path (str): The path to the TMD parameters.
+    """
 
     def requires(self):
         """Required input tasks."""
@@ -687,11 +694,7 @@ def output(self):
 
 
 class OverwriteCustomParameters(WorkflowTask):
-    """Overwrite parameters with custom parameters.
-
-    Attributes:
-        tmd_parameters_path (str): The path to the TMD parameters.
-    """
+    """Overwrite parameters with custom parameters."""
 
     custom_tmd_parameters_path = luigi.PathParameter(
         default="neurots_input/tmd_parameters_overwriten.json",

src/version.py

@@ -1,2 +1,2 @@
 """Package version."""
-VERSION = "1.0.1"
+VERSION = "1.0.2.dev0"

tests/__init__.py

@@ -1 +1,15 @@
 """Tests suite for the synthesis-workflow package."""
+from configparser import ConfigParser
+
+
+def get_config_parser(cfg_path):
+    """Return a config parser filed with values from the given file."""
+    params = ConfigParser()
+    params.read(cfg_path)
+    return params
+
+
+def export_config(params, filepath):
+    """Export params to a file."""
+    with open(filepath, "w", encoding="utf-8") as configfile:
+        params.write(configfile)

tests/conftest.py

@@ -3,7 +3,7 @@
 import json
 import os
 import shutil
-from configparser import ConfigParser
+from copy import deepcopy
 from pathlib import Path
 from subprocess import check_call
 
@@ -12,10 +12,14 @@
 import luigi
 import numpy as np
 import pytest
+from luigi_tools.task import WorkflowTask
 from neurocollage.planes import get_layer_annotation
 
 from synthesis_workflow.tasks import config
 
+from . import export_config
+from . import get_config_parser
+
 dir_content_diff.pandas.register()
 dir_content_diff_plugins.voxcell.register()
 
@@ -24,12 +28,6 @@
 DATA = TEST_ROOT / "data"
 
 
-def export_config(params, filepath):
-    """Export params to a file."""
-    with open(filepath, "w", encoding="utf-8") as configfile:
-        params.write(configfile)
-
-
 @pytest.fixture
 def root_dir():
     """The root directory."""
@@ -90,13 +88,6 @@ def tmp_working_dir(tmp_path):
     os.chdir(cwd)
 
 
-def get_config_parser(cfg_path):
-    """Return a config parser filed with values from the given file."""
-    params = ConfigParser()
-    params.read(cfg_path)
-    return params
-
-
 @pytest.fixture
 def small_O1_params():
     """Parameters for the small O1 case."""
@@ -176,3 +167,25 @@ def small_O1_working_directory(tmp_working_dir, small_O1_params, small_O1):
 
     # Reset luigi config
     luigi_config.clear()
+
+
+@pytest.fixture
+def WorkflowTask_exception_event():
+    """Fixture to catch exception from tasks deriving from WorkflowTask.
+
+    The events of the tasks are reset afterwards.
+    """
+    # pylint: disable=protected-access
+    current_callbacks = deepcopy(luigi.Task._event_callbacks)
+
+    failed_task = []
+    exceptions = []
+
+    @WorkflowTask.event_handler(luigi.Event.FAILURE)
+    def check_exception(task, exception):
+        failed_task.append(str(task))
+        exceptions.append(str(exception))
+
+    yield failed_task, exceptions
+
+    luigi.Task._event_callbacks = current_callbacks

tests/test_vacuum_workflow.py

@@ -1,11 +1,16 @@
 """Tests for workflows module."""
+import shutil
+
 import luigi
 import numpy as np
 import pytest
 from dir_content_diff import assert_equal_trees
 
 from synthesis_workflow.tasks.workflows import ValidateVacuumSynthesis
 
+from . import export_config
+from . import get_config_parser
+
 
 @pytest.mark.xdist_group("group_vacuum")
 def test_ValidateVacuumSynthesis(vacuum_working_directory, data_dir):
@@ -46,3 +51,33 @@ def test_ValidateVacuumSynthesis(vacuum_working_directory, data_dir):
             },
         },
     )
+
+
+@pytest.mark.xdist_group("group_vacuum")
+def test_neuronDB(vacuum_working_directory, data_dir, WorkflowTask_exception_event):
+    """Test the synthesis workflow with NeuronDB file not located in the morphology folder."""
+    root_dir = vacuum_working_directory[0].parent
+
+    # Create a new NeuronDB file outside the directory that contain the morphologies
+    neurondb_path = data_dir / "input_cells" / "neuronDB.xml"
+    new_neurondb_path = root_dir / "neuronDB.xml"
+    shutil.copyfile(neurondb_path, new_neurondb_path)
+
+    # Update config with new NeuronDB path
+    params = get_config_parser(root_dir / "luigi.cfg")
+    params["BuildMorphsDF"]["neurondb_path"] = (new_neurondb_path).as_posix()
+    export_config(params, root_dir / "luigi.cfg")
+
+    luigi_config = luigi.configuration.get_config()
+    luigi_config.read(root_dir / "luigi.cfg")
+
+    # Run the workflow
+    assert not luigi.build([ValidateVacuumSynthesis()], local_scheduler=True)
+
+    failed_task, exceptions = WorkflowTask_exception_event
+    assert len(failed_task) == 1
+    assert failed_task[0].startswith("BuildMorphsDF(")
+    assert exceptions == [
+        "The following morphologies extracted from the MorphDB file do not exist: "
+        "['C270106A', 'C170797A-P2', 'rat_20160908_E3_LH2_cell2', 'sm080625a1-6_idD']"
+    ]

tox.ini

@@ -96,4 +96,6 @@ allowlist_externals =
     make
     dot
 # set warnings as errors using the -W sphinx option
-commands = make html SPHINXOPTS=-W
+commands =
+    make clean
+    make html SPHINXOPTS=-W