Use Gitlab registry in CI and remove several warnings

.gitlab-ci.yml

@@ -19,22 +19,16 @@ include:
 
 .tox-template:
   # Using the same image as DVF
-  image: docker-registry-default.ocp.bbp.epfl.ch/bbp-ou-cells/data-validation-framework:latest
+  image: bbpgitlab.epfl.ch:5050/neuromath/ci/data-validation-framework:latest
 
 .pytest-template:
-  # Using Kubernetes runner with custom cpu count and memory amount
-  tags:
-    - kubernetes
   variables:
     KUBERNETES_CPU_LIMIT: 4
     KUBERNETES_CPU_REQUEST: 4
     KUBERNETES_MEMORY_LIMIT: 4Gi
     KUBERNETES_MEMORY_REQUEST: 4Gi
 
 lint:
-   # Using Kubernetes runner with custom memory amount
-   tags:
-      - kubernetes
    variables:
       KUBERNETES_CPU_LIMIT: 1
       KUBERNETES_CPU_REQUEST: 1

requirements/test.pip

@@ -1,4 +1,5 @@
 dir-content-diff>=0.2
+dir-content-diff-plugins
 pytest
 pytest-cov
 pytest-html

setup.py

@@ -44,7 +44,7 @@
     package_dir={"": "src"},
     python_requires=">=3.8",
     install_requires=reqs,
-    test_requires=test_reqs,
+    tests_require=test_reqs,
     extras_require={"docs": doc_reqs, "test": test_reqs},
     classifiers=[
         "Development Status :: 2 - Pre-Alpha",

src/synthesis_workflow/tasks/validation.py

@@ -81,6 +81,9 @@ def output(self):
         return MorphsDfLocalTarget(PathConfig().synth_morphs_df_path)
 
 
+@copy_params(
+    nb_jobs=ParamRef(RunnerConfig),
+)
 class PlotMorphometrics(WorkflowTask):
     """Plot cell morphometrics for two groups of cells so they can be easily compared.
 
@@ -141,6 +144,7 @@ def run(self):
             comp_label=self.comp_label,
             normalize=self.normalize,
             config_features=self.config_features,
+            n_workers=self.nb_jobs,
         )
 
     def output(self):

src/synthesis_workflow/vacuum_synthesis.py

@@ -80,7 +80,8 @@ def grow_vacuum_morphologies(
     otherwise 'M1-M5' from TNS are allowed.
     """
     global_gid = 0
-    vacuum_synth_morphs_df = pd.DataFrame()
+    # vacuum_synth_morphs_df = pd.DataFrame()
+    rows = []
     for mtype in tqdm(mtypes):
         tmd_parameters[mtype]["diameter_params"]["method"] = diametrizer
         tmd_distributions["mtypes"][mtype]["diameter"]["method"] = diametrizer
@@ -112,7 +113,8 @@ def grow_vacuum_morphologies(
             )
             for gid in gids
         ):
-            vacuum_synth_morphs_df = vacuum_synth_morphs_df.append(row)
+            rows.append(row)
+    vacuum_synth_morphs_df = pd.concat(rows)
     return vacuum_synth_morphs_df
 
 

src/synthesis_workflow/validation.py

@@ -83,23 +83,26 @@ def get_features_df(morphologies_mtypes: Dict, features_config: Dict, n_workers:
             (see ``neurom.apps.morph_stats.extract_dataframe``)
         n_workers (int) : number of workers for feature extractions
     """
-    features_df = pd.DataFrame()
+    rows = []
     for mtype, morphology_folders in morphologies_mtypes.items():
         features_df_tmp = extract_dataframe(
             morphology_folders, features_config, n_workers=n_workers
         )
         features_df_tmp["mtype"] = mtype
-        features_df = features_df.append(features_df_tmp.replace(0, np.nan))
+        rows.append(features_df_tmp.replace(0, np.nan))
+    features_df = pd.concat(rows)
     return features_df
 
 
-def _get_features_df_all_mtypes(morphs_df, features_config, morphology_path):
+def _get_features_df_all_mtypes(morphs_df, features_config, morphology_path, n_workers=None):
     """Wrapper for morph-validator functions."""
+    if n_workers is None:
+        n_workers = os.cpu_count()
     morphs_df_dict = {mtype: df[morphology_path] for mtype, df in morphs_df.groupby("mtype")}
     with warnings.catch_warnings():
         # Ignore some Numpy warnings
         warnings.simplefilter("ignore", category=RuntimeWarning)
-        return get_features_df(morphs_df_dict, features_config, n_workers=os.cpu_count())
+        return get_features_df(morphs_df_dict, features_config, n_workers=n_workers)
 
 
 def get_feature_configs(config_types="default"):
@@ -158,15 +161,16 @@ def get_feature_configs(config_types="default"):
 
 def _expand_lists(data):
     """Convert list element of dataframe to duplicated rows with float values."""
-    data_expanded = pd.DataFrame()
+    new_rows = []
     for row_id in data.index:
         if isinstance(data.loc[row_id, "value"], list):
             for value in data.loc[row_id, "value"]:
                 new_row = data.loc[row_id].copy()
                 new_row["value"] = value
-                data_expanded = data_expanded.append(new_row)
+                new_rows.append(new_row)
         else:
-            data_expanded = data_expanded.append(data.loc[row_id].copy())
+            new_rows.append(data.loc[row_id].copy())
+    data_expanded = pd.DataFrame(new_rows)
     return data_expanded
 
 
@@ -247,6 +251,7 @@ def plot_morphometrics(
     comp_label="comp",
     normalize=False,
     config_features=None,
+    n_workers=None,
 ):
     """Plot morphometrics.
 
@@ -260,14 +265,22 @@ def plot_morphometrics(
         comp_label (str): label for the compared morphologies
         normalize (bool): normalize data if set to True
         config_features (dict): mapping of features to plot
+        n_workers (int): the number of workers used to compute morphology features
     """
     if config_features is None:
         config_features = get_feature_configs(config_types="synthesis")
         # config_features["neurite"].update({"y_distances": ["min", "max"]})
 
-    base_features_df = _get_features_df_all_mtypes(base_morphs_df, config_features, base_key)
+    L.debug("Get features from base morphologies")
+    base_features_df = _get_features_df_all_mtypes(
+        base_morphs_df, config_features, base_key, n_workers=n_workers
+    )
     base_features_df["label"] = base_label
-    comp_features_df = _get_features_df_all_mtypes(comp_morphs_df, config_features, comp_key)
+
+    L.debug("Get features from compared morphologies")
+    comp_features_df = _get_features_df_all_mtypes(
+        comp_morphs_df, config_features, comp_key, n_workers=n_workers
+    )
     comp_features_df["label"] = comp_label
 
     base_features_df = base_features_df[
@@ -280,6 +293,7 @@ def plot_morphometrics(
     all_features_df = pd.concat([base_features_df, comp_features_df])
     ensure_dir(output_path)
     with DisableLogger():
+        L.debug("Plot violin figure to %s", str(output_path))
         plot_violin_features(
             all_features_df,
             ["basal_dendrite", "apical_dendrite"],

tests/conftest.py

@@ -8,16 +8,16 @@
 from subprocess import call
 
 import dir_content_diff.pandas
-import dir_content_diff.voxcell
+import dir_content_diff_plugins.voxcell
 import luigi
 import numpy as np
 import pytest
 
 from synthesis_workflow.tasks import config
 from synthesis_workflow.tools import get_layer_tags
 
-dir_content_diff.pandas.register_pandas()
-dir_content_diff.voxcell.register_voxcell()
+dir_content_diff.pandas.register()
+dir_content_diff_plugins.voxcell.register()
 
 
 TEST_ROOT = Path(__file__).parent

tests/test_O1_workflow.py

@@ -1,11 +1,13 @@
 """Tests for workflows module."""
 import luigi
 import numpy as np
+import pytest
 from dir_content_diff import assert_equal_trees
 
 from synthesis_workflow.tasks.workflows import ValidateSynthesis
 
 
+@pytest.mark.xdist_group("group_O1")
 def test_ValidateSynthesis(small_O1_working_directory, data_dir):
     """Test the synthesis workflow in simple atlas"""
     np.random.seed(0)
@@ -51,6 +53,9 @@ def test_ValidateSynthesis(small_O1_working_directory, data_dir):
                 "tolerance": 2e-3,
                 "absolute_tolerance": 1e-15,
             },
-            "validation/morphology_validation_reports/validation_results.json": {"tolerance": 2e-3},
+            "validation/morphology_validation_reports/validation_results.json": {
+                "tolerance": 2e-3,
+                "absolute_tolerance": 1e-12,
+            },
         },
     )

tests/test_vacuum_workflow.py

@@ -1,11 +1,13 @@
 """Tests for workflows module."""
 import luigi
 import numpy as np
+import pytest
 from dir_content_diff import assert_equal_trees
 
 from synthesis_workflow.tasks.workflows import ValidateVacuumSynthesis
 
 
+@pytest.mark.xdist_group("group_vacuum")
 def test_ValidateVacuumSynthesis(vacuum_working_directory, data_dir):
     """Test the synthesis workflow in vacuum"""
     np.random.seed(0)

tox.ini

@@ -20,8 +20,8 @@ setenv =
     COVERAGE_FILE = {env:COVERAGE_FILE:.coverage-{envname}}
 extras = test
 commands = pytest \
-    -n 4 \
-    --dist loadfile \
+    -n {env:PYTEST_NPROCS:3} \
+    --dist loadgroup \
     --basetemp={envtmpdir} \
     --cov={envsitepackagesdir}/{[base]name} \
     --cov={envsitepackagesdir}/{[base]morphval} \
@@ -85,6 +85,7 @@ commands =
     black {[base]files}
 
 [testenv:docs]
+basepython=python3.8
 changedir = doc
 extras = docs
 allowlist_externals =