[ENH] Expose n_procs option to Adni2BIDS Oasis2BIDS and `Aibl2B…

…IDS` (#1009)

* Add get_n_procs function

* expose n_procs to adni-to-bids

* expose n_procs to aibl-to-bids

* expose n_procs to oasis-to-bids

* add unit tests

* take into account review comments

* update unit tests

* Apply suggestions from code review

Co-authored-by: Ghislain Vaillant <ghisvail@users.noreply.github.com>

---------

Co-authored-by: Ghislain Vaillant <ghisvail@users.noreply.github.com>

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_av45_fbb_pet.py

@@ -10,6 +10,7 @@ def convert_adni_av45_fbb_pet(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: int = 1,
 ):
     """Convert AV-45 and Florbetaben PET images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_av45_fbb_pet(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, default=1
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -53,7 +59,13 @@ def convert_adni_av45_fbb_pet(
     cprint(
         "Paths of AV45 and Florbetaben PET images found. Exporting images into BIDS ..."
     )
-    paths_to_bids(images, destination_dir, "av45_fbb", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images,
+        destination_dir,
+        "av45_fbb",
+        mod_to_update=mod_to_update,
+        n_procs=n_procs,
+    )
     cprint(msg="AV45 and Florbetaben PET conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_dwi.py

@@ -10,6 +10,7 @@ def convert_adni_dwi(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert DW images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_dwi(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -52,7 +58,9 @@ def convert_adni_dwi(
     images = compute_dwi_paths(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of DWI images found. Exporting images into BIDS ...")
     # dwi_paths_to_bids(images, dest_dir)
-    paths_to_bids(images, destination_dir, "dwi", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "dwi", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="DWI conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_fdg_pet.py

@@ -49,6 +49,7 @@ def _convert_adni_fdg_pet(
     preprocessing_step: ADNIPreprocessingStep,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert FDG PET images of ADNI into BIDS format.
 
@@ -75,6 +76,11 @@ def _convert_adni_fdg_pet(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from pathlib import Path
 
@@ -97,7 +103,9 @@ def _convert_adni_fdg_pet(
     )
     cprint("Paths of FDG PET images found. Exporting images into BIDS ...")
     modality = _get_modality_from_adni_preprocessing_step(preprocessing_step)
-    paths_to_bids(images, destination_dir, modality, mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, modality, mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="FDG PET conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_flair.py

@@ -10,6 +10,7 @@ def convert_adni_flair(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert FLAIR images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_flair(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -52,7 +58,9 @@ def convert_adni_flair(
     images = compute_flair_paths(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of FLAIR images found. Exporting images into BIDS ...")
     # flair_paths_to_bids(images, dest_dir)
-    paths_to_bids(images, destination_dir, "flair", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "flair", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="FLAIR conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_fmri.py

@@ -10,6 +10,7 @@ def convert_adni_fmri(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert fMR images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_fmri(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -52,7 +58,9 @@ def convert_adni_fmri(
     images = compute_fmri_path(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of fMRI images found. Exporting images into BIDS ...")
     # fmri_paths_to_bids(dest_dir, images)
-    paths_to_bids(images, destination_dir, "fmri", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "fmri", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="fMRI conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_pib_pet.py

@@ -10,6 +10,7 @@ def convert_adni_pib_pet(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert PIB PET images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_pib_pet(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -51,7 +57,9 @@ def convert_adni_pib_pet(
     )
     images = compute_pib_pet_paths(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of PIB PET images found. Exporting images into BIDS ...")
-    paths_to_bids(images, destination_dir, "pib", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "pib", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="PIB PET conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_t1.py

@@ -10,6 +10,7 @@ def convert_adni_t1(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert T1 MR images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_t1(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os import path
 
@@ -51,7 +57,9 @@ def convert_adni_t1(
     )
     images = compute_t1_paths(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of T1 images found. Exporting images into BIDS ...")
-    paths_to_bids(images, destination_dir, "t1", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "t1", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="T1 conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_tau_pet.py

@@ -10,6 +10,7 @@ def convert_adni_tau_pet(
     conversion_dir: PathLike,
     subjects: Optional[List[str]] = None,
     mod_to_update: bool = False,
+    n_procs: Optional[int] = 1,
 ):
     """Convert Tau PET images of ADNI into BIDS format.
 
@@ -33,6 +34,11 @@ def convert_adni_tau_pet(
     mod_to_update : bool
         If True, pre-existing images in the BIDS directory
         will be erased and extracted again.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1
     """
     from os import path
 
@@ -51,7 +57,9 @@ def convert_adni_tau_pet(
     )
     images = compute_tau_pet_paths(source_dir, csv_dir, subjects, conversion_dir)
     cprint("Paths of TAU PET images found. Exporting images into BIDS ...")
-    paths_to_bids(images, destination_dir, "tau", mod_to_update=mod_to_update)
+    paths_to_bids(
+        images, destination_dir, "tau", mod_to_update=mod_to_update, n_procs=n_procs
+    )
     cprint(msg="TAU PET conversion done.", lvl="debug")
 
 

clinica/iotools/converters/adni_to_bids/adni_to_bids.py

@@ -173,6 +173,7 @@ def convert_images(
         subjs_list_path=None,
         modalities=None,
         force_new_extraction=False,
+        n_procs: Optional[int] = 1,
     ):
         """Convert the images of ADNI.
 
@@ -261,4 +262,5 @@ def convert_images(
                     conversion_dir=conversion_dir,
                     subjects=subjs_list,
                     mod_to_update=force_new_extraction,
+                    n_procs=n_procs,
                 )

clinica/iotools/converters/adni_to_bids/adni_to_bids_cli.py

@@ -2,6 +2,7 @@
 
 import click
 
+from clinica import option
 from clinica.iotools.converters import cli_param
 
 ALL_MODALITIES = ("T1", "PET_FDG", "PET_AMYLOID", "PET_TAU", "DWI", "FLAIR", "fMRI")
@@ -35,6 +36,8 @@
 @click.option(
     "-xml", "--xml_path", help="Path to the root directory containing the xml metadata."
 )
+@option.global_option_group
+@option.n_procs
 def cli(
     dataset_directory: str,
     clinical_data_directory: str,
@@ -44,6 +47,7 @@ def cli(
     clinical_data_only: bool = False,
     force_new_extraction: bool = False,
     modalities: List[str] = ALL_MODALITIES,
+    n_procs: Optional[int] = None,
 ) -> None:
     """ADNI to BIDS converter.
 
@@ -69,6 +73,7 @@ def cli(
             subjects_list,
             modalities,
             force_new_extraction,
+            n_procs=n_procs,
         )
 
     adni_to_bids.convert_clinical_data(

clinica/iotools/converters/adni_to_bids/adni_utils.py

@@ -1226,17 +1226,20 @@ def find_image_path(images, source_dir, modality, prefix, id_field):
     return images
 
 
-def paths_to_bids(images, bids_dir, modality, mod_to_update=False):
+def paths_to_bids(
+    images, bids_dir, modality, mod_to_update=False, n_procs: Optional[int] = 1
+):
     """Images in the list are converted and copied to directory in BIDS format.
 
     Args:
         images: List of images metadata and paths
         bids_dir: Path to the output BIDS directory
         modality: Imaging modality
         mod_to_update: If True, pre-existing images in the BIDS directory will be erased and extracted agai n.
+        n_procs: int, optional. The number of processes to use for conversion. Default=1.
     """
     from functools import partial
-    from multiprocessing import Pool, cpu_count
+    from multiprocessing import Pool
 
     if modality.lower() not in [
         "t1",
@@ -1256,7 +1259,7 @@ def paths_to_bids(images, bids_dir, modality, mod_to_update=False):
 
     images_list = list([data for _, data in images.iterrows()])
 
-    with Pool(processes=max(cpu_count() - 1, 1)) as pool:
+    with Pool(processes=n_procs) as pool:
         create_file_ = partial(
             create_file,
             modality=modality,

clinica/iotools/converters/aibl_to_bids/aibl_to_bids.py

@@ -2,6 +2,7 @@
 Convert the AIBL dataset (https://www.aibl.csiro.au/) into BIDS.
 """
 from pathlib import Path
+from typing import Optional
 
 
 def convert(
@@ -10,6 +11,7 @@ def convert(
     output_dataset: Path,
     overwrite: bool = False,
     clinical_data_only: bool = False,
+    n_procs: Optional[int] = 1,
 ) -> None:
     """Convert the AIBL dataset (https://www.aibl.csiro.au/) in a BIDS dataset.
 
@@ -32,6 +34,11 @@ def convert(
         If True, only convert the clinical data without the imaging data.
         If False, everything is converted.
         Default=False.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from clinica.utils.check_dependency import check_dcm2niix
 
@@ -45,6 +52,7 @@ def convert(
             input_clinical_data,
             output_dataset,
             overwrite,
+            n_procs=n_procs,
         )
 
     _convert_clinical_data(input_clinical_data, output_dataset)
@@ -55,6 +63,7 @@ def _convert_images(
     input_clinical_data: Path,
     output_dataset: Path,
     overwrite: bool = False,
+    n_procs: Optional[int] = 1,
 ) -> None:
     """Conversion of the AIBL imaging data in BIDS.
 
@@ -72,6 +81,11 @@ def _convert_images(
     overwrite : bool, optional
         Overwrites previously written nifti and json files.
         Default=False.
+
+    n_procs : int, optional
+        The requested number of processes.
+        If specified, it should be between 1 and the number of available CPUs.
+        Default=1.
     """
     from os.path import exists
 
@@ -85,6 +99,7 @@ def _convert_images(
             output_dataset,
             modality,
             overwrite=overwrite,
+            n_procs=n_procs,
         )
         for modality in Modality
     ]