[ADNI] Handle reading new format of clinical csv

clinica/iotools/bids_utils.py

@@ -68,6 +68,7 @@ def create_participants_df(
     import numpy as np
     import pandas as pd
 
+    from clinica.iotools.converters.adni_to_bids.adni_utils import load_clinical_csv
     from clinica.utils.stream import cprint
 
     fields_bids = ["participant_id"]
@@ -111,7 +112,9 @@ def create_participants_df(
                 if file_ext == ".xlsx":
                     file_to_read = pd.read_excel(file_to_read_path, sheet_name=sheet)
                 elif file_ext == ".csv":
-                    file_to_read = pd.read_csv(file_to_read_path)
+                    file_to_read = load_clinical_csv(
+                        clinical_data_dir, location.split(".")[0]
+                    )
                 prev_location = location
                 prev_sheet = sheet
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_av45_fbb_pet.py

@@ -44,12 +44,14 @@ def convert_adni_av45_fbb_pet(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if not subjects:
-        adni_merge_path = path.join(csv_dir, "ADNIMERGE.csv")
-        adni_merge = pd.read_csv(adni_merge_path, sep=",", low_memory=False)
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
         subjects = list(adni_merge.PTID.unique())
 
     cprint(
@@ -89,6 +91,7 @@ def compute_av45_fbb_pet_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     from clinica.iotools.converters.adni_to_bids.adni_utils import (
         find_image_path,
         get_images_pet,
+        load_clinical_csv,
     )
 
     pet_amyloid_col = [
@@ -108,14 +111,11 @@ def compute_av45_fbb_pet_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     pet_amyloid_dfs_list = []
 
     # Loading needed .csv files
-    av45qc = pd.read_csv(path.join(csv_dir, "AV45QC.csv"), sep=",", low_memory=False)
-    amyqc = pd.read_csv(path.join(csv_dir, "AMYQC.csv"), sep=",", low_memory=False)
-    pet_meta_list = pd.read_csv(
-        path.join(csv_dir, "PET_META_LIST.csv"), sep=",", low_memory=False
-    )
+    av45qc = load_clinical_csv(csv_dir, "AV45QC")
+    amyqc = load_clinical_csv(csv_dir, "AMYQC")
+    pet_meta_list = load_clinical_csv(csv_dir, "PET_META_LIST")
 
     for subj in subjs_list:
-
         # PET images metadata for subject
         subject_pet_meta = pet_meta_list[pet_meta_list["Subject"] == subj]
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_dwi.py

@@ -44,12 +44,14 @@ def convert_adni_dwi(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if not subjects:
-        adni_merge_path = path.join(csv_dir, "ADNIMERGE.csv")
-        adni_merge = pd.read_csv(adni_merge_path, sep=",", low_memory=False)
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
         subjects = list(adni_merge.PTID.unique())
 
     cprint(
@@ -82,6 +84,7 @@ def compute_dwi_paths(source_dir, csv_dir, subjs_list, conversion_dir):
 
     from clinica.iotools.converters.adni_to_bids.adni_utils import (
         find_image_path,
+        load_clinical_csv,
         visits_to_timepoints,
     )
 
@@ -100,18 +103,11 @@ def compute_dwi_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     dwi_dfs_list = []
 
     # Loading needed .csv files
-    adni_merge = pd.read_csv(
-        path.join(csv_dir, "ADNIMERGE.csv"), sep=",", low_memory=False
-    )
-
-    mayo_mri_qc = pd.read_csv(
-        path.join(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15.csv"),
-        sep=",",
-        low_memory=False,
-    )
+    adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
+    mayo_mri_qc = load_clinical_csv(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15")
     mayo_mri_qc = mayo_mri_qc[mayo_mri_qc.series_type == "DTI"]
 
-    mri_list = pd.read_csv(path.join(csv_dir, "MRILIST.csv"), sep=",", low_memory=False)
+    mri_list = load_clinical_csv(csv_dir, "MRILIST")
 
     # Selecting only DTI images that are not Multiband, processed or enhanced images
     mri_list = mri_list[mri_list.SEQUENCE.str.contains("dti", case=False, na=False)]
@@ -123,7 +119,6 @@ def compute_dwi_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     ]
 
     for subj in subjs_list:
-
         # Filter ADNIMERGE, MRI_LIST and QC for only one subject and sort the rows/visits by examination date
         adnimerge_subj = adni_merge[adni_merge.PTID == subj]
         adnimerge_subj = adnimerge_subj.sort_values("EXAMDATE")

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_fdg_pet.py

@@ -86,13 +86,15 @@ def _convert_adni_fdg_pet(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if subjects is None:
-        adni_merge = pd.read_csv(
-            Path(csv_dir) / "ADNIMERGE.csv", sep=",", low_memory=False
-        )
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
+
         subjects = list(adni_merge.PTID.unique())
     cprint(
         "Calculating paths of FDG PET images. "
@@ -227,7 +229,9 @@ def _load_df_with_column_check(
     csv_dir: Path, filename: str, required_columns: Set[str]
 ) -> pd.DataFrame:
     """Load the requested CSV file in a dataframe and check that the requested columns are present."""
-    df = pd.read_csv(csv_dir / filename, sep=",", low_memory=False)
+    from clinica.iotools.converters.adni_to_bids.adni_utils import load_clinical_csv
+
+    df = load_clinical_csv(csv_dir, filename)
     if not required_columns.issubset(set(df.columns)):
         raise ValueError(
             f"Missing column(s) from {filename} file."
@@ -238,17 +242,17 @@ def _load_df_with_column_check(
 
 _get_pet_qc_df = partial(
     _load_df_with_column_check,
-    filename="PETQC.csv",
+    filename="PETQC",
     required_columns={"PASS", "RID"},
 )
 _get_qc_adni_3_df = partial(
     _load_df_with_column_check,
-    filename="PETC3.csv",
+    filename="PETC3",
     required_columns={"SCANQLTY", "RID", "SCANDATE"},
 )
 _get_meta_list_df = partial(
     _load_df_with_column_check,
-    filename="PET_META_LIST.csv",
+    filename="PET_META_LIST",
     required_columns={"Subject"},
 )
 

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_flair.py

@@ -44,12 +44,14 @@ def convert_adni_flair(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if not subjects:
-        adni_merge_path = path.join(csv_dir, "ADNIMERGE.csv")
-        adni_merge = pd.read_csv(adni_merge_path, sep=",", low_memory=False)
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
         subjects = list(adni_merge.PTID.unique())
 
     cprint(
@@ -82,6 +84,7 @@ def compute_flair_paths(source_dir, csv_dir, subjs_list, conversion_dir):
 
     from clinica.iotools.converters.adni_to_bids.adni_utils import (
         find_image_path,
+        load_clinical_csv,
         visits_to_timepoints,
     )
 
@@ -101,18 +104,10 @@ def compute_flair_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     flair_dfs_list = []
 
     # Loading needed .csv files
-    adni_merge = pd.read_csv(
-        path.join(csv_dir, "ADNIMERGE.csv"), sep=",", low_memory=False
-    )
-
-    mayo_mri_qc = pd.read_csv(
-        path.join(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15.csv"),
-        sep=",",
-        low_memory=False,
-    )
+    adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
+    mayo_mri_qc = load_clinical_csv(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15")
     mayo_mri_qc = mayo_mri_qc[mayo_mri_qc.series_type == "AFL"]
-
-    mri_list = pd.read_csv(path.join(csv_dir, "MRILIST.csv"), sep=",", low_memory=False)
+    mri_list = load_clinical_csv(csv_dir, "MRILIST")
 
     # Selecting FLAIR DTI images that are not MPR
     mri_list = mri_list[mri_list.SEQUENCE.str.contains("flair", case=False, na=False)]
@@ -124,7 +119,6 @@ def compute_flair_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     ]
 
     for subj in subjs_list:
-
         # Filter ADNIMERGE, MRI_LIST and QC for only one subject and sort the rows/visits by examination date
         adnimerge_subj = adni_merge[adni_merge.PTID == subj]
         adnimerge_subj = adnimerge_subj.sort_values("EXAMDATE")

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_fmri.py

@@ -44,12 +44,14 @@ def convert_adni_fmri(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if not subjects:
-        adni_merge_path = path.join(csv_dir, "ADNIMERGE.csv")
-        adni_merge = pd.read_csv(adni_merge_path, sep=",", low_memory=False)
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
         subjects = list(adni_merge.PTID.unique())
 
     cprint(
@@ -82,6 +84,7 @@ def compute_fmri_path(source_dir, csv_dir, subjs_list, conversion_dir):
 
     from clinica.iotools.converters.adni_to_bids.adni_utils import (
         find_image_path,
+        load_clinical_csv,
         visits_to_timepoints,
     )
 
@@ -100,29 +103,21 @@ def compute_fmri_path(source_dir, csv_dir, subjs_list, conversion_dir):
     fmri_dfs_list = []
 
     # Loading needed .csv files
-    adni_merge = pd.read_csv(
-        path.join(csv_dir, "ADNIMERGE.csv"), sep=",", low_memory=False
-    )
+    adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
+
+    mayo_mri_qc = load_clinical_csv(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15")
 
-    mayo_mri_qc = pd.read_csv(
-        path.join(csv_dir, "MAYOADIRL_MRI_IMAGEQC_12_08_15.csv"),
-        sep=",",
-        low_memory=False,
-    )
     mayo_mri_qc = mayo_mri_qc[mayo_mri_qc.series_type == "fMRI"]
     mayo_mri_qc.columns = [x.upper() for x in mayo_mri_qc.columns]
 
-    mayo_mri_qc3 = pd.read_csv(
-        path.join(csv_dir, "MAYOADIRL_MRI_QUALITY_ADNI3.csv"), sep=",", low_memory=False
-    )
+    mayo_mri_qc3 = load_clinical_csv(csv_dir, "MAYOADIRL_MRI_QUALITY_ADNI3")
     mayo_mri_qc3 = mayo_mri_qc3[mayo_mri_qc3.SERIES_TYPE == "EPB"]
 
     # Concatenating visits in both QC files
     mayo_mri_qc = pd.concat(
         [mayo_mri_qc, mayo_mri_qc3], axis=0, ignore_index=True, sort=False
     )
-
-    mri_list = pd.read_csv(path.join(csv_dir, "MRILIST.csv"), sep=",", low_memory=False)
+    mri_list = load_clinical_csv(csv_dir, "MRILIST")
 
     # Selecting only fMRI images that are not Multiband
     mri_list = mri_list[
@@ -137,7 +132,6 @@ def compute_fmri_path(source_dir, csv_dir, subjs_list, conversion_dir):
 
     # We will convert the images for each subject in the subject list
     for subj in subjs_list:
-
         # Filter ADNIMERGE, MRI_LIST and QC for only one subject and sort the rows/visits by examination date
         adnimerge_subj = adni_merge[adni_merge.PTID == subj]
         adnimerge_subj = adnimerge_subj.sort_values("EXAMDATE")

clinica/iotools/converters/adni_to_bids/adni_modalities/adni_pib_pet.py

@@ -44,12 +44,14 @@ def convert_adni_pib_pet(
 
     import pandas as pd
 
-    from clinica.iotools.converters.adni_to_bids.adni_utils import paths_to_bids
+    from clinica.iotools.converters.adni_to_bids.adni_utils import (
+        load_clinical_csv,
+        paths_to_bids,
+    )
     from clinica.utils.stream import cprint
 
     if not subjects:
-        adni_merge_path = path.join(csv_dir, "ADNIMERGE.csv")
-        adni_merge = pd.read_csv(adni_merge_path, sep=",", low_memory=False)
+        adni_merge = load_clinical_csv(csv_dir, "ADNIMERGE")
         subjects = list(adni_merge.PTID.unique())
 
     cprint(
@@ -82,6 +84,7 @@ def compute_pib_pet_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     from clinica.iotools.converters.adni_to_bids.adni_utils import (
         find_image_path,
         get_images_pet,
+        load_clinical_csv,
     )
     from clinica.utils.pet import Tracer
 
@@ -101,13 +104,10 @@ def compute_pib_pet_paths(source_dir, csv_dir, subjs_list, conversion_dir):
     pet_pib_dfs_list = []
 
     # Loading needed .csv files
-    pibqc = pd.read_csv(path.join(csv_dir, "PIBQC.csv"), sep=",", low_memory=False)
-    pet_meta_list = pd.read_csv(
-        path.join(csv_dir, "PET_META_LIST.csv"), sep=",", low_memory=False
-    )
+    pibqc = load_clinical_csv(csv_dir, "PIBQC")
+    pet_meta_list = load_clinical_csv(csv_dir, "PET_META_LIST")
 
     for subj in subjs_list:
-
         # PET images metadata for subject
         subject_pet_meta = pet_meta_list[pet_meta_list["Subject"] == subj]