From 8991f7bf24bec99bb0ac0d5f8c3e93d643c157fb Mon Sep 17 00:00:00 2001 From: laemtl Date: Sat, 1 Apr 2023 03:57:45 +0000 Subject: [PATCH 1/2] cand_id subject_id support --- python/bids_import.py | 105 ++++++++++++++++++++++++++++------------ python/lib/candidate.py | 56 +++++++++------------ python/lib/eeg.py | 16 ++++-- python/lib/exitcode.py | 18 ++++--- 4 files changed, 122 insertions(+), 73 deletions(-) diff --git a/python/bids_import.py b/python/bids_import.py index faecce66d..68607a08c 100755 --- a/python/bids_import.py +++ b/python/bids_import.py @@ -26,30 +26,31 @@ # sys.tracebacklimit = 0 def main(): - bids_dir = '' - verbose = False - createcand = False - createvisit = False - profile = '' + bids_dir = '' + verbose = False + createcand = False + createvisit = False + idsvalidation = False + profile = '' long_options = [ "help", "profile=", "directory=", - "createcandidate", "createsession", "verbose" + "createcandidate", "createsession", "idsvalidation", "verbose" ] usage = ( '\n' 'usage : bids_import -d -p \n\n' 'options: \n' - '\t-p, --profile : name of the python database config file in ' - 'dicom-archive/.loris-mri\n' + '\t-p, --profile : name of the python database config file in dicom-archive/.loris-mri\n' '\t-d, --directory : BIDS directory to parse & insert into LORIS\n' '\t-c, --createcandidate: to create BIDS candidates in LORIS (optional)\n' '\t-s, --createsession : to create BIDS sessions in LORIS (optional)\n' + '\t-i, --idsvalidation : to validate BIDS directory for a matching pscid/candid pair (optional)\n' '\t-v, --verbose : be verbose\n' ) try: - opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csv', long_options) + opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csiv', long_options) except getopt.GetoptError: print(usage) sys.exit(lib.exitcode.GETOPT_FAILURE) @@ -68,12 +69,14 @@ def main(): createcand = True elif opt in ('-s', '--createsession'): createvisit = True + elif opt in ('-i', '--idsvalidation'): + idsvalidation = True # input error checking and load config_file file config_file = input_error_checking(profile, bids_dir, usage) # read and insert BIDS data - read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit) + read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit, idsvalidation) def input_error_checking(profile, bids_dir, usage): @@ -127,20 +130,22 @@ def input_error_checking(profile, bids_dir, usage): return config_file -def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit): +def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit, idsvalidation): """ Read the provided BIDS structure and import it into the database. - :param bids_dir : path to the BIDS directory - :type bids_dir : str - :param config_file: path to the config file with database connection information - :type config_file: str - :param verbose : flag for more printing if set - :type verbose : bool - :param createcand : allow database candidate creation if it did not exist already - :type createcand : bool - :param createvisit: allow database visit creation if it did not exist already - :type createvisit: bool + :param bids_dir : path to the BIDS directory + :type bids_dir : str + :param config_file : path to the config file with database connection information + :type config_file : str + :param verbose : flag for more printing if set + :type verbose : bool + :param createcand : allow database candidate creation if it did not exist already + :type createcand : bool + :param createvisit : allow database visit creation if it did not exist already + :type createvisit : bool + :param idsvalidation: allow pscid/candid validation in the BIDS directory name' + :type idsvalidation: bool """ # database connection @@ -155,6 +160,10 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit # making sure that there is a final / in data_dir data_dir = data_dir if data_dir.endswith('/') else data_dir + "/" + # Validate that pscid and candid matches + if idsvalidation: + validateids(bids_dir, db, verbose) + # load the BIDS directory bids_reader = BidsReader(bids_dir, verbose) if not bids_reader.participants_info \ @@ -180,8 +189,12 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit # greps BIDS candidate's info from LORIS (creates the candidate if it # does not exist yet in LORIS and the createcand flag is set to true) loris_cand_info = grep_or_create_candidate_db_info( - bids_reader, bids_id, db, createcand, loris_bids_root_dir, verbose + bids_reader, bids_id, db, createcand, verbose ) + + # create the candidate's directory in the LORIS BIDS import directory + lib.utilities.create_dir(loris_bids_root_dir + "sub-" + bids_id, verbose) + cand_id = loris_cand_info['CandID'] center_id = loris_cand_info['RegistrationCenterID'] project_id = loris_cand_info['RegistrationProjectID'] @@ -248,6 +261,33 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit db.disconnect() +def validateids(bids_dir, db, verbose): + """ + Validate that pscid and candid matches + + :param bids_dir : path to the BIDS directory + :type bids_dir : str + :param db : database handler object + :type db : object + :param verbose : flag for more printing if set + :type verbose : bool + """ + + bids_folder = bids_dir.rstrip('/').split('/')[-1] + bids_folder_parts = bids_folder.split('_') + psc_id = bids_folder_parts[0] + cand_id = bids_folder_parts[1] + + candidate = Candidate(verbose, cand_id=cand_id) + loris_cand_info = candidate.get_candidate_info_from_loris(db) + + if not loris_cand_info: + print("ERROR: could not find a candidate with cand_id " + cand_id + ".") + sys.exit(lib.exitcode.CANDID_NOT_FOUND) + if loris_cand_info['PSCID'] != psc_id: + print("ERROR: cand_id " + cand_id + " and psc_id " + psc_id + " do not match.") + sys.exit(lib.exitcode.PSCID_MISMATCH) + def create_loris_bids_directory(bids_reader, data_dir, verbose): """ Creates the LORIS BIDS import root directory (with name and BIDS version) @@ -314,8 +354,7 @@ def create_loris_bids_directory(bids_reader, data_dir, verbose): return loris_bids_dirname -def grep_or_create_candidate_db_info(bids_reader, bids_id, db, - createcand, loris_bids_dir, verbose): +def grep_or_create_candidate_db_info(bids_reader, bids_id, db, createcand, verbose): """ Greps (or creates if candidate does not exist and createcand is true) the BIDS candidate in the LORIS candidate's table and return a list of @@ -323,14 +362,12 @@ def grep_or_create_candidate_db_info(bids_reader, bids_id, db, :param bids_reader : BIDS information handler object :type bids_reader : object - :param bids_id : bids_id to be used as PSCID + :param bids_id : bids_id to be used (CandID or PSCID) :type bids_id : str :param db : database handler object :type db : object :param createcand : if true, creates the candidate in LORIS :type createcand : bool - :param loris_bids_dir: LORIS BIDS import root directory to copy data - :type loris_bids_dir: str :param verbose : if true, prints out information while executing :type verbose : bool @@ -339,16 +376,24 @@ def grep_or_create_candidate_db_info(bids_reader, bids_id, db, :rtype: list """ - candidate = Candidate(verbose, psc_id=bids_id) + candidate = Candidate(verbose=verbose, cand_id=bids_id) loris_cand_info = candidate.get_candidate_info_from_loris(db) + if not loris_cand_info: + candidate = Candidate(verbose, psc_id=bids_id) + loris_cand_info = candidate.get_candidate_info_from_loris(db) + if not loris_cand_info and createcand: loris_cand_info = candidate.create_candidate( db, bids_reader.participants_info ) + if not loris_cand_info: + print("Creating candidate failed. Cannot importing the files.\n") + sys.exit(lib.exitcode.CANDIDATE_CREATION_FAILURE) - # create the candidate's directory in the LORIS BIDS import directory - lib.utilities.create_dir(loris_bids_dir + "sub-" + bids_id, verbose) + if not loris_cand_info: + print("Candidate " + bids_id + " not found. You can retry with the --createcandidate option.\n") + sys.exit(lib.exitcode.CANDIDATE_NOT_FOUND) return loris_cand_info diff --git a/python/lib/candidate.py b/python/lib/candidate.py index 43161198e..aa36395cf 100644 --- a/python/lib/candidate.py +++ b/python/lib/candidate.py @@ -72,7 +72,12 @@ def create_candidate(self, db, participants_info): :rtype: dict """ - self.cand_id = self.generate_cand_id(db) + if not self.psc_id: + print("Cannot create a candidate without a PSCID.\n") + sys.exit(lib.exitcode.CANDIDATE_CREATION_FAILURE) + + if not self.cand_id: + self.cand_id = self.generate_cand_id(db) for row in participants_info: if not row['participant_id'] == self.psc_id: @@ -83,12 +88,11 @@ def create_candidate(self, db, participants_info): if 'age' in row: self.age = row['age'] - # three steps to find site: + # two steps to find site: # 1. try matching full name from 'site' column in participants.tsv in db # 2. try extracting alias from pscid - # 3. try finding previous site in candidate table - if 'site' in row and row['site'].lower() not in ("null", ""): + if 'site' in row: # search site id in psc table by its full name site_info = db.pselect( "SELECT CenterID FROM psc WHERE Name = %s", @@ -104,20 +108,9 @@ def create_candidate(self, db, participants_info): if site['Alias'] in row['participant_id']: self.center_id = site['CenterID'] - if self.center_id is None: - # try to find participant site in db - candidate_site_project = db.pselect( - "SELECT RegistrationCenterID FROM candidate WHERE pscid = %s", - [self.psc_id, ] - ) - if len(candidate_site_project) > 0: - self.center_id = candidate_site_project[0]['RegistrationCenterID'] - - # two steps to find project: - # 1. find full name in 'project' column in participants.tsv - # 2. find previous in candidate table + # try to find full name in 'project' column in participants.tsv - if 'project' in row and row['project'].lower() not in ("null", ""): + if 'project' in row: # search project id in Project table by its full name project_info = db.pselect( "SELECT ProjectID FROM Project WHERE Name = %s", @@ -126,15 +119,6 @@ def create_candidate(self, db, participants_info): if len(project_info) > 0: self.project_id = project_info[0]['ProjectID'] - if self.project_id is None: - # try to find participant project - candidate_site_project = db.pselect( - "SELECT RegistrationProjectID FROM candidate WHERE pscid = %s", - [self.psc_id, ] - ) - if len(candidate_site_project) > 0: - self.center_id = candidate_site_project[0]['RegistrationProjectID'] - if not self.center_id: print("ERROR: could not determine site for " + self.psc_id + "." + " Please check that your psc table contains a site with an" @@ -171,13 +155,12 @@ def create_candidate(self, db, participants_info): values=insert_val ) - loris_cand_info = self.get_candidate_info_from_loris(db) + return self.get_candidate_info_from_loris(db) - return loris_cand_info def get_candidate_info_from_loris(self, db): """ - Grep candidate information from the candidate table using PSCID. + Grep candidate information from the candidate table using the PSCID or CandID. :param db: database handler object :type db: object @@ -186,10 +169,17 @@ def get_candidate_info_from_loris(self, db): :rtype: dict """ - loris_cand_info = db.pselect( - "SELECT * FROM candidate WHERE PSCID = %s", - (self.psc_id,), - ) + loris_cand_info = None + if self.cand_id: + loris_cand_info = db.pselect( + "SELECT * FROM candidate WHERE CandID = %s", + (self.cand_id,), + ) + elif self.psc_id: + loris_cand_info = db.pselect( + "SELECT * FROM candidate WHERE PSCID = %s", + (self.psc_id,), + ) return loris_cand_info[0] if loris_cand_info else None diff --git a/python/lib/eeg.py b/python/lib/eeg.py index 00fc830cb..b1b1dfa2c 100755 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -1,6 +1,7 @@ """Deals with EEG BIDS datasets and register them into the database.""" import os +import sys import json import getpass from pyblake2 import blake2b @@ -163,11 +164,20 @@ def get_loris_cand_info(self): :rtype: list """ - candidate = Candidate(verbose=self.verbose, psc_id=self.bids_sub_id) + candidate = Candidate(verbose=self.verbose, cand_id=self.bids_sub_id) loris_cand_info = candidate.get_candidate_info_from_loris(self.db) + if not loris_cand_info: + candidate = Candidate(verbose=self.verbose, psc_id=self.bids_sub_id) + loris_cand_info = candidate.get_candidate_info_from_loris(self.db) + + if not loris_cand_info: + print("Candidate " + self.bids_sub_id + " not found. You can retry with the --createcandidate option.\n") + sys.exit(lib.exitcode.CANDIDATE_NOT_FOUND) + return loris_cand_info + def get_loris_session_id(self): """ Greps the LORIS session.ID corresponding to the BIDS visit. Note, @@ -189,8 +199,8 @@ def get_loris_session_id(self): loris_vl_info = session.get_session_info_from_loris() if not loris_vl_info: - message = "ERROR: visit label " + visit_label + "does not exist in " + \ - "the session table for candidate " + self.cand_id + \ + message = "ERROR: visit label " + visit_label + " does not exist in " + \ + "the session table for candidate " + str(self.cand_id) + \ "\nPlease make sure the visit label is created in the " + \ "database or run bids_import.py with the -s option -s if " + \ "you wish that the insertion pipeline creates the visit " + \ diff --git a/python/lib/exitcode.py b/python/lib/exitcode.py index cd57bdb0a..0a1c866fa 100644 --- a/python/lib/exitcode.py +++ b/python/lib/exitcode.py @@ -33,13 +33,17 @@ # -- Common file manipulation failures (exit codes from 60 to 79) -EXTRACTION_FAILURE = 60 # if archive extraction failed -FILE_TYPE_CHECK_FAILURE = 61 # if different file type from what's expected -INVALID_DICOM = 62 # if DICOM is invalid -MISSING_FILES = 63 # if there are missing files from what's expected -UNREADABLE_FILE = 64 # if could not properly read a file content -COPY_FAILURE = 65 # if copy failed -CREATE_DIR_FAILURE = 66 # if dir creation failed +EXTRACTION_FAILURE = 60 # if archive extraction failed +FILE_TYPE_CHECK_FAILURE = 61 # if different file type from what's expected +INVALID_DICOM = 62 # if DICOM is invalid +MISSING_FILES = 63 # if there are missing files from what's expected +UNREADABLE_FILE = 64 # if could not properly read a file content +COPY_FAILURE = 65 # if copy failed +CREATE_DIR_FAILURE = 66 # if dir creation failed +CANDID_NOT_FOUND = 67 # if candidate's cand_id not found in LORIS' DB +PSCID_MISMATCH = 68 # if candidate's cand_id and psc_id do not match +CANDIDATE_CREATION_FAILURE = 69 # if candidate creation failed +CANDIDATE_NOT_FOUND = 70 # if candidate not found in LORIS' DB # -- Other common generic failures (exit codes from 80 to 149) CLEANUP_FAILURE = 80 # if cleanup after script execution failed From 7d2c3e48d7284d1e0c4c4a61c9758a72222518f8 Mon Sep 17 00:00:00 2001 From: Laetitia Fesselier Date: Fri, 14 Apr 2023 08:57:53 -0400 Subject: [PATCH 2/2] Clean redundant exit code --- python/bids_import.py | 2 +- python/lib/exitcode.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/bids_import.py b/python/bids_import.py index 68607a08c..ac2740f5c 100755 --- a/python/bids_import.py +++ b/python/bids_import.py @@ -286,7 +286,7 @@ def validateids(bids_dir, db, verbose): sys.exit(lib.exitcode.CANDID_NOT_FOUND) if loris_cand_info['PSCID'] != psc_id: print("ERROR: cand_id " + cand_id + " and psc_id " + psc_id + " do not match.") - sys.exit(lib.exitcode.PSCID_MISMATCH) + sys.exit(lib.exitcode.CANDIDATE_MISMATCH) def create_loris_bids_directory(bids_reader, data_dir, verbose): """ diff --git a/python/lib/exitcode.py b/python/lib/exitcode.py index 0a1c866fa..2f1e2dbcf 100644 --- a/python/lib/exitcode.py +++ b/python/lib/exitcode.py @@ -41,9 +41,8 @@ COPY_FAILURE = 65 # if copy failed CREATE_DIR_FAILURE = 66 # if dir creation failed CANDID_NOT_FOUND = 67 # if candidate's cand_id not found in LORIS' DB -PSCID_MISMATCH = 68 # if candidate's cand_id and psc_id do not match -CANDIDATE_CREATION_FAILURE = 69 # if candidate creation failed -CANDIDATE_NOT_FOUND = 70 # if candidate not found in LORIS' DB +CANDIDATE_CREATION_FAILURE = 68 # if candidate creation failed +CANDIDATE_NOT_FOUND = 69 # if candidate not found in LORIS' DB # -- Other common generic failures (exit codes from 80 to 149) CLEANUP_FAILURE = 80 # if cleanup after script execution failed