Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

candID subject_id support #912

Merged
merged 2 commits into from
Apr 14, 2023
Merged

candID subject_id support #912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8991f7b
cand_id subject_id support
laemtl Apr 1, 2023
7d2c3e4
Clean redundant exit code
laemtl Apr 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 75 additions & 30 deletions python/bids_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,30 +26,31 @@
# sys.tracebacklimit = 0

def main():
bids_dir = ''
verbose = False
createcand = False
createvisit = False
profile = ''
bids_dir = ''
verbose = False
createcand = False
createvisit = False
idsvalidation = False
profile = ''

long_options = [
"help", "profile=", "directory=",
"createcandidate", "createsession", "verbose"
"createcandidate", "createsession", "idsvalidation", "verbose"
]
usage = (
'\n'
'usage : bids_import -d <bids_directory> -p <profile> \n\n'
'options: \n'
'\t-p, --profile : name of the python database config file in '
'dicom-archive/.loris-mri\n'
'\t-p, --profile : name of the python database config file in dicom-archive/.loris-mri\n'
'\t-d, --directory : BIDS directory to parse & insert into LORIS\n'
'\t-c, --createcandidate: to create BIDS candidates in LORIS (optional)\n'
'\t-s, --createsession : to create BIDS sessions in LORIS (optional)\n'
'\t-i, --idsvalidation : to validate BIDS directory for a matching pscid/candid pair (optional)\n'
'\t-v, --verbose : be verbose\n'
)

try:
opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csv', long_options)
opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csiv', long_options)
except getopt.GetoptError:
print(usage)
sys.exit(lib.exitcode.GETOPT_FAILURE)
Expand All @@ -68,12 +69,14 @@ def main():
createcand = True
elif opt in ('-s', '--createsession'):
createvisit = True
elif opt in ('-i', '--idsvalidation'):
idsvalidation = True

# input error checking and load config_file file
config_file = input_error_checking(profile, bids_dir, usage)

# read and insert BIDS data
read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit)
read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit, idsvalidation)


def input_error_checking(profile, bids_dir, usage):
Expand Down Expand Up @@ -127,20 +130,22 @@ def input_error_checking(profile, bids_dir, usage):
return config_file


def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit):
def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit, idsvalidation):
"""
Read the provided BIDS structure and import it into the database.

:param bids_dir : path to the BIDS directory
:type bids_dir : str
:param config_file: path to the config file with database connection information
:type config_file: str
:param verbose : flag for more printing if set
:type verbose : bool
:param createcand : allow database candidate creation if it did not exist already
:type createcand : bool
:param createvisit: allow database visit creation if it did not exist already
:type createvisit: bool
:param bids_dir : path to the BIDS directory
:type bids_dir : str
:param config_file : path to the config file with database connection information
:type config_file : str
:param verbose : flag for more printing if set
:type verbose : bool
:param createcand : allow database candidate creation if it did not exist already
:type createcand : bool
:param createvisit : allow database visit creation if it did not exist already
:type createvisit : bool
:param idsvalidation: allow pscid/candid validation in the BIDS directory name'
:type idsvalidation: bool
"""

# database connection
Expand All @@ -155,6 +160,10 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit
# making sure that there is a final / in data_dir
data_dir = data_dir if data_dir.endswith('/') else data_dir + "/"

# Validate that pscid and candid matches
if idsvalidation:
validateids(bids_dir, db, verbose)

# load the BIDS directory
bids_reader = BidsReader(bids_dir, verbose)
if not bids_reader.participants_info \
Expand All @@ -180,8 +189,12 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit
# greps BIDS candidate's info from LORIS (creates the candidate if it
# does not exist yet in LORIS and the createcand flag is set to true)
loris_cand_info = grep_or_create_candidate_db_info(
bids_reader, bids_id, db, createcand, loris_bids_root_dir, verbose
bids_reader, bids_id, db, createcand, verbose
)

# create the candidate's directory in the LORIS BIDS import directory
lib.utilities.create_dir(loris_bids_root_dir + "sub-" + bids_id, verbose)

cand_id = loris_cand_info['CandID']
center_id = loris_cand_info['RegistrationCenterID']
project_id = loris_cand_info['RegistrationProjectID']
Expand Down Expand Up @@ -248,6 +261,33 @@ def read_and_insert_bids(bids_dir, config_file, verbose, createcand, createvisit
db.disconnect()


def validateids(bids_dir, db, verbose):
"""
Validate that pscid and candid matches

:param bids_dir : path to the BIDS directory
:type bids_dir : str
:param db : database handler object
:type db : object
:param verbose : flag for more printing if set
:type verbose : bool
"""

bids_folder = bids_dir.rstrip('/').split('/')[-1]
bids_folder_parts = bids_folder.split('_')
psc_id = bids_folder_parts[0]
cand_id = bids_folder_parts[1]

candidate = Candidate(verbose, cand_id=cand_id)
loris_cand_info = candidate.get_candidate_info_from_loris(db)

if not loris_cand_info:
print("ERROR: could not find a candidate with cand_id " + cand_id + ".")
sys.exit(lib.exitcode.CANDID_NOT_FOUND)
if loris_cand_info['PSCID'] != psc_id:
print("ERROR: cand_id " + cand_id + " and psc_id " + psc_id + " do not match.")
sys.exit(lib.exitcode.CANDIDATE_MISMATCH)

def create_loris_bids_directory(bids_reader, data_dir, verbose):
"""
Creates the LORIS BIDS import root directory (with name and BIDS version)
Expand Down Expand Up @@ -314,23 +354,20 @@ def create_loris_bids_directory(bids_reader, data_dir, verbose):
return loris_bids_dirname


def grep_or_create_candidate_db_info(bids_reader, bids_id, db,
createcand, loris_bids_dir, verbose):
def grep_or_create_candidate_db_info(bids_reader, bids_id, db, createcand, verbose):
"""
Greps (or creates if candidate does not exist and createcand is true) the
BIDS candidate in the LORIS candidate's table and return a list of
candidates with their related fields from the database.

:param bids_reader : BIDS information handler object
:type bids_reader : object
:param bids_id : bids_id to be used as PSCID
:param bids_id : bids_id to be used (CandID or PSCID)
:type bids_id : str
:param db : database handler object
:type db : object
:param createcand : if true, creates the candidate in LORIS
:type createcand : bool
:param loris_bids_dir: LORIS BIDS import root directory to copy data
:type loris_bids_dir: str
:param verbose : if true, prints out information while executing
:type verbose : bool

Expand All @@ -339,16 +376,24 @@ def grep_or_create_candidate_db_info(bids_reader, bids_id, db,
:rtype: list
"""

candidate = Candidate(verbose, psc_id=bids_id)
candidate = Candidate(verbose=verbose, cand_id=bids_id)
loris_cand_info = candidate.get_candidate_info_from_loris(db)

if not loris_cand_info:
candidate = Candidate(verbose, psc_id=bids_id)
loris_cand_info = candidate.get_candidate_info_from_loris(db)

if not loris_cand_info and createcand:
loris_cand_info = candidate.create_candidate(
db, bids_reader.participants_info
)
if not loris_cand_info:
print("Creating candidate failed. Cannot importing the files.\n")
sys.exit(lib.exitcode.CANDIDATE_CREATION_FAILURE)

# create the candidate's directory in the LORIS BIDS import directory
lib.utilities.create_dir(loris_bids_dir + "sub-" + bids_id, verbose)
if not loris_cand_info:
print("Candidate " + bids_id + " not found. You can retry with the --createcandidate option.\n")
sys.exit(lib.exitcode.CANDIDATE_NOT_FOUND)

return loris_cand_info

Expand Down
56 changes: 23 additions & 33 deletions python/lib/candidate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,12 @@ def create_candidate(self, db, participants_info):
:rtype: dict
"""

self.cand_id = self.generate_cand_id(db)
if not self.psc_id:
print("Cannot create a candidate without a PSCID.\n")
sys.exit(lib.exitcode.CANDIDATE_CREATION_FAILURE)

if not self.cand_id:
self.cand_id = self.generate_cand_id(db)

for row in participants_info:
if not row['participant_id'] == self.psc_id:
Expand All @@ -83,12 +88,11 @@ def create_candidate(self, db, participants_info):
if 'age' in row:
self.age = row['age']

# three steps to find site:
# two steps to find site:
# 1. try matching full name from 'site' column in participants.tsv in db
# 2. try extracting alias from pscid
# 3. try finding previous site in candidate table

if 'site' in row and row['site'].lower() not in ("null", ""):
if 'site' in row:
# search site id in psc table by its full name
site_info = db.pselect(
"SELECT CenterID FROM psc WHERE Name = %s",
Expand All @@ -104,20 +108,9 @@ def create_candidate(self, db, participants_info):
if site['Alias'] in row['participant_id']:
self.center_id = site['CenterID']

if self.center_id is None:
# try to find participant site in db
candidate_site_project = db.pselect(
"SELECT RegistrationCenterID FROM candidate WHERE pscid = %s",
[self.psc_id, ]
)
if len(candidate_site_project) > 0:
self.center_id = candidate_site_project[0]['RegistrationCenterID']

# two steps to find project:
# 1. find full name in 'project' column in participants.tsv
# 2. find previous in candidate table
# try to find full name in 'project' column in participants.tsv

if 'project' in row and row['project'].lower() not in ("null", ""):
if 'project' in row:
# search project id in Project table by its full name
project_info = db.pselect(
"SELECT ProjectID FROM Project WHERE Name = %s",
Expand All @@ -126,15 +119,6 @@ def create_candidate(self, db, participants_info):
if len(project_info) > 0:
self.project_id = project_info[0]['ProjectID']

if self.project_id is None:
# try to find participant project
candidate_site_project = db.pselect(
"SELECT RegistrationProjectID FROM candidate WHERE pscid = %s",
[self.psc_id, ]
)
if len(candidate_site_project) > 0:
self.center_id = candidate_site_project[0]['RegistrationProjectID']

if not self.center_id:
print("ERROR: could not determine site for " + self.psc_id + "."
+ " Please check that your psc table contains a site with an"
Expand Down Expand Up @@ -171,13 +155,12 @@ def create_candidate(self, db, participants_info):
values=insert_val
)

loris_cand_info = self.get_candidate_info_from_loris(db)
return self.get_candidate_info_from_loris(db)

return loris_cand_info

def get_candidate_info_from_loris(self, db):
"""
Grep candidate information from the candidate table using PSCID.
Grep candidate information from the candidate table using the PSCID or CandID.

:param db: database handler object
:type db: object
Expand All @@ -186,10 +169,17 @@ def get_candidate_info_from_loris(self, db):
:rtype: dict
"""

loris_cand_info = db.pselect(
"SELECT * FROM candidate WHERE PSCID = %s",
(self.psc_id,),
)
loris_cand_info = None
if self.cand_id:
loris_cand_info = db.pselect(
"SELECT * FROM candidate WHERE CandID = %s",
(self.cand_id,),
)
elif self.psc_id:
loris_cand_info = db.pselect(
"SELECT * FROM candidate WHERE PSCID = %s",
(self.psc_id,),
)

return loris_cand_info[0] if loris_cand_info else None

Expand Down
16 changes: 13 additions & 3 deletions python/lib/eeg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Deals with EEG BIDS datasets and register them into the database."""

import os
import sys
import json
import getpass
from pyblake2 import blake2b
Expand Down Expand Up @@ -163,11 +164,20 @@ def get_loris_cand_info(self):
:rtype: list
"""

candidate = Candidate(verbose=self.verbose, psc_id=self.bids_sub_id)
candidate = Candidate(verbose=self.verbose, cand_id=self.bids_sub_id)
loris_cand_info = candidate.get_candidate_info_from_loris(self.db)

if not loris_cand_info:
candidate = Candidate(verbose=self.verbose, psc_id=self.bids_sub_id)
loris_cand_info = candidate.get_candidate_info_from_loris(self.db)

if not loris_cand_info:
print("Candidate " + self.bids_sub_id + " not found. You can retry with the --createcandidate option.\n")
sys.exit(lib.exitcode.CANDIDATE_NOT_FOUND)

return loris_cand_info


def get_loris_session_id(self):
"""
Greps the LORIS session.ID corresponding to the BIDS visit. Note,
Expand All @@ -189,8 +199,8 @@ def get_loris_session_id(self):
loris_vl_info = session.get_session_info_from_loris()

if not loris_vl_info:
message = "ERROR: visit label " + visit_label + "does not exist in " + \
"the session table for candidate " + self.cand_id + \
message = "ERROR: visit label " + visit_label + " does not exist in " + \
"the session table for candidate " + str(self.cand_id) + \
"\nPlease make sure the visit label is created in the " + \
"database or run bids_import.py with the -s option -s if " + \
"you wish that the insertion pipeline creates the visit " + \
Expand Down
17 changes: 10 additions & 7 deletions python/lib/exitcode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,16 @@


# -- Common file manipulation failures (exit codes from 60 to 79)
EXTRACTION_FAILURE = 60 # if archive extraction failed
FILE_TYPE_CHECK_FAILURE = 61 # if different file type from what's expected
INVALID_DICOM = 62 # if DICOM is invalid
MISSING_FILES = 63 # if there are missing files from what's expected
UNREADABLE_FILE = 64 # if could not properly read a file content
COPY_FAILURE = 65 # if copy failed
CREATE_DIR_FAILURE = 66 # if dir creation failed
EXTRACTION_FAILURE = 60 # if archive extraction failed
FILE_TYPE_CHECK_FAILURE = 61 # if different file type from what's expected
INVALID_DICOM = 62 # if DICOM is invalid
MISSING_FILES = 63 # if there are missing files from what's expected
UNREADABLE_FILE = 64 # if could not properly read a file content
COPY_FAILURE = 65 # if copy failed
CREATE_DIR_FAILURE = 66 # if dir creation failed
CANDID_NOT_FOUND = 67 # if candidate's cand_id not found in LORIS' DB
CANDIDATE_CREATION_FAILURE = 68 # if candidate creation failed
CANDIDATE_NOT_FOUND = 69 # if candidate not found in LORIS' DB

# -- Other common generic failures (exit codes from 80 to 149)
CLEANUP_FAILURE = 80 # if cleanup after script execution failed
Expand Down