Merge 24.1.8 into main

VERSION

@@ -1 +1 @@
-24.1.7
+24.1.8

python/lib/aws_s3.py

@@ -99,3 +99,22 @@ def download_file(self, s3_object_name, destination_file):
             self.s3_bucket_obj.download_file(s3_file_name, destination_file)
         except ClientError as err:
             raise Exception(f"{s3_object_name} download failure = {format(err)}")
+
+    def delete_file(self, s3_object_name):
+        """
+        Function to delete a s3 file or directory.
+
+        :param s3_object_name: name of the s3 file or directory
+         :type s3_object_name: str
+        """
+
+        s3_bucket_prefix = f"s3://{self.bucket_name}/"
+        s3_file_name = s3_object_name[len(s3_bucket_prefix):] \
+            if s3_object_name.startswith(s3_bucket_prefix) else s3_object_name
+        objects_to_delete = [{'Key': obj.key} for obj in
+                             self.s3_bucket_obj.objects.filter(Prefix=s3_file_name)]
+        self.s3_bucket_obj.delete_objects(
+            Delete={
+                'Objects': objects_to_delete
+            }
+        )

python/lib/database_lib/mri_violations_log.py

@@ -50,7 +50,7 @@ def insert_violations_log(self, field_value_dict):
             get_last_id=False
         )
 
-    def get_excluded_violations_for_tarchive_id(self, tarchive_id, severity=None):
+    def get_violations_for_tarchive_id(self, tarchive_id, severity=None):
         """
         Get the list of violations logged in `mri_violations_log` with excluded severity for a given `TarchiveID`.
 

python/lib/dcm2bids_imaging_pipeline_lib/dicom_archive_loader_pipeline.py

@@ -419,7 +419,7 @@ def _get_summary_of_insertion(self):
             self.tarchive_id
         )
         protocol_violations_list = [v["minc_location"] for v in prot_viol_results] if prot_viol_results else None
-        excl_viol_results = self.imaging_obj.mri_viol_log_db_obj.get_excluded_violations_for_tarchive_id(
+        excl_viol_results = self.imaging_obj.mri_viol_log_db_obj.get_violations_for_tarchive_id(
             self.tarchive_id, "exclude"
         )
         excluded_violations_list = [v["MincFile"] for v in excl_viol_results] if excl_viol_results else None

python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py

@@ -117,8 +117,8 @@ def __init__(self, loris_getopt_obj, script_name):
         # ---------------------------------------------------------------------------------------------
         # Determine acquisition protocol (or register into mri_protocol_violated_scans and exits)
         # ---------------------------------------------------------------------------------------------
+        self.scan_type_id, self.mri_protocol_group_id = self._determine_acquisition_protocol()
         if not self.loris_scan_type:
-            self.scan_type_id, self.mri_protocol_group_id = self._determine_acquisition_protocol()
             if not self.scan_type_id:
                 self._move_to_trashbin()
                 self._register_protocol_violated_scan()
@@ -129,7 +129,6 @@ def __init__(self, loris_getopt_obj, script_name):
             else:
                 self.scan_type_name = self.imaging_obj.get_scan_type_name_from_id(self.scan_type_id)
         else:
-            self.scan_type_id = self.imaging_obj.get_scan_type_id_from_scan_type_name(self.loris_scan_type)
             if not self.scan_type_id:
                 self._move_to_trashbin()
                 self._register_protocol_violated_scan()
@@ -154,6 +153,8 @@ def __init__(self, loris_getopt_obj, script_name):
         # ---------------------------------------------------------------------------------------------
         # Run extra file checks to determine possible protocol violations
         # ---------------------------------------------------------------------------------------------
+        self.warning_violations_list = []
+        self.exclude_violations_list = []
         if not self.bypass_extra_checks:
             self.violations_summary = self.imaging_obj.run_extra_file_checks(
                 self.session_obj.session_info_dict['ProjectID'],
@@ -162,8 +163,8 @@ def __init__(self, loris_getopt_obj, script_name):
                 self.scan_type_id,
                 self.json_file_dict
             )
-        self.warning_violations_list = self.violations_summary['warning']
-        self.exclude_violations_list = self.violations_summary['exclude']
+            self.warning_violations_list = self.violations_summary['warning']
+            self.exclude_violations_list = self.violations_summary['exclude']
 
         # ---------------------------------------------------------------------------------------------
         # Register files in the proper tables
@@ -350,7 +351,9 @@ def _determine_acquisition_protocol(self):
             self.scanner_id
         )
 
-        protocol_info = self.imaging_obj.get_acquisition_protocol_info(protocols_list, nifti_name, scan_param)
+        protocol_info = self.imaging_obj.get_acquisition_protocol_info(
+            protocols_list, nifti_name, scan_param, self.loris_scan_type
+        )
         self.log_info(protocol_info['error_message'], is_error="N", is_verbose="Y")
 
         return protocol_info['scan_type_id'], protocol_info['mri_protocol_group_id']
@@ -396,7 +399,7 @@ def _move_to_assembly_and_insert_file_info(self):
         self.log_info(message, is_error='N', is_verbose='Y')
 
         # add an entry in the violations log table if there is a warning violation associated to the file
-        if self.violations_summary['warning']:
+        if self.warning_violations_list:
             message = f"Inserting warning violations related to {self.assembly_nifti_rel_path}." \
                       f"  List of violations found: {self.warning_violations_list}"
             self.log_info(message, is_error='N', is_verbose='Y')

python/lib/dcm2bids_imaging_pipeline_lib/push_imaging_files_to_s3_pipeline.py

@@ -166,10 +166,10 @@ def _get_list_of_files_from_mri_violations_log(self):
         parameter_file).
         """
 
-        exclude_entries = self.imaging_obj.mri_viol_log_db_obj.get_excluded_violations_for_tarchive_id(
+        exclude_entries = self.imaging_obj.mri_viol_log_db_obj.get_violations_for_tarchive_id(
             self.tarchive_id, "exclude"
         )
-        warning_entries = self.imaging_obj.mri_viol_log_db_obj.get_excluded_violations_for_tarchive_id(
+        warning_entries = self.imaging_obj.mri_viol_log_db_obj.get_violations_for_tarchive_id(
             self.tarchive_id, "warning"
         )
 

python/lib/eeg.py

@@ -456,14 +456,15 @@ def fetch_and_insert_eeg_files(self, derivatives=False):
             result         = physiological.grep_file_id_from_hash(blake2)
             physio_file_id = result['PhysiologicalFileID'] if result else None
             eeg_path       = result['FilePath']            if result else None
+
             if not physio_file_id:
                 # grep the modality ID from physiological_modality table
                 modality_id = self.db.grep_id_from_lookup_table(
-                    id_field_name       = 'PhysiologicalModalityID',
-                    table_name          = 'physiological_modality',
-                    where_field_name    = 'PhysiologicalModality',
-                    where_value         = self.bids_modality,
-                    insert_if_not_found = False
+                    id_field_name='PhysiologicalModalityID',
+                    table_name='physiological_modality',
+                    where_field_name='PhysiologicalModality',
+                    where_value=self.bids_modality,
+                    insert_if_not_found=False
                 )
 
                 # copy the eeg_file to the LORIS BIDS import directory
@@ -474,36 +475,38 @@ def fetch_and_insert_eeg_files(self, derivatives=False):
                 # insert the file along with its information into
                 # physiological_file and physiological_parameter_file tables
                 eeg_file_info = {
-                    'FileType'       : file_type,
-                    'FilePath'       : eeg_path,
-                    'SessionID'      : self.session_id,
+                    'FileType': file_type,
+                    'FilePath': eeg_path,
+                    'SessionID': self.session_id,
                     'AcquisitionTime': eeg_acq_time,
-                    'InsertedByUser' : getpass.getuser(),
+                    'InsertedByUser': getpass.getuser(),
                     'PhysiologicalOutputTypeID': output_type_id,
-                    'PhysiologicalModalityID'  : modality_id
+                    'PhysiologicalModalityID': modality_id
                 }
                 physio_file_id = physiological.insert_physiological_file(
                     eeg_file_info, eeg_file_data
                 )
-
+            
             # if the EEG file was a set file, then update the filename for the .set
             # and .fdt files in the .set file so it can find the proper file for
             # visualization and analyses
-            if file_type == 'set':
+            file_paths_updated = file_type != 'set'
+            if not file_paths_updated:
                 set_full_path = os.path.join(self.data_dir, eeg_path)
                 fdt_full_path = eeg_file_data['fdt_file'] if 'fdt_file' in eeg_file_data.keys() else None
 
                 if fdt_full_path:
                     fdt_full_path = os.path.join(self.data_dir, eeg_file_data['fdt_file'])
-                utilities.update_set_file_path_info(set_full_path, fdt_full_path)
+                file_paths_updated = utilities.update_set_file_path_info(set_full_path, fdt_full_path)
 
-            inserted_eegs.append({
-                'file_id': physio_file_id,
-                'file_path': eeg_path,
-                'eegjson_file_path': eegjson_file_path,
-                'fdt_file_path': fdt_file_path,
-                'original_file_data': eeg_file,
-            })
+            if file_paths_updated:
+                inserted_eegs.append({
+                    'file_id': physio_file_id,
+                    'file_path': eeg_path,
+                    'eegjson_file_path': eegjson_file_path,
+                    'fdt_file_path': fdt_file_path,
+                    'original_file_data': eeg_file,
+                })
 
         return inserted_eegs
 
@@ -688,9 +691,34 @@ def fetch_and_insert_event_files(
                 # get the blake2b hash of the task events file
                 blake2 = blake2b(event_data_file.path.encode('utf-8')).hexdigest()
                 # insert event data in the database
-                physiological.insert_event_file(
-                    event_data, event_path, physiological_file_id, blake2
-                )
+                # TODO: Temporary
+                # TODO: should not have two different insert_event_file methods
+                # TODO: remove this TRY/CATCH and the legacy method when
+                # TODO: all LORIS code will support additional events.
+                insert_fallback = False
+                msg = ""
+                try:
+                    physiological.insert_event_file(
+                        event_data, event_path, physiological_file_id, blake2
+                    )
+                except NameError:
+                    # when the insert_event_file function does not exist
+                    msg = "WARNING: function 'insert_event_file' not found. Using fallback method."
+                    insert_fallback = True
+                except Exception as ex:
+                    # when event table structure is still the old one
+                    if ex.args[0] and ex.args[0].startswith("Insert query failure: "):
+                        msg = "WARNING: error during DB insert. Using fallback method."
+                        insert_fallback = True
+                    else:
+                        # re-raise other errors from db insert
+                        raise ex
+                # insert fallback, call legacy method
+                if insert_fallback:
+                    print(msg)
+                    physiological.insert_event_file_legacy(
+                        event_data, event_path, physiological_file_id, blake2
+                    )
 
                 event_paths.extend([event_path])
 
@@ -723,7 +751,7 @@ def fetch_and_insert_event_files(
 
                     if not event_metadata_file:
                         message = '\nWARNING: no events metadata files (event.json) associated' \
-                                  'with physiological file ID ' + physiological_file_id
+                                  'with physiological file ID ' + str(physiological_file_id)
                         print(message)
                     else:
                         # copy the event file to the LORIS BIDS import directory

python/lib/imaging.py

@@ -236,7 +236,20 @@ def insert_protocol_violated_scan(self, patient_name, cand_id, psc_id, tarchive_
          :type mri_protocol_group_id: int
         """
 
-        phase_encoding_dir = scan_param["PhaseEncodingDirection"] if "PhaseEncodingDirection" in scan_param else None
+        series_uid = scan_param["SeriesInstanceUID"] if "SeriesInstanceUID" in scan_param.keys() else None
+        image_type = str(scan_param["ImageType"]) if "ImageType" in scan_param.keys() else None
+        echo_number = repr(scan_param["EchoNumber"]) if "EchoNumber" in scan_param.keys() else None
+        phase_encoding_dir = scan_param["PhaseEncodingDirection"] \
+            if "PhaseEncodingDirection" in scan_param.keys() else None
+
+        # if there is already an entry for this violation in mri_protocol_violated_scans, do not insert anything
+        existing_prot_viol_scans = self.mri_prot_viol_scan_db_obj.get_protocol_violations_for_tarchive_id(tarchive_id)
+        for row in existing_prot_viol_scans:
+            if row['SeriesUID'] == series_uid \
+                    and row['PhaseEncodingDirection'] == phase_encoding_dir \
+                    and row['image_type'] == image_type \
+                    and row['EchoNumber'] == echo_number:
+                return
 
         info_to_insert_dict = {
             "CandID": cand_id,
@@ -257,10 +270,10 @@ def insert_protocol_violated_scan(self, patient_name, cand_id, psc_id, tarchive_
             "ystep_range": scan_param["ystep"] if "ystep" in scan_param.keys() else None,
             "zstep_range": scan_param["zstep"] if "zstep" in scan_param.keys() else None,
             "time_range": scan_param["time"] if "time" in scan_param.keys() else None,
-            "SeriesUID": scan_param["SeriesInstanceUID"] if "SeriesInstanceUID" in scan_param.keys() else None,
-            "image_type": str(scan_param["ImageType"]) if "ImageType" in scan_param.keys() else None,
+            "SeriesUID": series_uid,
+            "image_type": image_type,
             "PhaseEncodingDirection": phase_encoding_dir,
-            "EchoNumber": repr(scan_param["EchoNumber"]) if "EchoNumber" in scan_param else None,
+            "EchoNumber": echo_number,
             "MriProtocolGroupID": mri_protocol_group_id if mri_protocol_group_id else None
         }
         self.mri_prot_viol_scan_db_obj.insert_protocol_violated_scans(info_to_insert_dict)
@@ -272,6 +285,35 @@ def insert_mri_violations_log(self, info_to_insert_dict):
         :param info_to_insert_dict: dictionary with the information to be inserted in mri_violations_log
          :type info_to_insert_dict: dict
         """
+
+        series_uid = info_to_insert_dict["SeriesUID"]
+        echo_number = repr(info_to_insert_dict["EchoNumber"])
+        phase_encoding_dir = info_to_insert_dict["PhaseEncodingDirection"]
+        echo_time = info_to_insert_dict['EchoTime']
+        scan_type = info_to_insert_dict['Scan_type']
+        severity = info_to_insert_dict['Severity']
+        header = info_to_insert_dict['Header']
+        value = info_to_insert_dict['Value']
+        valid_regex = info_to_insert_dict['ValidRegex']
+        valid_range = info_to_insert_dict['ValidRange']
+
+        # if there is already an entry for this violation in mri_violations_log, do not insert anything
+        existing_viol_logs = self.mri_viol_log_db_obj.get_violations_for_tarchive_id(
+            info_to_insert_dict['TarchiveID']
+        )
+        for row in existing_viol_logs:
+            if str(row['SeriesUID']) == str(series_uid) \
+                    and str(row['PhaseEncodingDirection']) == str(phase_encoding_dir) \
+                    and str(row['EchoNumber']) == str(echo_number) \
+                    and str(row['Scan_type']) == str(scan_type) \
+                    and str(row['EchoTime']) == str(echo_time) \
+                    and str(row['Severity']) == str(severity) \
+                    and str(row['Header']) == str(header) \
+                    and str(row['Value']) == str(value) \
+                    and str(row['ValidRange']) == str(valid_range) \
+                    and str(row['ValidRegex']) == str(valid_regex):
+                return
+
         self.mri_viol_log_db_obj.insert_violations_log(info_to_insert_dict)
 
     def get_parameter_type_id(self, parameter_name):
@@ -567,7 +609,7 @@ def map_bids_param_to_loris_param(self, file_parameters):
 
         return file_parameters
 
-    def get_acquisition_protocol_info(self, protocols_list, nifti_name, scan_param):
+    def get_acquisition_protocol_info(self, protocols_list, nifti_name, scan_param, scan_type=None):
         """
         Get acquisition protocol information (scan_type_id or message to be printed in the log).
         - If the protocols list provided as input is empty, the scan_type_id will be set to None and proper message
@@ -608,7 +650,7 @@ def get_acquisition_protocol_info(self, protocols_list, nifti_name, scan_param):
 
         # look for matching protocols
         mri_protocol_group_id = protocols_list[0]['MriProtocolGroupID']
-        matching_protocols_list = self.look_for_matching_protocols(protocols_list, scan_param)
+        matching_protocols_list = self.look_for_matching_protocols(protocols_list, scan_param, scan_type)
 
         # if more than one protocol matching, return False, otherwise, return the scan type ID
         if not matching_protocols_list:
@@ -648,7 +690,7 @@ def get_bids_categories_mapping_for_scan_type_id(self, scan_type_id):
 
         return self.mri_prot_db_obj.get_bids_info_for_scan_type_id(scan_type_id)
 
-    def look_for_matching_protocols(self, protocols_list, scan_param):
+    def look_for_matching_protocols(self, protocols_list, scan_param, scan_type=None):
         """
         Look for matching protocols in protocols_list given scan parameters stored in scan_param.
 
@@ -661,9 +703,13 @@ def look_for_matching_protocols(self, protocols_list, scan_param):
          :rtype: list
         """
 
+        scan_type_id = self.get_scan_type_id_from_scan_type_name(scan_type) if scan_type else None
+
         matching_protocols_list = []
         for protocol in protocols_list:
-            if protocol['series_description_regex']:
+            if scan_type_id and protocol['Scan_type'] == scan_type_id:
+                matching_protocols_list.append(protocol['Scan_type'])
+            elif protocol['series_description_regex']:
                 if re.search(
                         rf"{protocol['series_description_regex']}", scan_param['SeriesDescription'], re.IGNORECASE
                 ):