Merge branch 'release/2.37.0'

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.36.0'
+__version__ = '2.37.0'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/extractors/biased_trials.py

@@ -185,8 +185,17 @@ class EphysTrials(BaseBpodTrialsExtractor):
 
     def _extract(self, extractor_classes=None, **kwargs) -> dict:
         extractor_classes = extractor_classes or []
+
+        # For iblrig v8 we use the biased trials table instead. ContrastLeft, ContrastRight and ProbabilityLeft are
+        # filled from the values in the bpod data itself rather than using the pregenerated session number
+        iblrig_version = self.settings.get('IBLRIG_VERSION', self.settings.get('IBLRIG_VERSION_TAG', '0'))
+        if version.parse(iblrig_version) >= version.parse('8.0.0'):
+            TrialsTable = TrialsTableBiased
+        else:
+            TrialsTable = TrialsTableEphys
+
         base = [GoCueTriggerTimes, StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes,
-                ErrorCueTriggerTimes, TrialsTableEphys, IncludedTrials, PhasePosQuiescence]
+                ErrorCueTriggerTimes, TrialsTable, IncludedTrials, PhasePosQuiescence]
         # Get all detected TTLs. These are stored for QC purposes
         self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials)
         # Exclude from trials table

ibllib/io/extractors/ephys_passive.py

@@ -86,22 +86,39 @@ def _load_passive_session_fixtures(session_path: str, task_collection: str = 'ra
     return fixture
 
 
-def _load_task_protocol(session_path: str, task_collection: str = 'raw_passive_data') -> str:
+def _load_task_version(session_path: str, task_collection: str = 'raw_passive_data') -> str:
     """Find the IBL rig version used for the session
 
     :param session_path: the path to a session
     :type session_path: str
     :return: ibl rig task protocol version
     :rtype: str
 
-    FIXME This function has a misleading name
     """
     settings = rawio.load_settings(session_path, task_collection=task_collection)
     ses_ver = settings["IBLRIG_VERSION"]
 
     return ses_ver
 
 
+def skip_task_replay(session_path: str, task_collection: str = 'raw_passive_data') -> bool:
+    """Find whether the task replay portion of the passive stimulus has been shown
+
+    :param session_path: the path to a session
+    :type session_path: str
+    :param task_collection: collection containing task data
+    :type task_collection: str
+    :return: whether or not the task replay has been run
+    :rtype: bool
+    """
+
+    settings = rawio.load_settings(session_path, task_collection=task_collection)
+    # Attempt to see if SKIP_EVENT_REPLAY is available, if not assume we do have task replay
+    skip_replay = settings.get('SKIP_EVENT_REPLAY', False)
+
+    return skip_replay
+
+
 def _load_passive_stim_meta() -> dict:
     """load_passive_stim_meta Loads the passive protocol metadata
 
@@ -536,7 +553,7 @@ def extract_task_replay(
     bpod = ephys_fpga.get_sync_fronts(sync, sync_map["bpod"], tmin=treplay[0], tmax=treplay[1])
     passiveValve_intervals = _extract_passiveValve_intervals(bpod)
 
-    task_version = _load_task_protocol(session_path, task_collection)
+    task_version = _load_task_version(session_path, task_collection)
     audio = ephys_fpga.get_sync_fronts(sync, sync_map["audio"], tmin=treplay[0], tmax=treplay[1])
     passiveTone_intervals, passiveNoise_intervals = _extract_passiveAudio_intervals(audio, task_version)
 
@@ -588,7 +605,7 @@ def extract_replay_debug(
     passiveValve_intervals = _extract_passiveValve_intervals(bpod)
     plot_valve_times(passiveValve_intervals, ax=ax)
 
-    task_version = _load_task_protocol(session_path, task_collection)
+    task_version = _load_task_version(session_path, task_collection)
     audio = ephys_fpga.get_sync_fronts(sync, sync_map["audio"], tmin=treplay[0])
     passiveTone_intervals, passiveNoise_intervals = _extract_passiveAudio_intervals(audio, task_version)
     plot_audio_times(passiveTone_intervals, passiveNoise_intervals, ax=ax)
@@ -647,13 +664,19 @@ def _extract(self, sync_collection: str = 'raw_ephys_data', task_collection: str
             log.error(f"Failed to extract RFMapping datasets: {e}")
             passiveRFM_times = None
 
-        try:
-            (passiveGabor_df, passiveStims_df,) = extract_task_replay(
-                self.session_path, sync_collection=sync_collection, task_collection=task_collection, sync=sync,
-                sync_map=sync_map, treplay=treplay)
-        except Exception as e:
-            log.error(f"Failed to extract task replay stimuli: {e}")
+        skip_replay = skip_task_replay(self.session_path, task_collection)
+        if not skip_replay:
+            try:
+                (passiveGabor_df, passiveStims_df,) = extract_task_replay(
+                    self.session_path, sync_collection=sync_collection, task_collection=task_collection, sync=sync,
+                    sync_map=sync_map, treplay=treplay)
+            except Exception as e:
+                log.error(f"Failed to extract task replay stimuli: {e}")
+                passiveGabor_df, passiveStims_df = (None, None)
+        else:
+            # If we don't have task replay then we set the treplay intervals to NaN in our passivePeriods_df dataset
             passiveGabor_df, passiveStims_df = (None, None)
+            passivePeriods_df.taskReplay = np.NAN
 
         if plot:
             f, ax = plt.subplots(1, 1)

ibllib/pipes/base_tasks.py

@@ -5,6 +5,8 @@
 from packaging import version
 from one.webclient import no_cache
 from iblutil.util import flatten
+import matplotlib.image
+from skimage.io import ImageCollection, imread
 
 from ibllib.pipes.tasks import Task
 import ibllib.io.session_params as sess_params
@@ -411,6 +413,48 @@ def rename_files(self, symlink_old=False):
             if symlink_old:
                 old_path.symlink_to(new_path)
 
+    @staticmethod
+    def _is_animated_gif(snapshot: Path) -> bool:
+        """
+        Test if image is an animated GIF file.
+
+        Parameters
+        ----------
+        snapshot : pathlib.Path
+            An image filepath to test.
+
+        Returns
+        -------
+        bool
+            True if image is an animated GIF.
+
+        Notes
+        -----
+        This could be achieved more succinctly with `from PIL import Image; Image.open(snapshot).is_animated`,
+        however despite being an indirect dependency, the Pillow library is not in the requirements,
+        whereas skimage is.
+        """
+        return snapshot.suffix == '.gif' and len(ImageCollection(str(snapshot))) > 1
+
+    @staticmethod
+    def _save_as_png(snapshot: Path) -> Path:
+        """
+        Save an image to PNG format.
+
+        Parameters
+        ----------
+        snapshot : pathlib.Path
+            An image filepath to convert.
+
+        Returns
+        -------
+        pathlib.Path
+            The new PNG image filepath.
+        """
+        img = imread(snapshot, as_gray=True)
+        matplotlib.image.imsave(snapshot.with_suffix('.png'), img, cmap='gray')
+        return snapshot.with_suffix('.png')
+
     def register_snapshots(self, unlink=False, collection=None):
         """
         Register any photos in the snapshots folder to the session. Typically imaging users will
@@ -431,14 +475,20 @@ def register_snapshots(self, unlink=False, collection=None):
         -------
         list of dict
             The newly registered Alyx notes.
+
+        Notes
+        -----
+        - Animated GIF files are not resized and therefore may take up significant space on the database.
+        - TIFF files are converted to PNG format before upload. The original file is not replaced.
+        - JPEG and PNG files are resized by Alyx.
         """
         collection = getattr(self, 'device_collection', None) if collection is None else collection
         collection = collection or ''  # If not defined, use no collection
         if collection and '*' in collection:
             collection = [p.name for p in self.session_path.glob(collection)]
             # Check whether folders on disk contain '*'; this is to stop an infinite recursion
             assert not any('*' in c for c in collection), 'folders containing asterisks not supported'
-        # If more that one collection exists, register snapshots in each collection
+        # If more than one collection exists, register snapshots in each collection
         if collection and not isinstance(collection, str):
             return flatten(filter(None, [self.register_snapshots(unlink, c) for c in collection]))
         snapshots_path = self.session_path.joinpath(*filter(None, (collection, 'snapshots')))
@@ -452,14 +502,20 @@ def register_snapshots(self, unlink=False, collection=None):
         note = dict(user=self.one.alyx.user, content_type='session', object_id=eid, text='')
 
         notes = []
-        exts = ('.jpg', '.jpeg', '.png', '.tif', '.tiff')
+        exts = ('.jpg', '.jpeg', '.png', '.tif', '.tiff', '.gif')
         for snapshot in filter(lambda x: x.suffix.lower() in exts, snapshots_path.glob('*.*')):
+            if snapshot.suffix in ('.tif', '.tiff') and not snapshot.with_suffix('.png').exists():
+                _logger.debug('converting "%s" to png...', snapshot.relative_to(self.session_path))
+                snapshot = self._save_as_png(snapshot_tif := snapshot)
+                if unlink:
+                    snapshot_tif.unlink()
             _logger.debug('Uploading "%s"...', snapshot.relative_to(self.session_path))
             if snapshot.with_suffix('.txt').exists():
                 with open(snapshot.with_suffix('.txt'), 'r') as txt_file:
                     note['text'] = txt_file.read().strip()
             else:
                 note['text'] = ''
+            note['width'] = 'orig' if self._is_animated_gif(snapshot) else None
             with open(snapshot, 'rb') as img_file:
                 files = {'image': img_file}
                 notes.append(self.one.alyx.rest('notes', 'create', data=note, files=files))

ibllib/pipes/behavior_tasks.py

@@ -184,17 +184,17 @@ def signature(self):
         signature = {
             'input_files': [],
             'output_files': [('_iblrig_taskSettings.raw.*', self.collection, True),
-                             ('_iblrig_encoderEvents.raw*', self.collection, True),
-                             ('_iblrig_encoderPositions.raw*', self.collection, True),
-                             ('_iblrig_encoderTrialInfo.raw*', self.collection, True),
-                             ('_iblrig_stimPositionScreen.raw*', self.collection, True),
-                             ('_iblrig_syncSquareUpdate.raw*', self.collection, True),
+                             ('_iblrig_encoderEvents.raw*', self.collection, False),
+                             ('_iblrig_encoderPositions.raw*', self.collection, False),
+                             ('_iblrig_encoderTrialInfo.raw*', self.collection, False),
+                             ('_iblrig_stimPositionScreen.raw*', self.collection, False),
+                             ('_iblrig_syncSquareUpdate.raw*', self.collection, False),
                              ('_iblrig_RFMapStim.raw*', self.collection, True)]
         }
         return signature
 
 
-class PassiveTask(base_tasks.BehaviourTask):
+class PassiveTaskNidq(base_tasks.BehaviourTask):
     priority = 90
     job_size = 'small'
 
@@ -208,10 +208,10 @@ def signature(self):
                             (f'_{self.sync_namespace}_sync.times.*', self.sync_collection, True),
                             ('*.wiring.json', self.sync_collection, False),
                             ('*.meta', self.sync_collection, False)],
-            'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, True),
+            'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, False),
                              ('_ibl_passivePeriods.intervalsTable.csv', self.output_collection, True),
                              ('_ibl_passiveRFM.times.npy', self.output_collection, True),
-                             ('_ibl_passiveStims.table.csv', self.output_collection, True)]
+                             ('_ibl_passiveStims.table.csv', self.output_collection, False)]
         }
         return signature
 
@@ -240,10 +240,10 @@ def signature(self):
                             (f'_{self.sync_namespace}_sync.channels.*', self.sync_collection, False),
                             (f'_{self.sync_namespace}_sync.polarities.*', self.sync_collection, False),
                             (f'_{self.sync_namespace}_sync.times.*', self.sync_collection, False)],
-            'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, True),
+            'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, False),
                              ('_ibl_passivePeriods.intervalsTable.csv', self.output_collection, True),
                              ('_ibl_passiveRFM.times.npy', self.output_collection, True),
-                             ('_ibl_passiveStims.table.csv', self.output_collection, True)]
+                             ('_ibl_passiveStims.table.csv', self.output_collection, False)]
         }
         return signature