Merge branch 'release/2.32.3'

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.32.2'
+__version__ = '2.32.3'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/extractors/base.py

@@ -161,6 +161,9 @@ def extract(self, bpod_trials=None, settings=None, **kwargs):
             self.settings = {"IBLRIG_VERSION": "100.0.0"}
         elif self.settings.get("IBLRIG_VERSION", "") == "":
             self.settings["IBLRIG_VERSION"] = "100.0.0"
+        # Get all detected TTLs. These are stored for QC purposes
+        self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials)
+
         return super(BaseBpodTrialsExtractor, self).extract(**kwargs)
 
     @property

ibllib/io/extractors/ephys_fpga.py

@@ -46,6 +46,7 @@
 import spikeglx
 import ibldsp.utils
 import one.alf.io as alfio
+from one.alf.files import filename_parts
 from iblutil.util import Bunch
 from iblutil.spacer import Spacer
 
@@ -816,6 +817,34 @@ def _extract(self, sync=None, chmap=None, sync_collection='raw_ephys_data',
         assert self.var_names == tuple(out.keys())
         return out
 
+    def _is_trials_object_attribute(self, var_name, variable_length_vars=None):
+        """
+        Check if variable name is expected to have the same length as trials.intervals.
+
+        Parameters
+        ----------
+        var_name : str
+            The variable name to check.
+        variable_length_vars : list
+            Set of variable names that are not expected to have the same length as trials.intervals.
+            This list may be passed by superclasses.
+
+        Returns
+        -------
+        bool
+            True if variable is a trials dataset.
+
+        Examples
+        --------
+        >>> assert self._is_trials_object_attribute('stimOnTrigger_times') is True
+        >>> assert self._is_trials_object_attribute('wheel_position') is False
+        """
+        save_name = self.save_names[self.var_names.index(var_name)] if var_name in self.var_names else None
+        if save_name:
+            return filename_parts(save_name)[1] == 'trials'
+        else:
+            return var_name not in (variable_length_vars or [])
+
     def build_trials(self, sync, chmap, display=False, **kwargs):
         """
         Extract task related event times from the sync.
@@ -914,7 +943,10 @@ def build_trials(self, sync, chmap, display=False, **kwargs):
         # Add the Bpod trial events, converting the timestamp fields to FPGA time.
         # NB: The trial intervals are by default a Bpod rsync field.
         out.update({k: self.bpod_trials[k][ibpod] for k in self.bpod_fields})
-        out.update({k: self.bpod2fpga(self.bpod_trials[k][ibpod]) for k in self.bpod_rsync_fields})
+        for k in self.bpod_rsync_fields:
+            # Some personal projects may extract non-trials object datasets that may not have 1 event per trial
+            idx = ibpod if self._is_trials_object_attribute(k) else np.arange(len(self.bpod_trials[k]), dtype=int)
+            out[k] = self.bpod2fpga(self.bpod_trials[k][idx])
         out.update({k: fpga_trials[k][ifpga] for k in fpga_trials.keys()})
 
         if display:  # pragma: no cover

ibllib/io/raw_data_loaders.py

@@ -926,6 +926,23 @@ def patch_settings(session_path, collection='raw_behavior_data',
     -------
     dict
         The modified settings.
+
+    Examples
+    --------
+    File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location.
+    >>> patch_settings('/data/subject/2020-01-01/002', number='001')
+    >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/')
+
+    File is moved into new collection within the same session, then patched.
+    >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/')
+    >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01')
+
+    Update subject, date and number.
+    >>> new_session_path = Path('/data/foobar/2024-02-24/002')
+    >>> old_session_path = Path('/data/baz/2024-02-23/001')
+    >>> patch_settings(old_session_path, collection='raw_task_data_00',
+    ...     subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1])
+    >>> shutil.move(old_session_path, new_session_path)
     """
     settings = load_settings(session_path, collection)
     if not settings:

ibllib/pipes/behavior_tasks.py

@@ -311,6 +311,8 @@ def _run(self, update=True, save=True):
 
     def extract_behaviour(self, **kwargs):
         self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection)
+        _logger.info('Bpod trials extractor: %s.%s',
+                     self.extractor.__module__, self.extractor.__class__.__name__)
         self.extractor.default_path = self.output_collection
         return self.extractor.extract(task_collection=self.collection, **kwargs)
 
@@ -453,12 +455,11 @@ def run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None):
         if plot_qc:
             _logger.info('Creating Trials QC plots')
             try:
-                # TODO needs to be adapted for chained protocols
                 session_id = self.one.path2eid(self.session_path)
-                plot_task = BehaviourPlots(session_id, self.session_path, one=self.one)
+                plot_task = BehaviourPlots(
+                    session_id, self.session_path, one=self.one, task_collection=self.output_collection)
                 _ = plot_task.run()
                 self.plot_tasks.append(plot_task)
-
             except Exception:
                 _logger.error('Could not create Trials QC Plot')
                 _logger.error(traceback.format_exc())

ibllib/plots/figures.py

@@ -71,35 +71,58 @@ def remove_axis_outline(ax):
 
 
 class BehaviourPlots(ReportSnapshot):
-    """
-    Behavioural plots
-    """
-
-    signature = {
-        'input_files': [
-            ('*trials.table.pqt', 'alf', True),
-        ],
-        'output_files': [
-            ('psychometric_curve.png', 'snapshot/behaviour', True),
-            ('chronometric_curve.png', 'snapshot/behaviour', True),
-            ('reaction_time_with_trials.png', 'snapshot/behaviour', True)
-        ]
-    }
+    """Behavioural plots."""
+
+    @property
+    def signature(self):
+        signature = {
+            'input_files': [
+                ('*trials.table.pqt', self.trials_collection, True),
+            ],
+            'output_files': [
+                ('psychometric_curve.png', 'snapshot/behaviour', True),
+                ('chronometric_curve.png', 'snapshot/behaviour', True),
+                ('reaction_time_with_trials.png', 'snapshot/behaviour', True)
+            ]
+        }
+        return signature
 
     def __init__(self, eid, session_path=None, one=None, **kwargs):
+        """
+        Generate and upload behaviour plots.
+
+        Parameters
+        ----------
+        eid : str, uuid.UUID
+            An experiment UUID.
+        session_path : pathlib.Path
+            A session path.
+        one : one.api.One
+            An instance of ONE for registration to Alyx.
+        trials_collection : str
+            The location of the trials data (default: 'alf').
+        kwargs
+            Arguments for ReportSnapshot constructor.
+        """
         self.one = one
         self.eid = eid
         self.session_path = session_path or self.one.eid2path(self.eid)
+        self.trials_collection = kwargs.pop('trials_collection', 'alf')
         super(BehaviourPlots, self).__init__(self.session_path, self.eid, one=self.one,
                                              **kwargs)
-        self.output_directory = self.session_path.joinpath('snapshot', 'behaviour')
+        # Output directory should mirror trials collection, sans 'alf' part
+        self.output_directory = self.session_path.joinpath(
+            'snapshot', 'behaviour', self.trials_collection.removeprefix('alf').strip('/'))
         self.output_directory.mkdir(exist_ok=True, parents=True)
 
     def _run(self):
 
         output_files = []
-        trials = alfio.load_object(self.session_path.joinpath('alf'), 'trials')
-        title = '_'.join(list(self.session_path.parts[-3:]))
+        trials = alfio.load_object(self.session_path.joinpath(self.trials_collection), 'trials')
+        if self.one:
+            title = self.one.path2ref(self.session_path, as_dict=False)
+        else:
+            title = '_'.join(list(self.session_path.parts[-3:]))
 
         fig, ax = training.plot_psychometric(trials, title=title, figsize=(8, 6))
         set_axis_label_size(ax)
@@ -127,9 +150,7 @@ def _run(self):
 
 # TODO put into histology and alignment pipeline
 class HistologySlices(ReportSnapshotProbe):
-    """
-    Plots coronal and sagittal slice showing electrode locations
-    """
+    """Plots coronal and sagittal slice showing electrode locations."""
 
     def _run(self):
 

ibllib/tests/extractors/test_ephys_fpga.py

@@ -524,6 +524,23 @@ def test_time_fields(self):
         fields = ephys_fpga.FpgaTrials._time_fields(expected + ('position', 'timebase', 'fooBaz'))
         self.assertCountEqual(expected, fields)
 
+    def test_is_trials_object_attribute(self):
+        """Test for FpgaTrials._is_trials_object_attribute method."""
+        extractor = ephys_fpga.FpgaTrials('subject/2020-01-01/001')
+        # Should assume this is a trials attribute if no save name defined
+        self.assertTrue(extractor._is_trials_object_attribute('stimOnTrigger_times'))
+        # Save name not trials attribute
+        self.assertFalse(extractor._is_trials_object_attribute('wheel_position'))
+        # Save name is trials attribute
+        self.assertTrue(extractor._is_trials_object_attribute('table'))
+        # Check with toy variables
+        extractor.var_names += ('foo_bar',)
+        extractor.save_names += (None,)
+        self.assertTrue(extractor._is_trials_object_attribute('foo_bar'))
+        self.assertFalse(extractor._is_trials_object_attribute('foo_bar', variable_length_vars='foo_bar'))
+        extractor.save_names = extractor.save_names[:-1] + ('_ibl_foo.bar_times.csv',)
+        self.assertFalse(extractor._is_trials_object_attribute('foo_bar'))
+
 
 if __name__ == '__main__':
     unittest.main(exit=False, verbosity=2)

release_notes.md

@@ -1,16 +1,20 @@
 ## Release Notes 2.32
 
 ## features
-- SDSC patcher automatically support revisons
+- SDSC patcher automatically support revisions
 
-## others
+## other
 - Add extra key to alignment qc with manual resolution for channel upload
-- 
+
+#### 2.32.3
+- FpgaTrials supports alignment of Bpod datasets not part of trials object
+- Support chained protocols in BehaviourPlots task
+
 ## Release Notes 2.31
 
 ### features
-- training status uses new extractor map
-- refactor neurodsp to ibldsp
+- Training status uses new extractor map
+- Refactor neurodsp to ibldsp
 - ITI qc check for iblrig v8
 - Support habituationChoiceWorld extraction in iblrig v8.15.0