diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 248d75039..bda80755e 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.32.2' +__version__ = '2.32.3' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/extractors/base.py b/ibllib/io/extractors/base.py index 1b3717a89..a41a15401 100644 --- a/ibllib/io/extractors/base.py +++ b/ibllib/io/extractors/base.py @@ -161,6 +161,9 @@ def extract(self, bpod_trials=None, settings=None, **kwargs): self.settings = {"IBLRIG_VERSION": "100.0.0"} elif self.settings.get("IBLRIG_VERSION", "") == "": self.settings["IBLRIG_VERSION"] = "100.0.0" + # Get all detected TTLs. These are stored for QC purposes + self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials) + return super(BaseBpodTrialsExtractor, self).extract(**kwargs) @property diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index e44239117..d8d803535 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -46,6 +46,7 @@ import spikeglx import ibldsp.utils import one.alf.io as alfio +from one.alf.files import filename_parts from iblutil.util import Bunch from iblutil.spacer import Spacer @@ -816,6 +817,34 @@ def _extract(self, sync=None, chmap=None, sync_collection='raw_ephys_data', assert self.var_names == tuple(out.keys()) return out + def _is_trials_object_attribute(self, var_name, variable_length_vars=None): + """ + Check if variable name is expected to have the same length as trials.intervals. + + Parameters + ---------- + var_name : str + The variable name to check. + variable_length_vars : list + Set of variable names that are not expected to have the same length as trials.intervals. + This list may be passed by superclasses. + + Returns + ------- + bool + True if variable is a trials dataset. + + Examples + -------- + >>> assert self._is_trials_object_attribute('stimOnTrigger_times') is True + >>> assert self._is_trials_object_attribute('wheel_position') is False + """ + save_name = self.save_names[self.var_names.index(var_name)] if var_name in self.var_names else None + if save_name: + return filename_parts(save_name)[1] == 'trials' + else: + return var_name not in (variable_length_vars or []) + def build_trials(self, sync, chmap, display=False, **kwargs): """ Extract task related event times from the sync. @@ -914,7 +943,10 @@ def build_trials(self, sync, chmap, display=False, **kwargs): # Add the Bpod trial events, converting the timestamp fields to FPGA time. # NB: The trial intervals are by default a Bpod rsync field. out.update({k: self.bpod_trials[k][ibpod] for k in self.bpod_fields}) - out.update({k: self.bpod2fpga(self.bpod_trials[k][ibpod]) for k in self.bpod_rsync_fields}) + for k in self.bpod_rsync_fields: + # Some personal projects may extract non-trials object datasets that may not have 1 event per trial + idx = ibpod if self._is_trials_object_attribute(k) else np.arange(len(self.bpod_trials[k]), dtype=int) + out[k] = self.bpod2fpga(self.bpod_trials[k][idx]) out.update({k: fpga_trials[k][ifpga] for k in fpga_trials.keys()}) if display: # pragma: no cover diff --git a/ibllib/io/raw_data_loaders.py b/ibllib/io/raw_data_loaders.py index ca9a83cca..36bed4abe 100644 --- a/ibllib/io/raw_data_loaders.py +++ b/ibllib/io/raw_data_loaders.py @@ -926,6 +926,23 @@ def patch_settings(session_path, collection='raw_behavior_data', ------- dict The modified settings. + + Examples + -------- + File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location. + >>> patch_settings('/data/subject/2020-01-01/002', number='001') + >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/') + + File is moved into new collection within the same session, then patched. + >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/') + >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01') + + Update subject, date and number. + >>> new_session_path = Path('/data/foobar/2024-02-24/002') + >>> old_session_path = Path('/data/baz/2024-02-23/001') + >>> patch_settings(old_session_path, collection='raw_task_data_00', + ... subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1]) + >>> shutil.move(old_session_path, new_session_path) """ settings = load_settings(session_path, collection) if not settings: diff --git a/ibllib/pipes/behavior_tasks.py b/ibllib/pipes/behavior_tasks.py index 001f3bfed..faa3c4423 100644 --- a/ibllib/pipes/behavior_tasks.py +++ b/ibllib/pipes/behavior_tasks.py @@ -311,6 +311,8 @@ def _run(self, update=True, save=True): def extract_behaviour(self, **kwargs): self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection) + _logger.info('Bpod trials extractor: %s.%s', + self.extractor.__module__, self.extractor.__class__.__name__) self.extractor.default_path = self.output_collection return self.extractor.extract(task_collection=self.collection, **kwargs) @@ -453,12 +455,11 @@ def run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None): if plot_qc: _logger.info('Creating Trials QC plots') try: - # TODO needs to be adapted for chained protocols session_id = self.one.path2eid(self.session_path) - plot_task = BehaviourPlots(session_id, self.session_path, one=self.one) + plot_task = BehaviourPlots( + session_id, self.session_path, one=self.one, task_collection=self.output_collection) _ = plot_task.run() self.plot_tasks.append(plot_task) - except Exception: _logger.error('Could not create Trials QC Plot') _logger.error(traceback.format_exc()) diff --git a/ibllib/plots/figures.py b/ibllib/plots/figures.py index 369709db1..e1977beb9 100644 --- a/ibllib/plots/figures.py +++ b/ibllib/plots/figures.py @@ -71,35 +71,58 @@ def remove_axis_outline(ax): class BehaviourPlots(ReportSnapshot): - """ - Behavioural plots - """ - - signature = { - 'input_files': [ - ('*trials.table.pqt', 'alf', True), - ], - 'output_files': [ - ('psychometric_curve.png', 'snapshot/behaviour', True), - ('chronometric_curve.png', 'snapshot/behaviour', True), - ('reaction_time_with_trials.png', 'snapshot/behaviour', True) - ] - } + """Behavioural plots.""" + + @property + def signature(self): + signature = { + 'input_files': [ + ('*trials.table.pqt', self.trials_collection, True), + ], + 'output_files': [ + ('psychometric_curve.png', 'snapshot/behaviour', True), + ('chronometric_curve.png', 'snapshot/behaviour', True), + ('reaction_time_with_trials.png', 'snapshot/behaviour', True) + ] + } + return signature def __init__(self, eid, session_path=None, one=None, **kwargs): + """ + Generate and upload behaviour plots. + + Parameters + ---------- + eid : str, uuid.UUID + An experiment UUID. + session_path : pathlib.Path + A session path. + one : one.api.One + An instance of ONE for registration to Alyx. + trials_collection : str + The location of the trials data (default: 'alf'). + kwargs + Arguments for ReportSnapshot constructor. + """ self.one = one self.eid = eid self.session_path = session_path or self.one.eid2path(self.eid) + self.trials_collection = kwargs.pop('trials_collection', 'alf') super(BehaviourPlots, self).__init__(self.session_path, self.eid, one=self.one, **kwargs) - self.output_directory = self.session_path.joinpath('snapshot', 'behaviour') + # Output directory should mirror trials collection, sans 'alf' part + self.output_directory = self.session_path.joinpath( + 'snapshot', 'behaviour', self.trials_collection.removeprefix('alf').strip('/')) self.output_directory.mkdir(exist_ok=True, parents=True) def _run(self): output_files = [] - trials = alfio.load_object(self.session_path.joinpath('alf'), 'trials') - title = '_'.join(list(self.session_path.parts[-3:])) + trials = alfio.load_object(self.session_path.joinpath(self.trials_collection), 'trials') + if self.one: + title = self.one.path2ref(self.session_path, as_dict=False) + else: + title = '_'.join(list(self.session_path.parts[-3:])) fig, ax = training.plot_psychometric(trials, title=title, figsize=(8, 6)) set_axis_label_size(ax) @@ -127,9 +150,7 @@ def _run(self): # TODO put into histology and alignment pipeline class HistologySlices(ReportSnapshotProbe): - """ - Plots coronal and sagittal slice showing electrode locations - """ + """Plots coronal and sagittal slice showing electrode locations.""" def _run(self): diff --git a/ibllib/tests/extractors/test_ephys_fpga.py b/ibllib/tests/extractors/test_ephys_fpga.py index a5d4ef254..006a4f85f 100644 --- a/ibllib/tests/extractors/test_ephys_fpga.py +++ b/ibllib/tests/extractors/test_ephys_fpga.py @@ -524,6 +524,23 @@ def test_time_fields(self): fields = ephys_fpga.FpgaTrials._time_fields(expected + ('position', 'timebase', 'fooBaz')) self.assertCountEqual(expected, fields) + def test_is_trials_object_attribute(self): + """Test for FpgaTrials._is_trials_object_attribute method.""" + extractor = ephys_fpga.FpgaTrials('subject/2020-01-01/001') + # Should assume this is a trials attribute if no save name defined + self.assertTrue(extractor._is_trials_object_attribute('stimOnTrigger_times')) + # Save name not trials attribute + self.assertFalse(extractor._is_trials_object_attribute('wheel_position')) + # Save name is trials attribute + self.assertTrue(extractor._is_trials_object_attribute('table')) + # Check with toy variables + extractor.var_names += ('foo_bar',) + extractor.save_names += (None,) + self.assertTrue(extractor._is_trials_object_attribute('foo_bar')) + self.assertFalse(extractor._is_trials_object_attribute('foo_bar', variable_length_vars='foo_bar')) + extractor.save_names = extractor.save_names[:-1] + ('_ibl_foo.bar_times.csv',) + self.assertFalse(extractor._is_trials_object_attribute('foo_bar')) + if __name__ == '__main__': unittest.main(exit=False, verbosity=2) diff --git a/release_notes.md b/release_notes.md index 72539b790..b04af15fa 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,16 +1,20 @@ ## Release Notes 2.32 ## features -- SDSC patcher automatically support revisons +- SDSC patcher automatically support revisions -## others +## other - Add extra key to alignment qc with manual resolution for channel upload -- + +#### 2.32.3 +- FpgaTrials supports alignment of Bpod datasets not part of trials object +- Support chained protocols in BehaviourPlots task + ## Release Notes 2.31 ### features -- training status uses new extractor map -- refactor neurodsp to ibldsp +- Training status uses new extractor map +- Refactor neurodsp to ibldsp - ITI qc check for iblrig v8 - Support habituationChoiceWorld extraction in iblrig v8.15.0