From 9b1cad81d433b0f7dfa9fab347389172e9421272 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Tue, 21 May 2024 19:18:54 +0300 Subject: [PATCH 1/3] - Flag to allow session registration without water administrations - Support extraction of repNum for advancedChoiceWorld --- ibllib/__init__.py | 2 +- ibllib/io/extractors/training_trials.py | 22 ++++++++++------------ ibllib/oneibl/registration.py | 9 +++++++-- ibllib/tests/extractors/test_extractors.py | 14 +++++++------- release_notes.md | 4 ++++ 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 2218bebc7..b823c2288 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.35.1' +__version__ = '2.35.2' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/extractors/training_trials.py b/ibllib/io/extractors/training_trials.py index 477942fd9..2164ffccd 100644 --- a/ibllib/io/extractors/training_trials.py +++ b/ibllib/io/extractors/training_trials.py @@ -1,5 +1,6 @@ import logging import numpy as np +from itertools import accumulate from packaging import version from one.alf.io import AlfBunch @@ -123,19 +124,16 @@ def _extract(self): def get_trial_repeat(trial): if 'debias_trial' in trial: return trial['debias_trial'] - else: + elif 'contrast' in trial and isinstance(trial['contrast'], dict): return trial['contrast']['type'] == 'RepeatContrast' + else: + # For advanced choice world before version 8.19.0 there was no 'debias_trial' field + # and no debiasing protocol applied, so simply return False + assert self.settings['PYBPOD_PROTOCOL'].startswith('_iblrig_tasks_advancedChoiceWorld') + return False - trial_repeated = np.array(list(map(get_trial_repeat, self.bpod_trials))).astype(int) - repNum = trial_repeated.copy() - c = 0 - for i in range(len(trial_repeated)): - if trial_repeated[i] == 0: - c = 0 - repNum[i] = 0 - continue - c += 1 - repNum[i] = c + trial_repeated = np.fromiter(map(get_trial_repeat, self.bpod_trials), int) + repNum = np.fromiter(accumulate(trial_repeated, lambda x, y: x + y if y else 0), int) return repNum @@ -163,7 +161,7 @@ class FeedbackTimes(BaseBpodTrialsExtractor): **Optional:** saves _ibl_trials.feedback_times.npy Gets reward and error state init times vectors, - checks if theintersection of nans is empty, then + checks if the intersection of nans is empty, then merges the 2 vectors. """ save_names = '_ibl_trials.feedback_times.npy' diff --git a/ibllib/oneibl/registration.py b/ibllib/oneibl/registration.py index 456fa8c2e..849baee8f 100644 --- a/ibllib/oneibl/registration.py +++ b/ibllib/oneibl/registration.py @@ -173,7 +173,7 @@ class IBLRegistrationClient(RegistrationClient): Object that keeps the ONE instance and provides method to create sessions and register data. """ - def register_session(self, ses_path, file_list=True, projects=None, procedures=None): + def register_session(self, ses_path, file_list=True, projects=None, procedures=None, register_reward=True): """ Register an IBL Bpod session in Alyx. @@ -188,11 +188,16 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N The project(s) to which the experiment belongs (optional). procedures : str, list An optional list of procedures, e.g. 'Behavior training/tasks'. + register_reward : bool + If true, register all water administrations in the settings files, if no admins already + present for this session. Returns ------- dict An Alyx session record. + list of dict, None + Alyx file records (or None if file_list is False). Notes ----- @@ -321,7 +326,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N _logger.info(session['url'] + ' ') # create associated water administration if not found - if not session['wateradmin_session_related'] and any(task_data): + if register_reward and not session['wateradmin_session_related'] and any(task_data): for md, d in filter(all, zip(settings, task_data)): _, _end_time = _get_session_times(ses_path, md, d) user = md.get('PYBPOD_CREATOR') diff --git a/ibllib/tests/extractors/test_extractors.py b/ibllib/tests/extractors/test_extractors.py index 437c43c07..04558f96a 100644 --- a/ibllib/tests/extractors/test_extractors.py +++ b/ibllib/tests/extractors/test_extractors.py @@ -207,21 +207,21 @@ def test_get_choice(self): self.assertTrue(all(choice[trial_nogo]) == 0) def test_get_repNum(self): - # TODO: Test its sawtooth # TRAINING SESSIONS rn = training_trials.RepNum( self.training_lt5['path']).extract()[0] self.assertTrue(isinstance(rn, np.ndarray)) - for i in range(3): - self.assertTrue(i in rn) + expected = [0, 1, 2, 0] + np.testing.assert_array_equal(rn, expected) # -- version >= 5.0.0 rn = training_trials.RepNum( self.training_ge5['path']).extract()[0] self.assertTrue(isinstance(rn, np.ndarray)) - for i in range(4): - self.assertTrue(i in rn) - # BIASED SESSIONS have no repeted trials + expected = [0, 0, 1, 2, 3, 0, 0, 0, 1, 2, 0, 1] + np.testing.assert_array_equal(rn, expected) + + # BIASED SESSIONS have no repeated trials def test_get_rewardVolume(self): # TRAINING SESSIONS @@ -243,7 +243,7 @@ def test_get_rewardVolume(self): rv = biased_trials.RewardVolume( self.biased_ge5['path']).extract()[0] self.assertTrue(isinstance(rv, np.ndarray)) - # Test if all non zero rewards are of the same value + # Test if all non-zero rewards are of the same value self.assertTrue(all([x == max(rv) for x in rv if x != 0])) def test_get_feedback_times_ge5(self): diff --git a/release_notes.md b/release_notes.md index fb90ddb7e..de27e6a2a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -10,6 +10,10 @@ #### 2.35.1 - Ensure no REST cache used when searching sessions in IBLRegistationClient +#### 2.35.2 +- Flag to allow session registration without water administrations +- Support extraction of repNum for advancedChoiceWorld + ## Release Note 2.34.0 ### features From 2371521a4fd7567a99e4fe73a92611a9808eba1c Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 22 May 2024 13:30:08 +0300 Subject: [PATCH 2/3] fix failing tests due to dependencies --- brainbox/metrics/single_units.py | 2 +- brainbox/plot_base.py | 2 +- ibllib/io/__init__.py | 1 + ibllib/io/extractors/__init__.py | 4 +++ ibllib/io/extractors/training_wheel.py | 34 ++++++++++++++++++---- ibllib/io/extractors/video_motion.py | 13 +++++---- ibllib/tests/extractors/test_extractors.py | 6 ++-- requirements.txt | 2 +- 8 files changed, 47 insertions(+), 17 deletions(-) diff --git a/brainbox/metrics/single_units.py b/brainbox/metrics/single_units.py index 5e8336651..9ead993a1 100644 --- a/brainbox/metrics/single_units.py +++ b/brainbox/metrics/single_units.py @@ -980,7 +980,7 @@ def quick_unit_metrics(spike_clusters, spike_times, spike_amps, spike_depths, r.amp_median[ir] = np.array(10 ** (camp['log_amps'].median() / 20)) r.amp_std_dB[ir] = np.array(camp['log_amps'].std()) srp = metrics.slidingRP_all(spikeTimes=spike_times, spikeClusters=spike_clusters, - **{'sampleRate': 30000, 'binSizeCorr': 1 / 30000}) + sampleRate=30000, binSizeCorr=1 / 30000) r.slidingRP_viol[ir] = srp['value'] # loop over each cluster to compute the rest of the metrics diff --git a/brainbox/plot_base.py b/brainbox/plot_base.py index e6515287b..5645a0b55 100644 --- a/brainbox/plot_base.py +++ b/brainbox/plot_base.py @@ -640,7 +640,7 @@ def scatter_xyc_plot(x, y, c, cmap=None, clim=None, rgb=False): data.set_clim(clim=clim) if rgb: norm = matplotlib.colors.Normalize(vmin=data.clim[0], vmax=data.clim[1], clip=True) - mapper = cm.ScalarMappable(norm=norm, cmap=cm.get_cmap(cmap)) + mapper = cm.ScalarMappable(norm=norm, cmap=plt.get_cmap(cmap)) cluster_color = np.array([mapper.to_rgba(col) for col in c]) data.set_color(color=cluster_color) diff --git a/ibllib/io/__init__.py b/ibllib/io/__init__.py index e69de29bb..0acbe2d74 100644 --- a/ibllib/io/__init__.py +++ b/ibllib/io/__init__.py @@ -0,0 +1 @@ +"""Loaders for unprocessed IBL video and task data, and parameter files.""" diff --git a/ibllib/io/extractors/__init__.py b/ibllib/io/extractors/__init__.py index e69de29bb..f817c2306 100644 --- a/ibllib/io/extractors/__init__.py +++ b/ibllib/io/extractors/__init__.py @@ -0,0 +1,4 @@ +"""IBL rig data pre-processing functions. + +Extractor classes for loading raw rig data and returning ALF compliant pre-processed data. +""" diff --git a/ibllib/io/extractors/training_wheel.py b/ibllib/io/extractors/training_wheel.py index 1d77d42e2..05bc105e7 100644 --- a/ibllib/io/extractors/training_wheel.py +++ b/ibllib/io/extractors/training_wheel.py @@ -1,3 +1,4 @@ +"""Extractors for the wheel position, velocity, and detected movement.""" import logging from collections.abc import Sized @@ -323,16 +324,35 @@ def extract_wheel_moves(re_ts, re_pos, display=False): def extract_first_movement_times(wheel_moves, trials, min_qt=None): """ Extracts the time of the first sufficiently large wheel movement for each trial. + To be counted, the movement must occur between go cue / stim on and before feedback / response time. The movement onset is sometimes just before the cue (occurring in the gap between quiescence end and cue start, or during the quiescence period but sub- - threshold). The movement is sufficiently large if it is greater than or equal to THRESH - :param wheel_moves: dictionary of detected wheel movement onsets and peak amplitudes for - use in extracting each trial's time of first movement. + threshold). The movement is sufficiently large if it is greater than or equal to THRESH. + + :param wheel_moves: :param trials: dictionary of trial data - :param min_qt: the minimum quiescence period, if None a default is used - :return: numpy array of first movement times, bool array indicating whether movement - crossed response threshold, and array of indices for wheel_moves arrays + :param min_qt: + :return: numpy array of + + Parameters + ---------- + wheel_moves : dict + Dictionary of detected wheel movement onsets and peak amplitudes for use in extracting each + trial's time of first movement. + trials : dict + Dictionary of trial data. + min_qt : float + The minimum quiescence period in seconds, if None a default is used. + + Returns + ------- + numpy.array + First movement times. + numpy.array + Bool array indicating whether movement crossed response threshold. + numpy.array + Indices for wheel_moves arrays. """ THRESH = .1 # peak amp should be at least .1 rad; ~1/3rd of the distance to threshold MIN_QT = .2 # default minimum enforced quiescence period @@ -371,6 +391,8 @@ def extract_first_movement_times(wheel_moves, trials, min_qt=None): class Wheel(BaseBpodTrialsExtractor): """ + Wheel extractor. + Get wheel data from raw files and converts positions into radians mathematical convention (anti-clockwise = +) and timestamps into seconds relative to Bpod clock. **Optional:** saves _ibl_wheel.times.npy and _ibl_wheel.position.npy diff --git a/ibllib/io/extractors/video_motion.py b/ibllib/io/extractors/video_motion.py index 144305125..6ca26863b 100644 --- a/ibllib/io/extractors/video_motion.py +++ b/ibllib/io/extractors/video_motion.py @@ -42,7 +42,7 @@ def find_nearest(array, value): class MotionAlignment: roi = {'left': ((800, 1020), (233, 1096)), 'right': ((426, 510), (104, 545)), 'body': ((402, 481), (31, 103))} - def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs): + def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), stream=False, **kwargs): self.one = one or ONE() self.eid = eid self.session_path = kwargs.pop('session_path', None) or self.one.eid2path(eid) @@ -51,7 +51,10 @@ def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs self.trials = self.wheel = self.camera_times = None raw_cam_path = self.session_path.joinpath('raw_video_data') camera_path = list(raw_cam_path.glob('_iblrig_*Camera.raw.*')) - self.video_paths = {vidio.label_from_path(x): x for x in camera_path} + if stream: + self.video_paths = vidio.url_from_eid(self.eid) + else: + self.video_paths = {vidio.label_from_path(x): x for x in camera_path} self.data = Bunch() self.alignment = Bunch() @@ -107,8 +110,8 @@ def load_data(self, download=False): if download: self.data.wheel = self.one.load_object(self.eid, 'wheel') self.data.trials = self.one.load_object(self.eid, 'trials') - cam = self.one.load(self.eid, ['camera.times'], dclass_output=True) - self.data.camera_times = {vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url)} + cam, det = self.one.load_datasets(self.eid, ['*Camera.times*']) + self.data.camera_times = {vidio.label_from_path(d['rel_path']): ts for ts, d in zip(cam, det)} else: alf_path = self.session_path / 'alf' wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent @@ -329,7 +332,7 @@ def animate(i): data['im'].set_data(frame) mkr = find_nearest(wheel.timestamps[wheel_mask], t_x) - data['marker'].set_data(wheel.timestamps[wheel_mask][mkr], wheel.position[wheel_mask][mkr]) + data['marker'].set_data([wheel.timestamps[wheel_mask][mkr]], [wheel.position[wheel_mask][mkr]]) return data['im'], data['ln'], data['marker'] diff --git a/ibllib/tests/extractors/test_extractors.py b/ibllib/tests/extractors/test_extractors.py index 04558f96a..92de8afb1 100644 --- a/ibllib/tests/extractors/test_extractors.py +++ b/ibllib/tests/extractors/test_extractors.py @@ -237,14 +237,14 @@ def test_get_rewardVolume(self): rv = biased_trials.RewardVolume( self.biased_lt5['path']).extract()[0] self.assertTrue(isinstance(rv, np.ndarray)) - # Test if all non zero rewards are of the same value - self.assertTrue(all([x == max(rv) for x in rv if x != 0])) + # Test if all non-zero rewards are of the same value + self.assertTrue(all(x == max(rv) for x in rv if x != 0)) # -- version >= 5.0.0 rv = biased_trials.RewardVolume( self.biased_ge5['path']).extract()[0] self.assertTrue(isinstance(rv, np.ndarray)) # Test if all non-zero rewards are of the same value - self.assertTrue(all([x == max(rv) for x in rv if x != 0])) + self.assertTrue(all(x == max(rv) for x in rv if x != 0)) def test_get_feedback_times_ge5(self): # TRAINING SESSIONS diff --git a/requirements.txt b/requirements.txt index 29094696f..955ba650c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,5 @@ mtscomp>=1.0.1 ONE-api~=2.7rc1 phylib>=2.4 psychofit -slidingRP>=1.0.0 # steinmetz lab refractory period metrics +slidingRP>=1.1.1 # steinmetz lab refractory period metrics pyqt5 From 863ba6246e1a55ec123a50b7f74317f582a8c94e Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 22 May 2024 14:00:15 +0300 Subject: [PATCH 3/3] Update release notes --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index de27e6a2a..db09f5e07 100644 --- a/release_notes.md +++ b/release_notes.md @@ -13,6 +13,7 @@ #### 2.35.2 - Flag to allow session registration without water administrations - Support extraction of repNum for advancedChoiceWorld +- Support matplotlib v3.9; min slidingRP version now 1.1.1 ## Release Note 2.34.0