diff --git a/brainbox/behavior/dlc.py b/brainbox/behavior/dlc.py index c7c42be92..bf85302e0 100644 --- a/brainbox/behavior/dlc.py +++ b/brainbox/behavior/dlc.py @@ -9,7 +9,7 @@ import scipy.interpolate as interpolate from scipy.stats import zscore -from neurodsp.smooth import smooth_interpolate_savgol +from ibldsp.smooth import smooth_interpolate_savgol from iblutil.numerical import bincount2D import brainbox.behavior.wheel as bbox_wheel diff --git a/brainbox/metrics/electrode_drift.py b/brainbox/metrics/electrode_drift.py index 3b2ac3bc1..f82ae9e55 100644 --- a/brainbox/metrics/electrode_drift.py +++ b/brainbox/metrics/electrode_drift.py @@ -1,6 +1,6 @@ import numpy as np -from neurodsp import smooth, utils, fourier +from ibldsp import smooth, utils, fourier from iblutil.numerical import bincount2D diff --git a/brainbox/metrics/single_units.py b/brainbox/metrics/single_units.py index 12a8f7593..5e8336651 100644 --- a/brainbox/metrics/single_units.py +++ b/brainbox/metrics/single_units.py @@ -981,7 +981,7 @@ def quick_unit_metrics(spike_clusters, spike_times, spike_amps, spike_depths, r.amp_std_dB[ir] = np.array(camp['log_amps'].std()) srp = metrics.slidingRP_all(spikeTimes=spike_times, spikeClusters=spike_clusters, **{'sampleRate': 30000, 'binSizeCorr': 1 / 30000}) - r.slidingRP_viol[srp['cidx']] = srp['value'] + r.slidingRP_viol[ir] = srp['value'] # loop over each cluster to compute the rest of the metrics for ic in np.arange(nclust): diff --git a/examples/archive/ibllib/synchronisation_ephys.py b/examples/archive/ibllib/synchronisation_ephys.py index b3580477f..e2ab8e4fb 100644 --- a/examples/archive/ibllib/synchronisation_ephys.py +++ b/examples/archive/ibllib/synchronisation_ephys.py @@ -1,4 +1,4 @@ -import neurodsp.utils +import ibldsp.utils import spikeglx import ibllib.io.extractors.ephys_fpga @@ -15,7 +15,7 @@ # if the data is needed as well, loop over the file # raw data contains raw ephys traces, while raw_sync contains the 16 sync traces -wg = neurodsp.utils.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1) +wg = ibldsp.utils.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1) for first, last in wg.firstlast: rawdata, rawsync = sr.read_samples(first, last) wg.print_progress() diff --git a/examples/loading_data/loading_raw_ephys_data.ipynb b/examples/loading_data/loading_raw_ephys_data.ipynb index 572c8de12..979e01c4c 100644 --- a/examples/loading_data/loading_raw_ephys_data.ipynb +++ b/examples/loading_data/loading_raw_ephys_data.ipynb @@ -236,7 +236,7 @@ "metadata": {}, "outputs": [], "source": [ - "from neurodsp.voltage import destripe\n", + "from ibldsp.voltage import destripe\n", "# Reminder : If not done before, remove first the sync channel from raw data\n", "# Apply destriping algorithm to data\n", "destriped = destripe(raw_ap, fs=sr_ap.fs)" @@ -445,7 +445,7 @@ "source": [ "## Useful modules\n", "* [ibllib.io.spikeglx](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/spikeglx.html)\n", - "* [ibllib.voltage.dsp](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/neurodsp.voltage.html)\n", + "* [ibllib.voltage.dsp](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/ibldsp.voltage.html)\n", "* [brainbox.io.spikeglx.stream](https://int-brain-lab.github.io/iblenv/_autosummary/brainbox.io.spikeglx.html#brainbox.io.spikeglx.stream)\n", "* [viewephys](https://github.com/oliche/viewephys) to visualise raw data snippets (Note: this package is not within `ibllib` but standalone)" ] diff --git a/examples/one/histology/coverage_map.py b/examples/one/histology/coverage_map.py index 31d60aa74..71ecfb213 100644 --- a/examples/one/histology/coverage_map.py +++ b/examples/one/histology/coverage_map.py @@ -1,7 +1,7 @@ import matplotlib.pyplot as plt import numpy as np from one.api import ONE -from neurodsp.utils import fcn_cosine +from ibldsp.utils import fcn_cosine import iblatlas.atlas as atlas from ibllib.pipes.histology import coverage diff --git a/ibllib/__init__.py b/ibllib/__init__.py index e736602e6..f4a1a12b6 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.30.0' +__version__ = '2.31.0' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/ephys/ephysqc.py b/ibllib/ephys/ephysqc.py index 6b8607ce9..65ec99f7e 100644 --- a/ibllib/ephys/ephysqc.py +++ b/ibllib/ephys/ephysqc.py @@ -11,7 +11,7 @@ from iblutil.util import Bunch import spikeglx import neuropixel -from neurodsp import fourier, utils, voltage +from ibldsp import fourier, utils, voltage from tqdm import tqdm from brainbox.io.spikeglx import Streamer diff --git a/ibllib/ephys/spikes.py b/ibllib/ephys/spikes.py index b0ee9295f..f81e2e503 100644 --- a/ibllib/ephys/spikes.py +++ b/ibllib/ephys/spikes.py @@ -142,7 +142,7 @@ def ks2_to_alf(ks_path, bin_path, out_path, bin_file=None, ampfactor=1, label=No """ m = ephysqc.phy_model_from_ks2_path(ks2_path=ks_path, bin_path=bin_path, bin_file=bin_file) ac = phylib.io.alf.EphysAlfCreator(m) - ac.convert(out_path, label=label, force=force, ampfactor=ampfactor) + ac.convert(out_path, label=label, force=force, ampfactor=float(ampfactor)) def ks2_to_tar(ks_path, out_path, force=False): diff --git a/ibllib/ephys/sync_probes.py b/ibllib/ephys/sync_probes.py index c02926fb6..3f3411479 100644 --- a/ibllib/ephys/sync_probes.py +++ b/ibllib/ephys/sync_probes.py @@ -199,7 +199,7 @@ def sync_probe_front_times(t, tref, sr, display=False, type='smooth', tol=2.0): to the sampling rate of digital channels. The residual is fit using frequency domain smoothing """ - import neurodsp.fourier + import ibldsp.fourier CAMERA_UPSAMPLING_RATE_HZ = 300 PAD_LENGTH_SECS = 60 STAT_LENGTH_SECS = 30 # median length to compute padding value @@ -214,7 +214,7 @@ def sync_probe_front_times(t, tref, sr, display=False, type='smooth', tol=2.0): res_filt = np.pad(res_upsamp, lpad, mode='median', stat_length=CAMERA_UPSAMPLING_RATE_HZ * STAT_LENGTH_SECS) fbounds = [0.001, 0.002] - res_filt = neurodsp.fourier.lp(res_filt, 1 / CAMERA_UPSAMPLING_RATE_HZ, fbounds)[lpad[0]:-lpad[1]] + res_filt = ibldsp.fourier.lp(res_filt, 1 / CAMERA_UPSAMPLING_RATE_HZ, fbounds)[lpad[0]:-lpad[1]] tout = np.arange(0, np.max(tref) + SYNC_SAMPLING_RATE_SECS, 20) sync_points = np.c_[tout, np.polyval(pol, tout) + np.interp(tout, t_upsamp, res_filt)] if display: diff --git a/ibllib/io/extractors/camera.py b/ibllib/io/extractors/camera.py index a44010821..5a2786b49 100644 --- a/ibllib/io/extractors/camera.py +++ b/ibllib/io/extractors/camera.py @@ -10,7 +10,7 @@ import matplotlib.pyplot as plt from iblutil.util import range_str -import neurodsp.utils as dsp +import ibldsp.utils as dsp from ibllib.plots import squares, vertical_lines from ibllib.io.video import assert_valid_label, VideoStreamer from iblutil.numerical import within_ranges diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index 009f68c52..e44239117 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -44,7 +44,7 @@ from packaging import version import spikeglx -import neurodsp.utils +import ibldsp.utils import one.alf.io as alfio from iblutil.util import Bunch from iblutil.spacer import Spacer @@ -160,11 +160,11 @@ def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False, parts=''): file_ftcp = Path(output_path).joinpath(f'fronts_times_channel_polarity{uuid.uuid4()}.bin') # loop over chunks of the raw ephys file - wg = neurodsp.utils.WindowGenerator(sr.ns, int(SYNC_BATCH_SIZE_SECS * sr.fs), overlap=1) + wg = ibldsp.utils.WindowGenerator(sr.ns, int(SYNC_BATCH_SIZE_SECS * sr.fs), overlap=1) fid_ftcp = open(file_ftcp, 'wb') for sl in wg.slice: ss = sr.read_sync(sl) - ind, fronts = neurodsp.utils.fronts(ss, axis=0) + ind, fronts = ibldsp.utils.fronts(ss, axis=0) # a = sr.read_sync_analog(sl) sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :], fronts.astype(np.double)] sav.tofile(fid_ftcp) @@ -775,7 +775,7 @@ def _extract(self, sync=None, chmap=None, sync_collection='raw_ephys_data', bpod_start = self.bpod_trials['intervals'][:, 0] if len(t_trial_start) > len(bpod_start) / 2: # if least half the trial start TTLs detected _logger.warning('Attempting to get protocol period from aligning trial start TTLs') - fcn, *_ = neurodsp.utils.sync_timestamps(bpod_start, t_trial_start) + fcn, *_ = ibldsp.utils.sync_timestamps(bpod_start, t_trial_start) buffer = 2.5 # the number of seconds to include before/after task start, end = fcn(self.bpod_trials['intervals'].flat[[0, -1]]) tmin = min(sync['times'][0], start - buffer) @@ -1202,7 +1202,7 @@ def sync_bpod_clock(bpod_trials, fpga_trials, sync_field): bpod_fpga_timestamps[i] = trials[sync_field] # Sync the two timestamps - fcn, drift, ibpod, ifpga = neurodsp.utils.sync_timestamps(*bpod_fpga_timestamps, return_indices=True) + fcn, drift, ibpod, ifpga = ibldsp.utils.sync_timestamps(*bpod_fpga_timestamps, return_indices=True) # If it's drifting too much throw warning or error _logger.info('N trials: %i bpod, %i FPGA, %i merged, sync %.5f ppm', diff --git a/ibllib/io/extractors/fibrephotometry.py b/ibllib/io/extractors/fibrephotometry.py index e9cb60321..d11a5856e 100644 --- a/ibllib/io/extractors/fibrephotometry.py +++ b/ibllib/io/extractors/fibrephotometry.py @@ -25,7 +25,7 @@ from ibllib.io.extractors.base import BaseExtractor from ibllib.io.raw_daq_loaders import load_channels_tdms, load_raw_daq_tdms from ibllib.io.extractors.training_trials import GoCueTriggerTimes -from neurodsp.utils import rises, sync_timestamps +from ibldsp.utils import rises, sync_timestamps _logger = logging.getLogger(__name__) diff --git a/ibllib/io/extractors/habituation_trials.py b/ibllib/io/extractors/habituation_trials.py index 655ea2de1..8d5a34a5e 100644 --- a/ibllib/io/extractors/habituation_trials.py +++ b/ibllib/io/extractors/habituation_trials.py @@ -2,6 +2,8 @@ import logging import numpy as np +from packaging import version + import ibllib.io.raw_data_loaders as raw from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes from ibllib.io.extractors.biased_trials import ContrastLR @@ -26,10 +28,13 @@ def _extract(self) -> dict: """ Extract the Bpod trial events. - The Bpod state machine for this task has extremely misleading names! The 'iti' state is - actually the delay between valve open and trial end (the stimulus is still present during - this period), and the 'trial_start' state is actually the ITI during which there is a 1s - Bpod TTL and gray screen period. + For iblrig versions < 8.13 the Bpod state machine for this task had extremely misleading names! + The 'iti' state was actually the delay between valve close and trial end (the stimulus is + still present during this period), and the 'trial_start' state is actually the ITI during + which there is a 1s Bpod TTL and gray screen period. + + In version 8.13 and later, the 'iti' state was renamed to 'post_reward' and 'trial_start' + was renamed to 'iti'. Returns ------- @@ -57,12 +62,17 @@ def _extract(self) -> dict: bpod_trials=self.bpod_trials, settings=self.settings, task_collection=self.task_collection) """ - The 'trial_start' state is in fact the 1s grey screen period, therefore the first timestamp - is really the end of the previous trial and also the stimOff trigger time. The second - timestamp is the true trial start time. + The 'trial_start'/'iti' state is in fact the 1s grey screen period, therefore the first + timestamp is really the end of the previous trial and also the stimOff trigger time. The + second timestamp is the true trial start time. This state was renamed in version 8.13. """ + state_names = self.bpod_trials[0]['behavior_data']['States timestamps'].keys() + rig_version = version.parse(self.settings['IBLRIG_VERSION']) + legacy_state_machine = 'post_reward' not in state_names and 'trial_start' in state_names + + key = 'iti' if (rig_version >= version.parse('8.13') and not legacy_state_machine) else 'trial_start' (_, *ends), starts = zip(*[ - t['behavior_data']['States timestamps']['trial_start'][-1] for t in self.bpod_trials] + t['behavior_data']['States timestamps'][key][-1] for t in self.bpod_trials] ) # StimOffTrigger times diff --git a/ibllib/io/extractors/mesoscope.py b/ibllib/io/extractors/mesoscope.py index 20b349eb0..ade22240b 100644 --- a/ibllib/io/extractors/mesoscope.py +++ b/ibllib/io/extractors/mesoscope.py @@ -105,7 +105,7 @@ class TimelineTrials(FpgaTrials): timeline = None """one.alf.io.AlfBunch: The timeline data object.""" - sync_field = 'itiIn_times' # trial start events + sync_field = 'itiIn_times' """str: The trial event to synchronize (must be present in extracted trials).""" def __init__(self, *args, sync_collection='raw_sync_data', **kwargs): diff --git a/ibllib/io/extractors/training_audio.py b/ibllib/io/extractors/training_audio.py index 8d14fb00d..a3c57ac15 100644 --- a/ibllib/io/extractors/training_audio.py +++ b/ibllib/io/extractors/training_audio.py @@ -9,8 +9,8 @@ from scipy.io import wavfile -from neurodsp.utils import WindowGenerator -from neurodsp import fourier +from ibldsp.utils import WindowGenerator +from ibldsp import fourier import ibllib.io.raw_data_loaders as ioraw from ibllib.io.extractors.training_trials import GoCueTimes diff --git a/ibllib/io/extractors/training_wheel.py b/ibllib/io/extractors/training_wheel.py index 2f1aded8c..1d77d42e2 100644 --- a/ibllib/io/extractors/training_wheel.py +++ b/ibllib/io/extractors/training_wheel.py @@ -4,7 +4,7 @@ import numpy as np from scipy import interpolate -from neurodsp.utils import sync_timestamps +from ibldsp.utils import sync_timestamps from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes import ibllib.io.raw_data_loaders as raw from ibllib.misc import structarr diff --git a/ibllib/io/extractors/video_motion.py b/ibllib/io/extractors/video_motion.py index 482aaf44b..929f18b88 100644 --- a/ibllib/io/extractors/video_motion.py +++ b/ibllib/io/extractors/video_motion.py @@ -15,7 +15,7 @@ from pathlib import Path from joblib import Parallel, delayed, cpu_count -from neurodsp.utils import WindowGenerator +from ibldsp.utils import WindowGenerator from one.api import ONE import ibllib.io.video as vidio from iblutil.util import Bunch diff --git a/ibllib/io/extractors/widefield.py b/ibllib/io/extractors/widefield.py index 0bccf92c1..e5417c94f 100644 --- a/ibllib/io/extractors/widefield.py +++ b/ibllib/io/extractors/widefield.py @@ -11,8 +11,17 @@ from ibllib.io.extractors.ephys_fpga import get_sync_fronts, get_sync_and_chn_map from ibllib.io.video import get_video_meta -import wfield.cli as wfield_cli -from labcams.io import parse_cam_log +_logger = logging.getLogger('ibllib') + +try: + import wfield.cli as wfield_cli +except ImportError: + _logger.warning('wfield not installed') + +try: + from labcams.io import parse_cam_log +except ImportError: + _logger.warning('labcams not installed') _logger = logging.getLogger('ibllib') diff --git a/ibllib/io/raw_daq_loaders.py b/ibllib/io/raw_daq_loaders.py index 8ac58c3e7..54aa92cba 100644 --- a/ibllib/io/raw_daq_loaders.py +++ b/ibllib/io/raw_daq_loaders.py @@ -6,7 +6,7 @@ import nptdms import numpy as np -import neurodsp.utils +import ibldsp.utils import one.alf.io as alfio import one.alf.exceptions as alferr from one.alf.spec import to_alf @@ -134,7 +134,7 @@ def load_sync_tdms(path, sync_map, fs=None, threshold=2.5, floor_percentile=10): logger.info(f'estimated analogue channel DC Offset approx. {np.mean(offset):.2f}') analogue -= offset ttl = analogue > threshold - ind, sign = neurodsp.utils.fronts(ttl.astype(int)) + ind, sign = ibldsp.utils.fronts(ttl.astype(int)) try: # attempt to get the times from the meta data times = np.vstack([ch.time_track() for ch in raw_channels]) times = times[tuple(ind)] @@ -276,8 +276,8 @@ def extract_sync_timeline(timeline, chmap=None, floor_percentile=10, threshold=N step = threshold.get(label) if isinstance(threshold, dict) else threshold if step is None: step = np.max(raw - offset) / 2 - iup = neurodsp.utils.rises(raw - offset, step=step, analog=True) - idown = neurodsp.utils.falls(raw - offset, step=step, analog=True) + iup = ibldsp.utils.rises(raw - offset, step=step, analog=True) + idown = ibldsp.utils.falls(raw - offset, step=step, analog=True) pol = np.r_[np.ones_like(iup), -np.ones_like(idown)].astype('i1') ind = np.r_[iup, idown] diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py index 9d6695cc8..a6a67fffc 100644 --- a/ibllib/pipes/dynamic_pipeline.py +++ b/ibllib/pipes/dynamic_pipeline.py @@ -2,6 +2,23 @@ The principal function here is `make_pipeline` which reads an `_ibl_experiment.description.yaml` file and determines the set of tasks required to preprocess the session. + +In the experiment description file there is a 'tasks' key that defines each task protocol and the +location of the raw data (i.e. task collection). The protocol subkey may contain an 'extractors' +field that should contain a list of dynamic pipeline task class names for extracting the task data. +These must be subclasses of the :class:`ibllib.pipes.base_tasks.DynamicTask` class. If the +extractors key is absent or empty, the tasks are chosen based on the sync label and protocol name. + +NB: The standard behvaiour extraction task classes (e.g. +:class:`ibllib.pipes.behaviour_tasks.ChoiceWorldTrialsBpod` and :class:`ibllib.pipes.behaviour_tasks.ChoiceWorldTrialsNidq`) +handle the clock synchronization, behaviour plots and QC. This is typically independent of the Bpod +trials extraction (i.e. extraction of trials data from the Bpod raw data, in Bpod time). The Bpod +trials extractor class is determined by the :func:`ibllib.io.extractors.base.protocol2extractor` +map. IBL protocols may be added to the ibllib.io.extractors.task_extractor_map.json file, while +non-IBL ones should be in projects.base.task_extractor_map.json file located in the personal +projects repo. The Bpod trials extractor class must be a subclass of the +:class:`ibllib.io.extractors.base.BaseBpodTrialsExtractor` class, and located in either the +personal projects repo or in :py:mod:`ibllib.io.extractors.bpod_trials` module. """ import logging import re diff --git a/ibllib/pipes/ephys_preprocessing.py b/ibllib/pipes/ephys_preprocessing.py index 9cfaa22e3..d13906e15 100644 --- a/ibllib/pipes/ephys_preprocessing.py +++ b/ibllib/pipes/ephys_preprocessing.py @@ -18,7 +18,7 @@ import packaging.version import one.alf.io as alfio -from neurodsp.utils import rms +from ibldsp.utils import rms import spikeglx from ibllib.misc import check_nvidia_driver diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index 7affc7139..925839073 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -10,7 +10,7 @@ import pandas as pd import spikeglx import neuropixel -from neurodsp.utils import rms +from ibldsp.utils import rms import one.alf.io as alfio from ibllib.misc import check_nvidia_driver diff --git a/ibllib/pipes/histology.py b/ibllib/pipes/histology.py index ccf7ade22..a07b99985 100644 --- a/ibllib/pipes/histology.py +++ b/ibllib/pipes/histology.py @@ -11,7 +11,7 @@ from ibllib.ephys.spikes import probes_description as extract_probes from ibllib.qc import base -from neurodsp.utils import fcn_cosine +from ibldsp.utils import fcn_cosine _logger = logging.getLogger(__name__) diff --git a/ibllib/pipes/training_status.py b/ibllib/pipes/training_status.py index 87bd019ec..963191150 100644 --- a/ibllib/pipes/training_status.py +++ b/ibllib/pipes/training_status.py @@ -1,30 +1,29 @@ -import one.alf.io as alfio -from one.alf.exceptions import ALFObjectNotFound - -from ibllib.io.raw_data_loaders import load_bpod -from ibllib.oneibl.registration import _get_session_times -from ibllib.io.extractors.base import get_session_extractor_type -from ibllib.io.session_params import read_params -from ibllib.io.extractors.bpod_trials import get_bpod_extractor - -from iblutil.util import setup_logger -from ibllib.plots.snapshot import ReportSnapshot -from iblutil.numerical import ismember -from brainbox.behavior import training +import logging +from pathlib import Path +from datetime import datetime +from itertools import chain import numpy as np import pandas as pd -from pathlib import Path +from iblutil.numerical import ismember +import one.alf.io as alfio +from one.alf.exceptions import ALFObjectNotFound import matplotlib.pyplot as plt import matplotlib.dates as mdates from matplotlib.lines import Line2D -from datetime import datetime import seaborn as sns import boto3 from botocore.exceptions import ProfileNotFound, ClientError -from itertools import chain -logger = setup_logger(__name__) +from ibllib.io.raw_data_loaders import load_bpod +from ibllib.oneibl.registration import _get_session_times +from ibllib.io.extractors.base import get_session_extractor_type, get_bpod_extractor_class +from ibllib.io.session_params import read_params +from ibllib.io.extractors.bpod_trials import get_bpod_extractor +from ibllib.plots.snapshot import ReportSnapshot +from brainbox.behavior import training + +logger = logging.getLogger(__name__) TRAINING_STATUS = {'untrainable': (-4, (0, 0, 0, 0)), @@ -265,7 +264,7 @@ def get_latest_training_information(sess_path, one): save_dataframe(df, subj_path) # Now go through the backlog and compute the training status for sessions. If for example one was missing as it is cumulative - # we need to go through and compute all the back log + # we need to go through and compute all the backlog # Find the earliest date in missing dates that we need to recompute the training status for missing_status = find_earliest_recompute_date(df.drop_duplicates('date').reset_index(drop=True)) for date in missing_status: @@ -313,13 +312,28 @@ def find_earliest_recompute_date(df): return df[first_index:].date.values -def compute_training_status(df, compute_date, one, force=True, task_collection='raw_behavior_data'): +def compute_training_status(df, compute_date, one, force=True): """ - Compute the training status for compute date based on training from that session and two previous days - :param df: training dataframe - :param compute_date: date to compute training on - :param one: ONE instance + Compute the training status for compute date based on training from that session and two previous days. + + When true and if the session trials can't be found, will attempt to re-extract from disk. :return: + + Parameters + ---------- + df : pandas.DataFrame + A training data frame, e.g. one generated from :func:`get_training_info_for_session`. + compute_date : str, datetime.datetime, pandas.Timestamp + The date to compute training on. + one : one.api.One + An instance of ONE for loading trials data. + force : bool + When true and if the session trials can't be found, will attempt to re-extract from disk. + + Returns + ------- + pandas.DataFrame + The input data frame with a 'training_status' column populated for `compute_date`. """ # compute_date = str(one.path2ref(session_path)['date']) @@ -431,11 +445,34 @@ def compute_session_duration_delay_location(sess_path, collections=None, **kwarg def get_data_collection(session_path): - """ - Returns the location of the raw behavioral data and extracted trials data for the session path. If - multiple locations in one session (e.g for dynamic) returns all of these - :param session_path: path of session - :return: + """Return the location of the raw behavioral data and extracted trials data for a given session. + + For multiple locations in one session (e.g. chained protocols), returns all collections. + Passive protocols are excluded. + + Parameters + ---------- + session_path : pathlib.Path + A session path in the form subject/date/number. + + Returns + ------- + list of str + A list of sub-directory names that contain raw behaviour data. + list of str + A list of sub-directory names that contain ALF trials data. + + Examples + -------- + An iblrig v7 session + + >>> get_data_collection(Path(r'C:/data/subject/2023-01-01/001')) + ['raw_behavior_data'], ['alf'] + + An iblrig v8 session where two protocols were run + + >>> get_data_collection(Path(r'C:/data/subject/2023-01-01/001')) + ['raw_task_data_00', 'raw_task_data_01], ['alf/task_00', 'alf/task_01'] """ experiment_description = read_params(session_path) collections = [] @@ -522,10 +559,22 @@ def get_sess_dict(session_path, one, protocol, alf_collections=None, raw_collect def get_training_info_for_session(session_paths, one, force=True): """ - Extract the training information needed for plots for each session - :param session_paths: list of session paths on same date - :param one: ONE instance - :return: + Extract the training information needed for plots for each session. + + Parameters + ---------- + session_paths : list of pathlib.Path + List of session paths on same date. + one : one.api.One + An ONE instance. + force : bool + When true and if the session trials can't be found, will attempt to re-extract from disk. + + Returns + ------- + list of dict + A list of dictionaries the length of `session_paths` containing individual and aggregate + performance information. """ # return list of dicts to add @@ -535,7 +584,12 @@ def get_training_info_for_session(session_paths, one, force=True): session_path = Path(session_path) protocols = [] for c in collections: - protocols.append(get_session_extractor_type(session_path, task_collection=c)) + try: + prot = get_bpod_extractor_class(session_path, task_collection=c) + prot = prot[:-6].lower() + except Exception: + prot = get_session_extractor_type(session_path, task_collection=c) + protocols.append(prot) un_protocols = np.unique(protocols) # Example, training, training, biased - training would be combined, biased not @@ -554,8 +608,8 @@ def get_training_info_for_session(session_paths, one, force=True): sess_dict = get_sess_dict(session_path, one, prot, alf_collections=alf, raw_collections=raw, force=force) else: prot = un_protocols[0] - sess_dict = get_sess_dict(session_path, one, prot, alf_collections=alf_collections, raw_collections=collections, - force=force) + sess_dict = get_sess_dict( + session_path, one, prot, alf_collections=alf_collections, raw_collections=collections, force=force) if sess_dict is not None: sess_dicts.append(sess_dict) diff --git a/ibllib/pipes/widefield_tasks.py b/ibllib/pipes/widefield_tasks.py index adf49f9eb..1a1ad225b 100644 --- a/ibllib/pipes/widefield_tasks.py +++ b/ibllib/pipes/widefield_tasks.py @@ -17,10 +17,14 @@ from ibllib.io.video import get_video_meta from ibllib.plots.snapshot import ReportSnapshot -import labcams.io _logger = logging.getLogger(__name__) +try: + import labcams.io +except ImportError: + _logger.warning('labcams not installed') + class WidefieldRegisterRaw(base_tasks.WidefieldTask, base_tasks.RegisterRawDataTask): diff --git a/ibllib/plots/figures.py b/ibllib/plots/figures.py index 34a444e9b..369709db1 100644 --- a/ibllib/plots/figures.py +++ b/ibllib/plots/figures.py @@ -12,7 +12,7 @@ import scipy.signal import matplotlib.pyplot as plt -from neurodsp import voltage +from ibldsp import voltage from ibllib.plots.snapshot import ReportSnapshotProbe, ReportSnapshot from one.api import ONE import one.alf.io as alfio @@ -614,7 +614,7 @@ def raw_destripe(raw, fs, t0, i_plt, n_plt, ''' # Import - from neurodsp import voltage + from ibldsp import voltage from ibllib.plots import Density # Init fig diff --git a/ibllib/plots/misc.py b/ibllib/plots/misc.py index 2a561ae8d..022e0c43f 100644 --- a/ibllib/plots/misc.py +++ b/ibllib/plots/misc.py @@ -4,7 +4,7 @@ import matplotlib.pyplot as plt import scipy -import neurodsp as dsp +import ibldsp as dsp def wiggle(w, fs=1, gain=0.71, color='k', ax=None, fill=True, linewidth=0.5, t0=0, clip=2, sf=None, diff --git a/ibllib/qc/camera.py b/ibllib/qc/camera.py index 8cf993573..821f891d2 100644 --- a/ibllib/qc/camera.py +++ b/ibllib/qc/camera.py @@ -43,7 +43,6 @@ import matplotlib.pyplot as plt import pandas as pd from matplotlib.patches import Rectangle -from labcams import parse_cam_log import one.alf.io as alfio from one.util import filter_datasets @@ -65,6 +64,11 @@ _log = logging.getLogger(__name__) +try: + from labcams import parse_cam_log +except ImportError: + _log.warning('labcams not installed') + class CameraQC(base.QC): """A class for computing camera QC metrics""" diff --git a/ibllib/qc/task_metrics.py b/ibllib/qc/task_metrics.py index cd67ef0ac..faee83ca0 100644 --- a/ibllib/qc/task_metrics.py +++ b/ibllib/qc/task_metrics.py @@ -2,6 +2,9 @@ This module runs a list of quality control metrics on the behaviour data. +NB: The QC should be loaded using :meth:`ibllib.pipes.base_tasks.BehaviourTask.run_qc` and not +instantiated directly. + Examples -------- Running on a rig computer and updating QC fields in Alyx: @@ -49,13 +52,14 @@ """ import logging import sys +import warnings +from packaging import version from datetime import datetime, timedelta from inspect import getmembers, isfunction -from functools import reduce +from functools import reduce, wraps from collections.abc import Sized import numpy as np -from packaging import version from scipy.stats import chisquare from brainbox.behavior.wheel import cm_to_rad, traces_by_trial @@ -67,31 +71,99 @@ _log = logging.getLogger(__name__) +BWM_CRITERIA = { + 'default': {'PASS': 0.99, 'WARNING': 0.90, 'FAIL': 0}, # Note: WARNING was 0.95 prior to Aug 2022 + '_task_stimOff_itiIn_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_positive_feedback_stimOff_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_negative_feedback_stimOff_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_wheel_move_during_closed_loop': {'PASS': 0.99, 'WARNING': 0}, + '_task_response_stimFreeze_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_detected_wheel_moves': {'PASS': 0.99, 'WARNING': 0}, + '_task_trial_length': {'PASS': 0.99, 'WARNING': 0}, + '_task_goCue_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_errorCue_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_stimOn_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_stimOff_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_stimFreeze_delays': {'PASS': 0.99, 'WARNING': 0}, + '_task_iti_delays': {'NOT_SET': 0}, + '_task_passed_trial_checks': {'NOT_SET': 0} +} + + +def _static_check(func): + """Log a warning when static method called with class instead of object. + + The TaskQC 'criteria' attribute is now varies depending on the task version and may be changed + in subclasses depending on hardware and task protocol. Therefore the + TaskQC.compute_session_status_from_dict method should be called with an object instance in + order use the correct criteria. + """ + @wraps(func) + def inner(*args, **kwargs): + warnings.warn('TaskQC.compute_session_status_from_dict is deprecated. ' + 'Use ibllib.qc.task_metrics.compute_session_status_from_dict instead', DeprecationWarning) + if not args: # allow function to raise on missing param + return compute_session_status_from_dict(*args, **kwargs) + if not isinstance(args[0], TaskQC): + _log.warning( + 'Calling TaskQC.compute_session_status_from_dict as a static method yields inconsistent results.' + ) + if len(args) == 1 and kwargs.get('criteria', None) is None: + kwargs['criteria'] = BWM_CRITERIA # old behaviour + else: + if kwargs.get('criteria', None) is None and len(args) == 2: + kwargs['criteria'] = args[0].criteria # ensure we use the obj's modified criteria + args = args[1:] + return compute_session_status_from_dict(*args, **kwargs) + return inner + + +def compute_session_status_from_dict(results, criteria=None): + """ + Given a dictionary of results, computes the overall session QC for each key and aggregates + in a single value + + Parameters + ---------- + results : dict + A dictionary of QC keys containing (usually scalar) values. + criteria : dict + A dictionary of qc keys containing map of PASS, WARNING, FAIL thresholds. + + Returns + ------- + str + Overall session QC outcome as a string. + dict + A map of QC tests and their outcomes. + """ + if not criteria: + criteria = {'default': BWM_CRITERIA['default']} + indices = np.zeros(len(results), dtype=int) + for i, k in enumerate(results): + if k in criteria.keys(): + indices[i] = TaskQC.thresholding(results[k], thresholds=criteria[k]) + else: + indices[i] = TaskQC.thresholding(results[k], thresholds=criteria['default']) + + def key_map(x): + return 'NOT_SET' if x < 0 else list(criteria['default'].keys())[x] + # Criteria map is in order of severity so the max index is our overall QC outcome + session_outcome = key_map(max(indices)) + outcomes = dict(zip(results.keys(), map(key_map, indices))) + return session_outcome, outcomes + + class TaskQC(base.QC): """A class for computing task QC metrics""" - criteria = dict() - criteria['default'] = {'PASS': 0.99, 'WARNING': 0.90, 'FAIL': 0} # Note: WARNING was 0.95 prior to Aug 2022 - criteria['_task_stimOff_itiIn_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_positive_feedback_stimOff_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_negative_feedback_stimOff_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_wheel_move_during_closed_loop'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_response_stimFreeze_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_detected_wheel_moves'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_trial_length'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_goCue_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_errorCue_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_stimOn_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_stimOff_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_stimFreeze_delays'] = {'PASS': 0.99, 'WARNING': 0} - criteria['_task_iti_delays'] = {'NOT_SET': 0} - criteria['_task_passed_trial_checks'] = {'NOT_SET': 0} + criteria = BWM_CRITERIA extractor = None """ibllib.qc.task_extractors.TaskQCExtractor: A task extractor object containing raw and extracted data.""" @staticmethod - def _thresholding(qc_value, thresholds=None): + def thresholding(qc_value, thresholds=None): """ Computes the outcome of a single key by applying thresholding. :param qc_value: proportion of passing qcs, between 0 and 1 @@ -99,9 +171,8 @@ def _thresholding(qc_value, thresholds=None): (cf. TaskQC.criteria attribute) :return: int where -1: NOT_SET, 0: PASS, 1: WARNING, 2: FAIL """ + thresholds = thresholds or {} MAX_BOUND, MIN_BOUND = (1, 0) - if not thresholds: - thresholds = TaskQC.criteria['default'].copy() if qc_value is None or np.isnan(qc_value): return int(-1) elif (qc_value > MAX_BOUND) or (qc_value < MIN_BOUND): @@ -134,6 +205,9 @@ def __init__(self, session_path_or_eid, **kwargs): self.metrics = None self.passed = None + # Criteria (initialize as outcomes vary by class, task, and hardware) + self.criteria = BWM_CRITERIA.copy() + def load_data(self, bpod_only=False, download_data=True): """Extract the data from raw data files. @@ -268,11 +342,15 @@ def run(self, update=False, namespace='task', **kwargs): self.update(outcome, namespace) return outcome, results + @_static_check @staticmethod def compute_session_status_from_dict(results, criteria=None): """ - Given a dictionary of results, computes the overall session QC for each key and aggregates - in a single value + (DEPRECATED) Given a dictionary of results, computes the overall session QC for each key + and aggregates in a single value. + + NB: Use :func:`ibllib.qc.task_metrics.compute_session_status_from_dict` instead and always + pass in the criteria. Parameters ---------- @@ -288,20 +366,7 @@ def compute_session_status_from_dict(results, criteria=None): dict A map of QC tests and their outcomes. """ - indices = np.zeros(len(results), dtype=int) - criteria = criteria or TaskQC.criteria - for i, k in enumerate(results): - if k in criteria.keys(): - indices[i] = TaskQC._thresholding(results[k], thresholds=criteria[k]) - else: - indices[i] = TaskQC._thresholding(results[k], thresholds=criteria['default']) - - def key_map(x): - return 'NOT_SET' if x < 0 else list(TaskQC.criteria['default'].keys())[x] - # Criteria map is in order of severity so the max index is our overall QC outcome - session_outcome = key_map(max(indices)) - outcomes = dict(zip(results.keys(), map(key_map, indices))) - return session_outcome, outcomes + ... # no longer called by decorator def compute_session_status(self): """ @@ -321,7 +386,7 @@ def compute_session_status(self): # Get mean passed of each check, or None if passed is None or all NaN results = {k: None if v is None or np.isnan(v).all() else np.nanmean(v) for k, v in self.passed.items()} - session_outcome, outcomes = self.compute_session_status_from_dict(results, self.criteria) + session_outcome, outcomes = compute_session_status_from_dict(results, self.criteria) return session_outcome, results, outcomes @@ -347,6 +412,11 @@ def compute(self, download_data=None, **kwargs): metrics = {} passed = {} + # Modify criteria based on version + ver = self.extractor.settings.get('IBLRIG_VERSION', '') or '0.0.0' + is_v8 = version.parse(ver) >= version.parse('8.0.0') + self.criteria['_task_iti_delays'] = {'PASS': 0.99, 'WARNING': 0} if is_v8 else {'NOT_SET': 0} + # Check all reward volumes == 3.0ul check = prefix + 'reward_volumes' metrics[check] = data['rewardVolume'] @@ -419,6 +489,15 @@ def compute(self, download_data=None, **kwargs): passed[check] = p < 0.05 if len(data['phase']) >= 400 else None # skip if too few trials metrics[check] = metric + # Check that the period of gray screen between stim off and the start of the next trial is + # 1s +/- 10%. + check = prefix + 'iti_delays' + iti = (np.roll(data['stimOn_times'], -1) - data['stimOff_times'])[:-1] + metric = np.r_[np.nan_to_num(iti, nan=np.inf), np.nan] - 1. + passed[check] = np.abs(metric) <= 0.1 + passed[check][-1] = np.NaN + metrics[check] = metric + # Checks common to training QC checks = [check_goCue_delays, check_stimOn_goCue_delays, check_stimOn_delays, check_stimOff_delays] @@ -526,23 +605,24 @@ def check_stimOff_itiIn_delays(data, **_): def check_iti_delays(data, **_): - """ Check that the period of gray screen between stim off and the start of the next trial is - 0.5s +/- 200%. + """ Check that the period of grey screen between stim off and the start of the next trial is + 1s +/- 10%. - Metric: M = stimOff (n) - trialStart (n+1) - 0.5 - Criterion: |M| < 1 + Metric: M = stimOff (n) - trialStart (n+1) - 1. + Criterion: |M| < 0.1 Units: seconds [s] :param data: dict of trial data with keys ('stimOff_times', 'intervals') """ # Initialize array the length of completed trials + ITI = 1. metric = np.full(data['intervals'].shape[0], np.nan) passed = metric.copy() # Get the difference between stim off and the start of the next trial # Missing data are set to Inf, except for the last trial which is a NaN metric[:-1] = \ - np.nan_to_num(data['intervals'][1:, 0] - data['stimOff_times'][:-1] - 0.5, nan=np.inf) - passed[:-1] = np.abs(metric[:-1]) < .5 # Last trial is not counted + np.nan_to_num(data['intervals'][1:, 0] - data['stimOff_times'][:-1] - ITI, nan=np.inf) + passed[:-1] = np.abs(metric[:-1]) < (ITI / 10) # Last trial is not counted assert data['intervals'].shape[0] == len(metric) == len(passed) return metric, passed diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py index f8a2b33de..3b473503a 100644 --- a/ibllib/qc/task_qc_viewer/task_qc.py +++ b/ibllib/qc/task_qc_viewer/task_qc.py @@ -245,7 +245,7 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N qc = QcFrame(qc_or_session) else: # assumed to be eid or session path one = one or ONE(mode='local' if local else 'auto') - if not is_session_path(qc_or_session): + if not is_session_path(Path(qc_or_session)): eid = one.to_eid(qc_or_session) session_path = one.eid2path(eid) else: @@ -309,7 +309,7 @@ def qc_gui_cli(): # Parse parameters parser = argparse.ArgumentParser(description='Quick viewer to see the behaviour data from' 'choice world sessions.') - parser.add_argument('session', help='session uuid') + parser.add_argument('session', help='session uuid or path') parser.add_argument('--bpod', action='store_true', help='run QC on Bpod data only (no FPGA)') parser.add_argument('--local', action='store_true', help='run from disk location (lab server') args = parser.parse_args() # returns data from the options specified (echo) diff --git a/ibllib/tests/qc/test_task_metrics.py b/ibllib/tests/qc/test_task_metrics.py index 404876160..9fd10cb4c 100644 --- a/ibllib/tests/qc/test_task_metrics.py +++ b/ibllib/tests/qc/test_task_metrics.py @@ -13,12 +13,45 @@ class TestAggregateOutcome(unittest.TestCase): + + def test_deprecation_warning(self): + """Remove TaskQC.compute_session_status_from_dict after 2024-04-01.""" + from datetime import datetime + self.assertFalse(datetime.now() > datetime(2024, 4, 1), 'remove TaskQC.compute_session_status_from_dict method.') + qc_dict = {'_task_iti_delays': .99} + with self.assertWarns(DeprecationWarning), self.assertLogs(qcmetrics.__name__, 'WARNING'): + out = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict) + expected = ('NOT_SET', {'_task_iti_delays': 'NOT_SET'}) + self.assertEqual(expected, out, 'failed to use BWM criteria') + # Should handle criteria as input, both as arg and kwarg + criteria = {'_task_iti_delays': {'PASS': 0.9, 'FAIL': 0}, 'default': {'PASS': 0.9, 'WARNING': 0.4}} + out = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict, criteria=criteria) + expected = ('PASS', {'_task_iti_delays': 'PASS'}) + self.assertEqual(expected, out, 'failed to use BWM criteria') + out = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict, criteria) + self.assertEqual(expected, out, 'failed to use BWM criteria') + qc = qcmetrics.TaskQC('/foo/subject/2024-01-01/001', one=ONE(mode='local', **TEST_DB)) + self.assertRaises(TypeError, qcmetrics.TaskQC.compute_session_status_from_dict) + if getattr(self, 'assertNoLogs', False) is False: + self.skipTest('Python < 3.10') # py 3.8 + with self.assertWarns(DeprecationWarning), self.assertNoLogs(qcmetrics.__name__, 'WARNING'): + out = qc.compute_session_status_from_dict(qc_dict) + expected = ('NOT_SET', {'_task_iti_delays': 'NOT_SET'}) + self.assertEqual(expected, out, 'failed to use BWM criteria') + # Should handle criteria as input, both as arg and kwarg + criteria = {'_task_iti_delays': {'PASS': 0.9, 'FAIL': 0}, 'default': {'PASS': 0}} + out, _ = qc.compute_session_status_from_dict(qc_dict, criteria=criteria) + self.assertEqual('PASS', out) + out, _ = qc.compute_session_status_from_dict(qc_dict, criteria) + self.assertEqual('PASS', out) + self.assertRaises(TypeError, qc.compute_session_status_from_dict) + def test_outcome_from_dict_default(self): # For a task that has no costume thresholds, default is 0.99 PASS and 0.9 WARNING and 0 FAIL, # np.nan and None return not set qc_dict = {'gnap': .99, 'gnop': np.nan, 'gnip': None, 'gnep': 0.9, 'gnup': 0.89} expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', 'gnip': 'NOT_SET', 'gnep': 'WARNING', 'gnup': 'FAIL'} - outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict) + outcome, outcome_dict = qcmetrics.compute_session_status_from_dict(qc_dict, qcmetrics.BWM_CRITERIA) self.assertEqual(outcome, 'FAIL') self.assertEqual(expect, outcome_dict) @@ -26,7 +59,7 @@ def test_outcome_from_dict_stimFreeze_delays(self): # For '_task_stimFreeze_delays' the threshold are 0.99 PASS and 0 WARNING qc_dict = {'gnap': .99, 'gnop': np.nan, '_task_stimFreeze_delays': .1} expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', '_task_stimFreeze_delays': 'WARNING'} - outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict) + outcome, outcome_dict = qcmetrics.compute_session_status_from_dict(qc_dict, qcmetrics.BWM_CRITERIA) self.assertEqual(outcome, 'WARNING') self.assertEqual(expect, outcome_dict) @@ -34,7 +67,7 @@ def test_outcome_from_dict_iti_delays(self): # For '_task_iti_delays' the threshold is 0 NOT_SET qc_dict = {'gnap': .99, 'gnop': np.nan, '_task_iti_delays': .1} expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', '_task_iti_delays': 'NOT_SET'} - outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict) + outcome, outcome_dict = qcmetrics.compute_session_status_from_dict(qc_dict, qcmetrics.BWM_CRITERIA) self.assertEqual(outcome, 'PASS') self.assertEqual(expect, outcome_dict) @@ -42,7 +75,7 @@ def test_out_of_bounds(self): # When qc values are below 0 or above 1, give error qc_dict = {'gnap': 1.01, 'gnop': 0, 'gnip': 0.99} with self.assertRaises(ValueError) as e: - outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict) + outcome, outcome_dict = qcmetrics.compute_session_status_from_dict(qc_dict, qcmetrics.BWM_CRITERIA) self.assertTrue(e.exception.args[0] == 'Values out of bound') @@ -74,7 +107,7 @@ def load_fake_bpod_data(n=5): correct[np.argmax(choice == -1)] = 0 quiescence_length = 0.2 + np.random.standard_exponential(size=(n,)) - iti_length = 0.5 # inter-trial interval + iti_length = 1 # inter-trial interval # trial lengths include quiescence period, a couple small trigger delays and iti trial_lengths = quiescence_length + resp_feeback_delay + (trigg_delay * 4) + iti_length # add on 60s for nogos + feedback time (1 or 2s) + ~0.5s for other responses diff --git a/ibllib/tests/test_ephys.py b/ibllib/tests/test_ephys.py index 5e024c6f9..d852b34f5 100644 --- a/ibllib/tests/test_ephys.py +++ b/ibllib/tests/test_ephys.py @@ -7,7 +7,7 @@ from one.api import ONE import neuropixel -from neurodsp import voltage +from ibldsp import voltage from ibllib.ephys import ephysqc, spikes from ibllib.tests import TEST_DB diff --git a/release_notes.md b/release_notes.md index f3e2e321e..fc7956db3 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,15 @@ +## Release Notes 2.31 + +### features +- training status uses new extractor map +- refactor neurodsp to ibldsp +- ITI qc check for iblrig v8 +- Support habituationChoiceWorld extraction in iblrig v8.15.0 + +### bugfixes +- NP2 waveforms extracted with correct dtype +- Sorted cluster ids in single unit metrics + ## Release Notes 2.30 ### features diff --git a/requirements.txt b/requirements.txt index 37d9af4b1..7c6217aa0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,10 +24,9 @@ tqdm>=4.32.1 iblatlas>=0.4.0 ibl-neuropixel>=0.8.1 iblutil>=1.7.0 -labcams # widefield extractor mtscomp>=1.0.1 ONE-api>=2.6 phylib>=2.4 psychofit slidingRP>=1.0.0 # steinmetz lab refractory period metrics -wfield==0.3.7 # widefield extractor frozen for now (2023/07/15) until Joao fixes latest version +pyqt5 \ No newline at end of file diff --git a/setup.py b/setup.py index 0d83836ae..bd8272b20 100644 --- a/setup.py +++ b/setup.py @@ -59,5 +59,8 @@ def get_version(rel_path): 'task_qc = ibllib.qc.task_qc_viewer.task_qc:qc_gui_cli', ], }, + extras_require={ + 'wfield': ['wfield==0.3.7', 'labcams'], + }, scripts=[], )