From acc063f38866f4ce2b9ea4fce8220b6d151eb8ae Mon Sep 17 00:00:00 2001 From: Michele Fabbri Date: Thu, 3 Nov 2022 16:03:28 +0000 Subject: [PATCH 1/9] minor fix to windows pathing for alyx_username --- iblrig/misc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iblrig/misc.py b/iblrig/misc.py index 677177b59..0ca68b19e 100644 --- a/iblrig/misc.py +++ b/iblrig/misc.py @@ -42,7 +42,10 @@ def call_exp_desc_gui(): log.info("Attempting to launch experiment description form...") # determine alyx_username - alyx_prod_config_path = Path.home() / ".one" / ".alyx.internationalbrainlab.org" + if platform == "win32": + alyx_prod_config_path = Path.home() / "AppData" / "Roaming" / ".one" / ".alyx.internationalbrainlab.org" + else: + alyx_prod_config_path = Path.home() / ".one" / ".alyx.internationalbrainlab.org" with open(alyx_prod_config_path, "r") as f: data = json.load(f) alyx_username = data["ALYX_LOGIN"] From dcfdeb8d2efe56c50632005e6493c4ad518c0936 Mon Sep 17 00:00:00 2001 From: Michele Fabbri Date: Fri, 4 Nov 2022 10:05:33 +0000 Subject: [PATCH 2/9] removal of unnecessary ONE version check in purge_rig_data script --- scripts/ibllib/purge_rig_data.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/scripts/ibllib/purge_rig_data.py b/scripts/ibllib/purge_rig_data.py index cedfd4b32..7b4f02c7f 100644 --- a/scripts/ibllib/purge_rig_data.py +++ b/scripts/ibllib/purge_rig_data.py @@ -8,21 +8,12 @@ from fnmatch import fnmatch from pathlib import Path -import one +from one.alf.cache import iter_datasets, iter_sessions from one.alf.files import get_session_path from one.api import ONE log = logging.getLogger('iblrig') -try: # Verify ONE-api is at v1.13.0 or greater - assert tuple(map(int, one.__version__.split('.'))) >= (1, 13, 0) - from one.alf.cache import iter_datasets, iter_sessions -except (AssertionError, ImportError) as e: - if e is AssertionError: - log.error("The found version of ONE needs to be updated to run this script, please run a 'pip install -U ONE-api' from " - "the appropriate anaconda environment") - raise - def session_name(path, lab=None) -> str: """ @@ -62,8 +53,7 @@ def purge_local_data(local_folder, filename='*', lab=None, dry=False, one=None): log.info(f'Local files to remove: {len(to_remove)}') for f in to_remove: log.info(f'DELETE: {f}') - if not dry: - f.unlink() + f.unlink() if not dry else None return to_remove @@ -74,9 +64,7 @@ def purge_local_data(local_folder, filename='*', lab=None, dry=False, one=None): parser.add_argument( '-lab', required=False, default=None, help='Lab name, in case sessions conflict between labs. default: None', ) - parser.add_argument( - '--dry', required=False, default=False, action='store_true', help='Dry run? default: False', - ) + parser.add_argument('--dry', required=False, default=False, action='store_true', help='Dry run? default: False') args = parser.parse_args() purge_local_data(args.folder, args.file, lab=args.lab, dry=args.dry) - print('Done\n') + print('purge_rig_data script done\n') From 862c128b9b72a63ff8f6cb4975609cf73552541f Mon Sep 17 00:00:00 2001 From: Michele Fabbri Date: Fri, 4 Nov 2022 10:38:53 +0000 Subject: [PATCH 3/9] dev ci test --- .github/workflows/main.yml | 17 +++++++++++------ README.md | 6 ++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3075607ed..ac7576b5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,12 +55,17 @@ jobs: - name: iblrig unit tests shell: pwsh -l {0} run: | - echo "------ start info ------" - conda info - conda env list - python --version - echo "------ end info ------" - python -m unittest discover + cd test_iblrig + python -m unittest test_adaptive.py + python -m unittest test_alyx.py + python -m unittest test_ibllib_imports.py + python -m unittest test_params.py + python -m unittest test_path_helper.py + python -m unittest test_pybpod_config.py + # python -m unittest test_scripts.py + python -m unittest test_start_pybpod.py + python -m unittest test_task.py + # python -m unittest discover - name: Generate requirements_frozen.txt shell: pwsh -l {0} diff --git a/README.md b/README.md index 6ac14065a..667c2e30f 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,9 @@ conda activate ibllib pip install ibllib ``` -NOTE: ONE will need to be configured for your use case. Please review the ONE [documentation](https://int-brain-lab.github.io/ONE/) for specifics on how to accomplish this. Then run the following command or something similar for your specific setup: `python -c "from one.api import ONE; ONE()"` +NOTE: ONE will need to be configured for your use case. Please review the ONE +[documentation](https://int-brain-lab.github.io/ONE/) for specifics on how to accomplish this. Then run the following command or +something similar for your specific setup: `python -c "from one.api import ONE; ONE()"` ### Running pybpod To run pybpod and begin data acquisition: @@ -63,7 +65,7 @@ categorization of an experiment and cleanly define what projects and procedures the tasks listed in the `add_ex_desc_gui_to_tasks` script, run the following commands from the **Anaconda Powershell Prompt**: ```powershell conda activate iblrig -git clone -b develop https://github.com/int-brain-lab/iblscripts C:\iblscripts +git clone https://github.com/int-brain-lab/iblscripts C:\iblscripts pip install -r C:\iblscripts\deploy\project_procedure_gui\pp_requirements.txt ``` From 32d0b939c2e59cda2413fe66bd9dea2d77ea193b Mon Sep 17 00:00:00 2001 From: owinter Date: Fri, 28 Oct 2022 10:12:18 +0100 Subject: [PATCH 4/9] spacers utils - add a spacer to the passive standalone task --- iblrig/spacer.py | 93 +++++++++++++++++++ ...rig_tasks_passiveChoiceWorldIndependent.py | 11 +++ test_iblrig/test_spacers.py | 19 ++++ 3 files changed, 123 insertions(+) create mode 100644 iblrig/spacer.py create mode 100644 test_iblrig/test_spacers.py diff --git a/iblrig/spacer.py b/iblrig/spacer.py new file mode 100644 index 000000000..9bb3c9ba6 --- /dev/null +++ b/iblrig/spacer.py @@ -0,0 +1,93 @@ +""" +The purpose of this module is provide tools to generate and identify spacers. + +Spacers are sequences of up and down pulses with a specific, identifiable pattern. +They are generated with a chirp coding to reduce cross-correlaation sidelobes. +They are used to mark the beginning of a behaviour sequence within a session. + +Usage: + spacer = Spacer() + spacer.add_spacer_states(sma, t, next_state='first_state') + for i in range(ntrials): + sma.add_state( + state_name="first_state", + state_timer=tup, + state_change_conditions={"Tup": f"spacer_low_{i:02d}"}, + output_actions=[("BNC1", 255)], # To FPGA + ) +""" + +import numpy as np + + +class Spacer(object): + """ + Computes spacer up times using a chirp up and down pattern + Returns a list of times for the spacer states + Each time corresponds to an up time of the BNC1 signal + + dt_start: first spacer up time + dt_end: last spacer up time + n_pulses: number of spacer up times, one-sided (i.e. 8 means 16 - 1 spacers times) + tup: duration of the spacer up time + """ + def __init__(self, dt_start=.02, dt_end=.4, n_pulses=8, tup=.05): + self.dt_start = dt_start + self.dt_end = dt_end + self.n_pulses = n_pulses + self.tup = tup + + def __repr__(self): + return f"Spacer(dt_start={self.dt_start}, dt_end={self.dt_end}, n_pulses={self.n_pulses}, tup={self.tup})" + + @property + def times(self): + """ + Computes spacer up times using a chirp up and down pattern + :return: numpy arrays of times + """ + # upsweep + t = np.linspace(self.dt_start, self.dt_end, self.n_pulses) + self.tup + # downsweep + t = np.r_[t[:-1], np.flipud(t)] + t = np.cumsum(t) + return t + + def generate_template(self, fs=1000): + """ + Generates a spacer voltage template to cross-correlate with a voltage trace + from a DAQ to detect a voltage trace + :return: + """ + t = self.times + ns = int((t[-1] + self.tup * 10) * fs) + np.cumsum(np.linspace(self.dt_start, self.dt_end, self.n_pulses) + self.tup) + sig = np.zeros(ns, ) + sig[(t * fs).astype(np.int32)] = 1 + sig[((t + self.tup) * fs).astype(np.int32)] = -1 + sig = np.cumsum(sig) + return sig + + def add_spacer_states(self, sma, next_state="exit"): + """ + Add spacer states to a state machine + :param sma: pybpodapi.state_machine.StateMachine object + :param next_state: name of the state following the spacer states + :return: + """ + assert next_state is not None + t = self.times + for i, time in enumerate(t): + next_loop = f"spacer_low_{i:02d}" if i < len(t) - 1 else next_state + sma.add_state( + state_name=f"spacer_up_{i:02d}", + state_timer=self.tup, + state_change_conditions={"Tup": f"spacer_low_{i:02d}"}, + output_actions=[("BNC1", 255)], # To FPGA + ) + sma.add_state( + state_name=f"spacer_low_{i:02d}", + state_timer=time, + state_change_conditions={"Tup": next_loop}, + output_actions=[], + ) diff --git a/tasks/_iblrig_tasks_passiveChoiceWorldIndependent/_iblrig_tasks_passiveChoiceWorldIndependent.py b/tasks/_iblrig_tasks_passiveChoiceWorldIndependent/_iblrig_tasks_passiveChoiceWorldIndependent.py index 41a3b50f1..dafa2dbb1 100644 --- a/tasks/_iblrig_tasks_passiveChoiceWorldIndependent/_iblrig_tasks_passiveChoiceWorldIndependent.py +++ b/tasks/_iblrig_tasks_passiveChoiceWorldIndependent/_iblrig_tasks_passiveChoiceWorldIndependent.py @@ -10,6 +10,7 @@ import iblrig.frame2TTL as frame2TTL import iblrig.misc as misc import iblrig.params as params +import iblrig.spacer import task_settings import user_settings from iblrig.bpod_helper import BpodMessageCreator, bpod_lights @@ -97,7 +98,16 @@ def do_card_sound(card, sound_msg): ) popup("WARNING!", msg) # Locks +# Start task by sending a spacer signal on BNC1 +log.info("Starting task by sending a spacer signal on BNC1") +sma = StateMachine(bpod) +spacer = iblrig.spacer.Spacer() +spacer.add_spacer_states(sma, next_state="exit") +bpod.send_state_machine(sma) +bpod.run_state_machine(sma) # Locks until state machine 'exit' is reached + # Run the passive part i.e. spontaneous activity and RFMapping stim +log.info("Run the passive part ie. spontaneous activity and RFMapping stim") bonsai.start_passive_visual_stim( sph.SESSION_RAW_DATA_FOLDER, display_idx=sph.PARAMS["DISPLAY_IDX"], @@ -107,6 +117,7 @@ def do_card_sound(card, sound_msg): ) # Locks # start Bonsai stim workflow +log.info("Run the visual stimulus mapping") bonsai.start_visual_stim(sph) time.sleep(5) log.info("Starting replay of task stims") diff --git a/test_iblrig/test_spacers.py b/test_iblrig/test_spacers.py new file mode 100644 index 000000000..efda5be81 --- /dev/null +++ b/test_iblrig/test_spacers.py @@ -0,0 +1,19 @@ +import numpy as np +import unittest + +from iblrig.spacer import Spacer + + +class TestSpacer(unittest.TestCase): + + def test_spacer(self): + spacer = Spacer(dt_start=.02, dt_end=.4, n_pulses=8, tup=.05) + self.AssertEqual(spacer.times.size, 15) + sig = spacer.generate_template(fs=1000) + ac = np.correlate(sig, sig, 'full') / np.sum(sig**2) + # import matplotlib.pyplot as plt + # plt.plot(ac) + # plt.show() + ac[sig.size-100: sig.size + 100] = 0 # remove the main peak + # the autocorrelation side lobes should be less than 30% + assert np.max(ac) < .3 From f48c931c197ed7dbc64f1a55ec723f55adc5ca30 Mon Sep 17 00:00:00 2001 From: Michele Fabbri Date: Fri, 28 Oct 2022 11:50:55 +0100 Subject: [PATCH 5/9] bug fix for to avoid an infinite loop --- iblrig/spacer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iblrig/spacer.py b/iblrig/spacer.py index 9bb3c9ba6..03fb58c77 100644 --- a/iblrig/spacer.py +++ b/iblrig/spacer.py @@ -78,9 +78,9 @@ def add_spacer_states(self, sma, next_state="exit"): assert next_state is not None t = self.times for i, time in enumerate(t): - next_loop = f"spacer_low_{i:02d}" if i < len(t) - 1 else next_state + next_loop = f"spacer_high_{i + 1:02d}" if i < len(t) - 1 else next_state sma.add_state( - state_name=f"spacer_up_{i:02d}", + state_name=f"spacer_high_{i:02d}", state_timer=self.tup, state_change_conditions={"Tup": f"spacer_low_{i:02d}"}, output_actions=[("BNC1", 255)], # To FPGA From 861443473f5df2381ab9771d9b2edd91a58779d1 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 8 Nov 2022 11:58:06 +0000 Subject: [PATCH 6/9] fix spacers states --- iblrig/spacer.py | 10 +++++++--- test_iblrig/test_spacers.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/iblrig/spacer.py b/iblrig/spacer.py index 03fb58c77..1696c6621 100644 --- a/iblrig/spacer.py +++ b/iblrig/spacer.py @@ -49,7 +49,7 @@ def times(self): # upsweep t = np.linspace(self.dt_start, self.dt_end, self.n_pulses) + self.tup # downsweep - t = np.r_[t[:-1], np.flipud(t)] + t = np.r_[t, np.flipud(t[1:])] t = np.cumsum(t) return t @@ -68,7 +68,7 @@ def generate_template(self, fs=1000): sig = np.cumsum(sig) return sig - def add_spacer_states(self, sma, next_state="exit"): + def add_spacer_states(self, sma=None, next_state="exit"): """ Add spacer states to a state machine :param sma: pybpodapi.state_machine.StateMachine object @@ -77,7 +77,11 @@ def add_spacer_states(self, sma, next_state="exit"): """ assert next_state is not None t = self.times + dt = np.diff(t, append=t[-1] + self.tup) for i, time in enumerate(t): + if sma is None: + print(i, time, dt[i]) + continue next_loop = f"spacer_high_{i + 1:02d}" if i < len(t) - 1 else next_state sma.add_state( state_name=f"spacer_high_{i:02d}", @@ -87,7 +91,7 @@ def add_spacer_states(self, sma, next_state="exit"): ) sma.add_state( state_name=f"spacer_low_{i:02d}", - state_timer=time, + state_timer=dt[i], state_change_conditions={"Tup": next_loop}, output_actions=[], ) diff --git a/test_iblrig/test_spacers.py b/test_iblrig/test_spacers.py index efda5be81..701da9111 100644 --- a/test_iblrig/test_spacers.py +++ b/test_iblrig/test_spacers.py @@ -8,7 +8,7 @@ class TestSpacer(unittest.TestCase): def test_spacer(self): spacer = Spacer(dt_start=.02, dt_end=.4, n_pulses=8, tup=.05) - self.AssertEqual(spacer.times.size, 15) + np.testing.assert_equal(spacer.times.size, 15) sig = spacer.generate_template(fs=1000) ac = np.correlate(sig, sig, 'full') / np.sum(sig**2) # import matplotlib.pyplot as plt From 9dd39ecb0f89ec54063608ab9bdf3786db288590 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 8 Nov 2022 13:13:21 +0000 Subject: [PATCH 7/9] spacers should not be overlapping --- iblrig/spacer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iblrig/spacer.py b/iblrig/spacer.py index 1696c6621..1c6301b59 100644 --- a/iblrig/spacer.py +++ b/iblrig/spacer.py @@ -36,12 +36,13 @@ def __init__(self, dt_start=.02, dt_end=.4, n_pulses=8, tup=.05): self.dt_end = dt_end self.n_pulses = n_pulses self.tup = tup + assert np.all(np.diff(self.times) > self.tup), 'Spacers are overlapping' def __repr__(self): return f"Spacer(dt_start={self.dt_start}, dt_end={self.dt_end}, n_pulses={self.n_pulses}, tup={self.tup})" @property - def times(self): + def times(self, latency=0): """ Computes spacer up times using a chirp up and down pattern :return: numpy arrays of times @@ -61,7 +62,6 @@ def generate_template(self, fs=1000): """ t = self.times ns = int((t[-1] + self.tup * 10) * fs) - np.cumsum(np.linspace(self.dt_start, self.dt_end, self.n_pulses) + self.tup) sig = np.zeros(ns, ) sig[(t * fs).astype(np.int32)] = 1 sig[((t + self.tup) * fs).astype(np.int32)] = -1 @@ -77,7 +77,7 @@ def add_spacer_states(self, sma=None, next_state="exit"): """ assert next_state is not None t = self.times - dt = np.diff(t, append=t[-1] + self.tup) + dt = np.diff(t, append=t[-1] + self.tup * 2) for i, time in enumerate(t): if sma is None: print(i, time, dt[i]) @@ -91,7 +91,7 @@ def add_spacer_states(self, sma=None, next_state="exit"): ) sma.add_state( state_name=f"spacer_low_{i:02d}", - state_timer=dt[i], + state_timer=dt[i] - self.tup, state_change_conditions={"Tup": next_loop}, output_actions=[], ) From c72e1df205f02c3298dfd8cf83bb5d4f7388f9e9 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 8 Nov 2022 15:01:41 +0000 Subject: [PATCH 8/9] spacer detection method --- iblrig/spacer.py | 20 ++++++++++++++++++++ test_iblrig/test_spacers.py | 15 +++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/iblrig/spacer.py b/iblrig/spacer.py index 1c6301b59..e39283c42 100644 --- a/iblrig/spacer.py +++ b/iblrig/spacer.py @@ -95,3 +95,23 @@ def add_spacer_states(self, sma=None, next_state="exit"): state_change_conditions={"Tup": next_loop}, output_actions=[], ) + + def find_spacers(self, signal, threshold=0.9, fs=1000): + """ + Find spacers in a voltage time serie. Assumes that the signal is a digital signal between 0 and 1 + :param signal: + :param threshold: + :param fs: + :return: + """ + template = self.generate_template(fs=fs) + xcor = np.correlate(signal, template, mode="full") / np.sum(template) + idetect = np.where(xcor > threshold)[0] + iidetect = np.cumsum(np.diff(idetect, prepend=0) > 1) + nspacers = iidetect[-1] + tspacer = np.zeros(nspacers) + for i in range(nspacers): + ispacer = idetect[iidetect == i + 1] + imax = np.argmax(xcor[ispacer]) + tspacer[i] = (ispacer[imax] - template.size + 1) / fs + return tspacer diff --git a/test_iblrig/test_spacers.py b/test_iblrig/test_spacers.py index 701da9111..87d217c78 100644 --- a/test_iblrig/test_spacers.py +++ b/test_iblrig/test_spacers.py @@ -17,3 +17,18 @@ def test_spacer(self): ac[sig.size-100: sig.size + 100] = 0 # remove the main peak # the autocorrelation side lobes should be less than 30% assert np.max(ac) < .3 + + def test_find_spacers(self): + """ + Generates a fake signal with 2 spacers and finds them + :return: + """ + fs = 1000 + spacer = Spacer(dt_start=.02, dt_end=.4, n_pulses=8, tup=.05) + start_times = [4.38, 96.58] + template = spacer.generate_template(fs) + signal = np.zeros(int(start_times[-1] * fs + template.size * 2)) + for start_time in start_times: + signal[int(start_time * fs): int(start_time * fs) + template.size] = template + spacer_times = spacer.find_spacers(signal, fs=fs) + np.testing.assert_allclose(spacer_times, start_times) From 1de28aabcf1b6588d5b2f32da8cada9f3c81e819 Mon Sep 17 00:00:00 2001 From: Michele Fabbri Date: Tue, 8 Nov 2022 15:43:54 +0000 Subject: [PATCH 9/9] 7.1.0 release prep --- .github/workflows/main.yml | 1 + iblrig/__init__.py | 2 +- release_notes.md | 9 ++++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac7576b5c..eab9236e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,6 +64,7 @@ jobs: python -m unittest test_pybpod_config.py # python -m unittest test_scripts.py python -m unittest test_start_pybpod.py + python -m unittest test_spacers.py python -m unittest test_task.py # python -m unittest discover diff --git a/iblrig/__init__.py b/iblrig/__init__.py index 4d593b4c2..7ddf1ad5a 100644 --- a/iblrig/__init__.py +++ b/iblrig/__init__.py @@ -1,4 +1,4 @@ -__version__ = "7.0.5" +__version__ = "7.1.0" import logging import colorlog diff --git a/release_notes.md b/release_notes.md index 4cd726148..48b06fd68 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,9 +1,12 @@ # **Release notes** +## **Release Notes 7.1.0** +- spacer utility feature added for chaining multiple tasks + ## **Release Notes 7.0.5** - - minor fix to windows pathing for experiment description gui to grab alyx username - - removal of unnecessary ONE version check in purge_rig_data script - - added bpod BNC1 output to several training tasks +- minor fix to windows pathing for experiment description gui to grab alyx username +- removal of unnecessary ONE version check in purge_rig_data script +- added bpod BNC1 output to several training tasks ## **Release Notes 7.0.4** - modified call to experiment description gui to pass alyx username and subject information