From 15a935d5a26f506942af99d1fd1cb477850cd8be Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Tue, 5 Mar 2024 11:03:56 -0500 Subject: [PATCH 1/6] Change nate_opto Bpod extractor --- projects/task_extractor_map.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/task_extractor_map.json b/projects/task_extractor_map.json index 9b5d0dd..d10c0d4 100644 --- a/projects/task_extractor_map.json +++ b/projects/task_extractor_map.json @@ -1,4 +1,4 @@ { "samuel_cuedBiasedChoiceWorld": "BiasedTrials", - "nate_optoBiasedChoiceWorld": "EphysTrials" + "nate_optoBiasedChoiceWorld": "BiasedTrials" } \ No newline at end of file From 0eb23a98bf28709f2be623c7ddcb6aad2bb035c5 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Tue, 5 Mar 2024 16:28:20 -0500 Subject: [PATCH 2/6] Extractor for laser stimulation intervals in Nate's task --- .../nate_optoBiasedChoiceWorld/task.py | 6 +- .../test_nate_optoBiasedChoiceWorld.py | 1 + projects/extraction_tasks.py | 1 + projects/nate_optoBiasedChoiceWorld.py | 58 +++++++++++++++++++ projects/task_extractor_map.json | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 projects/nate_optoBiasedChoiceWorld.py diff --git a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py index 71ea719..5c27bfc 100644 --- a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py +++ b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py @@ -45,7 +45,6 @@ def add_state(self, **kwargs): class Session(BiasedChoiceWorldSession): protocol_name = 'nate_optoBiasedChoiceWorld' - extractor_tasks = ['TrialRegisterRaw', 'ChoiceWorldTrials', 'TrainingStatus'] def __init__( self, @@ -63,6 +62,11 @@ def __init__( self.task_params['OPTO_STIM_STATES'] = opto_stim_states self.task_params['PROBABILITY_OPTO_STIM'] = probability_opto_stim + # loads in the settings in order to determine the main sync and thus the pipeline extractor tasks + is_main_sync = self.hardware_settings.get('MAIN_SYNC', False) + trials_task = 'OptoTrialsBpod' if is_main_sync else 'OptoTrialsNidq' + self.extractor_tasks = ['TrialRegisterRaw', trials_task, 'TrainingStatus'] + # generates the opto stimulation for each trial self.trials_table['opto_stimulation'] = np.random.choice( [0, 1], p=[1 - probability_opto_stim, probability_opto_stim], size=NTRIALS_INIT diff --git a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/test_nate_optoBiasedChoiceWorld.py b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/test_nate_optoBiasedChoiceWorld.py index 0128d9d..1f1570e 100644 --- a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/test_nate_optoBiasedChoiceWorld.py +++ b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/test_nate_optoBiasedChoiceWorld.py @@ -7,3 +7,4 @@ assert task.task_params.get('PROBABILITY_OPTO_STIM', None) is not None +assert any(t.startswith('OptoTrials') for t in task.extractor_tasks or []) diff --git a/projects/extraction_tasks.py b/projects/extraction_tasks.py index 906be12..c9c32c1 100644 --- a/projects/extraction_tasks.py +++ b/projects/extraction_tasks.py @@ -7,3 +7,4 @@ """ from projects.neuromodulators import ChoiceWorldNeuromodulators from projects.samuel_cuedBiasedChoiceWorld import CuedBiasedTrials, CuedBiasedTrialsTimeline +from projects.nate_optoBiasedChoiceWorld import OptoTrialsNidq, OptoTrialsBpod diff --git a/projects/nate_optoBiasedChoiceWorld.py b/projects/nate_optoBiasedChoiceWorld.py new file mode 100644 index 0000000..22df67e --- /dev/null +++ b/projects/nate_optoBiasedChoiceWorld.py @@ -0,0 +1,58 @@ +"""Bpod extractor for nate_optoBiasedChoiceWorld task. + +This is the same as biasedChoiceWorld with the addition of one dataset, `laserStimulation.intervals`; The times the +laser was on. + +The pipeline task subclasses, OptoTrialsBpod and OptoTrialsNidq, aren't strictly necessary. They simply assert that the +laserStimulation datasets were indeed saved and registered by the Bpod extractor class. +""" +import numpy as np +import ibllib.io.raw_data_loaders as raw +from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes +from ibllib.io.extractors.bpod_trials import BiasedTrials +from ibllib.pipes.behavior_tasks import ChoiceWorldTrialsNidq, ChoiceWorldTrialsBpod + + +class OptoTrialsBpod(ChoiceWorldTrialsBpod): + """Extract bpod only trials and laser stimulation data.""" + + @property + def signature(self): + signature = super().signature + signature['output_files'].append(('*laserStimulation.intervals.npy', self.output_collection, True)) + return signature + + +class OptoTrialsNidq(ChoiceWorldTrialsNidq): + """Extract trials and laser stimulation data aligned to NI-DAQ clock.""" + + @property + def signature(self): + signature = super().signature + signature['output_files'].append(('*laserStimulation.intervals.npy', self.output_collection, True)) + return signature + + +class TrialsOpto(BaseBpodTrialsExtractor): + var_names = BiasedTrials.var_names + ('laser_intervals',) + save_names = BiasedTrials.save_names + ('_ibl_laserStimulation.intervals.npy',) + + def _extract(self, extractor_classes=None, **kwargs) -> dict: + assert {'OPTO_STOP_STATES', 'OPTO_TTL_STATES', 'PROBABILITY_OPTO_STIM'} <= set(self.settings) + # Get all detected TTLs. These are stored for QC purposes + self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials) + # Extract common biased choice world datasets + out, _ = run_extractor_classes( + [BiasedTrials], session_path=self.session_path, bpod_trials=self.bpod_trials, + settings=self.settings, save=False, task_collection=self.task_collection) + + # Extract laser dataset + out['laser_intervals'] = np.full((len(self.bpod_trials), 2), np.nan) + for i, trial in enumerate(self.bpod_trials): + states = trial['behavior_data']['States timestamps'] + # Assumes one of these states per trial: takes the timestamp of the first matching state + start = next((v[0][0] for k, v in states.items() if k in self.settings['OPTO_TTL_STATES']), np.nan) + stop = next((v[0][0] for k, v in states.items() if k in self.settings['OPTO_STOP_STATES']), np.nan) + out['laser_intervals'][i, :] = (start, stop) + + return {k: out[k] for k in self.var_names} # Ensures all datasets present and ordered diff --git a/projects/task_extractor_map.json b/projects/task_extractor_map.json index d10c0d4..0de06cc 100644 --- a/projects/task_extractor_map.json +++ b/projects/task_extractor_map.json @@ -1,4 +1,4 @@ { "samuel_cuedBiasedChoiceWorld": "BiasedTrials", - "nate_optoBiasedChoiceWorld": "BiasedTrials" + "nate_optoBiasedChoiceWorld": "projects.nate_optoBiasedChoiceWorld.TrialsOpto" } \ No newline at end of file From ca0593f0dea289ed1df1d25ab8d96aa2fee74c7c Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 6 Mar 2024 00:15:53 -0500 Subject: [PATCH 3/6] Support older nate_optoBiased version --- projects/nate_optoBiasedChoiceWorld.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/nate_optoBiasedChoiceWorld.py b/projects/nate_optoBiasedChoiceWorld.py index 22df67e..39b2b37 100644 --- a/projects/nate_optoBiasedChoiceWorld.py +++ b/projects/nate_optoBiasedChoiceWorld.py @@ -38,21 +38,26 @@ class TrialsOpto(BaseBpodTrialsExtractor): save_names = BiasedTrials.save_names + ('_ibl_laserStimulation.intervals.npy',) def _extract(self, extractor_classes=None, **kwargs) -> dict: - assert {'OPTO_STOP_STATES', 'OPTO_TTL_STATES', 'PROBABILITY_OPTO_STIM'} <= set(self.settings) + settings = self.settings.copy() + if 'OPTO_STIM_STATES' in settings: + # It seems older versions did not distinguish start and stop states + settings['OPTO_TTL_STATES'] = settings['OPTO_STIM_STATES'][:1] + settings['OPTO_STOP_STATES'] = settings['OPTO_STIM_STATES'][1:] + assert {'OPTO_STOP_STATES', 'OPTO_TTL_STATES', 'PROBABILITY_OPTO_STIM'} <= set(settings) # Get all detected TTLs. These are stored for QC purposes self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials) # Extract common biased choice world datasets out, _ = run_extractor_classes( [BiasedTrials], session_path=self.session_path, bpod_trials=self.bpod_trials, - settings=self.settings, save=False, task_collection=self.task_collection) + settings=settings, save=False, task_collection=self.task_collection) # Extract laser dataset out['laser_intervals'] = np.full((len(self.bpod_trials), 2), np.nan) for i, trial in enumerate(self.bpod_trials): states = trial['behavior_data']['States timestamps'] # Assumes one of these states per trial: takes the timestamp of the first matching state - start = next((v[0][0] for k, v in states.items() if k in self.settings['OPTO_TTL_STATES']), np.nan) - stop = next((v[0][0] for k, v in states.items() if k in self.settings['OPTO_STOP_STATES']), np.nan) + start = next((v[0][0] for k, v in states.items() if k in settings['OPTO_TTL_STATES']), np.nan) + stop = next((v[0][0] for k, v in states.items() if k in settings['OPTO_STOP_STATES']), np.nan) out['laser_intervals'][i, :] = (start, stop) return {k: out[k] for k in self.var_names} # Ensures all datasets present and ordered From 2da8f2a992bf8ea37a6b20b9c4e7dab720772dbc Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 6 Mar 2024 10:39:32 -0500 Subject: [PATCH 4/6] Only extract periods where laser is on --- iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py | 2 +- projects/nate_optoBiasedChoiceWorld.py | 7 ++++--- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py index 5c27bfc..a448a8e 100644 --- a/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py +++ b/iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py @@ -110,7 +110,7 @@ def extra_parser(): default=DEFAULTS['OPTO_STIM_STATES'], nargs='+', type=str, - help=f'list of the state machine states where opto stim should be delivered', + help='list of the state machine states where opto stim should be delivered', ) return parser diff --git a/projects/nate_optoBiasedChoiceWorld.py b/projects/nate_optoBiasedChoiceWorld.py index 39b2b37..0d97e15 100644 --- a/projects/nate_optoBiasedChoiceWorld.py +++ b/projects/nate_optoBiasedChoiceWorld.py @@ -52,12 +52,13 @@ def _extract(self, extractor_classes=None, **kwargs) -> dict: settings=settings, save=False, task_collection=self.task_collection) # Extract laser dataset - out['laser_intervals'] = np.full((len(self.bpod_trials), 2), np.nan) - for i, trial in enumerate(self.bpod_trials): + laser_intervals = [] + for trial in filter(lambda t: t['opto_stimulation'], self.bpod_trials): states = trial['behavior_data']['States timestamps'] # Assumes one of these states per trial: takes the timestamp of the first matching state start = next((v[0][0] for k, v in states.items() if k in settings['OPTO_TTL_STATES']), np.nan) stop = next((v[0][0] for k, v in states.items() if k in settings['OPTO_STOP_STATES']), np.nan) - out['laser_intervals'][i, :] = (start, stop) + laser_intervals.append((start, stop)) + out['laser_intervals'] = np.array(laser_intervals, dtype=np.float64) return {k: out[k] for k in self.var_names} # Ensures all datasets present and ordered diff --git a/pyproject.toml b/pyproject.toml index f461ff4..5ff5c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "project_extraction" -version = "0.2.0" +version = "0.2.1" description = "Custom extractors for satellite tasks" dynamic = [ "readme" ] keywords = [ "IBL", "neuro-science" ] From 6057ce9646fda9f276b39969c56f222c0b5e7fcb Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Wed, 6 Mar 2024 13:32:23 -0500 Subject: [PATCH 5/6] Workflow to assert version increase on merge --- .github/workflows/main.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/main.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..4682789 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,36 @@ +name: Version Check + +on: + pull_request: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and check version + run: | + git fetch --tags + python setup.py sdist bdist_wheel + current=$(git describe --tags --match="[0-9]*" HEAD) + new=$(pip show ibllib | awk '/^Version: / {sub("^Version: ", ""); print}') + python <= version.parse("$new"))) + EOP + RC=$? + echo "Exit code $RC" + - name: Push tag + run: | + git tag -a "$new" HEAD + git push origin "$new" From d6f3779a916e9eac7fd8814c68c3dc2ea6445d54 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Wed, 6 Mar 2024 14:54:34 -0500 Subject: [PATCH 6/6] Version check (#9) * Fix workflow * Only create tag on push * Push tags in deploy workflow --- .github/workflows/checks.yaml | 37 ++++++++++++++++++++++++++++++++++ .github/workflows/deploy.yaml | 38 +++++++++++++++++++++++++++++++++++ .github/workflows/main.yaml | 36 --------------------------------- pyproject.toml | 2 +- 4 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/checks.yaml create mode 100644 .github/workflows/deploy.yaml delete mode 100644 .github/workflows/main.yaml diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml new file mode 100644 index 0000000..4d7eeeb --- /dev/null +++ b/.github/workflows/checks.yaml @@ -0,0 +1,37 @@ +name: Version Check + +on: + pull_request: + branches: + - main + +jobs: + checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'pyproject.toml' + - name: Install dependencies + run: | + pip install --editable . + pip install packaging + - name: Fetch versions + run: | + git fetch origin --tags + previous_version=$(git describe --tags --match="[0-9]*" origin/main) + latest_version=$(pip show project_extraction | awk '/^Version: / {sub("^Version: ", ""); print}') + echo "Version tag on main: $previous_version" + echo "Version tag on this branch: $latest_version" + echo "PREVIOUS_VERSION=$previous_version" >> $GITHUB_ENV + echo "LATEST_VERSION=$latest_version" >> $GITHUB_ENV + - name: Assert version + run: | + python <= version.parse("$LATEST_VERSION"))) + EOP diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..8c53804 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,38 @@ +name: Version Tag + +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'pyproject.toml' + - name: Install dependencies + run: | + pip install --editable . + - name: Fetch version + run: | + latest_version=$(pip show project_extraction | awk '/^Version: / {sub("^Version: ", ""); print}') + echo "LATEST_VERSION=$latest_version" >> $GITHUB_ENV + - name: Create tag + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git tag -a "$LATEST_VERSION" HEAD -m "Version v$LATEST_VERSION" + - name: Push changes + uses: ad-m/github-push-action@master + with: + force: true + tags: true diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml deleted file mode 100644 index 4682789..0000000 --- a/.github/workflows/main.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Version Check - -on: - pull_request: - branches: - - main - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and check version - run: | - git fetch --tags - python setup.py sdist bdist_wheel - current=$(git describe --tags --match="[0-9]*" HEAD) - new=$(pip show ibllib | awk '/^Version: / {sub("^Version: ", ""); print}') - python <= version.parse("$new"))) - EOP - RC=$? - echo "Exit code $RC" - - name: Push tag - run: | - git tag -a "$new" HEAD - git push origin "$new" diff --git a/pyproject.toml b/pyproject.toml index 5ff5c27..134f002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "project_extraction" -version = "0.2.1" +version = "0.2.1.post0" description = "Custom extractors for satellite tasks" dynamic = [ "readme" ] keywords = [ "IBL", "neuro-science" ]