Merge branch 'main' into zapit

.github/workflows/checks.yaml

@@ -0,0 +1,37 @@
+name: Version Check
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: 'pyproject.toml'
+    - name: Install dependencies
+      run: |
+        pip install --editable .
+        pip install packaging
+    - name: Fetch versions
+      run: |
+        git fetch origin --tags
+        previous_version=$(git describe --tags --match="[0-9]*" origin/main)
+        latest_version=$(pip show project_extraction | awk '/^Version: / {sub("^Version: ", ""); print}')
+        echo "Version tag on main: $previous_version"
+        echo "Version tag on this branch: $latest_version"
+        echo "PREVIOUS_VERSION=$previous_version" >> $GITHUB_ENV
+        echo "LATEST_VERSION=$latest_version" >> $GITHUB_ENV
+    - name: Assert version
+      run: |
+        python <<EOP
+        import sys; from packaging import version
+        sys.exit(int(version.parse("$PREVIOUS_VERSION") >= version.parse("$LATEST_VERSION")))
+        EOP

.github/workflows/deploy.yaml

@@ -0,0 +1,38 @@
+name: Version Tag
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.head_ref }}
+        fetch-depth: 0
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: 'pyproject.toml'
+    - name: Install dependencies
+      run: |
+        pip install --editable .
+    - name: Fetch version
+      run: |
+        latest_version=$(pip show project_extraction | awk '/^Version: / {sub("^Version: ", ""); print}')
+        echo "LATEST_VERSION=$latest_version" >> $GITHUB_ENV
+    - name: Create tag
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "GitHub Action"
+        git tag -a "$LATEST_VERSION" HEAD -m "Version v$LATEST_VERSION"
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        force: true
+        tags: true

README.md

@@ -1,5 +1,5 @@
 # project_extraction
-Extraction code for IBL satellite projects eg. U19 / personal projects
+Extraction code for IBL satellite projects e.g. U19 / personal projects
 
 ## Installation
 Clone the repository, and install in place
@@ -8,4 +8,9 @@ pip install -e .
 ```
 
 ## Reference
-`ibllib/io/extractors/extractor_types.json`
+
+- `projects/extractor_types.json` - (DEPRECATED) For adding custom extractors for legacy pipeline tasks
+- `projects/task_extractor_map.json` - Map custom task protocols to Bpod trials extractor class
+- `projects/task_type_procedures.json` - Associate Alyx procedures to a custom task protocol
+- `projects/_template.py` - Example for creating a custom Bpod extractor, QC or DAQ sync task
+- `projects/_extraction_tasks.py` - Where to import pipeline tasks so they are readable by ibllib

iblrig_custom_tasks/nate_optoBiasedChoiceWorld/task.py

@@ -60,7 +60,6 @@ def add_state(self, **kwargs):
 
 class Session(BiasedChoiceWorldSession):
     protocol_name = 'nate_optoBiasedChoiceWorld'
-    extractor_tasks = ['TrialRegisterRaw', 'ChoiceWorldTrials', 'TrainingStatus']
 
     def __init__(
         self,
@@ -77,6 +76,11 @@ def __init__(
         self.task_params['OPTO_STOP_STATES'] = opto_stop_states
         self.task_params['PROBABILITY_OPTO_STIM'] = probability_opto_stim
 
+        # loads in the settings in order to determine the main sync and thus the pipeline extractor tasks
+        is_main_sync = self.hardware_settings.get('MAIN_SYNC', False)
+        trials_task = 'OptoTrialsBpod' if is_main_sync else 'OptoTrialsNidq'
+        self.extractor_tasks = ['TrialRegisterRaw', trials_task, 'TrainingStatus']
+
         # generates the opto stimulation for each trial
         self.trials_table['opto_stimulation'] = np.random.choice(
             [0, 1],

iblrig_custom_tasks/nate_optoBiasedChoiceWorld/test_nate_optoBiasedChoiceWorld.py

@@ -7,3 +7,4 @@
 
 
 assert task.task_params.get('PROBABILITY_OPTO_STIM', None) is not None
+assert any(t.startswith('OptoTrials') for t in task.extractor_tasks or [])

iblrig_custom_tasks/samuel_cuedBiasedChoiceWorld/task.py

@@ -3,6 +3,7 @@
 from pybpodapi.protocol import StateMachine
 
 import iblrig.misc
+from iblrig.path_helper import load_pydantic_yaml, HardwareSettings
 from iblrig.base_choice_world import BiasedChoiceWorldSession
 from iblrig.hardware import SOFTCODE
 from iblutil.util import setup_logger
@@ -14,12 +15,19 @@
 
 
 class Session(BiasedChoiceWorldSession):
-
-    protocol_name = "samuel_cuedBiasedChoiceWorld"
-    extractor_tasks = ['TrialRegisterRaw', 'ChoiceWorldTrials', 'TrainingStatus']  # SP not sure what extractors to put here
 
-    def __init__(self, *args, delay_secs=0, **kwargs): #SP _init_ should be the same as biasedChoiceWorld, so should it be specified?
+    protocol_name = 'samuel_cuedBiasedChoiceWorld'
+
+    def __init__(self, *args, delay_secs=0, **kwargs):  #SP _init_ should be the same as biasedChoiceWorld, so should it be specified?
+        # loads in the settings in order to determine the main sync and thus the pipeline extractor tasks
+        hardware_settings = load_pydantic_yaml(HardwareSettings, kwargs.get('file_hardware_settings'))
+        hardware_settings.update(kwargs.get('hardware_settings', {}))
+        is_main_sync = hardware_settings.get('MAIN_SYNC', False)
+        trials_task = 'CuedBiasedTrials' if is_main_sync else 'CuedBiasedTrialsTimeline'
+        self.extractor_tasks = ['TrialRegisterRaw', trials_task, 'TrainingStatus']
+
         super().__init__(**kwargs)
+
         self.task_params["SESSION_DELAY_START"] = delay_secs
         # init behaviour data
         self.movement_left = self.device_rotary_encoder.THRESHOLD_EVENTS[
@@ -203,7 +211,7 @@ def get_state_machine_trial(self, i):
         return sma
 
 
-if __name__ == "__main__":  # pragma: no cover
+if __name__ == '__main__':  # pragma: no cover
     kwargs = iblrig.misc.get_task_arguments(parents=[Session.extra_parser()])
     sess = Session(**kwargs)
     sess.run()

iblrig_custom_tasks/samuel_cuedBiasedChoiceWorld/task_validation.py

@@ -1,4 +1,4 @@
 # check that samuel_cuedBiasedChoiceWorld runs as intended
 # 1. verify time delay between audio cue and stimOn is ~equal to the INTERACTIVE_DELAY specified
 # 2. verify that the readout from rotary encoder is uninterrupted?
-# 3. anything else?
+# 3. anything else?

iblrig_custom_tasks/samuel_cuedBiasedChoiceWorld/test_samuel_cuedBiasedChoiceWorld.py

@@ -5,4 +5,4 @@
 task = Session(subject='toto')
 
 
-# todo olivier: put the code to do the graph and write the doc
+# todo olivier: put the code to do the graph and write the doc

projects/_template.py

@@ -1,9 +1,85 @@
 from collections import OrderedDict
 
-from ibllib.pipes import tasks
+from ibllib.pipes.tasks import Pipeline
+from ibllib.pipes.behavior_tasks import ChoiceWorldTrialsBpod
+import ibllib.io.raw_data_loaders as raw
+from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes
 
 
-class TemplateTask(tasks.Task):
+class TemplateBpodTrialsExtractor(BaseBpodTrialsExtractor):
+    """
+    Extract ALF trials files for a custom Bpod protocol.
+
+    Attributes
+    ----------
+    bpod_trials : list of dict
+        The Bpod trials loaded from the _iblrig_taskData.raw.jsonable file.
+    settings : dict
+        The Bpod settings loaded from the _iblrig_taskSettings.raw.json file.
+    session_path : pathlib.Path
+        The absolute session path.
+    task_collection : str
+        The raw task data collection
+
+    """
+    var_names = ('laser_intervals', 'laser_probability', 'intervals')
+    """tuple of str: The variable names of each extracted dataset. The dict returned by `_extract`
+    must use the values in `var_names` as its keys.
+    """
+
+    save_names = ('_ibl_trials.laserIntervals.npy', None, '_ibl_trials.intervals.npy')
+    """tuple of str: The dataset filenames of the var_names returned by `_extract`. The tuple length
+    must match `var_names`. None values are not automatically saved to file. This may be useful for
+    extracting data you don't with to save, but will use for QC or debugging.
+    """
+
+    def _extract(self, extractor_classes=None, **kwargs) -> dict:
+        """
+        Extract the Bpod trial events.
+
+        Saving of the datasets to file is handled by the superclass.
+
+        Returns
+        -------
+        dict
+            A dictionary of Bpod trial events. The keys are defined in the `var_names` attribute.
+
+        See Also
+        --------
+        ibllib.io.extractors.habituation_trials.HabituationTrials - A good example of how trials
+            data can be extracted from raw data.
+
+        Examples
+        --------
+        Get all detected TTLs. These should be stored for QC purposes
+
+        >>> self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials)
+
+        These are the frame2TTL pulses as a list of lists, one per trial
+
+        >>> ttls = [raw.get_port_events(tr, 'BNC1') for tr in self.bpod_trials]
+
+        Extract datasets common to your adapted protocol, e.g. contrast, stim on, feedback, etc.
+
+        >>> from ibllib.io.extractors.biased_trials import ContrastLR
+        >>> from ibllib.io.extractors.training_trials import FeedbackTimes, StimOnTriggerTimes, GoCueTimes
+        >>> training = [ContrastLR, FeedbackTimes, GoCueTimes, StimOnTriggerTimes]
+        >>> out, _ = run_extractor_classes(
+        ...     training, session_path=self.session_path, save=False,  bpod_trials=self.bpod_trials,
+        ...     settings=self.settings, task_collection=self.task_collection)
+        """
+        ...
+
+
+class TemplateTask(ChoiceWorldTrialsBpod):
+    """A template behaviour task.
+
+    If the task protocol is a simple Bpod-only task (without an extra DAQ), you do not need a
+    separate behaviour task. Instead, create a new ibllib.io.extractors.BaseBpodTrialsExtractor
+    subclass (see above). You may need to create a custom Task if you want to run your own QC,
+    however for this you can simply overload the `run_qc` method with your preferred QC class in
+    the kwargs (see `projects.samuel_cuedBiasedChoiceWorld` for example).
+    """
     cpu = 1
     io_charge = 90
     level = 1
@@ -12,11 +88,77 @@ class TemplateTask(tasks.Task):
         ('_ibl_trials.laserProbability.npy', 'alf', True),
         ('_ibl_trials.intervals.npy', 'alf', True)]
 
-    def _run(self):
-        pass
+    def extract_behavior(self, **kwargs):
+        """Extract the Bpod trials data.
+
+        Parameters
+        ----------
+        kwargs
+
+        Returns
+        -------
+        dict
+            The extracted trials datasets.
+        list of pathlib.Path
+            The saved dataset filepaths.
+        """
+        # First determine the extractor from the task protocol
+        bpod_trials, out_files = ChoiceWorldTrialsBpod._extract_behaviour(self, save=False, **kwargs)
+
+        ...  # further trials manipulation, etc.
+        """
+        Sync Bpod trials to DAQ, etc. For syncing trials data to a DAQ, see
+        ibllib.io.extractors.ephys_fpga.TrialsFpga and ibllib.pipes.behavior_tasks.ChoiceWorldTrialsNidq.
+        If using such a trials extractor, run it here and assign the object to `self.extractor`.
+        This can be accessed by `run_qc` to access the loaded raw IO data.
+        """
+        dsets = bpod_trials
+
+        return dsets, out_files
+
+    def run_qc(self, trials_data=None, update=True):
+        """
+        Run task QC.
+
+        Parameters
+        ----------
+        trials_data : dict
+            The extracted trials datasets.
+        update : bool
+            If True, updates Alyx with the QC outcomes.
+
+        Returns
+        -------
+        ibllib.qc.base.QC
+            The QC object.
+        """
+        if not self.extractor or trials_data is None:
+            trials_data, _ = self.extract_behaviour(save=False)
+        if not trials_data:
+            raise ValueError('No trials data found')
+
+        # Compile task data for QC
+        qc_extractor = TaskQCExtractor(self.session_path, lazy=True, sync_collection=self.sync_collection, one=self.one,
+                                       sync_type=self.sync, task_collection=self.collection)
+        qc_extractor.data = qc_extractor.rename_data(trials_data)
+        if type(self.extractor).__name__ == 'HabituationTrials':
+            qc = HabituationQC(self.session_path, one=self.one, log=_logger)
+        else:
+            qc = TaskQC(self.session_path, one=self.one, log=_logger)
+            qc_extractor.wheel_encoding = 'X1'
+        qc_extractor.settings = self.extractor.settings
+        qc_extractor.frame_ttls, qc_extractor.audio_ttls = load_bpod_fronts(
+            self.session_path, task_collection=self.collection)
+        qc.extractor = qc_extractor
+
+        # Aggregate and update Alyx QC fields
+        namespace = 'task' if self.protocol_number is None else f'task_{self.protocol_number:02}'
+        qc.run(update=update, namespace=namespace)
+        return qc
 
 
-class TemplatePipeline(tasks.Pipeline):
+class TemplatePipeline(Pipeline):
+    """(DEPRECATED) An optional legacy task pipeline."""
     label = __name__
 
     def __init__(self, session_path=None, **kwargs):

projects/base.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import logging
 import importlib
+import warnings
 
 from ibllib.pipes import tasks
 
@@ -10,15 +11,17 @@
 
 def get_pipeline(task_type) -> tasks.Pipeline:
     """
-    Get the pipeline Task from task type - returns None if the task is not found
+    (DEPRECATED) Get the pipeline Task from task type - returns None if the task is not found.
+
     :param task_type: string that should match the module name
     :return:
     """
+    warnings.warn('get_pipeline is deprecated. Use instructions in extraction_tasks.py instead.', DeprecationWarning)
     if isinstance(task_type, Path):
         task_type = get_task_extractor_type(task_type)
     try:
         mdl = importlib.import_module(f"projects.{task_type}")
     except ModuleNotFoundError:
-        _logger.error({f"Import error: projects.{task_type} not found: does this project exists ?"})
+        _logger.error({f"Import error: projects.{task_type} not found: does this project exist?"})
         return
     return mdl.__pipeline__

projects/extraction_tasks.py

@@ -1 +1,10 @@
+"""
+Custom behaviour extractor tasks for personal projects.
+
+The task class name(s) should be added to the
+
+NB: This may need changing in the future if one of these modules requires optional dependencies.
+"""
 from projects.neuromodulators import ChoiceWorldNeuromodulators
+from projects.samuel_cuedBiasedChoiceWorld import CuedBiasedTrials, CuedBiasedTrialsTimeline
+from projects.nate_optoBiasedChoiceWorld import OptoTrialsNidq, OptoTrialsBpod