More documentation for custom tasks

projects/_template.py

@@ -2,6 +2,25 @@
 
 For an example of how to modify Bpod trials extraction (with either Bpod only or unchanged DAQ time alignment) check out
 'projects.neuromodulators'.  For an example of custom task QC, see 'projects.samuel_cuedBiasedChoiceWorld'.
+
+===========
+Definitions
+===========
+
+- **Bpod trials extractor**: A class for extracting trials data from the raw session data.
+  This class saves trial and other session data to ALF dataset files. An extractor is associated to
+  a session via the task protocol name to extractor class name map file in `projects/task_extractor_map.json`.
+- **Pipeline task**: A pipeline task is instantiated by the preprocessing pipeline and uses the
+  Bpod trials extractor to extract the session datasets. It also performs DAQ time alignment, runs
+  QC checks and registers/uploads the data. It can also optionally download the raw session data
+  if re-running extractions from a remote location. A pipeline task is associated to a session via
+  the session class :py:attr:`iblrig.base_tasks.BaseSession.extractor_tasks` attribute.
+- **Session**: The iblrig Session class that defines a behaviour task protocol (e.g. the Bpod state
+  machine, task parameters, etc.)
+- **experiment description**: The file used by the pipeline to determine which protocol which run,
+  its pipeline extractor classes, raw data locations, and also which DAQ should be used for time
+  alignment. This file is generated automatically based on the iblrig hardware settings and session
+  protocol.
 """
 from collections import OrderedDict
 
@@ -15,6 +34,18 @@ class TemplateBpodTrialsExtractor(BaseBpodTrialsExtractor):
     """
     Extract ALF trials files for a custom Bpod protocol.
 
+    Bpod trials extractors should be a subclass of the
+    :class:`ibllib.io.extractors.base.BaseBpodTrialsExtractor` class and should be for extracting
+    Bpod trial events only. For separate DAQ time alignment, a second class should be used (see
+    :class:`ibllib.io.extractors.ephys_fpga.FpgaTrials` and
+    :class:`ibllib.io.extractors.mesoscope.TimelineTrials` for examples).
+
+    To associate this extractor to your task protocol, you must add it to the
+    `projects/task_extractor_map.json` file. Additionally you can associate `Alyx procedures`_ (e.g.
+    'Optical stimulation') to the protocol with the `projects/task_type_procedures.json` file.
+
+    .. _Alyx procedures: https://openalyx.internationalbrainlab.org/admin/actions/proceduretype/
+
     Attributes
     ----------
     bpod_trials : list of dict
@@ -24,7 +55,9 @@ class TemplateBpodTrialsExtractor(BaseBpodTrialsExtractor):
     session_path : pathlib.Path
         The absolute session path.
     task_collection : str
-        The raw task data collection
+        The raw task data collection, e.g. 'raw_task_data_00'.
+    default_path : str, pathlib.Path
+        The default output folder relative to `session_path` (default: 'alf').
 
     """
     var_names = ('laser_intervals', 'laser_probability', 'intervals')
@@ -42,7 +75,13 @@ def _extract(self, extractor_classes=None, **kwargs) -> dict:
         """
         Extract the Bpod trial events.
 
-        Saving of the datasets to file is handled by the superclass.
+        The :meth:`BaseBpodTrialsExtractor._extract` method should be subclassed for doing the
+        actual extraction, however it should be called indirectly via the
+        :meth:`BaseBpodTrialsExtractor.extract` method.  This superclass method has a `save` param
+        which when true (default) will save the `var_names` data as `save_names` in the
+        `default_path`.  The `_extract` method should return a map of `var_names` to extracted data,
+        and the superclass `extract` method will return these along with a list of full file paths
+        if save=True (otherwise None).
 
         Returns
         -------
@@ -56,6 +95,8 @@ def _extract(self, extractor_classes=None, **kwargs) -> dict:
 
         Examples
         --------
+        Below are some example snippets that you may want to put into the `_extract` method.
+
         Get all detected TTLs. These should be stored for QC purposes
 
         >>> self.frame2ttl, self.audio = raw.load_bpod_fronts(self.session_path, data=self.bpod_trials)
@@ -80,24 +121,57 @@ class TemplateTask(ChoiceWorldTrialsBpod):
     """A template behaviour task.
 
     If the task protocol is a simple Bpod-only task (without an extra DAQ), you do not need a
-    separate behaviour task. Instead, create a new ibllib.io.extractors.BaseBpodTrialsExtractor
+    separate behaviour task. Instead, create a new :class:`ibllib.io.extractors.BaseBpodTrialsExtractor`
     subclass (see above). You may need to create a custom Task if you want to run your own QC,
     however for this you can simply overload the `run_qc` method with your preferred QC class in
     the kwargs (see `projects.samuel_cuedBiasedChoiceWorld` for example).
+
+    To associate your task to the task protocol, add it to the
+    :py:attr:`iblrig.base_tasks.BaseSession.extractor_tasks` list in your Session class.  You can
+    have separate pipeline Task classes for each DAQ you use, in which case name your classes
+    whatever + sync label, e.g. 'TemplateTaskBpod', 'TemplateTaskNidq', 'TemplateTaskTimeline', etc.
+    and simply add 'TemplateTask' to the extractor_tasks list and the pipeline will import the
+    correct class depending on the DAQ used during that session.
     """
     cpu = 1
     io_charge = 90
     level = 1
-    signature = [
-        ('_ibl_trials.laserIntervals.npy', 'alf', True),
-        ('_ibl_trials.laserProbability.npy', 'alf', True),
-        ('_ibl_trials.intervals.npy', 'alf', True)]
 
-    def extract_behavior(self, **kwargs):
+    @property
+    def signature(self):
+        """Define input and outputs required for this task.
+
+        This property returns dict of input files (those expected in order to extract data), and
+        output files (those expected to be saved, registered and uploaded). The `collection` and
+        `output_collection` properties are typically set automatically based on the experiment
+        description file (defaults are 'raw_task_data_00' and 'alf', respectively).  The boolean
+        determines if the file is essential. An error if raised if these files aren't present
+        before or after running the task, respectively.
+
+        The input_files may have a forth, optional value: a boolean indicating whether the input
+        file should be registered and uploaded. This is useful if your task has extra raw data to
+        upload.
+        """
+        signature = {
+            'input_files': [
+                ('_iblrig_taskData.raw.*', self.collection, True),
+                ('_iblrig_taskSettings.raw.*', self.collection, True),
+                ('_iblrig_encoderEvents.raw*', self.collection, True),
+                ('_iblrig_encoderPositions.raw*', self.collection, True)],
+            'output_files': [
+                ('*trials.laserIntervals.npy', self.output_collection, True),
+                ('*trials.laserProbability.npy', self.output_collection, False),
+                ('*trials.intervals.npy', self.output_collection, False)]
+        }
+        return signature
+
+    def extract_behavior(self, save=True, **kwargs):
         """Extract the Bpod trials data.
 
         Parameters
         ----------
+        save : bool
+            Whether to save the extracted data to disk.
         kwargs
 
         Returns
@@ -108,7 +182,7 @@ def extract_behavior(self, **kwargs):
             The saved dataset filepaths.
         """
         # First determine the extractor from the task protocol
-        bpod_trials, out_files = ChoiceWorldTrialsBpod._extract_behaviour(self, save=False, **kwargs)
+        bpod_trials, out_files = ChoiceWorldTrialsBpod.extract_behaviour(self, save=save, **kwargs)
 
         ...  # further trials manipulation, etc.
         """

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name            = "project_extraction"
-version         = "0.2.6"
+version         = "0.2.7"
 description     = "Custom extractors for satellite tasks"
 dynamic         = [ "readme" ]
 keywords        = [ "IBL", "neuro-science" ]