Merge pull request #724 from int-brain-lab/iblrigv8dev

8.24.3

CHANGELOG.md

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+8.24.3
+------
+* fix: create the `raw_ephys_data` folder even if there are no probes (when running behavior sessions on ephys rig)
+* move some Qt related code to `iblqt` repository
+
 8.24.2
 ------
 * make Frame2TTL validation more robust

docs/source/faq.rst

@@ -121,3 +121,32 @@ Frame2TTL
       Look for a voltage step in Frame2TTL's output when the calibration routine switches from dark to light.
    #. If you *do* see the change in the TTL signal, the Bpod might be faulty. Try using a different Bpod unit.
    #. If you do *not* see the voltage step, the Frame2TTL might be faulty. Try using a different Frame2TTL unit.
+
+
+Move a mouse onto a previous training stage
+===========================================
+Training phases enfold according to an automated procedure, see the `mouse training protocol on Figshare <https://
+figshare.com/articles/preprint/A_standardized_and_reproducible_method_to_measure_decision-making
+_in_mice_Appendix_2_IBL_protocol_for_mice_training/11634729?file=38099442>`_ for a description of these phases.
+
+However, it is possible that one wants to overwrite the automated progression, and bring back an animal onto a previous
+training stage.
+
+In this case:
+
+1. Select the training protocol on the GUI
+2. Select the wanted training phase manually
+
+
+   .. figure:: img/training_phase_manual_update.png
+      :width: 100%
+      :class: with-border
+
+      Select the training phase manually using the arrows.
+
+On the next day of training, the "automatic" criteria will be computed assuming the training phase last used, i.e.
+the one you manually selected in this instance.
+
+For example, if you have a mouse on Training Phase 5, and move it manually to Phase 4 (which can be useful when
+trying to debias an animal), the next session on "automatic" mode will compute the stage progression from stage 4,
+and disregard the fact that the mouse was on Phase 5 beforehand.

docs/source/reference_developer_guide.rst

@@ -139,4 +139,4 @@ To write the documentation:
 To release the documentation onto the `website <https://int-brain-lab.github.io/iblrig>`_:
 
 * Wait for the next release, or
-* Manually trigger the GitHub action by clicking "Run Workflow" (select ``master``) `here <https://github.com/int-brain-lab/iblrig/actions/workflows/docs.yaml>`_
+* Manually trigger the GitHub action by clicking "Run Workflow" (select ``iblrigv8dev``) `here <https://github.com/int-brain-lab/iblrig/actions/workflows/documentation.yaml>`_

iblrig/__init__.py

@@ -6,7 +6,7 @@
 # 5) git tag the release in accordance to the version number below (after merge!)
 # >>> git tag 8.15.6
 # >>> git push origin --tags
-__version__ = '8.24.2'
+__version__ = '8.24.3'
 
 
 from iblrig.version_management import get_detailed_version_string

iblrig/base_tasks.py

@@ -1215,7 +1215,7 @@ def __init__(self, *_, remote_rigs=None, **kwargs):
         if isinstance(remote_rigs, list):
             # For now we flatten to list of remote rig names but could permit list of (name, URI) tuples
             remote_rigs = list(filter(None, flatten(remote_rigs)))
-            all_remote_rigs = net.get_remote_devices(iblrig_settings=kwargs.get('iblrig_settings', None))
+            all_remote_rigs = net.get_remote_devices(iblrig_settings=kwargs.get('iblrig_settings'))
             if not set(remote_rigs).issubset(all_remote_rigs.keys()):
                 raise ValueError('Selected remote rigs not in remote rigs list')
             remote_rigs = {k: v for k, v in all_remote_rigs.items() if k in remote_rigs}

iblrig/gui/tab_data.py

@@ -17,7 +17,7 @@
 )
 from PyQt5.QtWidgets import QHeaderView, QStyledItemDelegate, QWidget
 
-from iblrig.gui.tools import DataFrameTableModel
+from iblqt.core import DataFrameTableModel
 from iblrig.gui.ui_tab_data import Ui_TabData
 from iblrig.path_helper import get_local_and_remote_paths
 from iblrig.transfer_experiments import CopyState, SessionCopier
@@ -79,7 +79,7 @@ def __init__(self, *args, **kwargs):
 
         # create empty DataFrameTableModel
         data = pd.DataFrame(None, index=[], columns=[c.name for c in COLUMNS])
-        self.tableModel = DataFrameTableModel(df=data)
+        self.tableModel = DataFrameTableModel(dataFrame=data)
 
         # create filter proxy & assign it to view
         self.tableProxy = QSortFilterProxyModel()

iblrig/gui/tools.py

@@ -9,17 +9,12 @@
 from shutil import disk_usage
 from typing import Any
 
-import numpy as np
-import pandas as pd
 from PyQt5 import QtGui
 from PyQt5.QtCore import (
-    QAbstractTableModel,
-    QModelIndex,
     QObject,
     QRunnable,
     Qt,
     QThreadPool,
-    QVariant,
     pyqtProperty,
     pyqtSignal,
     pyqtSlot,
@@ -223,144 +218,6 @@ def run(self) -> None:
             self.signals.finished.emit()
 
 
-class DataFrameTableModel(QAbstractTableModel):
-    def __init__(self, *args, df: pd.DataFrame, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._dataFrame = df
-
-    def dataFrame(self):
-        return self._dataFrame
-
-    def setDataFrame(self, data_frame: pd.DataFrame):
-        self.beginResetModel()
-        self._dataFrame = data_frame.copy()
-        self.endResetModel()
-
-    dataFrame = pyqtProperty(pd.DataFrame, fget=dataFrame, fset=setDataFrame)
-
-    def headerData(self, section, orientation, role=...):
-        """
-        Get the header data for the specified section.
-
-        Parameters
-        ----------
-        section : int
-            The section index.
-        orientation : Qt.Orientation
-            The orientation of the header.
-        role : int, optional
-            The role of the header data.
-
-        Returns
-        -------
-        QVariant
-            The header data.
-        """
-        if role == Qt.DisplayRole:
-            if orientation == Qt.Horizontal:
-                return str(self._dataFrame.columns[section])
-            else:
-                return str(self._dataFrame.index[section])
-
-    def rowCount(self, parent=...):
-        """
-        Get the number of rows in the model.
-
-        Parameters
-        ----------
-        parent : QModelIndex, optional
-            The parent index.
-
-        Returns
-        -------
-        int
-            The number of rows.
-        """
-        if isinstance(parent, QModelIndex) and parent.isValid():
-            return 0
-        return self.dataFrame.shape[0]
-
-    def columnCount(self, parent=...):
-        """
-        Get the number of columns in the model.
-
-        Parameters
-        ----------
-        parent : QModelIndex, optional
-            The parent index.
-
-        Returns
-        -------
-        int
-            The number of columns.
-        """
-        if isinstance(parent, QModelIndex) and parent.isValid():
-            return 0
-        return self.dataFrame.shape[1]
-
-    def data(self, index, role=...):
-        """
-        Get the data for the specified index.
-
-        Parameters
-        ----------
-        index : QModelIndex
-            The index of the data.
-        role : int, optional
-            The role of the data.
-
-        Returns
-        -------
-        QVariant
-            The data for the specified index.
-        """
-        if index.isValid():
-            row = self._dataFrame.index[index.row()]
-            col = self._dataFrame.columns[index.column()]
-            dat = self._dataFrame.iloc[row][col]
-            if role == Qt.DisplayRole:
-                if isinstance(dat, np.generic):
-                    return dat.item()
-                return dat
-        return QVariant()
-
-    def sort(self, column, order=...):
-        """
-        Sort the data based on the specified column and order.
-
-        Parameters
-        ----------
-        column : int
-            The column index to sort by.
-        order : Qt.SortOrder, optional
-            The sort order.
-        """
-        self.layoutAboutToBeChanged.emit()
-        col_name = self._dataFrame.columns.values[column]
-        self._dataFrame.sort_values(by=col_name, ascending=order == Qt.AscendingOrder, inplace=True)
-        self._dataFrame.reset_index(inplace=True, drop=True)
-        self.layoutChanged.emit()
-
-    def setData(self, index, value, role=Qt.DisplayRole):
-        """
-        Set data at the specified index with the given value.
-
-        Parameters
-        ----------
-        index : QModelIndex
-            The index where the data will be set.
-        value : Any
-            The new value to be set at the specified index.
-        role : int, optional
-            The role of the data. Default is Qt.DisplayRole.
-        """
-        if index.isValid():
-            row = self._dataFrame.index[index.row()]
-            col = self._dataFrame.columns[index.column()]
-            self._dataFrame.at[row, col] = value
-            self.dataChanged.emit(index, index, [role])
-
-
 class RemoteDevicesListView(QListView):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

iblrig/test/test_hardware_mixins.py

@@ -109,12 +109,12 @@ def test_rotary_encoder_mixin(self):
             'RotaryEncoder1_3',
             'RotaryEncoder1_4',
         ]
-        assert {
+        assert session.device_rotary_encoder.THRESHOLD_EVENTS == {
             -35: 'RotaryEncoder1_1',
             35: 'RotaryEncoder1_2',
             -2: 'RotaryEncoder1_3',
             2: 'RotaryEncoder1_4',
-        } == session.device_rotary_encoder.THRESHOLD_EVENTS
+        }
         with self.assertRaises(ValueError):
             RotaryEncoderMixin.start_mixin_rotary_encoder(session)
 

iblrig/test/test_transfers.py

@@ -6,6 +6,9 @@
 from pathlib import Path
 from unittest import mock
 
+from packaging import version
+
+import ibllib
 import iblrig.commands
 import iblrig.path_helper
 import iblrig.raw_data_loaders
@@ -228,6 +231,52 @@ def test_behavior_ephys_video_copy(self):
         self.assertEqual(set(final_experiment_description['devices']['cameras'].keys()), {'left'})
         self.assertEqual(set(final_experiment_description['sync'].keys()), {'nidq'})
 
+    # Requires recent change to ibllib test fixture code supporting no probe ephys recording files
+    @unittest.skipIf(version.parse(ibllib.__version__) < version.parse('2.39'), 'ibllib < 2.39')
+    def test_ephys_no_probe(self):
+        """Test copying a session at ephys rig when no probes were used (DAQ only)."""
+        # First create a behavior session
+        task_kwargs = copy.deepcopy(self.session_kwargs)
+        task_kwargs['hardware_settings'].update(
+            {
+                'device_cameras': None,
+                'MAIN_SYNC': False,  # this is quite important for ephys sessions
+            }
+        )
+        session = _create_behavior_session(kwargs=task_kwargs, ntrials=50)
+        folder_session_ephys = Path(self.td.name).joinpath('ephys', 'Subjects', *session.paths.SESSION_FOLDER.parts[-3:])
+
+        # Create an ephys acquisition
+        n_probes = 0
+        # SpikeGLX then saves these files into the session folder
+        populate_raw_spikeglx(folder_session_ephys, model='3B', n_probes=n_probes)
+
+        # Test the copiers
+        sc = BehaviorCopier(session_path=session.paths.SESSION_FOLDER, remote_subjects_folder=session.paths.REMOTE_SUBJECT_FOLDER)
+        self.assertEqual('.status_pending', sc.glob_file_remote_copy_status().suffix)
+        self.assertEqual(1, sc.state)
+        sc.copy_collections()
+        self.assertEqual(2, sc.state)
+        self.assertEqual('.status_complete', sc.glob_file_remote_copy_status().suffix)
+        sc.copy_collections()
+        self.assertEqual(2, sc.state)
+        sc.finalize_copy(number_of_expected_devices=None)
+        self.assertEqual(2, sc.state)  # here we still don't have all devices so we stay in state 2
+
+        ec = EphysCopier(session_path=folder_session_ephys, remote_subjects_folder=session.paths.REMOTE_SUBJECT_FOLDER)
+        self.assertEqual(0, ec.state)
+        ec.initialize_experiment()
+        self.assertEqual(1, ec.state)
+        self.assertIn('sync', ec.experiment_description)
+        ec.copy_collections()
+        self.assertEqual(2, ec.state)
+        # this time it's all there and we move on
+        ec.finalize_copy(number_of_expected_devices=None)
+        self.assertEqual(3, ec.state)
+        final_experiment_description = session_params.read_params(ec.remote_session_path)
+        self.assertEqual(1, len(final_experiment_description['tasks']))
+        self.assertEqual(set(final_experiment_description['sync'].keys()), {'nidq'})
+
     def test_copy_snapshots(self):
         """Test copy of snapshots folder(s)."""
         # Create without task data

iblrig/transfer_experiments.py

@@ -553,8 +553,9 @@ def initialize_experiment(self, acquisition_description=None, nprobes=None, main
         self._experiment_description = acquisition_description
         super().initialize_experiment(acquisition_description=acquisition_description, **kwargs)
         # once the session folders have been initialized, create the probe folders
+        (ephys_path := self.session_path.joinpath('raw_ephys_data')).mkdir(exist_ok=True)
         for n in range(nprobes):
-            self.session_path.joinpath('raw_ephys_data', f'probe{n:02}').mkdir(exist_ok=True, parents=True)
+            ephys_path.joinpath(f'probe{n:02}').mkdir(exist_ok=True)
 
     def _copy_collections(self):
         """Here we overload the copy to be able to rename the probes properly and also create the insertions."""