wip test cases sorting

src/spikeglx.py

@@ -2,8 +2,6 @@
 import logging
 from pathlib import Path
 import re
-import shutil
-import time
 
 import numpy as np
 
@@ -16,7 +14,7 @@
 
 SAMPLE_SIZE = 2  # int16
 DEFAULT_BATCH_SIZE = 1e6
-_logger = logging.getLogger(__name__)
+_logger = logging.getLogger("ibllib")
 
 
 def _get_companion_file(sglx_file, pattern='.meta'):
@@ -279,9 +277,9 @@ def read(self, nsel=slice(0, 10000), csel=slice(None), sync=True):
         """
         if not self.is_open:
             raise IOError("Reader not open; call `open` before `read`")
-        if hasattr(self, '_raw_channel_order'):
-            csel = self._raw_channel_order[csel]
-        darray = self._raw[nsel, csel].astype(np.float32, copy=True)
+        if hasattr(self, 'raw_channel_order'):
+            csel = self.raw_channel_order[csel]
+        darray = self._raw[nsel, :].astype(np.float32, copy=True)[..., csel]
         darray *= self.channel_conversion_sample2v[self.type][csel]
         if sync:
             return darray, self.read_sync(nsel)
@@ -374,27 +372,6 @@ def compress_file(self, keep_original=True, **kwargs):
             self.file_bin = file_out
         return file_out
 
-    def decompress_to_scratch(self, scratch_dir=None):
-        """
-        Decompresses the file to a temporary directory
-        Copy over the metadata file
-        """
-        if scratch_dir is None:
-            bin_file = Path(self.file_bin).with_suffix('.bin')
-        else:
-            scratch_dir.mkdir(exist_ok=True, parents=True)
-            bin_file = Path(scratch_dir).joinpath(self.file_bin.name).with_suffix('.bin')
-            shutil.copy(self.file_meta_data, bin_file.with_suffix('.meta'))
-        if not bin_file.exists():
-            t0 = time.time()
-            _logger.info('File is compressed, decompressing to a temporary file...')
-            self.decompress_file(
-                keep_original=True, out=bin_file.with_suffix('.bin_temp'), check_after_decompress=False, overwrite=True
-            )
-            shutil.move(bin_file.with_suffix('.bin_temp'), bin_file)
-            _logger.info(f"Decompression complete: {time.time() - t0:.2f}s")
-        return bin_file
-
     def decompress_file(self, keep_original=True, **kwargs):
         """
         Decompresses a mtscomp file
@@ -656,7 +633,7 @@ def _split_geometry_into_shanks(th, meta_data):
     return th
 
 
-def _geometry_from_meta(meta_data, return_index=False):
+def _geometry_from_meta(meta_data):
     """
     Gets the geometry, ie. the full trace header for the recording
     :param meta_data: meta_data dictionary as read by ibllib.io.spikeglx.read_meta_data
@@ -669,7 +646,7 @@ def _geometry_from_meta(meta_data, return_index=False):
         th = neuropixel.trace_header(version=major_version)
         th["flag"] = th["x"] * 0 + 1.0
         if return_index:
-            return th, np.arange(th['shank'].shape[0])
+            return th, np.arange(nc)
         else:
             return th
     th = cm.copy()
@@ -680,7 +657,6 @@ def _geometry_from_meta(meta_data, return_index=False):
         # there is a 20um offset between the probe tip and the first site in the coordinate conversion
         if major_version == 1:
             th["x"] = 70 - (th["x"])
-
         th["y"] += 20
         th.update(neuropixel.xy2rc(th["x"], th["y"], version=major_version))
     else:
@@ -700,6 +676,15 @@ def _geometry_from_meta(meta_data, return_index=False):
     else:
         return th
 
+    if return_index:
+        # here we sort the channels by shank, row and -col, this preserves the original NP1
+        # order while still allowing to deal with creative imro tables in NP2
+        sort_keys = np.c_[-th['col'], th['row'], th['shank']]
+        inds = np.lexsort(sort_keys.T)
+        th = {k: v[inds] for k, v in th.items()}
+        return th, inds
+    else:
+        return th
 
 def read_geometry(meta_file):
     """

src/tests/unit/cpu/test_ibldsp.py

@@ -569,6 +569,7 @@ class TestRawDataFeatures(unittest.TestCase):
     def setUp(self):
         self.fixtures_path = Path(__file__).parent.joinpath("fixtures")
         self.tmpdir = Path(tempfile.gettempdir()) / "rawdata"
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
         self.tmpdir.mkdir()
         self.ns_ap = 38502
         self.nc = 385

src/tests/unit/cpu/test_spikeglx.py

@@ -437,7 +437,7 @@ def test_check_ephys_file(self):
 
     def assert_read_glx(self, tglx):
         with spikeglx.Reader(tglx["bin_file"]) as sr:
-            dexpected = sr.channel_conversion_sample2v[sr.type] * tglx["D"]
+            dexpected = sr.channel_conversion_sample2v[sr.type] * tglx["D"][:, sr.raw_channel_order]
             d, sync = sr.read_samples(0, tglx["ns"])
             # could be rounding errors with non-integer sampling rates
             self.assertTrue(sr.nsync == 1)