From dba56584c47c9c2a1fa17b64ad26cf15fb0dc83a Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 1 Nov 2024 16:27:18 +0200 Subject: [PATCH 1/2] Resolves #873 and #871 --- ibllib/__init__.py | 2 +- ibllib/io/raw_data_loaders.py | 1 + ibllib/io/session_params.py | 9 ++++++--- ibllib/tests/test_io.py | 6 ++++++ release_notes.md | 27 +++++++++++++++------------ 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 9da654573..ced9677a9 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.40.0' +__version__ = '2.40.1' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/ibllib/io/raw_data_loaders.py b/ibllib/io/raw_data_loaders.py index 36bed4abe..d2ed0f688 100644 --- a/ibllib/io/raw_data_loaders.py +++ b/ibllib/io/raw_data_loaders.py @@ -200,6 +200,7 @@ def load_embedded_frame_data(session_path, label: str, raw=False): """ Load the embedded frame count and GPIO for a given session. If the file doesn't exist, or is empty, None values are returned. + :param session_path: Absolute path of session folder :param label: The specific video to load, one of ('left', 'right', 'body') :param raw: If True the raw data are returned without preprocessing, otherwise frame count is diff --git a/ibllib/io/session_params.py b/ibllib/io/session_params.py index e9127e9ae..b5a1d0692 100644 --- a/ibllib/io/session_params.py +++ b/ibllib/io/session_params.py @@ -135,7 +135,7 @@ def read_params(path) -> dict: """ if (path := Path(path)).is_dir(): - yaml_file = next(path.glob('_ibl_experiment.description*'), None) + yaml_file = next(path.glob('_ibl_experiment.description*.yaml'), None) else: yaml_file = path if path.exists() else None if not yaml_file: @@ -165,6 +165,11 @@ def merge_params(a, b, copy=False): dict A merged dictionary consisting of fields from `a` and `b`. """ + def to_hashable(dict_item): + """Convert protocol -> dict map to hashable tuple of protocol + sorted key value pairs.""" + hashable = (dict_item[0], *chain.from_iterable(sorted(dict_item[1].items()))) + return tuple(tuple(x) if isinstance(x, list) else x for x in hashable) + if copy: a = deepcopy(a) for k in b: @@ -176,8 +181,6 @@ def merge_params(a, b, copy=False): # For tasks, keep order and skip duplicates # Assert tasks is a list of single value dicts assert (not prev or set(map(len, prev)) == {1}) and set(map(len, b[k])) == {1} - # Convert protocol -> dict map to hashable tuple of protocol + sorted key value pairs - to_hashable = lambda itm: (itm[0], *chain.from_iterable(sorted(itm[1].items()))) # noqa # Get the set of previous tasks prev_tasks = set(map(to_hashable, chain.from_iterable(map(dict.items, prev)))) tasks = chain.from_iterable(map(dict.items, b[k])) diff --git a/ibllib/tests/test_io.py b/ibllib/tests/test_io.py index 661c4d2a4..09711e3b4 100644 --- a/ibllib/tests/test_io.py +++ b/ibllib/tests/test_io.py @@ -589,6 +589,12 @@ def test_merge_params(self): # Test assertion on duplicate sync b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}} self.assertRaises(AssertionError, session_params.merge_params, a, b) + # Test how it handles the extractors key, which is an unhashable list + f = {'tasks': [{'fooChoiceWorld': {'collection': 'bar', 'sync_label': 'bpod', 'extractors': ['a', 'b']}}]} + g = session_params.merge_params(a, f, copy=True) + self.assertCountEqual(['devices', 'procedures', 'projects', 'sync', 'tasks', 'version'], g.keys()) + self.assertEqual(4, len(g['tasks'])) + self.assertDictEqual(f['tasks'][0], g['tasks'][-1]) def test_get_protocol_number(self): """Test ibllib.io.session_params.get_task_protocol_number function.""" diff --git a/release_notes.md b/release_notes.md index 97aa02a8b..de1077ce0 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,9 +1,12 @@ ## Release Note 2.40.0 ### features -- iblsorter >= 1.9 sorting tasks with waveform extraction and channel sorting +- iblsorter >= 1.9 sorting tasks with waveform extraction and channel sorting - s3 patcher prototype +#### 2.40.1 +- Bugfix: ibllib.io.sess_params.merge_params supports tasks extractors key + ## Release Note 2.39.0 ### features @@ -18,7 +21,7 @@ #### 2.39.1 - Bugfix: brainbox.metrics.single_unit.quick_unit_metrics fix for indexing of n_spike_below2 -- +- #### 2.39.2 - Bugfix: routing of protocol to extractor through the project repository checks that the target is indeed an extractor class. @@ -81,7 +84,7 @@ target is indeed an extractor class. - oneibl.register_datasets accounts for non existing sessions when checking protected dsets #### 2.34.1 -- Ensure mesoscope frame QC files are sorted before concatenating +- Ensure mesoscope frame QC files are sorted before concatenating - Look for SESSION_TEMPLATE_ID key of task settings for extraction of pre-generated choice world sequences - Download required ap.meta files when building pipeline for task_qc command @@ -145,7 +148,7 @@ target is indeed an extractor class. - Added ibllib.pipes.dynamic_pipeline.get_trials_tasks function ### bugfixes -- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 +- Fix ibllib.io.extractors.ephys_fpga.extract_all for python 3.8 ### other - Change behavior qc to pass if number of trials > 400 (from start) can be found for which easy trial performance > 0.9 @@ -198,7 +201,7 @@ target is indeed an extractor class. ### features - Training status pipeline now compatible with dynamic pipeline - Dynamic DLC task using description file -- Full photometry lookup table +- Full photometry lookup table ### bugfixes - fix for untrainable, unbiasable don't repopulate if already exists @@ -213,7 +216,7 @@ target is indeed an extractor class. - split swanson areas ### bugfixes - trainig plots -- fix datahandler on SDSC for ONEv2 +- fix datahandler on SDSC for ONEv2 ### Release Notes 2.23.0 2023-05-19 - quiescence period extraction @@ -227,7 +230,7 @@ target is indeed an extractor class. ### Release Notes 2.22.2 2023-05-03 ### bugfixes - training plots -- +- ### features - can change download path for atlas ### Release Notes 2.22.1 2023-05-02 @@ -331,7 +334,7 @@ target is indeed an extractor class. ### Release Notes 2.17.0 2022-10-04 - units quality metrics use latest algorithms for refractory period violations and noise cut-off - + ## Release Notes 2.16 ### Release Notes 2.16.1 2022-09-28 ### bugfixes @@ -350,7 +353,7 @@ target is indeed an extractor class. - SessionLoader error handling and bug fix ### Release Notes 2.15.2 - 2022-09-22 -- extraction pipeline: fix unpacking of empty arguments field from alyx dict that prevents running task +- extraction pipeline: fix unpacking of empty arguments field from alyx dict that prevents running task ### Release Notes 2.15.1 - 2022-09-21 - atlas: gene-expression backend and MRI Toronto atlas stretch and squeeze factors (Dan/Olivier) @@ -364,7 +367,7 @@ target is indeed an extractor class. - new modalities: - photometry extraction (Mainen lab) - widefield extraction (Churchland lab) - + #### bugfixes - Spike sorting task: parse new pykilosort log format - Session loader @@ -439,7 +442,7 @@ target is indeed an extractor class. ### Release Notes 2.10.6 2022-03-15 - Allow parent tasks to be 'Incomplete' to run task on local server -- Change one base_rul for dlc_qc_plot on cortexlab +- Change one base_rul for dlc_qc_plot on cortexlab ### Release Notes 2.10.5 2022-03-11 - Fix moot release accident @@ -480,7 +483,7 @@ target is indeed an extractor class. ### Release Notes 2.9.0 2022-01-24 - Adding EphysDLC task in ephys_preprocessing pipeline -- NOTE: requires DLC environment to be set up on local servers! +- NOTE: requires DLC environment to be set up on local servers! - Fixes to EphysPostDLC dlc_qc_plot ## Release Notes 2.8 From 01b6d080e9669a28a674f10e9da3c54566da3b0a Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 1 Nov 2024 16:39:34 +0200 Subject: [PATCH 2/2] Fix workflow files for new min python version --- .github/workflows/ibllib_ci.yml | 6 +++--- .github/workflows/python-publish.yml | 2 +- ibllib/tests/qc/test_critical_reasons.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ibllib_ci.yml b/.github/workflows/ibllib_ci.yml index 7ca648938..b6a2b1f45 100644 --- a/.github/workflows/ibllib_ci.yml +++ b/.github/workflows/ibllib_ci.yml @@ -18,12 +18,12 @@ jobs: max-parallel: 2 matrix: os: ["windows-latest", "ubuntu-latest"] - python-version: ["3.8", "3.11"] + python-version: ["3.10", "3.12"] exclude: - os: windows-latest - python-version: 3.8 + python-version: 3.10 - os: ubuntu-latest - python-version: 3.11 + python-version: 3.12 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index e3e79e129..b44d8962e 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/ibllib/tests/qc/test_critical_reasons.py b/ibllib/tests/qc/test_critical_reasons.py index 31f9ef732..6355c66f7 100644 --- a/ibllib/tests/qc/test_critical_reasons.py +++ b/ibllib/tests/qc/test_critical_reasons.py @@ -82,7 +82,7 @@ def test_note_already_existing(self): note = one.alyx.rest('notes', 'list', django=f'object_id,{eid}', no_cache=True) self.assertEqual(len(note), 1) - self.assertNotEquals(original_note_id, note[0]['id']) + self.assertNotEqual(original_note_id, note[0]['id']) def test_guiinput_ins(self): eid = self.ins_id # probe id