From 4fd0f71b0193690856b4e74cf2a13a40a17cdacc Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 10 Oct 2024 21:29:21 +0300 Subject: [PATCH] Remote database querying by default --- CHANGELOG.md | 3 +- docs/notebooks/one_modes.ipynb | 221 +++++++++++++++++---------------- one/api.py | 8 +- one/tests/test_one.py | 6 +- 4 files changed, 123 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd3b9fe5..0938337e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog ## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.10.0] -This version improves behaviour of loading revisions and loading datasets from list_datasets output. +This version fixes issues with Alyx authentication in silent mode, and improves behaviour of loading revisions. Additionally the default query mode has changed to 'remote'. ### Modified @@ -13,6 +13,7 @@ This version improves behaviour of loading revisions and loading datasets from l - bugfix in one.params.setup: remove all extrenuous parameters (i.e. TOKEN) when running setup in silent mode - warn user to reauthenticate when password is None in silent mode - always force authentication when password passed, even when token cached +- OneAlyx uses remote mode by default, instead of auto ### Added diff --git a/docs/notebooks/one_modes.ipynb b/docs/notebooks/one_modes.ipynb index a129c7d6..2830fbd8 100644 --- a/docs/notebooks/one_modes.ipynb +++ b/docs/notebooks/one_modes.ipynb @@ -2,6 +2,9 @@ "cells": [ { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "# ONE API modes\n", "## Online vs Offline\n", @@ -11,16 +14,19 @@ "made [via REST](./one_advanced/one_advanced.html). Other online methods include `pid2eid` and\n", "`describe_revision`.\n", "\n", - "When the mode is not specified, it is usually set to 'auto', unless a cache_dir is specified for\n", + "When the mode is not specified, it is usually set to 'remote', unless a cache_dir is specified for\n", "which no database has been configured." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 2, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -36,31 +42,32 @@ "from one.api import ONE\n", "\n", "one = ONE()\n", - "print(one, one.mode) # online, 'auto' mode\n", + "print(one, one.mode) # online, 'remote' mode\n", "assert not one.offline\n", "\n", "one_offline = ONE(cache_dir=r'C:\\data')\n", "print(one_offline, one_offline.mode) # offline, 'local' mode\n", "assert one_offline.offline" - ], + ] + }, + { + "cell_type": "markdown", "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } - } - }, - { - "cell_type": "markdown", + }, "source": [ "## Query modes\n", "In 'auto' mode, the list, search and load methods will use the local cache tables and not\n", - "connect to Alyx, however the option to use Alyx is always there. If the cache tables can't be\n", - "downloaded from Alyx, or authentication fails, the mode will fall back to 'local'.\n", + "connect to Alyx, however the option to use Alyx is always there. In this mode, an up-to-date\n", + "copy of the database session and dataset tables is automatically downloaded. If these cache\n", + "tables can't be downloaded from Alyx, or authentication fails, the mode will fall back to 'local'.\n", "\n", "If 'remote' mode is specified, ONE will only query the remote database and will not use the\n", "local cache tables. Avoiding the database whenever possible is recommended as it doesn't rely\n", - "on a stable internet connection and reduces the load on the remote Alyx.\n", + "on a stable internet connection and reduces the load on the remote database.\n", "\n", "While in 'remote' mode, the local cache may be used by providing the query_type='local' keyword\n", "argument to any method. Likewise, in 'auto'/'local' mode, a remote query can be made by\n", @@ -70,165 +77,171 @@ "NB: The 'remote' query type is not valid in offline mode as there is no database associated to\n", "the local cache directory.\n", "" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", "execution_count": 3, - "outputs": [], - "source": [ - "eids = one.search(lab='cortexlab', query_type='remote') # Search Alyx instead of the local cache" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "eids = one.search(lab='cortexlab', query_type='remote') # Search Alyx instead of the local cache" + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## REST caching\n", "In remote mode ONE makes a REST query instead of using the local cache tables. The results of\n", "the remote REST queries are also cached for 24 hours. This means that making the same remote\n", "REST query twice in a row will only hit the database once. The default cache expiry can be set\n", "by changing the relevant AlyxClient property:" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "from datetime import timedelta\n", - "one.alyx.default_expiry = timedelta(days=20) # Cache results for up to 20 days" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "one.alyx.default_expiry = timedelta(days=20) # Cache results for up to 20 days" + ] }, { "cell_type": "markdown", - "source": [ - "You can temporarily deactivate the REST cache using the `no_cache` function in `one.webclient`.\n", - "When in this context no REST responses are cached and any existing cache files are not used.\n", - "Use this when the most up-to-date information is required:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "You can temporarily deactivate the REST cache using the `no_cache` function in `one.webclient`.\n", + "When in this context no REST responses are cached and any existing cache files are not used.\n", + "Use this when the most up-to-date information is required:" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "with webclient.no_cache(one.alyx):\n", - " eids, det = one.search(lab='cortexlab', query_type='remote')" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "with webclient.no_cache(one.alyx):\n", + " eids, det = one.search(lab='cortexlab', query_type='remote')" + ] }, { "cell_type": "markdown", - "source": [ - "When calling the alyx `rest` method directly you can deactivate the cache with the `no_cache`\n", - "keyword argument:\n" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "When calling the alyx `rest` method directly you can deactivate the cache with the `no_cache`\n", + "keyword argument:\n" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "ses = one.alyx.rest('sessions', 'list', lab='cortexlab', no_cache=True)" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "ses = one.alyx.rest('sessions', 'list', lab='cortexlab', no_cache=True)" + ] }, { "cell_type": "markdown", - "source": [ - "Caching greatly improves performance and should be used whenever possible.\n", - "For more information on ONE REST queries, see [this guide](./one_advanced/one_advanced.html).\n", - "\n", - "You can turn off REST caching when instantiating ONE with the `cache_rest` keyword argument:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "Caching greatly improves performance and should be used whenever possible.\n", + "For more information on ONE REST queries, see [this guide](./one_advanced/one_advanced.html).\n", + "\n", + "You can turn off REST caching when instantiating ONE with the `cache_rest` keyword argument:" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "one = ONE(cache_rest=None, mode='remote')" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "one = ONE(cache_rest=None, mode='remote')" + ] }, { "cell_type": "markdown", - "source": [ - "## Refreshing the cache tables\n", - "By default ONE will try to update the cache tables once every 24 hours. This can be set by\n", - "changing the 'cache_expiry' property:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "## Refreshing the cache tables\n", + "While in 'auto' mode, ONE will try to update the cache tables once every 24 hours.\n", + "This can be changed by changing the 'cache_expiry' property:" + ] }, { "cell_type": "code", "execution_count": 4, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { - "text/plain": "{'expired': False,\n 'created_time': datetime.datetime(2021, 9, 14, 13, 0),\n 'loaded_time': datetime.datetime(2021, 9, 14, 18, 15, 54, 384591),\n 'raw': {'datasets': {'date_created': '2021-09-14 13:00', 'origin': 'alyx'},\n 'sessions': {'date_created': '2021-09-14 13:00', 'origin': 'alyx'}}}" + "text/plain": [ + "{'expired': False,\n", + " 'created_time': datetime.datetime(2021, 9, 14, 13, 0),\n", + " 'loaded_time': datetime.datetime(2021, 9, 14, 18, 15, 54, 384591),\n", + " 'raw': {'datasets': {'date_created': '2021-09-14 13:00', 'origin': 'alyx'},\n", + " 'sessions': {'date_created': '2021-09-14 13:00', 'origin': 'alyx'}}}" + ] }, "execution_count": 4, "metadata": {}, @@ -241,44 +254,44 @@ "\n", "# The time when the cache was generated can be found in the cache metadata:\n", "one._cache._meta" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "Note that the cache won't be downloaded if the remote cache hasn't been updated since the last\n", - "download. The cache can be explicitly refreshed in two ways:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "Note that the cache won't be downloaded if the remote cache hasn't been updated since the last\n", + "download. The cache can be explicitly refreshed in two ways:" + ] }, { "cell_type": "code", "execution_count": 5, - "outputs": [], - "source": [ - "loaded_time = one.refresh_cache('refresh') # Explicitly refresh the cache\n", - "eids = one.search(lab='cortexlab', query_type='refresh') # Calls `refresh_cache` before searching" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "loaded_time = one.refresh_cache('refresh') # Explicitly refresh the cache\n", + "eids = one.search(lab='cortexlab', query_type='refresh') # Calls `refresh_cache` before searching" + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Summary\n", "Mode overview:\n", @@ -297,13 +310,7 @@ "\n", "**I want to use a remote ONE query with up-to-the-minute information**\n", "Call the ONE method from within the `webclient.no_cache` context." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] } ], "metadata": { @@ -327,4 +334,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/one/api.py b/one/api.py index 1156e965..888b06e0 100644 --- a/one/api.py +++ b/one/api.py @@ -49,7 +49,7 @@ class One(ConversionMixin): uuid_filenames = None """bool: whether datasets on disk have a UUID in their filename.""" - def __init__(self, cache_dir=None, mode='auto', wildcards=True, tables_dir=None): + def __init__(self, cache_dir=None, mode='local', wildcards=True, tables_dir=None): """An API for searching and loading data on a local filesystem Parameters @@ -1541,7 +1541,7 @@ def setup(cache_dir=None, silent=False, **kwargs): @lru_cache(maxsize=1) -def ONE(*, mode='auto', wildcards=True, **kwargs): +def ONE(*, mode='remote', wildcards=True, **kwargs): """ONE API factory. Determine which class to instantiate depending on parameters passed. @@ -1579,7 +1579,7 @@ def ONE(*, mode='auto', wildcards=True, **kwargs): if kwargs.pop('offline', False): _logger.warning('the offline kwarg will probably be removed. ' 'ONE is now offline by default anyway') - warnings.warn('"offline" param will be removed; use mode="local"', DeprecationWarning) + warnings.warn('"offline" param will be removed; use mode="local"', FutureWarning) mode = 'local' if (any(x in kwargs for x in ('base_url', 'username', 'password')) or @@ -1598,7 +1598,7 @@ def ONE(*, mode='auto', wildcards=True, **kwargs): class OneAlyx(One): """An API for searching and loading data through the Alyx database.""" def __init__(self, username=None, password=None, base_url=None, cache_dir=None, - mode='auto', wildcards=True, tables_dir=None, **kwargs): + mode='remote', wildcards=True, tables_dir=None, **kwargs): """An API for searching and loading data through the Alyx database. Parameters diff --git a/one/tests/test_one.py b/one/tests/test_one.py index b4040d51..de8d5341 100644 --- a/one/tests/test_one.py +++ b/one/tests/test_one.py @@ -1419,7 +1419,7 @@ def tearDownClass(cls) -> None: class TestOneRemote(unittest.TestCase): """Test remote queries using OpenAlyx""" def setUp(self) -> None: - self.one = OneAlyx(**TEST_DB_2) + self.one = OneAlyx(**TEST_DB_2, mode='auto') self.eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a' self.pid = 'da8dfec1-d265-44e8-84ce-6ae9c109b8bd' # Set cache directory to a temp dir to ensure that we re-download files @@ -1634,7 +1634,7 @@ def setUp(self) -> None: self.patch = mock.patch('one.params.iopar.getfile', new=partial(util.get_file, self.tempdir.name)) self.patch.start() - self.one = OneAlyx(**TEST_DB_2, cache_dir=self.tempdir.name) + self.one = OneAlyx(**TEST_DB_2, cache_dir=self.tempdir.name, mode='auto') self.fid = '17ab5b57-aaf6-4016-9251-66daadc200c7' # File record of channels.brainLocation self.eid = 'aad23144-0e52-4eac-80c5-c4ee2decb198' @@ -1973,7 +1973,7 @@ def test_one_factory(self): self.assertIsInstance(one_obj, One) # The offline param was given, raise deprecation warning (via log) - with self.assertWarns(DeprecationWarning): + with self.assertWarns(FutureWarning): ONE(offline=True, cache_dir=self.tempdir.name) # Test setup with virtual ONE method