From dc96fa8362ba9b884d403ae916f46adad184dd87 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Tue, 2 Apr 2024 12:37:22 +0200 Subject: [PATCH 01/18] TST: use sybil for doctests --- audbackend/core/conftest.py | 10 ++++++++++ pyproject.toml | 1 - tests/requirements.txt | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 4d533f68..b3da2181 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,14 +1,24 @@ import datetime +import doctest import os import tempfile import pytest +import sybil +from sybil.parsers.rest import DocTestParser import audeer import audbackend +# Collect doctests +pytest_collect_file = sybil.Sybil( + parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], + patterns=["*.py"], +).pytest() + + class DoctestFileSystem(audbackend.backend.FileSystem): def _date( self, diff --git a/pyproject.toml b/pyproject.toml index f2f6c191..8d14f485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,6 @@ skip = './audbackend.egg-info,./build,.docs/api,./docs/_templates' cache_dir = '.cache/pytest' xfail_strict = true addopts = ''' - --doctest-plus --cov=audbackend --cov-fail-under=100 --cov-report term-missing diff --git a/tests/requirements.txt b/tests/requirements.txt index e9d7f60d..a57476c0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ audformat pytest pytest-cov -pytest-doctestplus +sybil From 2b2c980494f0ed4818f9574b406862b242a446ce Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Tue, 2 Apr 2024 15:51:03 +0200 Subject: [PATCH 02/18] Hand fixtures to sybil --- audbackend/core/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index b3da2181..9c357706 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -16,6 +16,7 @@ pytest_collect_file = sybil.Sybil( parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], patterns=["*.py"], + fixtures=["prepare_docstring_tests"], ).pytest() From f0204bcf1beeaed368966d7ed514f0936f6b9ecb Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 14 Nov 2024 21:17:35 +0100 Subject: [PATCH 03/18] Add filesystem_backend fixture --- audbackend/core/conftest.py | 12 +++++++++++- audbackend/core/interface/unversioned.py | 8 ++------ audbackend/core/interface/versioned.py | 8 ++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 9c357706..59132dd7 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -16,7 +16,7 @@ pytest_collect_file = sybil.Sybil( parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], patterns=["*.py"], - fixtures=["prepare_docstring_tests"], + fixtures=["prepare_docstring_tests", "filesystem_backend"], ).pytest() @@ -36,6 +36,16 @@ def _owner( return "doctest" +@pytest.fixture(scope="function") +def filesystem_backend(): + backend = audbackend.backend.FileSystem("host", "repo") + backend.open() + + yield backend + + backend.close() + + @pytest.fixture(scope="function", autouse=True) def prepare_docstring_tests(doctest_namespace): with tempfile.TemporaryDirectory() as tmp: diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index fe334d95..f4714ab5 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -151,9 +151,7 @@ def date( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -521,9 +519,7 @@ def owner( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index a620f4ce..6552d229 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -192,9 +192,7 @@ def date( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -699,9 +697,7 @@ def owner( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" From c242636ac0456b580aca795dce44650cd9038520 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 14 Nov 2024 22:00:47 +0100 Subject: [PATCH 04/18] Further work on sybil for docstrings --- audbackend/core/backend/minio.py | 2 +- audbackend/core/conftest.py | 78 ++++++++++++++---------- audbackend/core/errors.py | 3 +- audbackend/core/interface/base.py | 18 ++---- audbackend/core/interface/maven.py | 7 ++- audbackend/core/interface/unversioned.py | 43 ++++--------- audbackend/core/interface/versioned.py | 3 +- 7 files changed, 74 insertions(+), 80 deletions(-) diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py index a86be1fc..0b5ec80a 100644 --- a/audbackend/core/backend/minio.py +++ b/audbackend/core/backend/minio.py @@ -44,7 +44,7 @@ class Minio(Base): >>> backend = Minio(host, repository, authentication=auth) >>> try: ... with backend: - ... backend.put_file(file, "/sub/file.txt") + ... backend.put_file("src.txt", "/sub/file.txt") ... backend.ls() ... finally: ... Minio.delete(host, repository, authentication=auth) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 59132dd7..6e164634 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,7 +1,5 @@ import datetime import doctest -import os -import tempfile import pytest import sybil @@ -20,7 +18,15 @@ ).pytest() -class DoctestFileSystem(audbackend.backend.FileSystem): +class FileSystem(audbackend.backend.FileSystem): + def __init__( + self, + host: str, + repository: str, + ): + super().__init__(host, repository) + self.opened = True + def _date( self, path: str, @@ -38,34 +44,44 @@ def _owner( @pytest.fixture(scope="function") def filesystem_backend(): - backend = audbackend.backend.FileSystem("host", "repo") - backend.open() - - yield backend - - backend.close() + yield FileSystem("host", "repo") @pytest.fixture(scope="function", autouse=True) -def prepare_docstring_tests(doctest_namespace): - with tempfile.TemporaryDirectory() as tmp: - # Change to tmp dir - current_dir = os.getcwd() - os.chdir(tmp) - # Prepare backend - audeer.mkdir("host") - audbackend.backend.FileSystem.create("host", "repo") - # Provide example file `src.txt` - audeer.touch("src.txt") - # Provide DoctestFileSystem as FileSystem, - # and audbackend - # in docstring examples - doctest_namespace["DoctestFileSystem"] = DoctestFileSystem - doctest_namespace["audbackend"] = audbackend - - yield - - # Remove backend - audbackend.backend.FileSystem.delete("host", "repo") - # Change back to current dir - os.chdir(current_dir) +def prepare_docstring_tests(tmpdir, monkeypatch): + r"""Code to be run before each doctest.""" + # Change to tmp dir + monkeypatch.chdir(tmpdir) + + # Provide example file `src.txt` + audeer.touch("src.txt") + + # Prepare backend + audeer.mkdir("host", "repo") + + yield + + +# @pytest.fixture(scope="function", autouse=True) +# def prepare_docstring_tests(doctest_namespace): +# with tempfile.TemporaryDirectory() as tmp: +# # Change to tmp dir +# current_dir = os.getcwd() +# os.chdir(tmp) +# # Prepare backend +# audeer.mkdir("host") +# audbackend.backend.FileSystem.create("host", "repo") +# # Provide example file `src.txt` +# audeer.touch("src.txt") +# # Provide DoctestFileSystem as FileSystem, +# # and audbackend +# # in docstring examples +# doctest_namespace["DoctestFileSystem"] = DoctestFileSystem +# doctest_namespace["audbackend"] = audbackend +# +# yield +# +# # Remove backend +# audbackend.backend.FileSystem.delete("host", "repo") +# # Change back to current dir +# os.chdir(current_dir) diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index ab7bd2a2..fed5a5dc 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -7,8 +7,7 @@ class BackendError(Exception): .. Prepare backend and interface for docstring examples >>> import audeer - >>> audeer.rmdir("host", "repo") - >>> _ = audeer.mkdir("host", "repo") + >>> import audbackend Examples: >>> backend = audbackend.backend.FileSystem("host", "repo") diff --git a/audbackend/core/interface/base.py b/audbackend/core/interface/base.py index 836be8d9..e521bf4c 100644 --- a/audbackend/core/interface/base.py +++ b/audbackend/core/interface/base.py @@ -36,8 +36,7 @@ def backend(self) -> Backend: backend object .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.backend @@ -53,8 +52,7 @@ def host(self) -> str: Returns: host path .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.host @@ -83,8 +81,7 @@ def join( or if joined path contains invalid character .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.join("/", "file.txt") @@ -105,8 +102,7 @@ def repository(self) -> str: repository name .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.repository @@ -123,8 +119,7 @@ def sep(self) -> str: file separator .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.sep @@ -150,8 +145,7 @@ def split( does not match ``'[A-Za-z0-9/._-]+'`` .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) + >>> interface = Base(filesystem_backend) Examples: >>> interface.split("/") diff --git a/audbackend/core/interface/maven.py b/audbackend/core/interface/maven.py index 59c5c1db..a7fca22d 100644 --- a/audbackend/core/interface/maven.py +++ b/audbackend/core/interface/maven.py @@ -63,6 +63,9 @@ class Maven(Versioned): ... as extensions + .. + >>> import audbackend + Examples: >>> file = "src.txt" >>> backend = audbackend.backend.FileSystem("host", "repo") @@ -139,9 +142,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Maven(backend) + >>> interface = Maven(filesystem_backend) Examples: >>> file = "src.txt" diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index f4714ab5..cfcdfa64 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -15,6 +15,9 @@ class Unversioned(Base): Args: backend: backend object + .. + >>> import audbackend + Examples: >>> file = "src.txt" >>> backend = audbackend.backend.FileSystem("host", "repo") @@ -50,9 +53,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -106,9 +107,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -191,9 +190,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -262,9 +259,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -333,9 +328,7 @@ def get_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -401,9 +394,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -471,9 +462,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -588,9 +577,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -650,9 +637,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -687,9 +672,7 @@ def remove_file( or does not match ``'[A-Za-z0-9/._-]+'`` .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem_backend) Examples: >>> file = "src.txt" diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index 6552d229..edec2d2d 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -21,7 +21,8 @@ class Versioned(Base): Args: backend: backend object - .. Prepare backend and interface for docstring examples + .. + >>> import audbackend Examples: >>> file = "src.txt" From 7155cf717845258d0abc2e93da5fff91078c3ac3 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 09:17:23 +0100 Subject: [PATCH 05/18] Fix doctests --- audbackend/core/backend/minio.py | 4 +- audbackend/core/conftest.py | 117 +++++++++++++---------- audbackend/core/errors.py | 1 + audbackend/core/interface/maven.py | 7 ++ audbackend/core/interface/unversioned.py | 47 +++++++++ audbackend/core/interface/versioned.py | 92 +++++++++++------- 6 files changed, 179 insertions(+), 89 deletions(-) diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py index 0b5ec80a..ed180960 100644 --- a/audbackend/core/backend/minio.py +++ b/audbackend/core/backend/minio.py @@ -40,11 +40,11 @@ class Minio(Base): >>> auth = ("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") >>> repository = "my-data" + audeer.uid() >>> Minio.create(host, repository, authentication=auth) - >>> file = audeer.touch("file.txt") + >>> file = audeer.touch("src.txt") >>> backend = Minio(host, repository, authentication=auth) >>> try: ... with backend: - ... backend.put_file("src.txt", "/sub/file.txt") + ... backend.put_file(file, "/sub/file.txt") ... backend.ls() ... finally: ... Minio.delete(host, repository, authentication=auth) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 6e164634..0ee0b905 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,5 +1,6 @@ import datetime import doctest +import os import pytest import sybil @@ -14,74 +15,88 @@ pytest_collect_file = sybil.Sybil( parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], patterns=["*.py"], - fixtures=["prepare_docstring_tests", "filesystem_backend"], + fixtures=[ + "filesystem_backend", + "prepare_docstring_tests", + "clear", + ], ).pytest() class FileSystem(audbackend.backend.FileSystem): - def __init__( - self, - host: str, - repository: str, - ): + def __init__(self, host, repository): super().__init__(host, repository) self.opened = True - def _date( - self, - path: str, - ) -> str: + def _date(self, path): date = datetime.datetime(1991, 2, 20) date = audbackend.core.utils.date_format(date) return date - def _owner( - self, - path: str, - ) -> str: + def _owner(self, path): return "doctest" +@pytest.fixture(scope="session", autouse=True) +def clear(): + """Clear local files and filesystem backend. + + When using a tmpdir with the scope ``"function"`` or ``"class"``, + a new tmpdir is used each line + in the docstring tests. + When using the next greater scope ``"module"``, + the same tmpdir is used in the whole file, + which means there is no scope + that provides a new tmpdir + for each function/method + within a file. + To simulate this behavior, + we use the scope ``"module"``, + and provide this ``clear()`` function + to reset after a finished docstring. + + """ + + def clear_all(): + # Clear backend + audeer.rmdir("host", "repo") + audeer.mkdir("host", "repo") + # Clear local files + files = audeer.list_file_names(".", basenames=True) + files = [file for file in files if not file == "src.txt"] + for file in files: + os.remove(file) + + yield clear_all + + @pytest.fixture(scope="function") def filesystem_backend(): + """Filesystem backend with patched date and owner methods. + + The backend is also opened already. + + """ yield FileSystem("host", "repo") -@pytest.fixture(scope="function", autouse=True) -def prepare_docstring_tests(tmpdir, monkeypatch): +@pytest.fixture(scope="module", autouse=True) +def prepare_docstring_tests(tmpdir_factory): r"""Code to be run before each doctest.""" - # Change to tmp dir - monkeypatch.chdir(tmpdir) - - # Provide example file `src.txt` - audeer.touch("src.txt") - - # Prepare backend - audeer.mkdir("host", "repo") - - yield - - -# @pytest.fixture(scope="function", autouse=True) -# def prepare_docstring_tests(doctest_namespace): -# with tempfile.TemporaryDirectory() as tmp: -# # Change to tmp dir -# current_dir = os.getcwd() -# os.chdir(tmp) -# # Prepare backend -# audeer.mkdir("host") -# audbackend.backend.FileSystem.create("host", "repo") -# # Provide example file `src.txt` -# audeer.touch("src.txt") -# # Provide DoctestFileSystem as FileSystem, -# # and audbackend -# # in docstring examples -# doctest_namespace["DoctestFileSystem"] = DoctestFileSystem -# doctest_namespace["audbackend"] = audbackend -# -# yield -# -# # Remove backend -# audbackend.backend.FileSystem.delete("host", "repo") -# # Change back to current dir -# os.chdir(current_dir) + tmp_dir = tmpdir_factory.mktemp("tmp") + + try: + # Change to tmp dir + current_dir = os.getcwd() + os.chdir(tmp_dir) + + # Provide example file `src.txt` + audeer.touch("src.txt") + + # Prepare backend + audeer.mkdir("host", "repo") + + yield + + finally: + os.chdir(current_dir) diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index fed5a5dc..c49c9aa7 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -8,6 +8,7 @@ class BackendError(Exception): >>> import audeer >>> import audbackend + >>> _ = audeer.mkdir("host", "repo") Examples: >>> backend = audbackend.backend.FileSystem("host", "repo") diff --git a/audbackend/core/interface/maven.py b/audbackend/core/interface/maven.py index a7fca22d..4e82aaac 100644 --- a/audbackend/core/interface/maven.py +++ b/audbackend/core/interface/maven.py @@ -65,6 +65,7 @@ class Maven(Versioned): .. >>> import audbackend + >>> import audeer Examples: >>> file = "src.txt" @@ -79,6 +80,9 @@ class Maven(Versioned): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' + .. + >>> clear() + """ # noqa: E501 def __init__( @@ -162,6 +166,9 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] + .. + >>> clear() + """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index cfcdfa64..0da09bde 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -17,6 +17,17 @@ class Unversioned(Base): .. >>> import audbackend + >>> import audeer + + .. >>> def clear(): + .. ... # Clear backend + .. ... audeer.rmdir("host", "repo") + .. ... audeer.mkdir("host", "repo") + .. ... # Clear local files + .. ... files = audeer.list_file_names(".", basenames=True) + .. ... files = [file for file in files if not file == "src.txt"] + .. ... for file in files: + .. ... os.remove(file) Examples: >>> file = "src.txt" @@ -30,6 +41,9 @@ class Unversioned(Base): >>> interface.get_file("/file.txt", "dst.txt") '...dst.txt' + .. + >>> clear() + """ def checksum( @@ -64,6 +78,9 @@ def checksum( >>> interface.checksum("/file.txt") 'd41d8cd98f00b204e9800998ecf8427e' + .. + >>> clear() + """ return self.backend.checksum(path) @@ -118,6 +135,9 @@ def copy_file( >>> interface.exists("/copy.txt") True + .. + >>> clear() + """ self.backend.copy_file( src_path, @@ -158,6 +178,9 @@ def date( >>> interface.date("/file.txt") '1991-02-20' + .. + >>> clear() + """ return self.backend.date(path) @@ -200,6 +223,9 @@ def exists( >>> interface.exists("/file.txt") True + .. + >>> clear() + """ return self.backend.exists( path, @@ -268,6 +294,9 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".") ['src.txt'] + .. + >>> clear() + """ return self.backend.get_archive( src_path, @@ -339,6 +368,9 @@ def get_file( >>> os.path.exists("dst.txt") True + .. + >>> clear() + """ return self.backend.get_file( src_path, @@ -411,6 +443,9 @@ def ls( >>> interface.ls("/sub/") ['/sub/archive.zip'] + .. + >>> clear() + """ # noqa: E501 return self.backend.ls( path, @@ -475,6 +510,9 @@ def move_file( >>> interface.exists("/file.txt") False + .. + >>> clear() + """ self.backend.move_file( src_path, @@ -516,6 +554,7 @@ def owner( >>> interface.owner("/file.txt") 'doctest' + >>> clear() """ return self.backend.owner(path) @@ -587,6 +626,8 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz") True + >>> clear() + """ self.backend.put_archive( src_root, @@ -647,6 +688,9 @@ def put_file( >>> interface.exists("/file.txt") True + .. + >>> clear() + """ self.backend.put_file( src_path, @@ -683,5 +727,8 @@ def remove_file( >>> interface.exists("/file.txt") False + .. + >>> clear() + """ self.backend.remove_file(path) diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index edec2d2d..73a01855 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -37,6 +37,8 @@ class Versioned(Base): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' + .. + >>> clear() """ @@ -71,9 +73,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -84,6 +84,9 @@ def checksum( >>> interface.checksum("/file.txt", "1.0.0") 'd41d8cd98f00b204e9800998ecf8427e' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.checksum(path_with_version) @@ -136,9 +139,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -149,6 +150,9 @@ def copy_file( >>> interface.exists("/copy.txt", "1.0.0") True + .. + >>> clear() + """ if version is None: versions = self.versions(src_path) @@ -201,6 +205,9 @@ def date( >>> interface.date("/file.txt", "1.0.0") '1991-02-20' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.date(path_with_version) @@ -236,9 +243,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -248,6 +253,9 @@ def exists( >>> interface.exists("/file.txt", "1.0.0") True + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.exists( @@ -312,9 +320,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -323,6 +329,9 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".", "1.0.0") ['src.txt'] + .. + >>> clear() + """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_archive( @@ -388,9 +397,7 @@ def get_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -401,6 +408,9 @@ def get_file( >>> os.path.exists("dst.txt") True + .. + >>> clear() + """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_file( @@ -431,9 +441,7 @@ def latest_version( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -442,6 +450,9 @@ def latest_version( >>> interface.latest_version("/file.txt") '2.0.0' + .. + >>> clear() + """ vs = self.versions(path) return vs[-1] @@ -496,9 +507,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -518,6 +527,9 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] + .. + >>> clear() + """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( @@ -638,9 +650,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -653,6 +663,9 @@ def move_file( >>> interface.exists("/file.txt", "1.0.0") False + .. + >>> clear() + """ if version is None: versions = self.versions(src_path) @@ -706,6 +719,9 @@ def owner( >>> interface.owner("/file.txt", "1.0.0") 'doctest' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.owner(path_with_version) @@ -772,9 +788,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -784,6 +798,9 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz", "1.0.0") True + .. + >>> clear() + """ dst_path_with_version = self._path_with_version(dst_path, version) self.backend.put_archive( @@ -842,9 +859,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -854,6 +869,9 @@ def put_file( >>> interface.exists("/file.txt", "3.0.0") True + .. + >>> clear() + """ dst_path_with_version = self._path_with_version(dst_path, version) return self.backend.put_file( @@ -885,9 +903,7 @@ def remove_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -898,6 +914,9 @@ def remove_file( >>> interface.exists("/file.txt", "1.0.0") False + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) self.backend.remove_file(path_with_version) @@ -929,9 +948,7 @@ def versions( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -940,6 +957,9 @@ def versions( >>> interface.versions("/file.txt") ['1.0.0', '2.0.0'] + .. + >>> clear() + """ utils.check_path(path) From 2e513556a854b68e16e302e3f92bd41046e09b31 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 09:47:59 +0100 Subject: [PATCH 06/18] Add further ideas --- audbackend/core/conftest.py | 76 ++++++++++++++----- audbackend/core/interface/unversioned.py | 69 ++++-------------- audbackend/core/interface/versioned.py | 93 +++++++----------------- tests/conftest.py | 27 +++++++ 4 files changed, 124 insertions(+), 141 deletions(-) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 0ee0b905..ae2e48f7 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -9,6 +9,7 @@ import audeer import audbackend +from tests.conftest import filesystem # noqa: F401 # Collect doctests @@ -16,7 +17,10 @@ parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], patterns=["*.py"], fixtures=[ + "filesystem", "filesystem_backend", + "mock_date", + "mock_owner", "prepare_docstring_tests", "clear", ], @@ -37,6 +41,28 @@ def _owner(self, path): return "doctest" +@pytest.fixture(scope="function") +def mock_date(): + r"""Custom date method to return a fixed date.""" + + def date(path: str, version: str = None) -> str: + date = datetime.datetime(1991, 2, 20) + date = audbackend.core.utils.date_format(date) + return date + + yield date + + +@pytest.fixture(scope="function") +def mock_owner(): + r"""Custom owner method to return a fixed owner.""" + + def owner(path: str, version: str = None) -> str: + return "doctest" + + yield owner + + @pytest.fixture(scope="session", autouse=True) def clear(): """Clear local files and filesystem backend. @@ -80,23 +106,35 @@ def filesystem_backend(): yield FileSystem("host", "repo") -@pytest.fixture(scope="module", autouse=True) -def prepare_docstring_tests(tmpdir_factory): +@pytest.fixture(scope="function", autouse=True) +def prepare_docstring_tests(tmpdir, monkeypatch): r"""Code to be run before each doctest.""" - tmp_dir = tmpdir_factory.mktemp("tmp") - - try: - # Change to tmp dir - current_dir = os.getcwd() - os.chdir(tmp_dir) - - # Provide example file `src.txt` - audeer.touch("src.txt") - - # Prepare backend - audeer.mkdir("host", "repo") - - yield - - finally: - os.chdir(current_dir) + # Change to tmp dir + monkeypatch.chdir(tmpdir) + + # Provide example file `src.txt` + audeer.touch("src.txt") + + yield + + +# @pytest.fixture(scope="module", autouse=True) +# def prepare_docstring_tests(tmpdir_factory): +# r"""Code to be run before each doctest.""" +# tmp_dir = tmpdir_factory.mktemp("tmp") +# +# try: +# # Change to tmp dir +# current_dir = os.getcwd() +# os.chdir(tmp_dir) +# +# # Provide example file `src.txt` +# audeer.touch("src.txt") +# +# # Prepare backend +# audeer.mkdir("host", "repo") +# +# yield +# +# finally: +# os.chdir(current_dir) diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index 0da09bde..83097fbd 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -19,16 +19,6 @@ class Unversioned(Base): >>> import audbackend >>> import audeer - .. >>> def clear(): - .. ... # Clear backend - .. ... audeer.rmdir("host", "repo") - .. ... audeer.mkdir("host", "repo") - .. ... # Clear local files - .. ... files = audeer.list_file_names(".", basenames=True) - .. ... files = [file for file in files if not file == "src.txt"] - .. ... for file in files: - .. ... os.remove(file) - Examples: >>> file = "src.txt" >>> backend = audbackend.backend.FileSystem("host", "repo") @@ -67,7 +57,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -78,9 +68,6 @@ def checksum( >>> interface.checksum("/file.txt") 'd41d8cd98f00b204e9800998ecf8427e' - .. - >>> clear() - """ return self.backend.checksum(path) @@ -124,7 +111,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -135,9 +122,6 @@ def copy_file( >>> interface.exists("/copy.txt") True - .. - >>> clear() - """ self.backend.copy_file( src_path, @@ -170,7 +154,8 @@ def date( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) + >>> interface.date = mock_date Examples: >>> file = "src.txt" @@ -178,9 +163,6 @@ def date( >>> interface.date("/file.txt") '1991-02-20' - .. - >>> clear() - """ return self.backend.date(path) @@ -213,7 +195,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -223,9 +205,6 @@ def exists( >>> interface.exists("/file.txt") True - .. - >>> clear() - """ return self.backend.exists( path, @@ -285,7 +264,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -294,9 +273,6 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".") ['src.txt'] - .. - >>> clear() - """ return self.backend.get_archive( src_path, @@ -357,7 +333,7 @@ def get_file( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -368,9 +344,6 @@ def get_file( >>> os.path.exists("dst.txt") True - .. - >>> clear() - """ return self.backend.get_file( src_path, @@ -426,7 +399,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -443,9 +416,6 @@ def ls( >>> interface.ls("/sub/") ['/sub/archive.zip'] - .. - >>> clear() - """ # noqa: E501 return self.backend.ls( path, @@ -497,7 +467,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -510,9 +480,6 @@ def move_file( >>> interface.exists("/file.txt") False - .. - >>> clear() - """ self.backend.move_file( src_path, @@ -546,7 +513,8 @@ def owner( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) + >>> interface.owner = mock_owner Examples: >>> file = "src.txt" @@ -554,7 +522,6 @@ def owner( >>> interface.owner("/file.txt") 'doctest' - >>> clear() """ return self.backend.owner(path) @@ -616,7 +583,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -626,8 +593,6 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz") True - >>> clear() - """ self.backend.put_archive( src_root, @@ -678,7 +643,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -688,9 +653,6 @@ def put_file( >>> interface.exists("/file.txt") True - .. - >>> clear() - """ self.backend.put_file( src_path, @@ -716,7 +678,7 @@ def remove_file( or does not match ``'[A-Za-z0-9/._-]+'`` .. - >>> interface = Unversioned(filesystem_backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -727,8 +689,5 @@ def remove_file( >>> interface.exists("/file.txt") False - .. - >>> clear() - """ self.backend.remove_file(path) diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index 73a01855..3d8b0fe0 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -23,12 +23,13 @@ class Versioned(Base): .. >>> import audbackend + >>> backend = filesystem Examples: >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> # backend = audbackend.backend.FileSystem("host", "repo") + >>> # backend.open() + >>> interface = Versioned(filesystem) >>> interface.put_archive(".", "/sub/archive.zip", "1.0.0", files=[file]) >>> for version in ["1.0.0", "2.0.0"]: ... interface.put_file(file, "/file.txt", version) @@ -37,9 +38,6 @@ class Versioned(Base): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' - .. - >>> clear() - """ def __init__( @@ -73,7 +71,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -84,9 +82,6 @@ def checksum( >>> interface.checksum("/file.txt", "1.0.0") 'd41d8cd98f00b204e9800998ecf8427e' - .. - >>> clear() - """ path_with_version = self._path_with_version(path, version) return self.backend.checksum(path_with_version) @@ -139,7 +134,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -150,9 +145,6 @@ def copy_file( >>> interface.exists("/copy.txt", "1.0.0") True - .. - >>> clear() - """ if version is None: versions = self.versions(src_path) @@ -197,7 +189,8 @@ def date( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) + >>> interface.date = mock_date Examples: >>> file = "src.txt" @@ -205,9 +198,6 @@ def date( >>> interface.date("/file.txt", "1.0.0") '1991-02-20' - .. - >>> clear() - """ path_with_version = self._path_with_version(path, version) return self.backend.date(path_with_version) @@ -243,7 +233,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -253,9 +243,6 @@ def exists( >>> interface.exists("/file.txt", "1.0.0") True - .. - >>> clear() - """ path_with_version = self._path_with_version(path, version) return self.backend.exists( @@ -320,7 +307,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -329,9 +316,6 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".", "1.0.0") ['src.txt'] - .. - >>> clear() - """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_archive( @@ -397,20 +381,18 @@ def get_file( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: - >>> file = "src.txt" - >>> interface.put_file(file, "/file.txt", "1.0.0") - >>> os.path.exists("dst.txt") + >>> src_file = "src.txt" + >>> dst_file = "dst.txt" + >>> interface.put_file(src_file, "/file.txt", "1.0.0") + >>> os.path.exists(dst_file) False - >>> _ = interface.get_file("/file.txt", "dst.txt", "1.0.0") - >>> os.path.exists("dst.txt") + >>> dst_file = interface.get_file("/file.txt", dst_file, "1.0.0") + >>> os.path.exists(dst_file) True - .. - >>> clear() - """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_file( @@ -441,7 +423,7 @@ def latest_version( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -450,9 +432,6 @@ def latest_version( >>> interface.latest_version("/file.txt") '2.0.0' - .. - >>> clear() - """ vs = self.versions(path) return vs[-1] @@ -507,7 +486,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -527,9 +506,6 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] - .. - >>> clear() - """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( @@ -650,7 +626,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -663,9 +639,6 @@ def move_file( >>> interface.exists("/file.txt", "1.0.0") False - .. - >>> clear() - """ if version is None: versions = self.versions(src_path) @@ -711,7 +684,8 @@ def owner( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) + >>> interface.owner = mock_owner Examples: >>> file = "src.txt" @@ -719,9 +693,6 @@ def owner( >>> interface.owner("/file.txt", "1.0.0") 'doctest' - .. - >>> clear() - """ path_with_version = self._path_with_version(path, version) return self.backend.owner(path_with_version) @@ -788,7 +759,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -798,9 +769,6 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz", "1.0.0") True - .. - >>> clear() - """ dst_path_with_version = self._path_with_version(dst_path, version) self.backend.put_archive( @@ -859,7 +827,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -869,9 +837,6 @@ def put_file( >>> interface.exists("/file.txt", "3.0.0") True - .. - >>> clear() - """ dst_path_with_version = self._path_with_version(dst_path, version) return self.backend.put_file( @@ -903,7 +868,7 @@ def remove_file( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -914,9 +879,6 @@ def remove_file( >>> interface.exists("/file.txt", "1.0.0") False - .. - >>> clear() - """ path_with_version = self._path_with_version(path, version) self.backend.remove_file(path_with_version) @@ -948,7 +910,7 @@ def versions( RuntimeError: if backend was not opened .. - >>> interface = Versioned(filesystem_backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -957,9 +919,6 @@ def versions( >>> interface.versions("/file.txt") ['1.0.0', '2.0.0'] - .. - >>> clear() - """ utils.check_path(path) diff --git a/tests/conftest.py b/tests/conftest.py index 8dd54aac..fd31de8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,33 @@ def authentication(): del os.environ[key] +@pytest.fixture(scope="function") +def filesystem(tmpdir): + repo = f"unittest-{pytest.UID}-{audeer.uid()[:8]}" + host = audeer.mkdir(tmpdir, "host") + audeer.mkdir(host, repo) + backend = audbackend.backend.FileSystem(host, repo) + backend.opened = True + yield backend + + +# @pytest.fixture(scope="function") +# def filesystem(tmpdir): +# repo = f"unittest-{pytest.UID}-{audeer.uid()[:8]}" +# host = audeer.mkdir(tmpdir, "host") +# audeer.mkdir(host, repo) +# backend = audbackend.backend.Filesystem(host, repo) +# +# root = audeer.mkdir(tmpdir, f"unittest-{pytest.UID}-{audeer.uid()[:8]}") +# # Wrap "local" filesystem in "dir" filesystem +# # to return paths relatiove to root +# yield fsspec.filesystem( +# "dir", +# path=root, +# fs=fsspec.filesystem("local"), +# ) + + @pytest.fixture(scope="package", autouse=False) def hosts(tmpdir_factory): return { From c236661bffcbdafb1a86c4105f47b3d7b2a23419 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 11:08:06 +0100 Subject: [PATCH 07/18] Clean up doctests --- audbackend/core/conftest.py | 83 ++---------------------- audbackend/core/errors.py | 5 +- audbackend/core/interface/base.py | 19 +----- audbackend/core/interface/maven.py | 14 ++-- audbackend/core/interface/unversioned.py | 16 ++--- audbackend/core/interface/versioned.py | 22 +++---- audbackend/core/utils.py | 5 +- 7 files changed, 34 insertions(+), 130 deletions(-) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index ae2e48f7..c185daf9 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,6 +1,5 @@ import datetime import doctest -import os import pytest import sybil @@ -18,29 +17,14 @@ patterns=["*.py"], fixtures=[ "filesystem", - "filesystem_backend", "mock_date", "mock_owner", + "mock_repr", "prepare_docstring_tests", - "clear", ], ).pytest() -class FileSystem(audbackend.backend.FileSystem): - def __init__(self, host, repository): - super().__init__(host, repository) - self.opened = True - - def _date(self, path): - date = datetime.datetime(1991, 2, 20) - date = audbackend.core.utils.date_format(date) - return date - - def _owner(self, path): - return "doctest" - - @pytest.fixture(scope="function") def mock_date(): r"""Custom date method to return a fixed date.""" @@ -63,47 +47,10 @@ def owner(path: str, version: str = None) -> str: yield owner -@pytest.fixture(scope="session", autouse=True) -def clear(): - """Clear local files and filesystem backend. - - When using a tmpdir with the scope ``"function"`` or ``"class"``, - a new tmpdir is used each line - in the docstring tests. - When using the next greater scope ``"module"``, - the same tmpdir is used in the whole file, - which means there is no scope - that provides a new tmpdir - for each function/method - within a file. - To simulate this behavior, - we use the scope ``"module"``, - and provide this ``clear()`` function - to reset after a finished docstring. - - """ - - def clear_all(): - # Clear backend - audeer.rmdir("host", "repo") - audeer.mkdir("host", "repo") - # Clear local files - files = audeer.list_file_names(".", basenames=True) - files = [file for file in files if not file == "src.txt"] - for file in files: - os.remove(file) - - yield clear_all - - @pytest.fixture(scope="function") -def filesystem_backend(): - """Filesystem backend with patched date and owner methods. - - The backend is also opened already. - - """ - yield FileSystem("host", "repo") +def mock_repr(): + """Custom __repr__ method to return fixed string.""" + return 'audbackend.interface.FileSystem("host", "repo")' @pytest.fixture(scope="function", autouse=True) @@ -116,25 +63,3 @@ def prepare_docstring_tests(tmpdir, monkeypatch): audeer.touch("src.txt") yield - - -# @pytest.fixture(scope="module", autouse=True) -# def prepare_docstring_tests(tmpdir_factory): -# r"""Code to be run before each doctest.""" -# tmp_dir = tmpdir_factory.mktemp("tmp") -# -# try: -# # Change to tmp dir -# current_dir = os.getcwd() -# os.chdir(tmp_dir) -# -# # Provide example file `src.txt` -# audeer.touch("src.txt") -# -# # Prepare backend -# audeer.mkdir("host", "repo") -# -# yield -# -# finally: -# os.chdir(current_dir) diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index c49c9aa7..76c3fb32 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -8,10 +8,11 @@ class BackendError(Exception): >>> import audeer >>> import audbackend - >>> _ = audeer.mkdir("host", "repo") Examples: - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> try: ... interface = audbackend.interface.Unversioned(backend) diff --git a/audbackend/core/interface/base.py b/audbackend/core/interface/base.py index e521bf4c..25a83024 100644 --- a/audbackend/core/interface/base.py +++ b/audbackend/core/interface/base.py @@ -36,7 +36,9 @@ def backend(self) -> Backend: backend object .. - >>> interface = Base(filesystem_backend) + >>> import audbackend + >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> interface = Base(backend) Examples: >>> interface.backend @@ -51,9 +53,6 @@ def host(self) -> str: Returns: host path - .. - >>> interface = Base(filesystem_backend) - Examples: >>> interface.host 'host' @@ -80,9 +79,6 @@ def join( or does not start with ``'/'``, or if joined path contains invalid character - .. - >>> interface = Base(filesystem_backend) - Examples: >>> interface.join("/", "file.txt") '/file.txt' @@ -101,9 +97,6 @@ def repository(self) -> str: Returns: repository name - .. - >>> interface = Base(filesystem_backend) - Examples: >>> interface.repository 'repo' @@ -118,9 +111,6 @@ def sep(self) -> str: Returns: file separator - .. - >>> interface = Base(filesystem_backend) - Examples: >>> interface.sep '/' @@ -144,9 +134,6 @@ def split( ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - .. - >>> interface = Base(filesystem_backend) - Examples: >>> interface.split("/") ('/', '') diff --git a/audbackend/core/interface/maven.py b/audbackend/core/interface/maven.py index 4e82aaac..d951e6fd 100644 --- a/audbackend/core/interface/maven.py +++ b/audbackend/core/interface/maven.py @@ -68,10 +68,12 @@ class Maven(Versioned): >>> import audeer Examples: - >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> interface = Maven(backend) + >>> file = "src.txt" >>> interface.put_archive(".", "/sub/archive.zip", "1.0.0", files=[file]) >>> for version in ["1.0.0", "2.0.0"]: ... interface.put_file(file, "/file.txt", version) @@ -80,9 +82,6 @@ class Maven(Versioned): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' - .. - >>> clear() - """ # noqa: E501 def __init__( @@ -146,7 +145,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> interface = Maven(filesystem_backend) + >>> interface = Maven(filesystem) Examples: >>> file = "src.txt" @@ -166,9 +165,6 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] - .. - >>> clear() - """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index 83097fbd..60d21f12 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -20,10 +20,12 @@ class Unversioned(Base): >>> import audeer Examples: - >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> interface = Unversioned(backend) + >>> file = "src.txt" >>> interface.put_file(file, "/file.txt") >>> interface.put_archive(".", "/sub/archive.zip", files=[file]) >>> interface.ls() @@ -31,9 +33,6 @@ class Unversioned(Base): >>> interface.get_file("/file.txt", "dst.txt") '...dst.txt' - .. - >>> clear() - """ def checksum( @@ -338,11 +337,8 @@ def get_file( Examples: >>> file = "src.txt" >>> interface.put_file(file, "/file.txt") - >>> os.path.exists("dst.txt") - False - >>> _ = interface.get_file("/file.txt", "dst.txt") - >>> os.path.exists("dst.txt") - True + >>> interface.get_file("/file.txt", "dst.txt") + '...dst.txt' """ return self.backend.get_file( diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index 3d8b0fe0..610fda47 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -23,13 +23,15 @@ class Versioned(Base): .. >>> import audbackend - >>> backend = filesystem + >>> import audeer Examples: + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") + >>> backend.open() + >>> interface = Versioned(backend) >>> file = "src.txt" - >>> # backend = audbackend.backend.FileSystem("host", "repo") - >>> # backend.open() - >>> interface = Versioned(filesystem) >>> interface.put_archive(".", "/sub/archive.zip", "1.0.0", files=[file]) >>> for version in ["1.0.0", "2.0.0"]: ... interface.put_file(file, "/file.txt", version) @@ -384,14 +386,10 @@ def get_file( >>> interface = Versioned(filesystem) Examples: - >>> src_file = "src.txt" - >>> dst_file = "dst.txt" - >>> interface.put_file(src_file, "/file.txt", "1.0.0") - >>> os.path.exists(dst_file) - False - >>> dst_file = interface.get_file("/file.txt", dst_file, "1.0.0") - >>> os.path.exists(dst_file) - True + >>> file = "src.txt" + >>> interface.put_file(file, "/file.txt", "1.0.0") + >>> interface.get_file("/file.txt", "dst.txt", "1.0.0") + '...dst.txt' """ src_path_with_version = self._path_with_version(src_path, version) diff --git a/audbackend/core/utils.py b/audbackend/core/utils.py index 262d957b..af0ffcf1 100644 --- a/audbackend/core/utils.py +++ b/audbackend/core/utils.py @@ -115,10 +115,11 @@ def checksum(file: str) -> str: >>> hash = audformat.utils.hash(df, strict=True) >>> hash '9021a9b6e1e696ba9de4fe29346319b2' + >>> parquet_file = audeer.path("file.parquet") >>> table = pa.Table.from_pandas(df) >>> table = table.replace_schema_metadata({"hash": hash}) - >>> pq.write_table(table, "file.parquet", compression="snappy") - >>> checksum("file.parquet") + >>> pq.write_table(table, parquet_file, compression="snappy") + >>> checksum(parquet_file) '9021a9b6e1e696ba9de4fe29346319b2' """ From 6b92bc48221c243b35d246bc06908b7391e3fcd0 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 11:43:20 +0100 Subject: [PATCH 08/18] Test usage documentation as well --- docs/conftest.py | 16 +++ docs/developer-guide.rst | 154 +++++++++++---------------- docs/usage.rst | 218 ++++++++++++++------------------------- 3 files changed, 156 insertions(+), 232 deletions(-) create mode 100644 docs/conftest.py diff --git a/docs/conftest.py b/docs/conftest.py new file mode 100644 index 00000000..4dc5513a --- /dev/null +++ b/docs/conftest.py @@ -0,0 +1,16 @@ +from doctest import ELLIPSIS + +from sybil import Sybil +from sybil.parsers.rest import DocTestParser +from sybil.parsers.rest import PythonCodeBlockParser + +from audbackend.core.conftest import mock_date # noqa: F401 +from audbackend.core.conftest import mock_owner # noqa: F401 +from audbackend.core.conftest import prepare_docstring_tests # noqa: F401 + + +pytest_collect_file = Sybil( + parsers=[DocTestParser(optionflags=ELLIPSIS), PythonCodeBlockParser()], + pattern="*.rst", + fixtures=["mock_date", "mock_owner", "prepare_docstring_tests"], +).pytest() diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 9d4e02b5..2490fac8 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir(os.path.join("docs", "tmp-developer-guide")) - os.chdir(_tmp_root) - - .. _developer-guide: Developer guide @@ -62,9 +50,10 @@ we implement the following helper class. -.. jupyter-execute:: +.. code-block:: python import audbackend + import os import shelve class UserDB: @@ -94,7 +83,7 @@ helper class. Now, we implement the interface. -.. jupyter-execute:: +.. code-block:: python class UserContent(audbackend.interface.Base): @@ -122,27 +111,24 @@ Let's create a repository with our custom interface, and upload a file: -.. jupyter-execute:: - import audeer - - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() - interface = UserContent(backend) - - interface.add_user("audeering", "pa$$word") - audeer.touch("local.txt") - interface.upload("audeering", "pa$$word", "local.txt") - interface.ls("audeering") +>>> import audeer +>>> host = audeer.mkdir("host") +>>> audbackend.backend.FileSystem.create(host, "repo") +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() +>>> interface = UserContent(backend) +>>> interface.add_user("audeering", "pa$$word") +>>> file = audeer.touch("local.txt") +>>> interface.upload("audeering", "pa$$word", file) +>>> interface.ls("audeering") +['/audeering/local.txt'] At the end we clean up and delete our repo. -.. jupyter-execute:: - - backend.close() - audbackend.backend.FileSystem.delete("./host", "repo") +>>> backend.close() +>>> audbackend.backend.FileSystem.delete(host, "repo") .. _develop-new-backend: @@ -188,7 +174,7 @@ in the constructor: namely ``"//db"``. * ``_db``: connection object to the database. -.. jupyter-execute:: +.. code-block:: python import audbackend import os @@ -213,7 +199,7 @@ we will dynamically add the required methods one after another using a dedicated decorator: -.. jupyter-execute:: +.. code-block:: python import functools @@ -234,7 +220,7 @@ This is not mandatory and whether it is needed depends on the backend. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def __del__(self): @@ -257,7 +243,7 @@ stored on our backend: * ``date``: the date when the file was added * ``owner``: the owner of the file -.. jupyter-execute:: +.. code-block:: python import errno import os @@ -290,9 +276,7 @@ stored on our backend: Now we create a repository. -.. jupyter-execute:: - - SQLite.create("./host", "repo") +>>> SQLite.create(host, "repo") Before we can access the repository we add a method to open @@ -300,7 +284,7 @@ an existing database (or raise an error it is not found). -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _open( @@ -319,17 +303,15 @@ and access the repository we created. We then wrap the object with the :class:`audbackend.interface.Versioned` interface. -.. jupyter-execute:: - - backend = SQLite("./host", "repo") - backend.open() - interface = audbackend.interface.Versioned(backend) +>>> backend = SQLite(host, "repo") +>>> backend.open() +>>> interface = audbackend.interface.Versioned(backend) Next, we implement a method to check if a file exists. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _exists( @@ -347,12 +329,13 @@ if a file exists. result = db.execute(query).fetchone()[0] == 1 return result - interface.exists("/file.txt", "1.0.0") +>>> interface.exists("/file.txt", "1.0.0") +False And a method that uploads a file to our backend. -.. jupyter-execute:: +.. code-block:: python import datetime import getpass @@ -379,16 +362,15 @@ a file to our backend. Let's put a file on the backend. -.. jupyter-execute:: - - file = audeer.touch("file.txt") - interface.put_file(file, "/file.txt", "1.0.0") - interface.exists("/file.txt", "1.0.0") +>>> file = audeer.touch("file.txt") +>>> interface.put_file(file, "/file.txt", "1.0.0") +>>> interface.exists("/file.txt", "1.0.0") +True We need three more functions to access its meta information. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _checksum( @@ -404,9 +386,7 @@ to access its meta information. checksum = db.execute(query).fetchone()[0] return checksum - interface.checksum("/file.txt", "1.0.0") - -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _date( @@ -422,9 +402,7 @@ to access its meta information. date = db.execute(query).fetchone()[0] return date - interface.date("/file.txt", "1.0.0") - -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _owner( @@ -440,8 +418,6 @@ to access its meta information. owner = db.execute(query).fetchone()[0] return owner - interface.owner("/file.txt", "1.0.0") - Implementing a copy function is optional. But the default implementation will temporarily download the file @@ -449,7 +425,7 @@ and then upload it again. Hence, we provide a more efficient implementation. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _copy_file( @@ -473,13 +449,14 @@ we provide a more efficient implementation. data = (dst_path, checksum, content, date, owner) db.execute(query, data) - interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") - interface.exists("/copy/file.txt", "1.0.0") +>>> interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") +>>> interface.exists("/copy/file.txt", "1.0.0") +True Implementing a move function is also optional, but it is more efficient if we provide one. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _move_file( @@ -496,14 +473,15 @@ but it is more efficient if we provide one. """ db.execute(query) - interface.move_file("/copy/file.txt", "/move/file.txt", version="1.0.0") - interface.exists("/move/file.txt", "1.0.0") +>>> interface.move_file("/copy/file.txt", "/move/file.txt", version="1.0.0") +>>> interface.exists("/move/file.txt", "1.0.0") +True We implement a method to fetch a file from the backend. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _get_file( @@ -524,15 +502,14 @@ from the backend. Which we then use to download the file. -.. jupyter-execute:: - - file = interface.get_file("/file.txt", "local.txt", "1.0.0") +>>> interface.get_file("/file.txt", "local.txt", "1.0.0") +'...local.txt' To inspect the files on our backend we provide a listing method. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _ls( @@ -556,19 +533,16 @@ we provide a listing method. Let's test it. -.. jupyter-execute:: - - interface.ls("/") - -.. jupyter-execute:: - - interface.ls("/file.txt") +>>> interface.ls("/") +[('/file.txt', '1.0.0'), ('/move/file.txt', '1.0.0')] +>>> interface.ls("/file.txt") +[('/file.txt', '1.0.0')] To delete a file from our backend requires another method. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _remove_file( @@ -583,13 +557,14 @@ requires another method. """ db.execute(query) - interface.remove_file("/file.txt", "1.0.0") - interface.ls("/") +>>> interface.remove_file("/file.txt", "1.0.0") +>>> interface.ls("/") +[('/move/file.txt', '1.0.0')] We add a method to close the connection to a database and call it. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _close( @@ -597,7 +572,7 @@ to a database and call it. ): self._db.close() - backend.close() +>>> backend.close() Finally, we add a method that @@ -606,7 +581,7 @@ and removes the repository (or raises an error if the database does not exist). -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _delete( @@ -621,20 +596,11 @@ if the database does not exist). os.remove(self._path) os.rmdir(os.path.dirname(self._path)) - SQLite.delete("./host", "repo") +>>> SQLite.delete(host, "repo") And that's it, we have a fully functional backend. Voilà! -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: - - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) - - .. _SQLite: https://sqlite.org/index.html diff --git a/docs/usage.rst b/docs/usage.rst index 1150c9cc..b593e56d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir("docs", "tmp-usage") - os.chdir(_tmp_root) - - .. _usage: Usage @@ -39,12 +27,10 @@ To store data on a backend we need to create a repository first. We select the :class:`audbackend.backend.FileSystem` backend. -.. jupyter-execute:: - :hide-output: - - import audbackend - - audbackend.backend.FileSystem.create("./host", "repo") +>>> import audbackend +>>> import audeer +>>> host = audeer.mkdir("host") +>>> audbackend.backend.FileSystem.create(host, "repo") Once we have an existing repository, we can access it by instantiating the backend class. @@ -56,10 +42,8 @@ If you are unsure whether your backend requires this step, just do it always. -.. jupyter-execute:: - - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() After establishing a connection we could directly execute read and write operations @@ -72,9 +56,7 @@ Here, we use :class:`audbackend.interface.Unversioned`. It does not support versioning, i.e. exactly one file exists for a backend path. -.. jupyter-execute:: - - interface = audbackend.interface.Unversioned(backend) +>>> interface = audbackend.interface.Unversioned(backend) Now we can upload our first file to the repository. Note, @@ -82,59 +64,53 @@ it is important to provide an absolute path from the root of the repository by starting it with ``/``. -.. jupyter-execute:: - - import audeer - - file = audeer.touch("file.txt") - interface.put_file(file, "/file.txt") +>>> file = audeer.touch("file.txt") +>>> interface.put_file(file, "/file.txt") We check if the file exists in the repository. -.. jupyter-execute:: - - interface.exists("/file.txt") +>>> interface.exists("/file.txt") +True And access its meta information, like its checksum. -.. jupyter-execute:: - - interface.checksum("/file.txt") +>>> interface.checksum("/file.txt") +'d41d8cd98f00b204e9800998ecf8427e' Its creation date. -.. jupyter-execute:: +.. + >>> interface.date = mock_date - interface.date("/file.txt") +>>> interface.date("/file.txt") +'1991-02-20' Or the owner who uploaded the file. -.. jupyter-execute:: +.. + >>> interface.owner = mock_owner - interface.owner("/file.txt") +>>> interface.owner("/file.txt") +'doctest' We create a copy of the file and verify it exists. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt") - interface.exists("/copy/file.txt") +>>> interface.copy_file("/file.txt", "/copy/file.txt") +>>> interface.exists("/copy/file.txt") +True We move it to a new location. -.. jupyter-execute:: - - interface.move_file("/copy/file.txt", "/move/file.txt") - interface.exists("/copy/file.txt"), interface.exists("/move/file.txt") +>>> interface.move_file("/copy/file.txt", "/move/file.txt") +>>> interface.exists("/copy/file.txt"), interface.exists("/move/file.txt") +(False, True) We download the file and store it as ``local.txt``. -.. jupyter-execute:: - - file = interface.get_file("/file.txt", "local.txt") +>>> file = interface.get_file("/file.txt", "local.txt") It is possible to upload one or more files @@ -146,29 +122,24 @@ and store them as ``folder.zip`` under the sub-path ``/archives/`` in the repository. -.. jupyter-execute:: - - folder = audeer.mkdir("./folder") - audeer.touch(folder, "file1.txt") - audeer.touch(folder, "file2.txt") - interface.put_archive(folder, "/archives/folder.zip") +>>> folder = audeer.mkdir("./folder") +>>> _ = audeer.touch(folder, "file1.txt") +>>> _ = audeer.touch(folder, "file2.txt") +>>> interface.put_archive(folder, "/archives/folder.zip") When we download an archive it is automatically extracted, when using :meth:`audbackend.interface.Unversioned.get_archive` instead of :meth:`audbackend.interface.Unversioned.get_file`. -.. jupyter-execute:: - - paths = interface.get_archive("/archives/folder.zip", "downloaded_folder") - paths +>>> interface.get_archive("/archives/folder.zip", "downloaded_folder") +['file1.txt', 'file2.txt'] We can list all files in the repository. -.. jupyter-execute:: - - interface.ls("/") +>>> interface.ls("/") +['/archives/folder.zip', '/file.txt', '/move/file.txt'] If we provide a sub-path @@ -177,31 +148,25 @@ a list with files that start with the sub-path is returned. -.. jupyter-execute:: - - interface.ls("/archives/") +>>> interface.ls("/archives/") +['/archives/folder.zip'] We can remove files. -.. jupyter-execute:: - - interface.remove_file("/file.txt") - interface.remove_file("/archives/folder.zip") - interface.ls("/") +>>> interface.remove_file("/file.txt") +>>> interface.remove_file("/archives/folder.zip") +>>> interface.ls("/") +['/move/file.txt'] Finally, we close the connection to the backend. -.. jupyter-execute:: - - backend.close() +>>> backend.close() And delete the whole repository with all its content. -.. jupyter-execute:: - - audbackend.backend.FileSystem.delete("host", "repo") +>>> audbackend.backend.FileSystem.delete(host, "repo") Now, if we try to open the repository again, @@ -210,12 +175,11 @@ we will get an error for all backend classes as it depends on the implementation). -.. jupyter-execute:: - - try: - backend.open() - except audbackend.BackendError as ex: - display(str(ex.exception)) +>>> try: +... backend.open() +... except audbackend.BackendError as ex: +... print(str(ex.exception)) +[Errno 2] No such file or directory: .../host/repo/' .. _versioned-data-on-a-file-system: @@ -229,92 +193,70 @@ This time we access it with the :class:`audbackend.interface.Versioned` interface (which is also used by default). -.. jupyter-execute:: - - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() - interface = audbackend.interface.Versioned(backend) +>>> audbackend.backend.FileSystem.create(host, "repo") +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() +>>> interface = audbackend.interface.Versioned(backend) We then upload a file and assign version ``"1.0.0"`` to it. -.. jupyter-execute:: - - with open("file.txt", "w") as file: - file.write("Content v1.0.0") - interface.put_file("file.txt", "/file.txt", "1.0.0") +>>> file = audeer.path("file.txt") +>>> with open(file, "w") as fp: +... _ = fp.write("Content v1.0.0") +>>> interface.put_file(file, "/file.txt", "1.0.0") Now we change the file for version ``"2.0.0"``. -.. jupyter-execute:: - - with open("file.txt", "w") as file: - file.write("Content v2.0.0") - interface.put_file("file.txt", "/file.txt", "2.0.0") +>>> with open(file, "w") as fp: +... _ = fp.write("Content v2.0.0") +>>> interface.put_file(file, "/file.txt", "2.0.0") If we inspect the content of the repository it will return a list of tuples containing file name and version. -.. jupyter-execute:: - - interface.ls("/") +>>> interface.ls("/") +[('/file.txt', '1.0.0'), ('/file.txt', '2.0.0')] We can also inspect the available versions for a file. -.. jupyter-execute:: - - interface.versions("/file.txt") +>>> interface.versions("/file.txt") +['1.0.0', '2.0.0'] Or request it's latest version. -.. jupyter-execute:: - - interface.latest_version("/file.txt") +>>> interface.latest_version("/file.txt") +'2.0.0' We can copy a specific version of a file. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") - interface.ls("/copy/") +>>> interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") +>>> interface.ls("/copy/") +[('/copy/file.txt', '1.0.0')] Or all versions. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt") - interface.ls("/copy/") +>>> interface.copy_file("/file.txt", "/copy/file.txt") +>>> interface.ls("/copy/") +[('/copy/file.txt', '1.0.0'), ('/copy/file.txt', '2.0.0')] We move them to a new location. -.. jupyter-execute:: - - interface.move_file("/copy/file.txt", "/move/file.txt") - interface.ls("/move/") +>>> interface.move_file("/copy/file.txt", "/move/file.txt") +>>> interface.ls("/move/") +[('/move/file.txt', '1.0.0'), ('/move/file.txt', '2.0.0')] When downloading a file, we can select the desired version. -.. jupyter-execute:: - - path = interface.get_file("/file.txt", "local.txt", "1.0.0") - with open(path, "r") as file: - display(file.read()) +>>> path = interface.get_file("/file.txt", "local.txt", "1.0.0") +>>> with open(path, "r") as file: +... print(file.read()) +Content v1.0.0 When we are done, we close the connection to the repository. -.. jupyter-execute:: - - backend.close() - -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: - - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) +>>> backend.close() From 122546e93f767c5b0d683b803cade2f226d9c894 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 12:01:46 +0100 Subject: [PATCH 09/18] Move filesystem fixture --- audbackend/core/conftest.py | 25 ++++++++++++++++++++++++- tests/conftest.py | 27 --------------------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index c185daf9..80be51e0 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -8,7 +8,6 @@ import audeer import audbackend -from tests.conftest import filesystem # noqa: F401 # Collect doctests @@ -25,6 +24,30 @@ ).pytest() +@pytest.fixture(scope="function") +def filesystem(tmpdir): + """Filesystem backend. + + A repository with unique name is created + for the filesystem backend. + The filesystem backend is marked as opened + and returned. + + Args: + tmpdir: tmpdir fixture + + Returns: + filesystem backend object + + """ + repo = f"repo-{audeer.uid()[:8]}" + host = audeer.mkdir(tmpdir, "host") + audeer.mkdir(host, repo) + backend = audbackend.backend.FileSystem(host, repo) + backend.opened = True + yield backend + + @pytest.fixture(scope="function") def mock_date(): r"""Custom date method to return a fixed date.""" diff --git a/tests/conftest.py b/tests/conftest.py index fd31de8c..8dd54aac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,33 +45,6 @@ def authentication(): del os.environ[key] -@pytest.fixture(scope="function") -def filesystem(tmpdir): - repo = f"unittest-{pytest.UID}-{audeer.uid()[:8]}" - host = audeer.mkdir(tmpdir, "host") - audeer.mkdir(host, repo) - backend = audbackend.backend.FileSystem(host, repo) - backend.opened = True - yield backend - - -# @pytest.fixture(scope="function") -# def filesystem(tmpdir): -# repo = f"unittest-{pytest.UID}-{audeer.uid()[:8]}" -# host = audeer.mkdir(tmpdir, "host") -# audeer.mkdir(host, repo) -# backend = audbackend.backend.Filesystem(host, repo) -# -# root = audeer.mkdir(tmpdir, f"unittest-{pytest.UID}-{audeer.uid()[:8]}") -# # Wrap "local" filesystem in "dir" filesystem -# # to return paths relatiove to root -# yield fsspec.filesystem( -# "dir", -# path=root, -# fs=fsspec.filesystem("local"), -# ) - - @pytest.fixture(scope="package", autouse=False) def hosts(tmpdir_factory): return { From 96a341b8d44bfe957ba4b868c3b357c39d3d8fc2 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 12:12:41 +0100 Subject: [PATCH 10/18] Avoid creation of file outside tmpdir --- docs/developer-guide.rst | 2 +- docs/usage.rst | 6 ++++-- tests/test_interface_unversioned.py | 9 +++++---- tests/test_interface_versioned.py | 19 +++++-------------- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 2490fac8..f585dce6 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -502,7 +502,7 @@ from the backend. Which we then use to download the file. ->>> interface.get_file("/file.txt", "local.txt", "1.0.0") +>>> interface.get_file("/file.txt", audeer.path("local.txt"), "1.0.0") '...local.txt' To inspect the files diff --git a/docs/usage.rst b/docs/usage.rst index b593e56d..ef737154 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -110,7 +110,9 @@ We move it to a new location. We download the file and store it as ``local.txt``. ->>> file = interface.get_file("/file.txt", "local.txt") +>>> file = audeer.path("local.txt") +>>> interface.get_file("/file.txt", file) +'...local.txt' It is possible to upload one or more files @@ -251,7 +253,7 @@ We move them to a new location. When downloading a file, we can select the desired version. ->>> path = interface.get_file("/file.txt", "local.txt", "1.0.0") +>>> path = interface.get_file("/file.txt", audeer.path("local.txt"), "1.0.0") >>> with open(path, "r") as file: ... print(file.read()) Content v1.0.0 diff --git a/tests/test_interface_unversioned.py b/tests/test_interface_unversioned.py index ec153a1d..a539c9ce 100644 --- a/tests/test_interface_unversioned.py +++ b/tests/test_interface_unversioned.py @@ -788,11 +788,12 @@ def _checksum( interface.put_file(path, "/remote.txt", validate=True) assert interface.exists("/remote.txt") + local_file = audeer.path(tmpdir, "local.txt") with pytest.raises(InterruptedError, match=error_msg): - interface_bad.get_file("/remote.txt", "local.txt", validate=True) - assert not os.path.exists("local.txt") - interface.get_file("/remote.txt", "local.txt", validate=True) - assert os.path.exists("local.txt") + interface_bad.get_file("/remote.txt", local_file, validate=True) + assert not os.path.exists(local_file) + interface.get_file("/remote.txt", local_file, validate=True) + assert os.path.exists(local_file) with pytest.raises(InterruptedError, match=error_msg): interface_bad.copy_file("/remote.txt", "/copy.txt", validate=True) diff --git a/tests/test_interface_versioned.py b/tests/test_interface_versioned.py index e0c3b063..72db19a2 100644 --- a/tests/test_interface_versioned.py +++ b/tests/test_interface_versioned.py @@ -1158,21 +1158,12 @@ def _checksum( interface.put_file(path, "/remote.txt", "1.0.0", validate=True) assert interface.exists("/remote.txt", "1.0.0") + local_file = audeer.path(tmpdir, "local.txt") with pytest.raises(InterruptedError, match=error_msg): - interface_bad.get_file( - "/remote.txt", - "local.txt", - "1.0.0", - validate=True, - ) - assert not os.path.exists("local.txt") - interface.get_file( - "/remote.txt", - "local.txt", - "1.0.0", - validate=True, - ) - assert os.path.exists("local.txt") + interface_bad.get_file("/remote.txt", local_file, "1.0.0", validate=True) + assert not os.path.exists(local_file) + interface.get_file("/remote.txt", local_file, "1.0.0", validate=True) + assert os.path.exists(local_file) with pytest.raises(InterruptedError, match=error_msg): interface_bad.copy_file( From b700bc78b5c7324dba57abba55846b5a888b4bf0 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:01:45 +0100 Subject: [PATCH 11/18] Remove dependency on jupyer-sphinx --- docs/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3fe7fffd..7e70f5a1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,3 @@ -ipykernel -jupyter-sphinx sphinx sphinx-apipages sphinx-audeering-theme >=1.2.1 From b8a8da10b6e667bfd58d181bb74215e0efe066c2 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:11:41 +0100 Subject: [PATCH 12/18] Update legacy backend section --- docs/legacy.rst | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/docs/legacy.rst b/docs/legacy.rst index ad657fb2..c662ad33 100644 --- a/docs/legacy.rst +++ b/docs/legacy.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir(os.path.join("docs", "tmp")) - os.chdir(_tmp_root) - - .. _legacy-backends: Legacy backends @@ -44,34 +32,28 @@ that contain a dot in its file extension, you have to list those extensions explicitly. -.. jupyter-execute:: +.. code-block:: python import audbackend + import audeer - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") + host = audeer.mkdir("host") + audbackend.backend.FileSystem.create(host, "repo") + backend = audbackend.backend.FileSystem(host, "repo") backend.open() interface = audbackend.interface.Maven(backend, extensions=["tar.gz"]) -Afterwards we upload an TAR.GZ archive -and check that it is stored as expected. +Afterwards we upload an TAR.GZ archive. -.. jupyter-execute:: +.. code-block:: python - import audeer import tempfile with tempfile.TemporaryDirectory() as tmp: audeer.touch(audeer.path(tmp, "file.txt")) interface.put_archive(tmp, "/file.tar.gz", "1.0.0") - audeer.list_file_names("./host", recursive=True, basenames=True) - - -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: +And check that it is stored as expected. - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) +>>> audeer.list_file_names(host, recursive=True, basenames=True) +['repo/file/1.0.0/file-1.0.0.tar.gz'] From d27a2e6bcc5c92accdbfc5c6f315a3a0d7ebb24b Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:14:04 +0100 Subject: [PATCH 13/18] Remove jupyter-sphinx from docs/conf.py --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b249438d..948fda7b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ ] pygments_style = None extensions = [ - "jupyter_sphinx", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", From d9bdaaca9da2d4b816b61435bffb49a2ea81d499 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:25:26 +0100 Subject: [PATCH 14/18] Try to fix under MacOS --- docs/developer-guide.rst | 33 ++++++++++++++++++++++----------- docs/legacy.rst | 2 ++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index f585dce6..c1a190fe 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -52,9 +52,11 @@ helper class. .. code-block:: python - import audbackend import os - import shelve + import pickle + + import audbackend + class UserDB: r"""User database. @@ -65,20 +67,24 @@ helper class. """ def __init__(self, backend: audbackend.backend.Base): self.backend = backend + self.remote_file = "/.db.pkl" + self.local_file = audeer.path(".db.pkl") def __enter__(self) -> shelve.Shelf: - if self.backend.exists("/user.db"): - self.backend.get_file("/user.db", "~.db") - self._map = shelve.open("~.db", flag="w", writeback=True) + if self.backend.exists(self.remote_file): + self.backend.get_file(self.remote_file, self.local_file) + if os.path.exists(self.local_file): + with open(self.local_file, "rb") as file: + self._map = pickle.load(file) else: - self._map = shelve.open("~.db", writeback=True) + self._map = {} return self._map def __exit__(self, exc_type, exc_val, exc_tb): - self._map.close() - self.backend.put_file("~.db", "/user.db") - os.remove("~.db") - + with open(self.local_file, "wb") as file: + pickle.dump(self._map, file, protocol=pickle.HIGHEST_PROTOCOL) + self.backend.put_file(self.local_file, self.remote_file) + os.remove(self.local_file) Now, we implement the interface. @@ -176,9 +182,11 @@ in the constructor: .. code-block:: python - import audbackend import os + import audbackend + + class SQLite(audbackend.backend.Base): def __init__( @@ -203,6 +211,7 @@ using a dedicated decorator: import functools + def add_method(cls): def decorator(func): @functools.wraps(func) @@ -249,6 +258,7 @@ stored on our backend: import os import sqlite3 as sl + @add_method(SQLite) def _create( self, @@ -340,6 +350,7 @@ a file to our backend. import datetime import getpass + @add_method(SQLite) def _put_file( self, diff --git a/docs/legacy.rst b/docs/legacy.rst index c662ad33..60200d88 100644 --- a/docs/legacy.rst +++ b/docs/legacy.rst @@ -37,6 +37,7 @@ you have to list those extensions explicitly. import audbackend import audeer + host = audeer.mkdir("host") audbackend.backend.FileSystem.create(host, "repo") backend = audbackend.backend.FileSystem(host, "repo") @@ -49,6 +50,7 @@ Afterwards we upload an TAR.GZ archive. import tempfile + with tempfile.TemporaryDirectory() as tmp: audeer.touch(audeer.path(tmp, "file.txt")) interface.put_archive(tmp, "/file.tar.gz", "1.0.0") From 859ab10128fd0d765059515793af9f235ca092ab Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:30:26 +0100 Subject: [PATCH 15/18] Fix typo --- docs/developer-guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index c1a190fe..8c8a9c08 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -70,7 +70,7 @@ helper class. self.remote_file = "/.db.pkl" self.local_file = audeer.path(".db.pkl") - def __enter__(self) -> shelve.Shelf: + def __enter__(self) -> dict: if self.backend.exists(self.remote_file): self.backend.get_file(self.remote_file, self.local_file) if os.path.exists(self.local_file): From 5aff7a671a70990d911dec74676c5fd8e4d3b8ff Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:43:28 +0100 Subject: [PATCH 16/18] Try to fix under Windows --- docs/legacy.rst | 5 +++++ docs/usage.rst | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docs/legacy.rst b/docs/legacy.rst index 60200d88..39a252e5 100644 --- a/docs/legacy.rst +++ b/docs/legacy.rst @@ -57,5 +57,10 @@ Afterwards we upload an TAR.GZ archive. And check that it is stored as expected. +.. + >>> import platform + +.. skip: next if(platform.system() == "Windows") + >>> audeer.list_file_names(host, recursive=True, basenames=True) ['repo/file/1.0.0/file-1.0.0.tar.gz'] diff --git a/docs/usage.rst b/docs/usage.rst index ef737154..27416974 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -177,6 +177,11 @@ we will get an error for all backend classes as it depends on the implementation). +.. + >>> import platform + +.. skip: next if(platform.system() == "Windows") + >>> try: ... backend.open() ... except audbackend.BackendError as ex: From 8632f15c6ee2b7243198b6a9cec593b9053884fa Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:44:40 +0100 Subject: [PATCH 17/18] Add missing SkipParser --- docs/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/conftest.py b/docs/conftest.py index 4dc5513a..209f61a5 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,6 +3,7 @@ from sybil import Sybil from sybil.parsers.rest import DocTestParser from sybil.parsers.rest import PythonCodeBlockParser +from sybil.parsers.rest import SkipParser from audbackend.core.conftest import mock_date # noqa: F401 from audbackend.core.conftest import mock_owner # noqa: F401 @@ -10,7 +11,11 @@ pytest_collect_file = Sybil( - parsers=[DocTestParser(optionflags=ELLIPSIS), PythonCodeBlockParser()], + parsers=[ + DocTestParser(optionflags=ELLIPSIS), + PythonCodeBlockParser(), + SkipParser(), + ], pattern="*.rst", fixtures=["mock_date", "mock_owner", "prepare_docstring_tests"], ).pytest() From a185e8aae0b909722479f2d9f93894c3c7bc8abf Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 13:57:53 +0100 Subject: [PATCH 18/18] Update docs/usage.rst Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 27416974..a263e409 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -232,7 +232,7 @@ for a file. >>> interface.versions("/file.txt") ['1.0.0', '2.0.0'] -Or request it's latest version. +Or request its latest version. >>> interface.latest_version("/file.txt") '2.0.0'