From 2647892a817ec42585e12990d2d22587a9ef1df8 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 17:38:13 +0100 Subject: [PATCH] TST: use sybil for doctests (#250) * TST: use sybil for doctests * Hand fixtures to sybil * Add filesystem_backend fixture * Further work on sybil for docstrings * Fix doctests * Add further ideas * Clean up doctests * Test usage documentation as well * Move filesystem fixture * Avoid creation of file outside tmpdir * Remove dependency on jupyer-sphinx * Update legacy backend section * Remove jupyter-sphinx from docs/conf.py * Try to fix under MacOS * Fix typo * Try to fix under Windows * Add missing SkipParser * Update docs/usage.rst Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --------- Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- audbackend/core/backend/minio.py | 2 +- audbackend/core/conftest.py | 104 +++++++---- audbackend/core/errors.py | 7 +- audbackend/core/interface/base.py | 21 +-- audbackend/core/interface/maven.py | 14 +- audbackend/core/interface/unversioned.py | 67 +++---- audbackend/core/interface/versioned.py | 76 +++----- audbackend/core/utils.py | 5 +- docs/conf.py | 1 - docs/conftest.py | 21 +++ docs/developer-guide.rst | 189 +++++++++---------- docs/legacy.rst | 41 ++--- docs/requirements.txt | 2 - docs/usage.rst | 225 +++++++++-------------- pyproject.toml | 1 - tests/requirements.txt | 2 +- tests/test_interface_unversioned.py | 9 +- tests/test_interface_versioned.py | 19 +- 18 files changed, 355 insertions(+), 451 deletions(-) create mode 100644 docs/conftest.py diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py index a86be1fc..ed180960 100644 --- a/audbackend/core/backend/minio.py +++ b/audbackend/core/backend/minio.py @@ -40,7 +40,7 @@ class Minio(Base): >>> auth = ("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") >>> repository = "my-data" + audeer.uid() >>> Minio.create(host, repository, authentication=auth) - >>> file = audeer.touch("file.txt") + >>> file = audeer.touch("src.txt") >>> backend = Minio(host, repository, authentication=auth) >>> try: ... with backend: diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 4d533f68..80be51e0 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,50 +1,88 @@ import datetime -import os -import tempfile +import doctest import pytest +import sybil +from sybil.parsers.rest import DocTestParser import audeer import audbackend -class DoctestFileSystem(audbackend.backend.FileSystem): - def _date( - self, - path: str, - ) -> str: +# Collect doctests +pytest_collect_file = sybil.Sybil( + parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], + patterns=["*.py"], + fixtures=[ + "filesystem", + "mock_date", + "mock_owner", + "mock_repr", + "prepare_docstring_tests", + ], +).pytest() + + +@pytest.fixture(scope="function") +def filesystem(tmpdir): + """Filesystem backend. + + A repository with unique name is created + for the filesystem backend. + The filesystem backend is marked as opened + and returned. + + Args: + tmpdir: tmpdir fixture + + Returns: + filesystem backend object + + """ + repo = f"repo-{audeer.uid()[:8]}" + host = audeer.mkdir(tmpdir, "host") + audeer.mkdir(host, repo) + backend = audbackend.backend.FileSystem(host, repo) + backend.opened = True + yield backend + + +@pytest.fixture(scope="function") +def mock_date(): + r"""Custom date method to return a fixed date.""" + + def date(path: str, version: str = None) -> str: date = datetime.datetime(1991, 2, 20) date = audbackend.core.utils.date_format(date) return date - def _owner( - self, - path: str, - ) -> str: + yield date + + +@pytest.fixture(scope="function") +def mock_owner(): + r"""Custom owner method to return a fixed owner.""" + + def owner(path: str, version: str = None) -> str: return "doctest" + yield owner + + +@pytest.fixture(scope="function") +def mock_repr(): + """Custom __repr__ method to return fixed string.""" + return 'audbackend.interface.FileSystem("host", "repo")' + @pytest.fixture(scope="function", autouse=True) -def prepare_docstring_tests(doctest_namespace): - with tempfile.TemporaryDirectory() as tmp: - # Change to tmp dir - current_dir = os.getcwd() - os.chdir(tmp) - # Prepare backend - audeer.mkdir("host") - audbackend.backend.FileSystem.create("host", "repo") - # Provide example file `src.txt` - audeer.touch("src.txt") - # Provide DoctestFileSystem as FileSystem, - # and audbackend - # in docstring examples - doctest_namespace["DoctestFileSystem"] = DoctestFileSystem - doctest_namespace["audbackend"] = audbackend - - yield - - # Remove backend - audbackend.backend.FileSystem.delete("host", "repo") - # Change back to current dir - os.chdir(current_dir) +def prepare_docstring_tests(tmpdir, monkeypatch): + r"""Code to be run before each doctest.""" + # Change to tmp dir + monkeypatch.chdir(tmpdir) + + # Provide example file `src.txt` + audeer.touch("src.txt") + + yield diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index ab7bd2a2..76c3fb32 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -7,11 +7,12 @@ class BackendError(Exception): .. Prepare backend and interface for docstring examples >>> import audeer - >>> audeer.rmdir("host", "repo") - >>> _ = audeer.mkdir("host", "repo") + >>> import audbackend Examples: - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> try: ... interface = audbackend.interface.Unversioned(backend) diff --git a/audbackend/core/interface/base.py b/audbackend/core/interface/base.py index 836be8d9..25a83024 100644 --- a/audbackend/core/interface/base.py +++ b/audbackend/core/interface/base.py @@ -36,6 +36,7 @@ def backend(self) -> Backend: backend object .. + >>> import audbackend >>> backend = audbackend.backend.FileSystem("host", "repo") >>> interface = Base(backend) @@ -52,10 +53,6 @@ def host(self) -> str: Returns: host path - .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) - Examples: >>> interface.host 'host' @@ -82,10 +79,6 @@ def join( or does not start with ``'/'``, or if joined path contains invalid character - .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) - Examples: >>> interface.join("/", "file.txt") '/file.txt' @@ -104,10 +97,6 @@ def repository(self) -> str: Returns: repository name - .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) - Examples: >>> interface.repository 'repo' @@ -122,10 +111,6 @@ def sep(self) -> str: Returns: file separator - .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) - Examples: >>> interface.sep '/' @@ -149,10 +134,6 @@ def split( ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> interface = Base(backend) - Examples: >>> interface.split("/") ('/', '') diff --git a/audbackend/core/interface/maven.py b/audbackend/core/interface/maven.py index 59c5c1db..d951e6fd 100644 --- a/audbackend/core/interface/maven.py +++ b/audbackend/core/interface/maven.py @@ -63,11 +63,17 @@ class Maven(Versioned): ... as extensions + .. + >>> import audbackend + >>> import audeer + Examples: - >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> interface = Maven(backend) + >>> file = "src.txt" >>> interface.put_archive(".", "/sub/archive.zip", "1.0.0", files=[file]) >>> for version in ["1.0.0", "2.0.0"]: ... interface.put_file(file, "/file.txt", version) @@ -139,9 +145,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Maven(backend) + >>> interface = Maven(filesystem) Examples: >>> file = "src.txt" diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index fe334d95..60d21f12 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -15,11 +15,17 @@ class Unversioned(Base): Args: backend: backend object + .. + >>> import audbackend + >>> import audeer + Examples: - >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> interface = Unversioned(backend) + >>> file = "src.txt" >>> interface.put_file(file, "/file.txt") >>> interface.put_archive(".", "/sub/archive.zip", files=[file]) >>> interface.ls() @@ -50,9 +56,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -106,9 +110,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -151,9 +153,8 @@ def date( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) + >>> interface.date = mock_date Examples: >>> file = "src.txt" @@ -193,9 +194,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -264,9 +263,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -335,18 +332,13 @@ def get_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" >>> interface.put_file(file, "/file.txt") - >>> os.path.exists("dst.txt") - False - >>> _ = interface.get_file("/file.txt", "dst.txt") - >>> os.path.exists("dst.txt") - True + >>> interface.get_file("/file.txt", "dst.txt") + '...dst.txt' """ return self.backend.get_file( @@ -403,9 +395,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -473,9 +463,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -521,9 +509,8 @@ def owner( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) + >>> interface.owner = mock_owner Examples: >>> file = "src.txt" @@ -592,9 +579,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -654,9 +639,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" @@ -691,9 +674,7 @@ def remove_file( or does not match ``'[A-Za-z0-9/._-]+'`` .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Unversioned(backend) + >>> interface = Unversioned(filesystem) Examples: >>> file = "src.txt" diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index a620f4ce..610fda47 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -21,13 +21,17 @@ class Versioned(Base): Args: backend: backend object - .. Prepare backend and interface for docstring examples + .. + >>> import audbackend + >>> import audeer Examples: - >>> file = "src.txt" - >>> backend = audbackend.backend.FileSystem("host", "repo") + >>> host = audeer.mkdir("host") + >>> audbackend.backend.FileSystem.create(host, "repo") + >>> backend = audbackend.backend.FileSystem(host, "repo") >>> backend.open() >>> interface = Versioned(backend) + >>> file = "src.txt" >>> interface.put_archive(".", "/sub/archive.zip", "1.0.0", files=[file]) >>> for version in ["1.0.0", "2.0.0"]: ... interface.put_file(file, "/file.txt", version) @@ -36,7 +40,6 @@ class Versioned(Base): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' - """ def __init__( @@ -70,9 +73,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -135,9 +136,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -192,9 +191,8 @@ def date( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) + >>> interface.date = mock_date Examples: >>> file = "src.txt" @@ -237,9 +235,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -313,9 +309,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -389,18 +383,13 @@ def get_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" >>> interface.put_file(file, "/file.txt", "1.0.0") - >>> os.path.exists("dst.txt") - False - >>> _ = interface.get_file("/file.txt", "dst.txt", "1.0.0") - >>> os.path.exists("dst.txt") - True + >>> interface.get_file("/file.txt", "dst.txt", "1.0.0") + '...dst.txt' """ src_path_with_version = self._path_with_version(src_path, version) @@ -432,9 +421,7 @@ def latest_version( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -497,9 +484,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -639,9 +624,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -699,9 +682,8 @@ def owner( RuntimeError: if backend was not opened .. - >>> backend = DoctestFileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) + >>> interface.owner = mock_owner Examples: >>> file = "src.txt" @@ -775,9 +757,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -845,9 +825,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -888,9 +866,7 @@ def remove_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" @@ -932,9 +908,7 @@ def versions( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem) Examples: >>> file = "src.txt" diff --git a/audbackend/core/utils.py b/audbackend/core/utils.py index 262d957b..af0ffcf1 100644 --- a/audbackend/core/utils.py +++ b/audbackend/core/utils.py @@ -115,10 +115,11 @@ def checksum(file: str) -> str: >>> hash = audformat.utils.hash(df, strict=True) >>> hash '9021a9b6e1e696ba9de4fe29346319b2' + >>> parquet_file = audeer.path("file.parquet") >>> table = pa.Table.from_pandas(df) >>> table = table.replace_schema_metadata({"hash": hash}) - >>> pq.write_table(table, "file.parquet", compression="snappy") - >>> checksum("file.parquet") + >>> pq.write_table(table, parquet_file, compression="snappy") + >>> checksum(parquet_file) '9021a9b6e1e696ba9de4fe29346319b2' """ diff --git a/docs/conf.py b/docs/conf.py index b249438d..948fda7b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ ] pygments_style = None extensions = [ - "jupyter_sphinx", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", diff --git a/docs/conftest.py b/docs/conftest.py new file mode 100644 index 00000000..209f61a5 --- /dev/null +++ b/docs/conftest.py @@ -0,0 +1,21 @@ +from doctest import ELLIPSIS + +from sybil import Sybil +from sybil.parsers.rest import DocTestParser +from sybil.parsers.rest import PythonCodeBlockParser +from sybil.parsers.rest import SkipParser + +from audbackend.core.conftest import mock_date # noqa: F401 +from audbackend.core.conftest import mock_owner # noqa: F401 +from audbackend.core.conftest import prepare_docstring_tests # noqa: F401 + + +pytest_collect_file = Sybil( + parsers=[ + DocTestParser(optionflags=ELLIPSIS), + PythonCodeBlockParser(), + SkipParser(), + ], + pattern="*.rst", + fixtures=["mock_date", "mock_owner", "prepare_docstring_tests"], +).pytest() diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 9d4e02b5..8c8a9c08 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir(os.path.join("docs", "tmp-developer-guide")) - os.chdir(_tmp_root) - - .. _developer-guide: Developer guide @@ -62,10 +50,13 @@ we implement the following helper class. -.. jupyter-execute:: +.. code-block:: python + + import os + import pickle import audbackend - import shelve + class UserDB: r"""User database. @@ -76,25 +67,29 @@ helper class. """ def __init__(self, backend: audbackend.backend.Base): self.backend = backend - - def __enter__(self) -> shelve.Shelf: - if self.backend.exists("/user.db"): - self.backend.get_file("/user.db", "~.db") - self._map = shelve.open("~.db", flag="w", writeback=True) + self.remote_file = "/.db.pkl" + self.local_file = audeer.path(".db.pkl") + + def __enter__(self) -> dict: + if self.backend.exists(self.remote_file): + self.backend.get_file(self.remote_file, self.local_file) + if os.path.exists(self.local_file): + with open(self.local_file, "rb") as file: + self._map = pickle.load(file) else: - self._map = shelve.open("~.db", writeback=True) + self._map = {} return self._map def __exit__(self, exc_type, exc_val, exc_tb): - self._map.close() - self.backend.put_file("~.db", "/user.db") - os.remove("~.db") - + with open(self.local_file, "wb") as file: + pickle.dump(self._map, file, protocol=pickle.HIGHEST_PROTOCOL) + self.backend.put_file(self.local_file, self.remote_file) + os.remove(self.local_file) Now, we implement the interface. -.. jupyter-execute:: +.. code-block:: python class UserContent(audbackend.interface.Base): @@ -122,27 +117,24 @@ Let's create a repository with our custom interface, and upload a file: -.. jupyter-execute:: - - import audeer - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() - interface = UserContent(backend) - - interface.add_user("audeering", "pa$$word") - audeer.touch("local.txt") - interface.upload("audeering", "pa$$word", "local.txt") - interface.ls("audeering") +>>> import audeer +>>> host = audeer.mkdir("host") +>>> audbackend.backend.FileSystem.create(host, "repo") +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() +>>> interface = UserContent(backend) +>>> interface.add_user("audeering", "pa$$word") +>>> file = audeer.touch("local.txt") +>>> interface.upload("audeering", "pa$$word", file) +>>> interface.ls("audeering") +['/audeering/local.txt'] At the end we clean up and delete our repo. -.. jupyter-execute:: - - backend.close() - audbackend.backend.FileSystem.delete("./host", "repo") +>>> backend.close() +>>> audbackend.backend.FileSystem.delete(host, "repo") .. _develop-new-backend: @@ -188,11 +180,13 @@ in the constructor: namely ``"//db"``. * ``_db``: connection object to the database. -.. jupyter-execute:: +.. code-block:: python - import audbackend import os + import audbackend + + class SQLite(audbackend.backend.Base): def __init__( @@ -213,10 +207,11 @@ we will dynamically add the required methods one after another using a dedicated decorator: -.. jupyter-execute:: +.. code-block:: python import functools + def add_method(cls): def decorator(func): @functools.wraps(func) @@ -234,7 +229,7 @@ This is not mandatory and whether it is needed depends on the backend. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def __del__(self): @@ -257,12 +252,13 @@ stored on our backend: * ``date``: the date when the file was added * ``owner``: the owner of the file -.. jupyter-execute:: +.. code-block:: python import errno import os import sqlite3 as sl + @add_method(SQLite) def _create( self, @@ -290,9 +286,7 @@ stored on our backend: Now we create a repository. -.. jupyter-execute:: - - SQLite.create("./host", "repo") +>>> SQLite.create(host, "repo") Before we can access the repository we add a method to open @@ -300,7 +294,7 @@ an existing database (or raise an error it is not found). -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _open( @@ -319,17 +313,15 @@ and access the repository we created. We then wrap the object with the :class:`audbackend.interface.Versioned` interface. -.. jupyter-execute:: - - backend = SQLite("./host", "repo") - backend.open() - interface = audbackend.interface.Versioned(backend) +>>> backend = SQLite(host, "repo") +>>> backend.open() +>>> interface = audbackend.interface.Versioned(backend) Next, we implement a method to check if a file exists. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _exists( @@ -347,16 +339,18 @@ if a file exists. result = db.execute(query).fetchone()[0] == 1 return result - interface.exists("/file.txt", "1.0.0") +>>> interface.exists("/file.txt", "1.0.0") +False And a method that uploads a file to our backend. -.. jupyter-execute:: +.. code-block:: python import datetime import getpass + @add_method(SQLite) def _put_file( self, @@ -379,16 +373,15 @@ a file to our backend. Let's put a file on the backend. -.. jupyter-execute:: - - file = audeer.touch("file.txt") - interface.put_file(file, "/file.txt", "1.0.0") - interface.exists("/file.txt", "1.0.0") +>>> file = audeer.touch("file.txt") +>>> interface.put_file(file, "/file.txt", "1.0.0") +>>> interface.exists("/file.txt", "1.0.0") +True We need three more functions to access its meta information. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _checksum( @@ -404,9 +397,7 @@ to access its meta information. checksum = db.execute(query).fetchone()[0] return checksum - interface.checksum("/file.txt", "1.0.0") - -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _date( @@ -422,9 +413,7 @@ to access its meta information. date = db.execute(query).fetchone()[0] return date - interface.date("/file.txt", "1.0.0") - -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _owner( @@ -440,8 +429,6 @@ to access its meta information. owner = db.execute(query).fetchone()[0] return owner - interface.owner("/file.txt", "1.0.0") - Implementing a copy function is optional. But the default implementation will temporarily download the file @@ -449,7 +436,7 @@ and then upload it again. Hence, we provide a more efficient implementation. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _copy_file( @@ -473,13 +460,14 @@ we provide a more efficient implementation. data = (dst_path, checksum, content, date, owner) db.execute(query, data) - interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") - interface.exists("/copy/file.txt", "1.0.0") +>>> interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") +>>> interface.exists("/copy/file.txt", "1.0.0") +True Implementing a move function is also optional, but it is more efficient if we provide one. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _move_file( @@ -496,14 +484,15 @@ but it is more efficient if we provide one. """ db.execute(query) - interface.move_file("/copy/file.txt", "/move/file.txt", version="1.0.0") - interface.exists("/move/file.txt", "1.0.0") +>>> interface.move_file("/copy/file.txt", "/move/file.txt", version="1.0.0") +>>> interface.exists("/move/file.txt", "1.0.0") +True We implement a method to fetch a file from the backend. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _get_file( @@ -524,15 +513,14 @@ from the backend. Which we then use to download the file. -.. jupyter-execute:: - - file = interface.get_file("/file.txt", "local.txt", "1.0.0") +>>> interface.get_file("/file.txt", audeer.path("local.txt"), "1.0.0") +'...local.txt' To inspect the files on our backend we provide a listing method. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _ls( @@ -556,19 +544,16 @@ we provide a listing method. Let's test it. -.. jupyter-execute:: - - interface.ls("/") - -.. jupyter-execute:: - - interface.ls("/file.txt") +>>> interface.ls("/") +[('/file.txt', '1.0.0'), ('/move/file.txt', '1.0.0')] +>>> interface.ls("/file.txt") +[('/file.txt', '1.0.0')] To delete a file from our backend requires another method. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _remove_file( @@ -583,13 +568,14 @@ requires another method. """ db.execute(query) - interface.remove_file("/file.txt", "1.0.0") - interface.ls("/") +>>> interface.remove_file("/file.txt", "1.0.0") +>>> interface.ls("/") +[('/move/file.txt', '1.0.0')] We add a method to close the connection to a database and call it. -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _close( @@ -597,7 +583,7 @@ to a database and call it. ): self._db.close() - backend.close() +>>> backend.close() Finally, we add a method that @@ -606,7 +592,7 @@ and removes the repository (or raises an error if the database does not exist). -.. jupyter-execute:: +.. code-block:: python @add_method(SQLite) def _delete( @@ -621,20 +607,11 @@ if the database does not exist). os.remove(self._path) os.rmdir(os.path.dirname(self._path)) - SQLite.delete("./host", "repo") +>>> SQLite.delete(host, "repo") And that's it, we have a fully functional backend. Voilà! -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: - - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) - - .. _SQLite: https://sqlite.org/index.html diff --git a/docs/legacy.rst b/docs/legacy.rst index ad657fb2..39a252e5 100644 --- a/docs/legacy.rst +++ b/docs/legacy.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir(os.path.join("docs", "tmp")) - os.chdir(_tmp_root) - - .. _legacy-backends: Legacy backends @@ -44,34 +32,35 @@ that contain a dot in its file extension, you have to list those extensions explicitly. -.. jupyter-execute:: +.. code-block:: python import audbackend + import audeer + - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") + host = audeer.mkdir("host") + audbackend.backend.FileSystem.create(host, "repo") + backend = audbackend.backend.FileSystem(host, "repo") backend.open() interface = audbackend.interface.Maven(backend, extensions=["tar.gz"]) -Afterwards we upload an TAR.GZ archive -and check that it is stored as expected. +Afterwards we upload an TAR.GZ archive. -.. jupyter-execute:: +.. code-block:: python - import audeer import tempfile + with tempfile.TemporaryDirectory() as tmp: audeer.touch(audeer.path(tmp, "file.txt")) interface.put_archive(tmp, "/file.tar.gz", "1.0.0") - audeer.list_file_names("./host", recursive=True, basenames=True) +And check that it is stored as expected. +.. + >>> import platform -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: +.. skip: next if(platform.system() == "Windows") - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) +>>> audeer.list_file_names(host, recursive=True, basenames=True) +['repo/file/1.0.0/file-1.0.0.tar.gz'] diff --git a/docs/requirements.txt b/docs/requirements.txt index 3fe7fffd..7e70f5a1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,3 @@ -ipykernel -jupyter-sphinx sphinx sphinx-apipages sphinx-audeering-theme >=1.2.1 diff --git a/docs/usage.rst b/docs/usage.rst index 1150c9cc..a263e409 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,15 +1,3 @@ -.. set temporal working directory -.. jupyter-execute:: - :hide-code: - - import os - import audeer - - _cwd_root = os.getcwd() - _tmp_root = audeer.mkdir("docs", "tmp-usage") - os.chdir(_tmp_root) - - .. _usage: Usage @@ -39,12 +27,10 @@ To store data on a backend we need to create a repository first. We select the :class:`audbackend.backend.FileSystem` backend. -.. jupyter-execute:: - :hide-output: - - import audbackend - - audbackend.backend.FileSystem.create("./host", "repo") +>>> import audbackend +>>> import audeer +>>> host = audeer.mkdir("host") +>>> audbackend.backend.FileSystem.create(host, "repo") Once we have an existing repository, we can access it by instantiating the backend class. @@ -56,10 +42,8 @@ If you are unsure whether your backend requires this step, just do it always. -.. jupyter-execute:: - - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() After establishing a connection we could directly execute read and write operations @@ -72,9 +56,7 @@ Here, we use :class:`audbackend.interface.Unversioned`. It does not support versioning, i.e. exactly one file exists for a backend path. -.. jupyter-execute:: - - interface = audbackend.interface.Unversioned(backend) +>>> interface = audbackend.interface.Unversioned(backend) Now we can upload our first file to the repository. Note, @@ -82,59 +64,55 @@ it is important to provide an absolute path from the root of the repository by starting it with ``/``. -.. jupyter-execute:: - - import audeer - - file = audeer.touch("file.txt") - interface.put_file(file, "/file.txt") +>>> file = audeer.touch("file.txt") +>>> interface.put_file(file, "/file.txt") We check if the file exists in the repository. -.. jupyter-execute:: - - interface.exists("/file.txt") +>>> interface.exists("/file.txt") +True And access its meta information, like its checksum. -.. jupyter-execute:: - - interface.checksum("/file.txt") +>>> interface.checksum("/file.txt") +'d41d8cd98f00b204e9800998ecf8427e' Its creation date. -.. jupyter-execute:: +.. + >>> interface.date = mock_date - interface.date("/file.txt") +>>> interface.date("/file.txt") +'1991-02-20' Or the owner who uploaded the file. -.. jupyter-execute:: +.. + >>> interface.owner = mock_owner - interface.owner("/file.txt") +>>> interface.owner("/file.txt") +'doctest' We create a copy of the file and verify it exists. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt") - interface.exists("/copy/file.txt") +>>> interface.copy_file("/file.txt", "/copy/file.txt") +>>> interface.exists("/copy/file.txt") +True We move it to a new location. -.. jupyter-execute:: - - interface.move_file("/copy/file.txt", "/move/file.txt") - interface.exists("/copy/file.txt"), interface.exists("/move/file.txt") +>>> interface.move_file("/copy/file.txt", "/move/file.txt") +>>> interface.exists("/copy/file.txt"), interface.exists("/move/file.txt") +(False, True) We download the file and store it as ``local.txt``. -.. jupyter-execute:: - - file = interface.get_file("/file.txt", "local.txt") +>>> file = audeer.path("local.txt") +>>> interface.get_file("/file.txt", file) +'...local.txt' It is possible to upload one or more files @@ -146,29 +124,24 @@ and store them as ``folder.zip`` under the sub-path ``/archives/`` in the repository. -.. jupyter-execute:: - - folder = audeer.mkdir("./folder") - audeer.touch(folder, "file1.txt") - audeer.touch(folder, "file2.txt") - interface.put_archive(folder, "/archives/folder.zip") +>>> folder = audeer.mkdir("./folder") +>>> _ = audeer.touch(folder, "file1.txt") +>>> _ = audeer.touch(folder, "file2.txt") +>>> interface.put_archive(folder, "/archives/folder.zip") When we download an archive it is automatically extracted, when using :meth:`audbackend.interface.Unversioned.get_archive` instead of :meth:`audbackend.interface.Unversioned.get_file`. -.. jupyter-execute:: - - paths = interface.get_archive("/archives/folder.zip", "downloaded_folder") - paths +>>> interface.get_archive("/archives/folder.zip", "downloaded_folder") +['file1.txt', 'file2.txt'] We can list all files in the repository. -.. jupyter-execute:: - - interface.ls("/") +>>> interface.ls("/") +['/archives/folder.zip', '/file.txt', '/move/file.txt'] If we provide a sub-path @@ -177,31 +150,25 @@ a list with files that start with the sub-path is returned. -.. jupyter-execute:: - - interface.ls("/archives/") +>>> interface.ls("/archives/") +['/archives/folder.zip'] We can remove files. -.. jupyter-execute:: - - interface.remove_file("/file.txt") - interface.remove_file("/archives/folder.zip") - interface.ls("/") +>>> interface.remove_file("/file.txt") +>>> interface.remove_file("/archives/folder.zip") +>>> interface.ls("/") +['/move/file.txt'] Finally, we close the connection to the backend. -.. jupyter-execute:: - - backend.close() +>>> backend.close() And delete the whole repository with all its content. -.. jupyter-execute:: - - audbackend.backend.FileSystem.delete("host", "repo") +>>> audbackend.backend.FileSystem.delete(host, "repo") Now, if we try to open the repository again, @@ -210,12 +177,16 @@ we will get an error for all backend classes as it depends on the implementation). -.. jupyter-execute:: +.. + >>> import platform + +.. skip: next if(platform.system() == "Windows") - try: - backend.open() - except audbackend.BackendError as ex: - display(str(ex.exception)) +>>> try: +... backend.open() +... except audbackend.BackendError as ex: +... print(str(ex.exception)) +[Errno 2] No such file or directory: .../host/repo/' .. _versioned-data-on-a-file-system: @@ -229,92 +200,70 @@ This time we access it with the :class:`audbackend.interface.Versioned` interface (which is also used by default). -.. jupyter-execute:: - - audbackend.backend.FileSystem.create("./host", "repo") - backend = audbackend.backend.FileSystem("./host", "repo") - backend.open() - interface = audbackend.interface.Versioned(backend) +>>> audbackend.backend.FileSystem.create(host, "repo") +>>> backend = audbackend.backend.FileSystem(host, "repo") +>>> backend.open() +>>> interface = audbackend.interface.Versioned(backend) We then upload a file and assign version ``"1.0.0"`` to it. -.. jupyter-execute:: - - with open("file.txt", "w") as file: - file.write("Content v1.0.0") - interface.put_file("file.txt", "/file.txt", "1.0.0") +>>> file = audeer.path("file.txt") +>>> with open(file, "w") as fp: +... _ = fp.write("Content v1.0.0") +>>> interface.put_file(file, "/file.txt", "1.0.0") Now we change the file for version ``"2.0.0"``. -.. jupyter-execute:: - - with open("file.txt", "w") as file: - file.write("Content v2.0.0") - interface.put_file("file.txt", "/file.txt", "2.0.0") +>>> with open(file, "w") as fp: +... _ = fp.write("Content v2.0.0") +>>> interface.put_file(file, "/file.txt", "2.0.0") If we inspect the content of the repository it will return a list of tuples containing file name and version. -.. jupyter-execute:: - - interface.ls("/") +>>> interface.ls("/") +[('/file.txt', '1.0.0'), ('/file.txt', '2.0.0')] We can also inspect the available versions for a file. -.. jupyter-execute:: +>>> interface.versions("/file.txt") +['1.0.0', '2.0.0'] - interface.versions("/file.txt") +Or request its latest version. -Or request it's latest version. - -.. jupyter-execute:: - - interface.latest_version("/file.txt") +>>> interface.latest_version("/file.txt") +'2.0.0' We can copy a specific version of a file. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") - interface.ls("/copy/") +>>> interface.copy_file("/file.txt", "/copy/file.txt", version="1.0.0") +>>> interface.ls("/copy/") +[('/copy/file.txt', '1.0.0')] Or all versions. -.. jupyter-execute:: - - interface.copy_file("/file.txt", "/copy/file.txt") - interface.ls("/copy/") +>>> interface.copy_file("/file.txt", "/copy/file.txt") +>>> interface.ls("/copy/") +[('/copy/file.txt', '1.0.0'), ('/copy/file.txt', '2.0.0')] We move them to a new location. -.. jupyter-execute:: - - interface.move_file("/copy/file.txt", "/move/file.txt") - interface.ls("/move/") +>>> interface.move_file("/copy/file.txt", "/move/file.txt") +>>> interface.ls("/move/") +[('/move/file.txt', '1.0.0'), ('/move/file.txt', '2.0.0')] When downloading a file, we can select the desired version. -.. jupyter-execute:: - - path = interface.get_file("/file.txt", "local.txt", "1.0.0") - with open(path, "r") as file: - display(file.read()) +>>> path = interface.get_file("/file.txt", audeer.path("local.txt"), "1.0.0") +>>> with open(path, "r") as file: +... print(file.read()) +Content v1.0.0 When we are done, we close the connection to the repository. -.. jupyter-execute:: - - backend.close() - -.. reset working directory and clean up -.. jupyter-execute:: - :hide-code: - - import shutil - os.chdir(_cwd_root) - shutil.rmtree(_tmp_root) +>>> backend.close() diff --git a/pyproject.toml b/pyproject.toml index f2f6c191..8d14f485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,6 @@ skip = './audbackend.egg-info,./build,.docs/api,./docs/_templates' cache_dir = '.cache/pytest' xfail_strict = true addopts = ''' - --doctest-plus --cov=audbackend --cov-fail-under=100 --cov-report term-missing diff --git a/tests/requirements.txt b/tests/requirements.txt index e9d7f60d..a57476c0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ audformat pytest pytest-cov -pytest-doctestplus +sybil diff --git a/tests/test_interface_unversioned.py b/tests/test_interface_unversioned.py index ec153a1d..a539c9ce 100644 --- a/tests/test_interface_unversioned.py +++ b/tests/test_interface_unversioned.py @@ -788,11 +788,12 @@ def _checksum( interface.put_file(path, "/remote.txt", validate=True) assert interface.exists("/remote.txt") + local_file = audeer.path(tmpdir, "local.txt") with pytest.raises(InterruptedError, match=error_msg): - interface_bad.get_file("/remote.txt", "local.txt", validate=True) - assert not os.path.exists("local.txt") - interface.get_file("/remote.txt", "local.txt", validate=True) - assert os.path.exists("local.txt") + interface_bad.get_file("/remote.txt", local_file, validate=True) + assert not os.path.exists(local_file) + interface.get_file("/remote.txt", local_file, validate=True) + assert os.path.exists(local_file) with pytest.raises(InterruptedError, match=error_msg): interface_bad.copy_file("/remote.txt", "/copy.txt", validate=True) diff --git a/tests/test_interface_versioned.py b/tests/test_interface_versioned.py index e0c3b063..72db19a2 100644 --- a/tests/test_interface_versioned.py +++ b/tests/test_interface_versioned.py @@ -1158,21 +1158,12 @@ def _checksum( interface.put_file(path, "/remote.txt", "1.0.0", validate=True) assert interface.exists("/remote.txt", "1.0.0") + local_file = audeer.path(tmpdir, "local.txt") with pytest.raises(InterruptedError, match=error_msg): - interface_bad.get_file( - "/remote.txt", - "local.txt", - "1.0.0", - validate=True, - ) - assert not os.path.exists("local.txt") - interface.get_file( - "/remote.txt", - "local.txt", - "1.0.0", - validate=True, - ) - assert os.path.exists("local.txt") + interface_bad.get_file("/remote.txt", local_file, "1.0.0", validate=True) + assert not os.path.exists(local_file) + interface.get_file("/remote.txt", local_file, "1.0.0", validate=True) + assert os.path.exists(local_file) with pytest.raises(InterruptedError, match=error_msg): interface_bad.copy_file(