diff --git a/audbackend/backend/__init__.py b/audbackend/backend/__init__.py index 06be3a6a..01397be9 100644 --- a/audbackend/backend/__init__.py +++ b/audbackend/backend/__init__.py @@ -6,3 +6,7 @@ from audbackend.core.backend.artifactory import Artifactory except ImportError: # pragma: no cover pass +try: + from audbackend.core.backend.minio import Minio +except ImportError: # pragma: no cover + pass diff --git a/audbackend/core/api.py b/audbackend/core/api.py index 890fa65b..428c8ea5 100644 --- a/audbackend/core/api.py +++ b/audbackend/core/api.py @@ -233,3 +233,11 @@ def register( register("artifactory", Artifactory) except ImportError: # pragma: no cover pass + + # Register optional backends + try: + from audbackend.core.backend.minio import Minio + + register("minio", Minio) + except ImportError: # pragma: no cover + pass diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py new file mode 100644 index 00000000..18ff74bb --- /dev/null +++ b/audbackend/core/backend/minio.py @@ -0,0 +1,417 @@ +import configparser +import getpass +import mimetypes +import os +import tempfile +import typing + +import minio + +import audeer + +from audbackend.core import utils +from audbackend.core.backend.base import Base + + +class Minio(Base): + r"""Backend for MinIO. + + Args: + host: host address + repository: repository name + authentication: username, password / access key, secret key token tuple. + If ``None``, + it requests it by calling :meth:`get_authentication` + secure: if ``None``, + it looks in the config file for it, + compare :meth:`get_config`. + If it cannot find a matching entry, + it defaults to ``True``. + Needs to be ``True`` + when using TLS for the connection, + and ``False`` otherwise, + e.g. when using a `local MinIO server`_. + **kwargs: keyword arguments passed on to `minio.Minio`_ + + .. _local MinIO server: https://min.io/docs/minio/container/index.html + .. _minio.Minio: https://min.io/docs/minio/linux/developers/python/API.html + + Examples: + >>> host = "play.min.io" # playground provided by https://min.io + >>> auth = ("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") + >>> repository = "my-data" + audeer.uid() + >>> Minio.create(host, repository, authentication=auth) + >>> file = audeer.touch("file.txt") + >>> backend = Minio(host, repository, authentication=auth) + >>> try: + ... with backend: + ... backend.put_file(file, "/sub/file.txt") + ... backend.ls() + ... finally: + ... Minio.delete(host, repository, authentication=auth) + ['/sub/file.txt'] + + """ # noqa: E501 + + def __init__( + self, + host: str, + repository: str, + *, + authentication: typing.Tuple[str, str] = None, + secure: bool = None, + **kwargs, + ): + super().__init__(host, repository, authentication=authentication) + + if authentication is None: + self.authentication = self.get_authentication(host) + + if secure is None: + config = self.get_config(host) + secure = config.get("secure", True) + + # Open MinIO client + self._client = minio.Minio( + host, + access_key=self.authentication[0], + secret_key=self.authentication[1], + secure=secure, + **kwargs, + ) + + @classmethod + def get_authentication(cls, host: str) -> typing.Tuple[str, str]: + """Access and secret tokens for given host. + + Returns a authentication for MinIO server + as tuple. + + To get the authentication tuple, + the function looks first + for the two environment variables + ``MINIO_ACCESS_KEY`` and + ``MINIO_SECRET_KEY``. + Otherwise, + it tries to extract missing values + from a config file, + see :meth:`get_config`. + If no config file exists + or if it has missing entries, + ``None`` is returned + for the missing entries. + + Args: + host: hostname + + Returns: + access token tuple + + """ + config = cls.get_config(host) + access_key = os.getenv("MINIO_ACCESS_KEY", config.get("access_key")) + secret_key = os.getenv("MINIO_SECRET_KEY", config.get("secret_key")) + + return access_key, secret_key + + @classmethod + def get_config(cls, host: str) -> typing.Dict: + """Configuration of MinIO server. + + The default path of the config file is + :file:`~/.config/audbackend/minio.cfg`. + It can be overwritten with the environment variable + ``MINIO_CONFIG_FILE``. + + If no config file can be found, + or no entry for the requested host, + an empty dictionary is returned. + + The config file + expects one section per host, + e.g. + + .. code-block:: ini + + [play.min.io] + access_key = "Q3AM3UQ867SPQQA43P2F" + secret_key = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + + Args: + host: hostname + + Returns: + config entries as dictionary + + """ + config_file = os.getenv("MINIO_CONFIG_FILE", "~/.config/audbackend/minio.cfg") + config_file = audeer.path(config_file) + + if os.path.exists(config_file): + config = configparser.ConfigParser(allow_no_value=True) + config.read(config_file) + try: + config = dict(config.items(host)) + except configparser.NoSectionError: + config = {} + else: + config = {} + + return config + + def close( + self, + ): + r"""Close connection to backend. + + This will only change the status of + :attr:`audbackend.backend.Minio.opened` + as Minio handles closing the session itself. + + """ + if self.opened: + self.opened = False + + def _checksum( + self, + path: str, + ) -> str: + r"""MD5 checksum of file on backend.""" + path = self.path(path) + return self._client.stat_object(self.repository, path).etag + + def _collapse( + self, + path, + ): + r"""Convert to virtual path. + + + -> + / + + """ + path = f"/{path}" + return path.replace("/", self.sep) + + def _copy_file( + self, + src_path: str, + dst_path: str, + verbose: bool, + ): + r"""Copy file on backend.""" + src_path = self.path(src_path) + dst_path = self.path(dst_path) + # `copy_object()` has a maximum size limit of 5GB. + # We use 4.9GB to have some headroom + if self._size(src_path) / 1024 / 1024 / 1024 >= 4.9: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = audeer.path(tmp_dir, os.path.basename(src_path)) + self._get_file(src_path, tmp_path, verbose) + checksum = self._checksum(src_path) + self._put_file(tmp_path, dst_path, checksum, verbose) + else: + self._client.copy_object( + self.repository, + dst_path, + minio.commonconfig.CopySource(self.repository, src_path), + metadata=_metadata(), + ) + + def _create( + self, + ): + r"""Create repository.""" + if self._client.bucket_exists(self.repository): + utils.raise_file_exists_error(self.repository) + self._client.make_bucket(self.repository) + + def _date( + self, + path: str, + ) -> str: + r"""Get last modification date of file on backend.""" + path = self.path(path) + date = self._client.stat_object(self.repository, path).last_modified + date = utils.date_format(date) + return date + + def _delete( + self, + ): + r"""Delete repository and all its content.""" + # Delete all objects in bucket + objects = self._client.list_objects(self.repository, recursive=True) + for obj in objects: + self._client.remove_object(self.repository, obj.object_name) + # Delete bucket + self._client.remove_bucket(self.repository) + + def _exists( + self, + path: str, + ) -> bool: + r"""Check if file exists on backend.""" + path = self.path(path) + try: + self._client.stat_object(self.repository, path) + except minio.error.S3Error: + return False + return True + + def _get_file( + self, + src_path: str, + dst_path: str, + verbose: bool, + ): + r"""Get file from backend.""" + src_path = self.path(src_path) + src_size = self._client.stat_object(self.repository, src_path).size + chunk = 4 * 1024 + with audeer.progress_bar(total=src_size, disable=not verbose) as pbar: + desc = audeer.format_display_message( + f"Download {os.path.basename(str(src_path))}", pbar=True + ) + pbar.set_description_str(desc) + pbar.refresh() + + dst_size = 0 + try: + response = self._client.get_object(self.repository, src_path) + with open(dst_path, "wb") as dst_fp: + while src_size > dst_size: + data = response.read(chunk) + n_data = len(data) + if n_data > 0: + dst_fp.write(data) + dst_size += n_data + pbar.update(n_data) + except Exception as e: # pragma: no cover + raise RuntimeError(f"Error downloading file: {e}") + finally: + response.close() + response.release_conn() + + def _ls( + self, + path: str, + ) -> typing.List[str]: + r"""List all files under sub-path.""" + path = self.path(path) + objects = self._client.list_objects( + self.repository, + prefix=path, + recursive=True, + ) + return [self._collapse(obj.object_name) for obj in objects] + + def _move_file( + self, + src_path: str, + dst_path: str, + verbose: bool, + ): + r"""Move file on backend.""" + self._copy_file(src_path, dst_path, verbose) + self._remove_file(src_path) + + def _open( + self, + ): + r"""Open connection to backend.""" + # At the moment, session management is handled automatically. + # If we want to manage this ourselves, + # we need to use the `http_client` argument of `minio.Minio` + if not self._client.bucket_exists(self.repository): + utils.raise_file_not_found_error(self.repository) + + def _owner( + self, + path: str, + ) -> str: + r"""Get owner of file on backend.""" + path = self.path(path) + # NOTE: + # we use a custom metadata entry to track the owner + # as stats.owner_name is always empty. + meta = self._client.stat_object(self.repository, path).metadata + return meta["x-amz-meta-owner"] if "x-amz-meta-owner" in meta else "" + + def path( + self, + path: str, + ) -> str: + r"""Convert to backend path. + + Args: + path: path on backend + + Returns: + path + + """ + path = path.replace(self.sep, "/") + if path.startswith("/"): + # /path -> path + path = path[1:] + return path + + def _put_file( + self, + src_path: str, + dst_path: str, + checksum: str, + verbose: bool, + ): + r"""Put file to backend.""" + dst_path = self.path(dst_path) + if verbose: # pragma: no cover + desc = audeer.format_display_message( + f"Deploy {src_path}", + pbar=False, + ) + print(desc, end="\r") + + content_type = mimetypes.guess_type(src_path)[0] or "application/octet-stream" + self._client.fput_object( + self.repository, + dst_path, + src_path, + content_type=content_type, + metadata=_metadata(), + ) + + if verbose: # pragma: no cover + # Clear progress line + print(audeer.format_display_message(" ", pbar=False), end="\r") + + def _remove_file( + self, + path: str, + ): + r"""Remove file from backend.""" + path = self.path(path) + # Enforce error if path does not exist + self._client.stat_object(self.repository, path) + self._client.remove_object(self.repository, path) + + def _size( + self, + path: str, + ) -> int: + r"""Get size of file on backend.""" + path = self.path(path) + size = self._client.stat_object(self.repository, path).size + return size + + +def _metadata(): + """Dictionary with owner entry. + + When uploaded as metadata to MinIO, + it can be accessed under ``stat_object(...).metadata["x-amz-meta-owner"]``. + + """ + return {"owner": getpass.getuser()} diff --git a/docs/api-src/audbackend.backend.rst b/docs/api-src/audbackend.backend.rst index 221e81a4..3075e3f8 100644 --- a/docs/api-src/audbackend.backend.rst +++ b/docs/api-src/audbackend.backend.rst @@ -14,6 +14,7 @@ backends are supported: Artifactory FileSystem + Minio Users can implement their own backend by deriving from diff --git a/pyproject.toml b/pyproject.toml index 78643310..fd0feea3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,12 @@ dynamic = ['version'] artifactory = [ 'dohq-artifactory >=0.10.0', ] +minio = [ + 'minio', +] all = [ 'dohq-artifactory >=0.10.0', + 'minio', ] diff --git a/tests/conftest.py b/tests/conftest.py index 3ea04592..e0e3d2eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,37 @@ # unittest-- pytest.UID = audeer.uid()[:8] +# Define static hosts +pytest.HOSTS = { + "artifactory": "https://audeering.jfrog.io/artifactory", + "minio": "play.min.io", +} + + +@pytest.fixture(scope="package", autouse=True) +def authentication(): + """Provide authentication tokens for supported backends.""" + if pytest.HOSTS["minio"] == "play.min.io": + defaults = { + key: os.environ.get(key, None) + for key in ["MINIO_ACCESS_KEY", "MINIO_SECRET_KEY"] + } + # MinIO credentials for the public read/write server + # at play.min.io, see + # https://min.io/docs/minio/linux/developers/python/minio-py.html + os.environ["MINIO_ACCESS_KEY"] = "Q3AM3UQ867SPQQA43P2F" + os.environ["MINIO_SECRET_KEY"] = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + else: + defaults = {} + + yield + + for key, value in defaults.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] + @pytest.fixture(scope="package", autouse=True) def register_single_folder(): @@ -31,8 +62,9 @@ def hosts(tmpdir_factory): return { # For tests based on backend names (deprecated), # like audbackend.access() - "artifactory": "https://audeering.jfrog.io/artifactory", + "artifactory": pytest.HOSTS["artifactory"], "file-system": str(tmpdir_factory.mktemp("host")), + "minio": pytest.HOSTS["minio"], "single-folder": str(tmpdir_factory.mktemp("host")), } @@ -45,7 +77,15 @@ def owner(request): hasattr(audbackend.backend, "Artifactory") and backend_cls == audbackend.backend.Artifactory ): - owner = backend_cls.get_authentication("audeering.jfrog.io/artifactory")[0] + host_wo_https = pytest.HOSTS["artifactory"][8:] + owner = backend_cls.get_authentication(host_wo_https)[0] + elif ( + hasattr(audbackend.backend, "Minio") and backend_cls == audbackend.backend.Minio + ): + if os.name == "nt": + owner = "runneradmin" + else: + owner = getpass.getuser() else: if os.name == "nt": owner = "Administrators" @@ -76,14 +116,18 @@ def interface(tmpdir_factory, request): """ backend_cls, interface_cls = request.param + artifactory = False if ( hasattr(audbackend.backend, "Artifactory") and backend_cls == audbackend.backend.Artifactory ): artifactory = True - host = "https://audeering.jfrog.io/artifactory" + host = pytest.HOSTS["artifactory"] + elif ( + hasattr(audbackend.backend, "Minio") and backend_cls == audbackend.backend.Minio + ): + host = pytest.HOSTS["minio"] else: - artifactory = False host = str(tmpdir_factory.mktemp("host")) repository = f"unittest-{pytest.UID}-{audeer.uid()[:8]}" diff --git a/tests/test_api.py b/tests/test_api.py index 1efb1d81..584c3d3d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,6 +20,12 @@ f"unittest-{audeer.uid()[:8]}", audbackend.backend.Artifactory, ), + ( + "minio", + "minio", + f"unittest-{audeer.uid()[:8]}", + audbackend.backend.Minio, + ), pytest.param( # backend does not exist "bad-backend", None, diff --git a/tests/test_backend_artifactory.py b/tests/test_backend_artifactory.py index 42108fa3..48d77ffc 100644 --- a/tests/test_backend_artifactory.py +++ b/tests/test_backend_artifactory.py @@ -70,14 +70,14 @@ def test_authentication(tmpdir, hosts, hide_credentials): backend.open() -@pytest.mark.parametrize("host", ["https://audeering.jfrog.io/artifactory"]) +@pytest.mark.parametrize("host", [pytest.HOSTS["artifactory"]]) @pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) def test_create_delete_repositories(host, repository): audbackend.backend.Artifactory.create(host, repository) audbackend.backend.Artifactory.delete(host, repository) -@pytest.mark.parametrize("host", ["https://audeering.jfrog.io/artifactory"]) +@pytest.mark.parametrize("host", [pytest.HOSTS["artifactory"]]) @pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) @pytest.mark.parametrize("authentication", [("non-existing", "non-existing")]) def test_errors(host, repository, authentication): diff --git a/tests/test_backend_minio.py b/tests/test_backend_minio.py new file mode 100644 index 00000000..6fc71e61 --- /dev/null +++ b/tests/test_backend_minio.py @@ -0,0 +1,344 @@ +import os + +import pytest + +import audeer + +import audbackend + + +@pytest.fixture(scope="function", autouse=False) +def hide_credentials(): + defaults = { + key: os.environ.get(key, None) + for key in [ + "MINIO_ACCESS_KEY", + "MINIO_SECRET_KEY", + "MINIO_CONFIG_FILE", + ] + } + for key, value in defaults.items(): + if value is not None: + del os.environ[key] + + yield + + for key, value in defaults.items(): + if value is not None: + os.environ[key] = value + elif key in os.environ: + del os.environ[key] + + +def test_authentication(tmpdir, hosts, hide_credentials): + host = hosts["minio"] + config_path = audeer.path(tmpdir, "config.cfg") + os.environ["MINIO_CONFIG_FILE"] = config_path + + # config file does not exist + + backend = audbackend.backend.Minio(host, "repository") + assert backend.authentication == (None, None) + + # config file is empty + + audeer.touch(config_path) + backend = audbackend.backend.Minio(host, "repository") + assert backend.authentication == (None, None) + + # config file entry without username and password + + with open(config_path, "w") as fp: + fp.write(f"[{host}]\n") + + backend = audbackend.backend.Minio(host, "repository") + assert backend.authentication == (None, None) + + # config file entry with username and password + + access_key = "bad" + secret_key = "bad" + with open(config_path, "w") as fp: + fp.write(f"[{host}]\n") + fp.write(f"access_key = {access_key}\n") + fp.write(f"secret_key = {secret_key}\n") + + backend = audbackend.backend.Minio(host, "repository") + assert backend.authentication == ("bad", "bad") + with pytest.raises(audbackend.BackendError): + backend.open() + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Minio, audbackend.interface.Unversioned)], + indirect=True, +) +@pytest.mark.parametrize( + "path, expected,", + [ + ("/text.txt", "text/plain"), + ], +) +def test_content_type(tmpdir, interface, path, expected): + r"""Test setting of content type. + + Args: + tmpdir: tmpdir fixture + interface: interface fixture + path: path of file on backend + expected: expected content type + + """ + tmp_path = audeer.touch(audeer.path(tmpdir, path[1:])) + interface.put_file(tmp_path, path) + stats = interface._backend._client.stat_object(interface.repository, path) + assert stats.content_type == expected + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Minio, audbackend.interface.Unversioned)], + indirect=True, +) +@pytest.mark.parametrize( + "src_path, dst_path,", + [ + ( + "/big.1.txt", + "/big.2.txt", + ), + ], +) +def test_copy_large_file(tmpdir, interface, src_path, dst_path): + r"""Test copying of large files. + + ``minio.Minio.copy_object()`` has a limit of 5 GB. + We mock the ``audbackend.backend.Minio._size()`` method + to return a value equivalent to 5 GB. + to trigger the fall back copy mechanism for large files, + without having to create a large file. + + Args: + tmpdir: tmpdir fixture + interface: interface fixture + src_path: source path of file on backend + dst_path: destination of copy operation on backend + + """ + tmp_path = audeer.touch(audeer.path(tmpdir, "big.1.txt")) + interface.put_file(tmp_path, src_path) + interface._backend._size = lambda x: 5 * 1024 * 1024 * 1024 + interface.copy_file(src_path, dst_path) + assert interface.exists(src_path) + assert interface.exists(dst_path) + + +@pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) +@pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) +def test_create_delete_repositories(host, repository): + audbackend.backend.Minio.create(host, repository) + audbackend.backend.Minio.delete(host, repository) + + +@pytest.mark.parametrize("host", [pytest.HOSTS["minio"]]) +@pytest.mark.parametrize("repository", [f"unittest-{pytest.UID}-{audeer.uid()[:8]}"]) +@pytest.mark.parametrize("authentication", [("bad-access", "bad-secret")]) +def test_errors(host, repository, authentication): + backend = audbackend.backend.Minio(host, repository, authentication=authentication) + with pytest.raises(audbackend.BackendError): + backend.open() + + +def test_get_config(tmpdir, hosts, hide_credentials): + r"""Test parsing of configuration. + + The `get_config()` class method is responsible + for parsing a Minio backend config file. + + Args: + tmpdir: tmpdir fixture + hosts: hosts fixture + hide_credentials: hide_credentials fixture + + """ + host = hosts["minio"] + config_path = audeer.path(tmpdir, "config.cfg") + os.environ["MINIO_CONFIG_FILE"] = config_path + + # config file does not exist + config = audbackend.backend.Minio.get_config(host) + assert config == {} + + # config file is empty + audeer.touch(config_path) + config = audbackend.backend.Minio.get_config(host) + assert config == {} + + # config file has different host + with open(config_path, "w") as fp: + fp.write(f"[{host}.abc]\n") + config = audbackend.backend.Minio.get_config(host) + assert config == {} + + # config file entry without variables + with open(config_path, "w") as fp: + fp.write(f"[{host}]\n") + config = audbackend.backend.Minio.get_config(host) + assert config == {} + + # config file entry with variables + access_key = "user" + secret_key = "pass" + secure = True + with open(config_path, "w") as fp: + fp.write(f"[{host}]\n") + fp.write(f"access_key = {access_key}\n") + fp.write(f"secret_key = {secret_key}\n") + fp.write(f"secure = {secure}\n") + config = audbackend.backend.Minio.get_config(host) + assert config["access_key"] == access_key + assert config["secret_key"] == secret_key + assert config["secure"] + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Minio, audbackend.interface.Maven)], + indirect=True, +) +@pytest.mark.parametrize( + "file, version, extensions, regex, expected", + [ + ( + "/file.tar.gz", + "1.0.0", + [], + False, + "/file.tar/1.0.0/file.tar-1.0.0.gz", + ), + ( + "/file.tar.gz", + "1.0.0", + ["tar.gz"], + False, + "/file/1.0.0/file-1.0.0.tar.gz", + ), + ( + "/.tar.gz", + "1.0.0", + ["tar.gz"], + False, + "/.tar/1.0.0/.tar-1.0.0.gz", + ), + ( + "/tar.gz", + "1.0.0", + ["tar.gz"], + False, + "/tar/1.0.0/tar-1.0.0.gz", + ), + ( + "/.tar.gz", + "1.0.0", + [], + False, + "/.tar/1.0.0/.tar-1.0.0.gz", + ), + ( + "/.tar", + "1.0.0", + [], + False, + "/.tar/1.0.0/.tar-1.0.0", + ), + ( + "/tar", + "1.0.0", + [], + False, + "/tar/1.0.0/tar-1.0.0", + ), + # test regex + ( + "/file.0.tar.gz", + "1.0.0", + [r"\d+.tar.gz"], + False, + "/file.0.tar/1.0.0/file.0.tar-1.0.0.gz", + ), + ( + "/file.0.tar.gz", + "1.0.0", + [r"\d+.tar.gz"], + True, + "/file/1.0.0/file-1.0.0.0.tar.gz", + ), + ( + "/file.99.tar.gz", + "1.0.0", + [r"\d+.tar.gz"], + True, + "/file/1.0.0/file-1.0.0.99.tar.gz", + ), + ( + "/file.prediction.99.tar.gz", + "1.0.0", + [r"prediction.\d+.tar.gz", r"truth.tar.gz"], + True, + "/file/1.0.0/file-1.0.0.prediction.99.tar.gz", + ), + ( + "/file.truth.tar.gz", + "1.0.0", + [r"prediction.\d+.tar.gz", r"truth.tar.gz"], + True, + "/file/1.0.0/file-1.0.0.truth.tar.gz", + ), + ( + "/file.99.tar.gz", + "1.0.0", + [r"(\d+.)?tar.gz"], + True, + "/file/1.0.0/file-1.0.0.99.tar.gz", + ), + ( + "/file.tar.gz", + "1.0.0", + [r"(\d+.)?tar.gz"], + True, + "/file/1.0.0/file-1.0.0.tar.gz", + ), + ], +) +def test_maven_file_structure( + tmpdir, interface, file, version, extensions, regex, expected +): + """Test using the Maven interface with a Minio backend. + + Args: + tmpdir: tmpdir fixture + interface: interface fixture, + which needs to be called with the Minio backend + and the Maven interface + file: file name + version: file version + extensions: extensions considered by the Maven interface + regex: if ``True``, + ``extensions`` are considered as a regex + expected: expected file structure on backend + + """ + interface.extensions = extensions + interface.regex = regex + + src_path = audeer.touch(audeer.path(tmpdir, "tmp")) + interface.put_file(src_path, file, version) + + url = str(interface.backend.path(expected)) + url_expected = str( + interface.backend.path(interface._path_with_version(file, version)) + ) + assert url_expected == url + assert interface.ls(file) == [(file, version)] + assert interface.ls() == [(file, version)] diff --git a/tests/test_interface_maven.py b/tests/test_interface_maven.py index 0c2ad182..25d6e9f5 100644 --- a/tests/test_interface_maven.py +++ b/tests/test_interface_maven.py @@ -11,12 +11,16 @@ from singlefolder import SingleFolder +# Backend-interface combinations to use in all tests +backend_interface_combinations = [ + (audbackend.backend.FileSystem, audbackend.interface.Maven), + (SingleFolder, audbackend.interface.Maven), +] + + @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Maven), - (SingleFolder, audbackend.interface.Maven), - ], + backend_interface_combinations, indirect=True, ) def test_errors(tmpdir, interface): @@ -75,10 +79,7 @@ def test_errors(tmpdir, interface): @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Maven), - (SingleFolder, audbackend.interface.Maven), - ], + backend_interface_combinations, indirect=True, ) @pytest.mark.parametrize( diff --git a/tests/test_interface_unversioned.py b/tests/test_interface_unversioned.py index 81a7c496..b5cf432c 100644 --- a/tests/test_interface_unversioned.py +++ b/tests/test_interface_unversioned.py @@ -15,6 +15,15 @@ from singlefolder import SingleFolder +# Backend-interface combinations to use in all tests +backend_interface_combinations = [ + (audbackend.backend.Artifactory, audbackend.interface.Unversioned), + (audbackend.backend.FileSystem, audbackend.interface.Unversioned), + (audbackend.backend.Minio, audbackend.interface.Unversioned), + (SingleFolder, audbackend.interface.Unversioned), +] + + @pytest.fixture(scope="function", autouse=False) def tree(tmpdir, request): r"""Create file tree.""" @@ -88,11 +97,7 @@ def tree(tmpdir, request): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): @@ -169,11 +174,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_copy(tmpdir, src_path, dst_path, interface): @@ -212,11 +213,7 @@ def test_copy(tmpdir, src_path, dst_path, interface): @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_errors(tmpdir, interface): @@ -554,11 +551,7 @@ def test_errors(tmpdir, interface): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_exists(tmpdir, path, interface): @@ -594,18 +587,8 @@ def test_exists(tmpdir, path, interface): @pytest.mark.parametrize( "interface, owner", [ - ( - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - audbackend.backend.Artifactory, - ), - ( - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - audbackend.backend.FileSystem, - ), - ( - (SingleFolder, audbackend.interface.Unversioned), - SingleFolder, - ), + ((backend, interface), backend) + for backend, interface in backend_interface_combinations ], indirect=True, ) @@ -620,6 +603,17 @@ def test_file(tmpdir, src_path, dst_path, owner, interface): interface.put_file(src_path, dst_path) assert interface.exists(dst_path) + # Download with already existing src_path + interface.get_file(dst_path, src_path) + assert os.path.exists(src_path) + assert interface.checksum(dst_path) == audeer.md5(src_path) + assert interface.owner(dst_path) == owner + date = datetime.datetime.today().strftime("%Y-%m-%d") + assert interface.date(dst_path) == date + + # Repeat, but remove src_path first + os.remove(src_path) + assert not os.path.exists(src_path) interface.get_file(dst_path, src_path) assert os.path.exists(src_path) assert interface.checksum(dst_path) == audeer.md5(src_path) @@ -633,11 +627,7 @@ def test_file(tmpdir, src_path, dst_path, owner, interface): @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_ls(tmpdir, interface): @@ -707,11 +697,7 @@ def test_ls(tmpdir, interface): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.Artifactory, audbackend.interface.Unversioned), - (audbackend.backend.FileSystem, audbackend.interface.Unversioned), - (SingleFolder, audbackend.interface.Unversioned), - ], + backend_interface_combinations, indirect=True, ) def test_move(tmpdir, src_path, dst_path, interface): diff --git a/tests/test_interface_versioned.py b/tests/test_interface_versioned.py index f42c0cd6..e0c3b063 100644 --- a/tests/test_interface_versioned.py +++ b/tests/test_interface_versioned.py @@ -15,6 +15,13 @@ from singlefolder import SingleFolder +# Backend-interface combinations to use in all tests +backend_interface_combinations = [ + (audbackend.backend.FileSystem, audbackend.interface.Versioned), + (SingleFolder, audbackend.interface.Versioned), +] + + @pytest.fixture(scope="function", autouse=False) def tree(tmpdir, request): r"""Create file tree.""" @@ -88,10 +95,7 @@ def tree(tmpdir, request): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): @@ -181,10 +185,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_copy(tmpdir, src_path, src_versions, dst_path, version, interface): @@ -237,10 +238,7 @@ def test_copy(tmpdir, src_path, src_versions, dst_path, version, interface): @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_errors(tmpdir, interface): @@ -678,10 +676,7 @@ def test_errors(tmpdir, interface): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_exists(tmpdir, path, version, interface): @@ -721,14 +716,8 @@ def test_exists(tmpdir, path, version, interface): @pytest.mark.parametrize( "interface, owner", [ - ( - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - audbackend.backend.FileSystem, - ), - ( - (SingleFolder, audbackend.interface.Versioned), - SingleFolder, - ), + ((backend, interface), backend) + for backend, interface in backend_interface_combinations ], indirect=True, ) @@ -756,10 +745,7 @@ def test_file(tmpdir, src_path, dst_path, version, interface, owner): @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) @pytest.mark.parametrize( @@ -1035,10 +1021,7 @@ def test_ls(tmpdir, interface, files, path, latest, pattern, expected): ) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_move(tmpdir, src_path, src_versions, dst_path, version, interface): @@ -1110,10 +1093,7 @@ def test_repr(): @pytest.mark.parametrize("dst_path", ["/file.ext", "/sub/file.ext"]) @pytest.mark.parametrize( "interface", - [ - (audbackend.backend.FileSystem, audbackend.interface.Versioned), - (SingleFolder, audbackend.interface.Versioned), - ], + backend_interface_combinations, indirect=True, ) def test_versions(tmpdir, dst_path, interface):