diff --git a/simplekv/net/boto3store.py b/simplekv/net/boto3store.py index 4d57e93a..73f404b6 100644 --- a/simplekv/net/boto3store.py +++ b/simplekv/net/boto3store.py @@ -21,7 +21,6 @@ def map_boto3_exceptions(key=None, exc_pass=()): raise IOError(str(ex)) -# todo: test this more thoroughly class Boto3SimpleKeyFile(io.RawIOBase): # see: https://alexwlchan.net/2019/02/working-with-large-s3-objects/ # author: Alex Chan, license: MIT @@ -30,7 +29,7 @@ def __init__(self, s3_object): self.position = 0 def __repr__(self): - return "<%s s3_object=%r>" % (type(self).__name__, self.s3_object) + return "<%s s3_object=%r >" % (type(self).__name__, self.s3_object) @property def size(self): diff --git a/tests/bucket_manager.py b/tests/bucket_manager.py index db0d3730..41af1cdf 100644 --- a/tests/bucket_manager.py +++ b/tests/bucket_manager.py @@ -32,6 +32,22 @@ def boto_bucket(access_key, secret_key, host, key.delete() bucket.delete() +@contextmanager +def boto3_bucket(access_key, secret_key, host, + bucket_name=None, **kwargs): + import boto3 + name = bucket_name or 'testrun-bucket-{}'.format(uuid()) + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=name) + s3_resource = boto3.resource('s3') + bucket = s3_resource.Bucket(name) + + yield bucket + + for key in bucket.objects.all(): + key.delete() + bucket.delete() + def load_boto_credentials(): # loaded from the same place tox.ini. here's a sample diff --git a/tests/test_boto3_store.py b/tests/test_boto3_store.py new file mode 100644 index 00000000..433ab8fa --- /dev/null +++ b/tests/test_boto3_store.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import os + +import pytest + +boto3 = pytest.importorskip('boto3') +from simplekv.net.botostore import BotoStore +from simplekv._compat import BytesIO + +from basic_store import BasicStore +from url_store import UrlStore +from bucket_manager import boto_credentials, boto3_bucket +from conftest import ExtendedKeyspaceTests +from simplekv.contrib import ExtendedKeyspaceMixin + + +@pytest.fixture(params=boto_credentials, + ids=[c['access_key'] for c in boto_credentials]) +def credentials(request): + return request.param + + +@pytest.yield_fixture() +def bucket(credentials): + with boto3_bucket(**credentials) as bucket: + yield bucket + + +class TestBotoStorage(BasicStore, UrlStore): + @pytest.fixture(params=[True, False]) + def reduced_redundancy(self, request): + return request.param + + @pytest.fixture + def storage_class(self, reduced_redundancy): + return 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD' + + @pytest.fixture(params=['', '/test-prefix']) + def prefix(self, request): + return request.param + + @pytest.fixture + def store(self, bucket, prefix, reduced_redundancy): + return BotoStore(bucket, prefix, reduced_redundancy=reduced_redundancy) + + # Disable max key length test as it leads to problems with minio + test_max_key_length = None + + def test_get_filename_nonexistant(self, store, key, tmp_path): + with pytest.raises(KeyError): + store.get_file(key, os.path.join(str(tmp_path), 'a')) + + def test_key_error_on_nonexistant_get_filename(self, store, key, tmp_path): + with pytest.raises(KeyError): + store.get_file(key, os.path.join(str(tmp_path), 'a')) + + def test_storage_class_put( + self, store, prefix, key, value, storage_class, bucket + ): + store.put(key, value) + + keyname = prefix + key + + if storage_class != 'STANDARD': + pytest.xfail('boto does not support checking the storage class?') + + assert bucket.Object(keyname).storage_class == storage_class + + def test_storage_class_putfile( + self, store, prefix, key, value, storage_class, bucket + ): + store.put_file(key, BytesIO(value)) + + keyname = prefix + key + + if storage_class != 'STANDARD': + pytest.xfail('boto does not support checking the storage class?') + assert bucket.Object(keyname).storage_class == storage_class + + +class TestExtendedKeyspaceBotoStore(TestBotoStorage, ExtendedKeyspaceTests): + @pytest.fixture + def store(self, bucket, prefix, reduced_redundancy): + class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, BotoStore): + pass + return ExtendedKeyspaceStore(bucket, prefix, + reduced_redundancy=reduced_redundancy)