Add the unit tests for the boto3 store

simplekv/net/boto3store.py

@@ -21,7 +21,6 @@ def map_boto3_exceptions(key=None, exc_pass=()):
         raise IOError(str(ex))
 
 
-# todo: test this more thoroughly
 class Boto3SimpleKeyFile(io.RawIOBase):
     # see: https://alexwlchan.net/2019/02/working-with-large-s3-objects/
     # author: Alex Chan, license: MIT
@@ -30,7 +29,7 @@ def __init__(self, s3_object):
         self.position = 0
 
     def __repr__(self):
-        return "<%s s3_object=%r>" % (type(self).__name__, self.s3_object)
+        return "<%s s3_object=%r >" % (type(self).__name__, self.s3_object)
 
     @property
     def size(self):

tests/bucket_manager.py

@@ -32,6 +32,22 @@ def boto_bucket(access_key, secret_key, host,
         key.delete()
     bucket.delete()
 
+@contextmanager
+def boto3_bucket(access_key, secret_key, host,
+                 bucket_name=None, **kwargs):
+    import boto3
+    name = bucket_name or 'testrun-bucket-{}'.format(uuid())
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=name)
+    s3_resource = boto3.resource('s3')
+    bucket = s3_resource.Bucket('zerosteiner.private')
+
+    yield bucket
+
+    for key in bucket.objects.all():
+        key.delete()
+    bucket.delete()
+
 
 def load_boto_credentials():
     # loaded from the same place tox.ini. here's a sample

tests/test_boto3_store.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+import os
+
+import pytest
+
+boto3 = pytest.importorskip('boto3')
+from simplekv.net.botostore import BotoStore
+from simplekv._compat import BytesIO
+
+from basic_store import BasicStore
+from url_store import UrlStore
+from bucket_manager import boto_credentials, boto3_bucket
+from conftest import ExtendedKeyspaceTests
+from simplekv.contrib import ExtendedKeyspaceMixin
+
+
+@pytest.fixture(params=boto_credentials,
+                ids=[c['access_key'] for c in boto_credentials])
+def credentials(request):
+    return request.param
+
+
+@pytest.yield_fixture()
+def bucket(credentials):
+    with boto3_bucket(**credentials) as bucket:
+        yield bucket
+
+
+class TestBotoStorage(BasicStore, UrlStore):
+    @pytest.fixture(params=[True, False])
+    def reduced_redundancy(self, request):
+        return request.param
+
+    @pytest.fixture
+    def storage_class(self, reduced_redundancy):
+        return 'REDUCED_REDUNDANCY' if reduced_redundancy else 'STANDARD'
+
+    @pytest.fixture(params=['', '/test-prefix'])
+    def prefix(self, request):
+        return request.param
+
+    @pytest.fixture
+    def store(self, bucket, prefix, reduced_redundancy):
+        return BotoStore(bucket, prefix, reduced_redundancy=reduced_redundancy)
+
+    # Disable max key length test as it leads to problems with minio
+    test_max_key_length = None
+
+    def test_get_filename_nonexistant(self, store, key, tmp_path):
+        with pytest.raises(KeyError):
+            store.get_file(key, os.path.join(str(tmp_path), 'a'))
+
+    def test_key_error_on_nonexistant_get_filename(self, store, key, tmp_path):
+        with pytest.raises(KeyError):
+            store.get_file(key, os.path.join(str(tmp_path), 'a'))
+
+    def test_storage_class_put(
+        self, store, prefix, key, value, storage_class, bucket
+    ):
+        store.put(key, value)
+
+        keyname = prefix + key
+
+        if storage_class != 'STANDARD':
+            pytest.xfail('boto does not support checking the storage class?')
+
+        assert bucket.Object(keyname).storage_class == storage_class
+
+    def test_storage_class_putfile(
+        self, store, prefix, key, value, storage_class, bucket
+    ):
+        store.put_file(key, BytesIO(value))
+
+        keyname = prefix + key
+
+        if storage_class != 'STANDARD':
+            pytest.xfail('boto does not support checking the storage class?')
+        assert bucket.Object(keyname).storage_class == storage_class
+
+
+class TestExtendedKeyspaceBotoStore(TestBotoStorage, ExtendedKeyspaceTests):
+    @pytest.fixture
+    def store(self, bucket, prefix, reduced_redundancy):
+        class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, BotoStore):
+            pass
+        return ExtendedKeyspaceStore(bucket, prefix,
+                                     reduced_redundancy=reduced_redundancy)