Fix some bugs in the boto3 store implementation

simplekv/net/boto3store.py

@@ -2,26 +2,23 @@
 # coding=utf8
 
 from .._compat import imap
-from .. import KeyValueStore, UrlMixin, CopyMixin
+from .. import KeyValueStore, CopyMixin
 from contextlib import contextmanager
 from shutil import copyfileobj
 import io
 
 
 @contextmanager
 def map_boto3_exceptions(key=None, exc_pass=()):
-    """Map boto-specific exceptions to the simplekv-API."""
+    """Map boto3-specific exceptions to the simplekv-API."""
     from botocore.exceptions import ClientError
 
-    #try:
-    #    yield
-    #except StorageResponseError as e:
-    #    if e.code == 'NoSuchKey':
-    #        raise KeyError(key)
-    #    raise IOError(str(e))
-    except (ClientError,) as e:
-        if e.__class__.__name__ not in exc_pass:
-            raise IOError(str(e))
+    try:
+        yield
+    except ClientError as ex:
+        if ex.response['Error']['Code'] == 'NoSuchKey':
+            raise KeyError(key)
+        raise IOError(str(ex))
 
 
 # todo: test this more thoroughly
@@ -81,27 +78,19 @@ def readable(self):
         return True
 
 
-class Boto3Store(KeyValueStore, UrlMixin, CopyMixin):
-    def __init__(self, bucket, prefix='', url_valid_time=0,
-                 reduced_redundancy=False, public=False, metadata=None):
+class Boto3Store(KeyValueStore, CopyMixin):
+    def __init__(self, bucket, prefix=''):
         self.prefix = prefix.strip().lstrip('/')
         self.bucket = bucket
-        self.reduced_redundancy = reduced_redundancy
-        self.public = public
-        self.url_valid_time = url_valid_time
-        self.metadata = metadata or {}
 
-    def __new_key(self, name):
-        # todo: test that this works if the object hasn't been created yet
-        key = self.bucket.Object(self.prefix + name)
-        key.put(Metadata=self.metadata)
-        return key
+    def __new_object(self, name):
+        return self.bucket.Object(self.prefix + name)
 
     def iter_keys(self, prefix=u""):
         with map_boto3_exceptions():
             prefix_len = len(self.prefix)
             return imap(lambda k: k.key[prefix_len:],
-                        self.bucket.objects.filter(self.prefix + prefix))
+                        self.bucket.objects.filter(Prefix=self.prefix + prefix))
 
     def _has_key(self, key):
         from botocore.exceptions import ClientError
@@ -115,65 +104,51 @@ def _has_key(self, key):
             return True
 
     def _delete(self, key):
-        # todo: handle the exception of the key does not exist
         self.bucket.Object(self.prefix + key).delete()
 
     def _get(self, key):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            obj = k.get()
-            return obj.get()['Body'].read()
+            obj = obj.get()
+            return obj['Body'].read()
 
     def _get_file(self, key, file):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            obj = k.get()
+            obj = obj.get()
             return copyfileobj(obj['Body'], file)
 
     def _get_filename(self, key, filename):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            obj = k.get()
+            obj = obj.get()
             with open(filename, 'wb') as file:
                 return copyfileobj(obj['Body'], file)
 
     def _open(self, key):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            return Boto3SimpleKeyFile(k)
+            return Boto3SimpleKeyFile(obj)
 
     def _copy(self, source, dest):
         if not self._has_key(source):
             raise KeyError(source)
+        obj = self.__new_object(dest)
         with map_boto3_exceptions(key=source):
-            # todo: test that this works as written being relative from self.bucket instead
-            # of being hard-coded like https://stackoverflow.com/a/32504096/2722961
-            self.bucket.Object(self.prefix + dest).copy_from(CopySource=self.prefix + source)
+            obj.copy_from(CopySource=self.bucket.name + '/' + self.prefix + source)
 
     def _put(self, key, data):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            k.put(Body=data)
+            obj.put(Body=data)
             return key
 
     def _put_file(self, key, file):
-        k = self.__new_key(key)
+        obj = self.__new_object(key)
         with map_boto3_exceptions(key=key):
-            k.put(Body=file)
+            obj.put(Body=file)
             return key
 
     def _put_filename(self, key, filename):
         with open(filename, 'rb') as file:
             return self._put(key, file)
-
-    def _url_for(self, key):
-        k = self.__new_key(key)
-        params = {
-            'Bucket' self.bucket.name,
-            'Key': k.key
-        }
-        # todo: finish this, see: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-presigned-urls.html
-        with map_boto3_exceptions(key=key):
-            return s3_client.generate_presigned_url('get_object',
-                                             Params=params,
-                                             ExpiresIn=self.url_valid_time)