Skip to content

Commit

Permalink
authentication interface and contexts for interface methods
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChrisSchinnerl
ChrisSchinnerl committed Sep 26, 2023
1 parent ad9e995 commit 0290dd5
Show file tree
Hide file tree
Showing 23 changed files with 238 additions and 228 deletions.
2 changes: 1 addition & 1 deletion awscli_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"testing"
"time"

"github.com/SiaFoundation/gofakes3"
"go.sia.tech/gofakes3"
)

func TestCLILsBuckets(t *testing.T) {
Expand Down
63 changes: 35 additions & 28 deletions backend.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package gofakes3

import (
"context"
"encoding/hex"
"io"
"net/http"
"time"

"github.com/aws/aws-sdk-go/aws/awserr"
Expand Down Expand Up @@ -136,7 +138,7 @@ type Backend interface {
// ListBuckets returns a list of all buckets owned by the authenticated
// sender of the request.
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html
ListBuckets() ([]BucketInfo, error)
ListBuckets(ctx context.Context) ([]BucketInfo, error)

// ListBucket returns the contents of a bucket. Backends should use the
// supplied prefix to limit the contents of the bucket and to sort the
Expand All @@ -157,18 +159,18 @@ type Backend interface {
// work fine if you ignore the pagination request, but this may not suit
// your application. Not all backends bundled with gofakes3 correctly
// support this pagination yet, but that will change.
ListBucket(name string, prefix *Prefix, page ListBucketPage) (*ObjectList, error)
ListBucket(ctx context.Context, name string, prefix *Prefix, page ListBucketPage) (*ObjectList, error)

// CreateBucket creates the bucket if it does not already exist. The name
// should be assumed to be a valid name.
//
// If the bucket already exists, a gofakes3.ResourceError with
// gofakes3.ErrBucketAlreadyExists MUST be returned.
CreateBucket(name string) error
CreateBucket(ctx context.Context, name string) error

// BucketExists should return a boolean indicating the bucket existence, or
// an error if the backend was unable to determine existence.
BucketExists(name string) (exists bool, err error)
BucketExists(ctx context.Context, name string) (exists bool, err error)

// DeleteBucket deletes a bucket if and only if it is empty.
//
Expand All @@ -178,7 +180,7 @@ type Backend interface {
// If the bucket does not exist, gofakes3.ErrNoSuchBucket MUST be returned.
//
// AWS does not validate the bucket's name for anything other than existence.
DeleteBucket(name string) error
DeleteBucket(ctx context.Context, name string) error

// GetObject must return a gofakes3.ErrNoSuchKey error if the object does
// not exist. See gofakes3.KeyNotFound() for a convenient way to create
Expand All @@ -193,7 +195,7 @@ type Backend interface {
// implementers MUST return ErrNotImplemented.
//
// If the backend is a VersionedBackend, GetObject retrieves the latest version.
GetObject(bucketName, objectName string, rangeRequest *ObjectRangeRequest) (*Object, error)
GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *ObjectRangeRequest) (*Object, error)

// HeadObject fetches the Object from the backend, but reading the Contents
// will return io.EOF immediately.
Expand All @@ -204,7 +206,7 @@ type Backend interface {
//
// HeadObject should return a NotFound() error if the object does not
// exist.
HeadObject(bucketName, objectName string) (*Object, error)
HeadObject(ctx context.Context, bucketName, objectName string) (*Object, error)

// DeleteObject deletes an object from the bucket.
//
Expand All @@ -222,18 +224,18 @@ type Backend interface {
// delete marker, which becomes the latest version of the object. If there
// isn't a null version, Amazon S3 does not remove any objects.
//
DeleteObject(bucketName, objectName string) (ObjectDeleteResult, error)
DeleteObject(ctx context.Context, bucketName, objectName string) (ObjectDeleteResult, error)

// PutObject should assume that the key is valid. The map containing meta
// may be nil.
//
// The size can be used if the backend needs to read the whole reader; use
// gofakes3.ReadAll() for this job rather than io.ReadAll().
PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64) (PutObjectResult, error)
PutObject(ctx context.Context, bucketName, key string, meta map[string]string, input io.Reader, size int64) (PutObjectResult, error)

DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error)
DeleteMulti(ctx context.Context, bucketName string, objects ...string) (MultiDeleteResult, error)

CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (CopyObjectResult, error)
CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (CopyObjectResult, error)
}

// VersionedBackend may be optionally implemented by a Backend in order to support
Expand All @@ -248,11 +250,11 @@ type VersionedBackend interface {
//
// If the bucket has never had versioning enabled, VersioningConfiguration MUST return
// empty strings (S300001).
VersioningConfiguration(bucket string) (VersioningConfiguration, error)
VersioningConfiguration(ctx context.Context, bucket string) (VersioningConfiguration, error)

// SetVersioningConfiguration must return a gofakes3.ErrNoSuchBucket error if the bucket
// does not exist. See gofakes3.BucketNotFound() for a convenient way to create one.
SetVersioningConfiguration(bucket string, v VersioningConfiguration) error
SetVersioningConfiguration(ctx context.Context, bucket string, v VersioningConfiguration) error

// GetObject must return a gofakes3.ErrNoSuchKey error if the object does
// not exist. See gofakes3.KeyNotFound() for a convenient way to create
Expand All @@ -271,6 +273,7 @@ type VersionedBackend interface {
// FIXME: s3assumer test; what happens when versionID is empty? Does it
// return the latest?
GetObjectVersion(
ctx context.Context,
bucketName, objectName string,
versionID VersionID,
rangeRequest *ObjectRangeRequest) (*Object, error)
Expand All @@ -284,7 +287,7 @@ type VersionedBackend interface {
//
// HeadObjectVersion should return a NotFound() error if the object does not
// exist.
HeadObjectVersion(bucketName, objectName string, versionID VersionID) (*Object, error)
HeadObjectVersion(ctx context.Context, bucketName, objectName string, versionID VersionID) (*Object, error)

// DeleteObjectVersion permanently deletes a specific object version.
//
Expand All @@ -294,10 +297,10 @@ type VersionedBackend interface {
// If the bucket exists and either the object does not exist (S300003) or
// the version does not exist (S300002), you MUST return an empty
// ObjectDeleteResult and a nil error.
DeleteObjectVersion(bucketName, objectName string, versionID VersionID) (ObjectDeleteResult, error)
DeleteObjectVersion(ctx context.Context, bucketName, objectName string, versionID VersionID) (ObjectDeleteResult, error)

// DeleteMultiVersions permanently deletes all of the specified Object Versions
DeleteMultiVersions(bucketName string, objects ...ObjectID) (MultiDeleteResult, error)
DeleteMultiVersions(ctx context.Context, bucketName string, objects ...ObjectID) (MultiDeleteResult, error)

// Backend implementers can assume the ListBucketVersionsPage is valid:
// KeyMarker and VersionIDMarker will either both be set, or both be unset. No
Expand All @@ -310,7 +313,7 @@ type VersionedBackend interface {
//
// The Backend MUST treat a nil prefix identically to a zero prefix, and a
// nil page identically to a zero page.
ListBucketVersions(bucketName string, prefix *Prefix, page *ListBucketVersionsPage) (*ListBucketVersionsResult, error)
ListBucketVersions(ctx context.Context, bucketName string, prefix *Prefix, page *ListBucketVersionsPage) (*ListBucketVersionsResult, error)
}

// MultipartBackend may be optionally implemented by a Backend in order to
Expand All @@ -319,27 +322,31 @@ type VersionedBackend interface {
// in-memory implementation which holds all parts in memory until the upload
// gets finalised and pushed to the backend.
type MultipartBackend interface {
CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error)
UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error)
CreateMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (UploadID, error)
UploadPart(ctx context.Context, bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error)

ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error)
ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error)
ListMultipartUploads(ctx context.Context, bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error)
ListParts(ctx context.Context, bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error)

AbortMultipartUpload(bucket, object string, id UploadID) error
CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error)
AbortMultipartUpload(ctx context.Context, bucket, object string, id UploadID) error
CompleteMultipartUpload(ctx context.Context, bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error)
}

type AuthenticatedBackend interface {
AuthenticationMiddleware(http.Handler) http.Handler
}

// CopyObject is a helper function useful for quickly implementing CopyObject on
// a backend that already supports GetObject and PutObject. This isn't very
// efficient so only use this if performance isn't important.
func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result CopyObjectResult, err error) {
c, err := db.GetObject(srcBucket, srcKey, nil)
func CopyObject(ctx context.Context, db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result CopyObjectResult, err error) {
c, err := db.GetObject(ctx, srcBucket, srcKey, nil)
if err != nil {
return
}
defer c.Contents.Close()

_, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size)
_, err = db.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size)
if err != nil {
return
}
Expand All @@ -350,9 +357,9 @@ func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta ma
}, nil
}

func MergeMetadata(db Backend, bucketName string, objectName string, meta map[string]string) error {
func MergeMetadata(ctx context.Context, db Backend, bucketName string, objectName string, meta map[string]string) error {
// get potential existing object to potentially carry metadata over
existingObj, err := db.GetObject(bucketName, objectName, nil)
existingObj, err := db.GetObject(ctx, bucketName, objectName, nil)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() != string(ErrNoSuchKey) {
return err
Expand Down
47 changes: 24 additions & 23 deletions backend/s3afero/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package s3afero

import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"fmt"
Expand All @@ -11,8 +12,8 @@ import (
"reflect"
"testing"

"github.com/SiaFoundation/gofakes3"
"github.com/spf13/afero"
"go.sia.tech/gofakes3"
)

func testingBackends(t *testing.T) []gofakes3.Backend {
Expand All @@ -26,7 +27,7 @@ func testingBackends(t *testing.T) []gofakes3.Backend {
if err != nil {
t.Fatal(err)
}
if err := multi.CreateBucket("test"); err != nil {
if err := multi.CreateBucket(context.Background(), "test"); err != nil {
t.Fatal(err)
}

Expand All @@ -44,14 +45,14 @@ func TestPutGet(t *testing.T) {
}

contents := []byte("contents")
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
t.Fatal(err)
}
hasher := md5.New()
hasher.Write(contents)
hash := hasher.Sum(nil)

obj, err := backend.GetObject("test", "yep", nil)
obj, err := backend.GetObject(context.Background(), "test", "yep", nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -90,14 +91,14 @@ func TestPutGetRange(t *testing.T) {

contents := []byte("contents")
expected := contents[1:7]
if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
t.Fatal(err)
}
hasher := md5.New()
hasher.Write(contents)
hash := hasher.Sum(nil)

obj, err := backend.GetObject("test", "yep", &gofakes3.ObjectRangeRequest{Start: 1, End: 6})
obj, err := backend.GetObject(context.Background(), "test", "yep", &gofakes3.ObjectRangeRequest{Start: 1, End: 6})
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -135,16 +136,16 @@ func TestPutListRoot(t *testing.T) {
}

contents1 := []byte("contents1")
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
t.Fatal(err)
}

contents2 := []byte("contents2")
if _, err := backend.PutObject("test", "bar", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "bar", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
t.Fatal(err)
}

result, err := backend.ListBucket("test",
result, err := backend.ListBucket(context.Background(), "test",
&gofakes3.Prefix{HasPrefix: true, HasDelimiter: true, Delimiter: "/"},
gofakes3.ListBucketPage{})
if err != nil {
Expand Down Expand Up @@ -181,17 +182,17 @@ func TestPutListDir(t *testing.T) {
}

contents1 := []byte("contents1")
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
t.Fatal(err)
}

contents2 := []byte("contents2")
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
t.Fatal(err)
}

{
result, err := backend.ListBucket("test",
result, err := backend.ListBucket(context.Background(), "test",
&gofakes3.Prefix{Prefix: "foo/", HasPrefix: true, HasDelimiter: true, Delimiter: "/"},
gofakes3.ListBucketPage{})
if err != nil {
Expand All @@ -203,7 +204,7 @@ func TestPutListDir(t *testing.T) {
}

{
result, err := backend.ListBucket("test",
result, err := backend.ListBucket(context.Background(), "test",
&gofakes3.Prefix{Prefix: "foo/bar", HasPrefix: true, HasDelimiter: true, Delimiter: "/"},
gofakes3.ListBucketPage{})
if err != nil {
Expand All @@ -227,15 +228,15 @@ func TestPutDelete(t *testing.T) {
}

contents := []byte("contents1")
if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo", meta, bytes.NewReader(contents), int64(len(contents))); err != nil {
t.Fatal(err)
}

if _, err := backend.DeleteObject("test", "foo"); err != nil {
if _, err := backend.DeleteObject(context.Background(), "test", "foo"); err != nil {
t.Fatal(err)
}

result, err := backend.ListBucket("test",
result, err := backend.ListBucket(context.Background(), "test",
&gofakes3.Prefix{HasPrefix: true, HasDelimiter: true, Delimiter: "/"},
gofakes3.ListBucketPage{})
if err != nil {
Expand All @@ -259,24 +260,24 @@ func TestPutDeleteMulti(t *testing.T) {
}

contents1 := []byte("contents1")
if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil {
t.Fatal(err)
}

contents2 := []byte("contents2")
if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
if _, err := backend.PutObject(context.Background(), "test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil {
t.Fatal(err)
}

deleteResult, err := backend.DeleteMulti("test", "foo/bar", "foo/baz")
deleteResult, err := backend.DeleteMulti(context.Background(), "test", "foo/bar", "foo/baz")
if err != nil {
t.Fatal(err)
}
if err := deleteResult.AsError(); err != nil {
t.Fatal(err)
}

bucketContents, err := backend.ListBucket("test",
bucketContents, err := backend.ListBucket(context.Background(), "test",
&gofakes3.Prefix{HasPrefix: true, HasDelimiter: true, Delimiter: "/"},
gofakes3.ListBucketPage{})
if err != nil {
Expand Down Expand Up @@ -308,13 +309,13 @@ func TestMultiCreateBucket(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if ok, _ := multi.BucketExists("test"); ok {
if ok, _ := multi.BucketExists(context.Background(), "test"); ok {
t.Fatal()
}
if err := multi.CreateBucket("test"); err != nil {
if err := multi.CreateBucket(context.Background(), "test"); err != nil {
t.Fatal(err)
}
if ok, _ := multi.BucketExists("test"); !ok {
if ok, _ := multi.BucketExists(context.Background(), "test"); !ok {
t.Fatal()
}
}
Loading

0 comments on commit 0290dd5

Please sign in to comment.