e2e: use official sdk in TestS3Basic

go.mod

@@ -3,6 +3,7 @@ module go.sia.tech/renterd
 go 1.22.5
 
 require (
+	github.com/aws/aws-sdk-go-v2/config v1.27.31
 	github.com/gabriel-vasile/mimetype v1.4.5
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/google/go-cmp v0.6.0
@@ -31,6 +32,18 @@ require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
 	github.com/aws/aws-sdk-go v1.55.5 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.30 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/cloudflare/cloudflare-go v0.101.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect

go.sum

@@ -4,6 +4,32 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmH
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
 github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
+github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI=
+github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/cloudflare/cloudflare-go v0.101.0 h1:SXWNSEDkbdY84iFIZGyTdWQwDfd98ljv0/4UubpleBQ=
 github.com/cloudflare/cloudflare-go v0.101.0/go.mod h1:xXQHnoXKR48JlWbFS42i2al3nVqimVhcYvKnIdXLw9g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/test/e2e/cluster.go

@@ -13,6 +13,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	s3aws "github.com/aws/aws-sdk-go/service/s3"
 	"github.com/minio/minio-go/v7"
 	"go.sia.tech/core/consensus"
 	"go.sia.tech/core/gateway"
@@ -61,6 +65,7 @@ type TestCluster struct {
 	Autopilot *autopilot.Client
 	Bus       *bus.Client
 	Worker    *worker.Client
+	S3Aws     *s3aws.S3
 	S3        *minio.Client
 	S3Core    *minio.Core
 
@@ -319,6 +324,18 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 	})
 	tt.OK(err)
 
+	mySession := session.Must(session.NewSession())
+	s3AWSClient := s3aws.New(mySession, aws.NewConfig().
+		WithEndpoint(s3Client.EndpointURL().String()).
+		WithRegion("dummy").
+		WithS3ForcePathStyle(true).
+		WithCredentials(credentials.NewCredentials(&credentials.StaticProvider{
+			Value: credentials.Value{
+				AccessKeyID:     test.S3AccessKeyID,
+				SecretAccessKey: test.S3SecretAccessKey,
+			},
+		})))
+
 	// Create bus.
 	busDir := filepath.Join(dir, "bus")
 	b, bShutdownFn, cm, bs, err := newTestBus(ctx, busDir, busCfg, dbCfg, wk, logger)
@@ -388,6 +405,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 		Bus:       busClient,
 		Worker:    workerClient,
 		S3:        s3Client,
+		S3Aws:     s3AWSClient,
 		S3Core:    s3Core,
 
 		workerShutdownFns:    workerShutdownFns,

internal/test/e2e/s3_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	s3aws "github.com/aws/aws-sdk-go/service/s3"
 	"github.com/google/go-cmp/cmp"
 	"github.com/minio/minio-go/v7"
 	rhpv2 "go.sia.tech/core/rhp/v2"
@@ -37,7 +38,7 @@ func TestS3Basic(t *testing.T) {
 	defer cluster.Shutdown()
 
 	// delete default bucket before testing.
-	s3 := cluster.S3
+	s3 := cluster.S3Aws
 	tt := cluster.tt
 	if err := cluster.Bus.DeleteBucket(context.Background(), api.DefaultBucketName); err != nil {
 		t.Fatal(err)
@@ -46,145 +47,164 @@ func TestS3Basic(t *testing.T) {
 	// create bucket
 	bucket := "bucket"
 	objPath := "obj#ct" // special char to check escaping
-	tt.OK(s3.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{}))
+	tt.OKAll(s3.CreateBucket(&s3aws.CreateBucketInput{
+		Bucket: &bucket,
+	}))
 
 	// list buckets
-	buckets, err := s3.ListBuckets(context.Background())
+	lbo, err := s3.ListBuckets(&s3aws.ListBucketsInput{})
 	tt.OK(err)
-	if len(buckets) != 1 {
+	if buckets := lbo.Buckets; len(buckets) != 1 {
 		t.Fatalf("unexpected number of buckets, %d != 1", len(buckets))
-	} else if buckets[0].Name != bucket {
-		t.Fatalf("unexpected bucket name, %s != %s", buckets[0].Name, bucket)
+	} else if *buckets[0].Name != bucket {
+		t.Fatalf("unexpected bucket name, %s != %s", *buckets[0].Name, bucket)
 	} else if buckets[0].CreationDate.IsZero() {
 		t.Fatal("expected non-zero creation date")
 	}
 
 	// exist buckets
-	exists, err := s3.BucketExists(context.Background(), bucket)
+	_, err = s3.HeadBucket(&s3aws.HeadBucketInput{Bucket: &bucket})
 	tt.OK(err)
-	if !exists {
-		t.Fatal("expected bucket to exist")
-	}
-	exists, err = s3.BucketExists(context.Background(), bucket+"nonexistent")
-	tt.OK(err)
-	if exists {
-		t.Fatal("expected bucket to not exist")
-	}
+	nonexistentBucket := "nonexistent"
+	_, err = s3.HeadBucket(&s3aws.HeadBucketInput{Bucket: &nonexistentBucket})
+	tt.AssertContains(err, "NotFound")
 
 	// add object to the bucket
 	data := frand.Bytes(10)
 	etag := md5.Sum(data)
-	uploadInfo, err := s3.PutObject(context.Background(), bucket, objPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
+	uploadInfo, err := s3.PutObject(&s3aws.PutObjectInput{
+		Body:   bytes.NewReader(data),
+		Bucket: &bucket,
+		Key:    &objPath,
+	})
 	tt.OK(err)
-	if uploadInfo.ETag != hex.EncodeToString(etag[:]) {
-		t.Fatalf("expected ETag %v, got %v", hex.EncodeToString(etag[:]), uploadInfo.ETag)
+	if *uploadInfo.ETag != api.FormatETag(hex.EncodeToString(etag[:])) {
+		t.Fatalf("expected ETag %v, got %v", hex.EncodeToString(etag[:]), *uploadInfo.ETag)
 	}
 	busObject, err := cluster.Bus.Object(context.Background(), bucket, objPath, api.GetObjectOptions{})
 	tt.OK(err)
 	if busObject.Object == nil {
 		t.Fatal("expected object to exist")
-	} else if busObject.Object.ETag != uploadInfo.ETag {
-		t.Fatalf("expected ETag %q, got %q", uploadInfo.ETag, busObject.Object.ETag)
+	} else if api.FormatETag(busObject.Object.ETag) != *uploadInfo.ETag {
+		t.Fatalf("expected ETag %v, got %v", *uploadInfo.ETag, busObject.Object.ETag)
 	}
 
-	_, err = s3.PutObject(context.Background(), bucket+"nonexistent", objPath, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
+	_, err = s3.PutObject(&s3aws.PutObjectInput{
+		Body:   bytes.NewReader(data),
+		Bucket: &nonexistentBucket,
+		Key:    &objPath,
+	})
 	tt.AssertIs(err, errBucketNotExists)
 
 	// get object
-	obj, err := s3.GetObject(context.Background(), bucket, objPath, minio.GetObjectOptions{})
+	obj, err := s3.GetObject(&s3aws.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &objPath,
+	})
 	tt.OK(err)
-	if b, err := io.ReadAll(obj); err != nil {
+	if b, err := io.ReadAll(obj.Body); err != nil {
 		t.Fatal(err)
 	} else if !bytes.Equal(b, data) {
 		t.Fatal("data mismatch")
-	} else if info, err := obj.Stat(); err != nil {
-		t.Fatal(err)
-	} else if info.ETag != uploadInfo.ETag {
-		t.Fatal("unexpected ETag:", info.ETag, uploadInfo.ETag)
+	} else if *obj.ETag != *uploadInfo.ETag {
+		t.Fatal("unexpected ETag:", *obj.ETag, *uploadInfo.ETag)
 	}
 
 	// stat object
-	info, err := s3.StatObject(context.Background(), bucket, objPath, minio.StatObjectOptions{})
+	info, err := s3.HeadObject(&s3aws.HeadObjectInput{
+		Bucket: &bucket,
+		Key:    &objPath,
+	})
 	tt.OK(err)
-	if info.Size != int64(len(data)) {
+	if *info.ContentLength != int64(len(data)) {
 		t.Fatal("size mismatch")
-	} else if info.ETag != uploadInfo.ETag {
+	} else if *info.ETag != *uploadInfo.ETag {
 		t.Fatal("unexpected ETag:", info.ETag)
 	}
 
 	// stat object that doesn't exist
-	_, err = s3.StatObject(context.Background(), bucket, "nonexistent", minio.StatObjectOptions{})
-	if err == nil || !strings.Contains(err.Error(), "The specified key does not exist") {
-		t.Fatal(err)
-	}
+	info, err = s3.HeadObject(&s3aws.HeadObjectInput{
+		Bucket: &nonexistentBucket,
+		Key:    &objPath,
+	})
+	tt.AssertContains(err, "NotFound")
 
 	// add another bucket
-	tt.OK(s3.MakeBucket(context.Background(), bucket+"2", minio.MakeBucketOptions{}))
+	bucket2 := "bucket2"
+	tt.OKAll(s3.CreateBucket(&s3aws.CreateBucketInput{
+		Bucket: &bucket2,
+	}))
 
 	// copy our object into the new bucket.
-	res, err := s3.CopyObject(context.Background(), minio.CopyDestOptions{
-		Bucket: bucket + "2",
-		Object: objPath,
-	}, minio.CopySrcOptions{
-		Bucket: bucket,
-		Object: objPath,
+	src := fmt.Sprintf("%s/%s", bucket, objPath)
+	res, err := s3.CopyObject(&s3aws.CopyObjectInput{
+		CopySource: &src,
+		Bucket:     &bucket2,
+		Key:        &objPath,
 	})
 	tt.OK(err)
-	if res.LastModified.IsZero() {
+	if res.CopyObjectResult.LastModified.IsZero() {
 		t.Fatal("expected LastModified to be non-zero")
-	} else if !res.LastModified.After(start.UTC()) {
+	} else if !res.CopyObjectResult.LastModified.After(start.UTC()) {
 		t.Fatal("expected LastModified to be after the start of our test")
-	} else if res.ETag == "" {
-		t.Fatal("expected ETag to be set")
+	} else if *res.CopyObjectResult.ETag != *uploadInfo.ETag {
+		t.Fatal("expected correct ETag to be set")
 	}
 
 	// get copied object
-	obj, err = s3.GetObject(context.Background(), bucket+"2", objPath, minio.GetObjectOptions{})
+	obj, err = s3.GetObject(&s3aws.GetObjectInput{
+		Bucket: &bucket2,
+		Key:    &objPath,
+	})
 	tt.OK(err)
-	if b, err := io.ReadAll(obj); err != nil {
+	if b, err := io.ReadAll(obj.Body); err != nil {
 		t.Fatal(err)
 	} else if !bytes.Equal(b, data) {
 		t.Fatal("data mismatch")
 	}
 
 	// assert deleting the bucket fails because it's not empty
-	err = s3.RemoveBucket(context.Background(), bucket)
+	_, err = s3.DeleteBucket(&s3aws.DeleteBucketInput{Bucket: &bucket})
 	tt.AssertIs(err, gofakes3.ErrBucketNotEmpty)
 
 	// assert deleting the bucket fails because it doesn't exist
-	err = s3.RemoveBucket(context.Background(), bucket+"nonexistent")
+	_, err = s3.DeleteBucket(&s3aws.DeleteBucketInput{Bucket: &nonexistentBucket})
 	tt.AssertIs(err, errBucketNotExists)
 
 	// remove the object
-	tt.OK(s3.RemoveObject(context.Background(), bucket, objPath, minio.RemoveObjectOptions{}))
+	tt.OKAll(s3.DeleteObject(&s3aws.DeleteObjectInput{Bucket: &bucket, Key: &objPath}))
 
 	// try to get object
-	obj, err = s3.GetObject(context.Background(), bucket, objPath, minio.GetObjectOptions{})
-	tt.OK(err)
-	_, err = io.ReadAll(obj)
-	tt.AssertContains(err, "The specified key does not exist")
+	obj, err = s3.GetObject(&s3aws.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &objPath,
+	})
+	tt.AssertContains(err, "NoSuchKey")
 
 	// add a few objects to the bucket.
-	tt.OKAll(s3.PutObject(context.Background(), bucket, "dir/", bytes.NewReader(frand.Bytes(10)), 10, minio.PutObjectOptions{}))
-	tt.OKAll(s3.PutObject(context.Background(), bucket, "dir/file", bytes.NewReader(frand.Bytes(10)), 10, minio.PutObjectOptions{}))
+	tmpObj1 := "dir/"
+	body := frand.Bytes(10)
+	tt.OKAll(s3.PutObject(&s3aws.PutObjectInput{
+		Body:   bytes.NewReader(body),
+		Bucket: &bucket,
+		Key:    &tmpObj1,
+	}))
+	tmpObj2 := "dir/file"
+	tt.OKAll(s3.PutObject(&s3aws.PutObjectInput{
+		Body:   bytes.NewReader(body),
+		Bucket: &bucket,
+		Key:    &tmpObj2,
+	}))
 
 	// delete them using the multi delete endpoint.
-	objectsCh := make(chan minio.ObjectInfo, 3)
-	objectsCh <- minio.ObjectInfo{Key: "dir/file"}
-	objectsCh <- minio.ObjectInfo{Key: "dir/"}
-	close(objectsCh)
-	results := s3.RemoveObjects(context.Background(), bucket, objectsCh, minio.RemoveObjectsOptions{})
-	for res := range results {
-		tt.OK(res.Err)
-	}
+	tt.OKAll(s3.DeleteObject(&s3aws.DeleteObjectInput{Bucket: &bucket, Key: &tmpObj1}))
+	tt.OKAll(s3.DeleteObject(&s3aws.DeleteObjectInput{Bucket: &bucket, Key: &tmpObj2}))
 
 	// delete bucket
-	tt.OK(s3.RemoveBucket(context.Background(), bucket))
-	exists, err = s3.BucketExists(context.Background(), bucket)
+	_, err = s3.DeleteBucket(&s3aws.DeleteBucketInput{Bucket: &bucket})
 	tt.OK(err)
-	if exists {
-		t.Fatal("expected bucket to not exist")
-	}
+	_, err = s3.HeadBucket(&s3aws.HeadBucketInput{Bucket: &bucket})
+	tt.AssertContains(err, "NotFound")
 }
 
 func TestS3ObjectMetadata(t *testing.T) {