From 8805b71c84d6f6cbf6a03c0e4a64ffa8eeb2b87c Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 10:22:58 +0200 Subject: [PATCH 1/8] gofake3: set ETag header in copyObject --- gofakes3.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gofakes3.go b/gofakes3.go index 11b913d..43b0ea4 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -779,6 +779,9 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h if srcObj.VersionID != "" { w.Header().Set("x-amz-version-id", string(srcObj.VersionID)) } + if result.ETag != "" { + w.Header().Set("ETag", result.ETag) + } return g.xmlEncoder(w).Encode(result) } From 961bab8f04c8e63c8a697b921b0a3264d10c6c21 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 10:29:13 +0200 Subject: [PATCH 2/8] gofake3: add ETag to PutObjectResult --- backend.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend.go b/backend.go index af749d2..bb5cc5e 100644 --- a/backend.go +++ b/backend.go @@ -125,6 +125,10 @@ type PutObjectResult struct { // created version ID. If versioning is not enabled, this should be // empty. VersionID VersionID + + // ETag is the value of the ETag header returned by the backend, stripped of + // its quotation marks. + ETag string } // Backend provides a set of operations to be implemented in order to support From 1a68115d034b80c9522691cf1291c4e93b416328 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 10:36:16 +0200 Subject: [PATCH 3/8] gofake3: use ETag from result if present --- gofakes3.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gofakes3.go b/gofakes3.go index 43b0ea4..b4d7468 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -725,7 +725,12 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r g.log.Print(LogInfo, "CREATED VERSION:", bucket, object, result.VersionID) w.Header().Set("x-amz-version-id", string(result.VersionID)) } - w.Header().Set("ETag", `"`+hex.EncodeToString(rdr.Sum(nil))+`"`) + + etag := result.ETag + if etag == "" { + etag = hex.EncodeToString(rdr.Sum(nil)) + } + w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag)) return nil } From a3a2ec03588fa661e4d9ec4fc44e6c482525f839 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 11:06:38 +0200 Subject: [PATCH 4/8] gofake3: add response objects --- backend.go | 20 ++++---------------- gofakes3.go | 24 ++++++++++++++++-------- messages.go | 25 +++++++++++++++++++++++++ uploader.go | 39 +++++++++++++++++++++++---------------- 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/backend.go b/backend.go index bb5cc5e..f5334b1 100644 --- a/backend.go +++ b/backend.go @@ -1,7 +1,6 @@ package gofakes3 import ( - "encoding/hex" "io" "time" @@ -120,17 +119,6 @@ func (p ListBucketPage) IsEmpty() bool { return p == ListBucketPage{} } -type PutObjectResult struct { - // If versioning is enabled on the bucket, this should be set to the - // created version ID. If versioning is not enabled, this should be - // empty. - VersionID VersionID - - // ETag is the value of the ETag header returned by the backend, stripped of - // its quotation marks. - ETag string -} - // Backend provides a set of operations to be implemented in order to support // gofakes3. // @@ -324,13 +312,13 @@ type VersionedBackend interface { // gets finalised and pushed to the backend. type MultipartBackend interface { CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error) - UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) + UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) AbortMultipartUpload(bucket, object string, id UploadID) error - CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error) + CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) } // CopyObject is a helper function useful for quickly implementing CopyObject on @@ -343,13 +331,13 @@ func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta ma } defer c.Contents.Close() - _, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + res, err := db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) if err != nil { return } return CopyObjectResult{ - ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + ETag: res.ETag, LastModified: NewContentTime(time.Now()), }, nil } diff --git a/gofakes3.go b/gofakes3.go index b4d7468..8fe0456 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -663,7 +663,11 @@ func (g *GoFakeS3) createObjectBrowserUpload(bucket string, w http.ResponseWrite w.Header().Set("x-amz-version-id", string(result.VersionID)) } - w.Header().Set("ETag", `"`+hex.EncodeToString(rdr.Sum(nil))+`"`) + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) + } + w.Header().Set("ETag", etag) return nil } @@ -728,9 +732,9 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r etag := result.ETag if etag == "" { - etag = hex.EncodeToString(rdr.Sum(nil)) + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) } - w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag)) + w.Header().Set("ETag", etag) return nil } @@ -953,7 +957,7 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload return err } - w.Header().Add("ETag", etag) + w.Header().Add("ETag", formatETag(etag)) return nil } @@ -974,17 +978,17 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - versionID, etag, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in) + res, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in) if err != nil { return err } - if versionID != "" { - w.Header().Set("x-amz-version-id", string(versionID)) + if res.VersionID != "" { + w.Header().Set("x-amz-version-id", string(res.VersionID)) } return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResult{ - ETag: etag, + ETag: res.ETag, Bucket: bucket, Key: object, }) @@ -1210,3 +1214,7 @@ func listBucketVersionsPageFromQuery(query url.Values) (page ListBucketVersionsP return page, nil } + +func formatETag(etag string) string { + return fmt.Sprintf("\"%s\"", etag) +} diff --git a/messages.go b/messages.go index 17138fe..6d51a2f 100644 --- a/messages.go +++ b/messages.go @@ -365,6 +365,20 @@ func (b *ListBucketVersionsResult) AddPrefix(prefix string) { b.CommonPrefixes = append(b.CommonPrefixes, CommonPrefix{Prefix: prefix}) } +type UploadPartResult struct { + ETag string `xml:"ETag,omitempty"` +} + +type CompleteMultipartResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + type ListMultipartUploadsResult struct { Bucket string `xml:"Bucket"` @@ -430,6 +444,17 @@ type ListMultipartUploadPartItem struct { Size int64 `xml:"Size"` } +// PutObjectResult contains the response from a PutObject operation. +type PutObjectResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + // CopyObjectResult contains the response from a CopyObject operation. type CopyObjectResult struct { XMLName xml.Name `xml:"CopyObjectResult"` diff --git a/uploader.go b/uploader.go index 48b3af2..c77cb85 100644 --- a/uploader.go +++ b/uploader.go @@ -367,17 +367,17 @@ func (u *uploader) AbortMultipartUpload(bucket, object string, id UploadID) erro return nil } -func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) { +func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) { body, err := io.ReadAll(input) if err != nil { - return "", err + return nil, err } if len(body) != int(contentLength) { - return "", ErrIncompleteBody + return nil, ErrIncompleteBody } mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", err + return nil, err } mpu.mu.Lock() @@ -387,7 +387,7 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int // from guaranteed unique input: hash := md5.New() hash.Write([]byte(body)) - etag = fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) + etag := fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) part := multipartUploadPart{ PartNumber: partNumber, @@ -399,13 +399,14 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int mpu.parts = append(mpu.parts, make([]*multipartUploadPart, partNumber-len(mpu.parts)+1)...) } mpu.parts[partNumber] = &part - return etag, nil + + return &UploadPartResult{ETag: formatETag(etag)}, nil } -func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (version VersionID, etag string, err error) { +func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) { mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", "", err + return nil, err } mpu.mu.Lock() @@ -417,23 +418,23 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i // end up uploading more parts than you need to assemble, so it should // probably just ignore that? if len(input.Parts) > mpuPartsLen { - return "", "", ErrInvalidPart + return nil, ErrInvalidPart } if !input.partsAreSorted() { - return "", "", ErrInvalidPartOrder + return nil, ErrInvalidPartOrder } var size int64 for _, inPart := range input.Parts { if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) } upPart := mpu.parts[inPart.PartNumber] if strings.Trim(inPart.ETag, "\"") != strings.Trim(upPart.ETag, "\"") { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) } size += int64(len(upPart.Body)) @@ -444,16 +445,22 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i body = append(body, mpu.parts[part.PartNumber].Body...) } - hash := fmt.Sprintf("%x", md5.Sum(body)) - result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body))) if err != nil { - return "", "", err + return nil, err + } + + etag := result.ETag + if etag == "" { + etag = formatETag(fmt.Sprintf("%x", md5.Sum(body))) } // if getUnlocked succeeded, so will this: u.buckets[bucket].remove(id) - return result.VersionID, hash, nil + return &CompleteMultipartResult{ + VersionID: result.VersionID, + ETag: etag, + }, nil } func (u *uploader) getUnlocked(bucket, object string, id UploadID) (mu *multipartUpload, err error) { From 6e69dffdbbb23caaf63fce0fc3b33dcf2a38e6b1 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 11:10:37 +0200 Subject: [PATCH 5/8] gofake3: rename CompleteMultipartUploadResult --- backend.go | 2 +- gofakes3.go | 9 ++++++--- messages.go | 4 ++-- uploader.go | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/backend.go b/backend.go index f5334b1..aba93b7 100644 --- a/backend.go +++ b/backend.go @@ -318,7 +318,7 @@ type MultipartBackend interface { ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) AbortMultipartUpload(bucket, object string, id UploadID) error - CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) + CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartUploadResult, error) } // CopyObject is a helper function useful for quickly implementing CopyObject on diff --git a/gofakes3.go b/gofakes3.go index 8fe0456..66c0284 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -788,9 +788,12 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h if srcObj.VersionID != "" { w.Header().Set("x-amz-version-id", string(srcObj.VersionID)) } - if result.ETag != "" { - w.Header().Set("ETag", result.ETag) + + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(srcObj.Hash)) } + w.Header().Set("ETag", etag) return g.xmlEncoder(w).Encode(result) } @@ -987,7 +990,7 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa w.Header().Set("x-amz-version-id", string(res.VersionID)) } - return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResult{ + return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResponse{ ETag: res.ETag, Bucket: bucket, Key: object, diff --git a/messages.go b/messages.go index 6d51a2f..1075421 100644 --- a/messages.go +++ b/messages.go @@ -69,7 +69,7 @@ func (c CompleteMultipartUploadRequest) partIDs() []int { return inParts } -type CompleteMultipartUploadResult struct { +type CompleteMultipartUploadResponse struct { Location string `xml:"Location"` Bucket string `xml:"Bucket"` Key string `xml:"Key"` @@ -369,7 +369,7 @@ type UploadPartResult struct { ETag string `xml:"ETag,omitempty"` } -type CompleteMultipartResult struct { +type CompleteMultipartUploadResult struct { // If versioning is enabled on the bucket, this should be set to the // created version ID. If versioning is not enabled, this should be // empty. diff --git a/uploader.go b/uploader.go index c77cb85..ad47be3 100644 --- a/uploader.go +++ b/uploader.go @@ -403,7 +403,7 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int return &UploadPartResult{ETag: formatETag(etag)}, nil } -func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) { +func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartUploadResult, error) { mpu, err := u.getUnlocked(bucket, object, id) if err != nil { return nil, err @@ -457,7 +457,7 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i // if getUnlocked succeeded, so will this: u.buckets[bucket].remove(id) - return &CompleteMultipartResult{ + return &CompleteMultipartUploadResult{ VersionID: result.VersionID, ETag: etag, }, nil From 0a4e022e41a6cb104d3179e56e8061f33c9b0821 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 11:21:35 +0200 Subject: [PATCH 6/8] gofake3: fix lint --- gofakes3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gofakes3.go b/gofakes3.go index 66c0284..9161334 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -955,12 +955,12 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload } } - etag, err := g.uploader.UploadPart(bucket, object, uploadID, int(partNumber), r.ContentLength, rdr) + res, err := g.uploader.UploadPart(bucket, object, uploadID, int(partNumber), r.ContentLength, rdr) if err != nil { return err } - w.Header().Add("ETag", formatETag(etag)) + w.Header().Add("ETag", res.ETag) return nil } From 8f864e3a98c5f3a2ee929e4f88e1f96df78ed38b Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 11:30:55 +0200 Subject: [PATCH 7/8] testing: fix TestCopyObject --- backend/s3mem/backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/s3mem/backend.go b/backend/s3mem/backend.go index 48937d1..f935467 100644 --- a/backend/s3mem/backend.go +++ b/backend/s3mem/backend.go @@ -257,6 +257,7 @@ func (db *Backend) PutObject(bucketName, objectName string, meta map[string]stri result.VersionID = item.versionID } + result.ETag = item.etag return result, nil } From d69e32b8d0cf7346b54635e20183e8dbda815bd9 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 26 Sep 2023 11:39:29 +0200 Subject: [PATCH 8/8] testing: fix TestListMultipartUploadParts --- uploader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uploader.go b/uploader.go index ad47be3..7c7a8be 100644 --- a/uploader.go +++ b/uploader.go @@ -387,7 +387,7 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int // from guaranteed unique input: hash := md5.New() hash.Write([]byte(body)) - etag := fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) + etag := formatETag(hex.EncodeToString(hash.Sum(nil))) part := multipartUploadPart{ PartNumber: partNumber, @@ -400,7 +400,7 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int } mpu.parts[partNumber] = &part - return &UploadPartResult{ETag: formatETag(etag)}, nil + return &UploadPartResult{ETag: etag}, nil } func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartUploadResult, error) {