From 9fe2e31ae0872b6de61eae888d4dd4fc88645c9c Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 2 Oct 2023 17:20:34 +0200 Subject: [PATCH 1/2] s3: smaller fixes for rclone --- internal/testing/s3_test.go | 11 +++++++++++ s3/backend.go | 6 +++++- stores/metadata.go | 17 ++++++++++++++++- stores/multipart.go | 7 +++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/internal/testing/s3_test.go b/internal/testing/s3_test.go index ef1fb17a8..a80dc1fbb 100644 --- a/internal/testing/s3_test.go +++ b/internal/testing/s3_test.go @@ -562,6 +562,17 @@ func TestS3MultipartUploads(t *testing.T) { t.Fatal("unexpected data:", string(data)) } + // Download again with range request. + b := make([]byte, 5) + downloadedObj, err = s3.GetObject(context.Background(), "multipart", "foo", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } else if _, err = downloadedObj.ReadAt(b, 5); err != nil { + t.Fatal(err) + } else if !bytes.Equal(b, []byte("world")) { + t.Fatal("unexpected data:", string(b)) + } + // Start a second multipart upload. uploadID, err = core.NewMultipartUpload(context.Background(), "multipart", "bar", minio.PutObjectOptions{}) if err != nil { diff --git a/s3/backend.go b/s3/backend.go index 4a84728a2..094f79779 100644 --- a/s3/backend.go +++ b/s3/backend.go @@ -229,7 +229,11 @@ func (s *s3) GetObject(ctx context.Context, bucketName, objectName string, range var opts []api.DownloadObjectOption if rangeRequest != nil { - opts = append(opts, api.DownloadWithRange(rangeRequest.Start, rangeRequest.End)) + length := int64(-1) + if rangeRequest.End >= 0 { + length = rangeRequest.End - rangeRequest.Start + 1 + } + opts = append(opts, api.DownloadWithRange(rangeRequest.Start, length)) } res, err := s.w.GetObject(ctx, bucketName, objectName, opts...) if err != nil && strings.Contains(err.Error(), api.ErrBucketNotFound.Error()) { diff --git a/stores/metadata.go b/stores/metadata.go index 908387bb2..8d193f223 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -1238,7 +1238,7 @@ func (s *SQLStore) AddPartialSlab(ctx context.Context, data []byte, minShards, t func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath, dstPath, mimeType string) (om api.ObjectMetadata, err error) { err = s.retryTransaction(func(tx *gorm.DB) error { var srcObj dbObject - err := tx.Where("objects.object_id = ? AND DBBucket.name = ?", srcPath, srcBucket). + err = tx.Where("objects.object_id = ? AND DBBucket.name = ?", srcPath, srcBucket). Joins("DBBucket"). Take(&srcObj). Error @@ -1251,6 +1251,21 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath return fmt.Errorf("failed to fetch src object health: %w", err) } + if srcBucket == dstBucket && srcPath == dstPath { + om = api.ObjectMetadata{ + Health: srcObjHealth, + MimeType: srcObj.MimeType, + ModTime: srcObj.CreatedAt.UTC(), + Name: srcObj.ObjectID, + Size: srcObj.Size, + } + return nil + } + _, err = deleteObject(tx, dstBucket, dstPath) + if err != nil { + return fmt.Errorf("failed to delete object: %w", err) + } + var srcSlices []dbSlice err = tx.Where("db_object_id = ?", srcObj.ID). Find(&srcSlices). diff --git a/stores/multipart.go b/stores/multipart.go index 08eaa1dd7..2dcb11b7a 100644 --- a/stores/multipart.go +++ b/stores/multipart.go @@ -313,6 +313,13 @@ func (s *SQLStore) CompleteMultipartUpload(ctx context.Context, bucket, path str if mu.DBBucket.Name != bucket { return fmt.Errorf("bucket name mismatch: %v != %v: %w", mu.DBBucket.Name, bucket, api.ErrBucketNotFound) } + + // Delete potentially existing object. + _, err := deleteObject(tx, bucket, path) + if err != nil { + return fmt.Errorf("failed to delete object: %w", err) + } + // Sort the parts. sort.Slice(mu.Parts, func(i, j int) bool { return mu.Parts[i].PartNumber < mu.Parts[j].PartNumber From c1c99480524cc15d39e691c3748a06d992fef6e9 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 3 Oct 2023 10:04:44 +0200 Subject: [PATCH 2/2] stores: handle mimetype --- stores/metadata.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stores/metadata.go b/stores/metadata.go index 8d193f223..5eb027098 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -1252,6 +1252,9 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath } if srcBucket == dstBucket && srcPath == dstPath { + // No copying is happening. We just update the metadata on the src + // object. + srcObj.MimeType = mimeType om = api.ObjectMetadata{ Health: srcObjHealth, MimeType: srcObj.MimeType, @@ -1259,7 +1262,7 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath Name: srcObj.ObjectID, Size: srcObj.Size, } - return nil + return tx.Save(&srcObj).Error } _, err = deleteObject(tx, dstBucket, dstPath) if err != nil {