Merge pull request #637 from SiaFoundation/chris/rclone-fixes

Various S3 fixes to get rclone mount working

internal/testing/s3_test.go

@@ -562,6 +562,17 @@ func TestS3MultipartUploads(t *testing.T) {
 		t.Fatal("unexpected data:", string(data))
 	}
 
+	// Download again with range request.
+	b := make([]byte, 5)
+	downloadedObj, err = s3.GetObject(context.Background(), "multipart", "foo", minio.GetObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	} else if _, err = downloadedObj.ReadAt(b, 5); err != nil {
+		t.Fatal(err)
+	} else if !bytes.Equal(b, []byte("world")) {
+		t.Fatal("unexpected data:", string(b))
+	}
+
 	// Start a second multipart upload.
 	uploadID, err = core.NewMultipartUpload(context.Background(), "multipart", "bar", minio.PutObjectOptions{})
 	if err != nil {

s3/backend.go

@@ -229,7 +229,11 @@ func (s *s3) GetObject(ctx context.Context, bucketName, objectName string, range
 
 	var opts []api.DownloadObjectOption
 	if rangeRequest != nil {
-		opts = append(opts, api.DownloadWithRange(rangeRequest.Start, rangeRequest.End))
+		length := int64(-1)
+		if rangeRequest.End >= 0 {
+			length = rangeRequest.End - rangeRequest.Start + 1
+		}
+		opts = append(opts, api.DownloadWithRange(rangeRequest.Start, length))
 	}
 	res, err := s.w.GetObject(ctx, bucketName, objectName, opts...)
 	if err != nil && strings.Contains(err.Error(), api.ErrBucketNotFound.Error()) {

stores/metadata.go

@@ -1238,7 +1238,7 @@ func (s *SQLStore) AddPartialSlab(ctx context.Context, data []byte, minShards, t
 func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath, dstPath, mimeType string) (om api.ObjectMetadata, err error) {
 	err = s.retryTransaction(func(tx *gorm.DB) error {
 		var srcObj dbObject
-		err := tx.Where("objects.object_id = ? AND DBBucket.name = ?", srcPath, srcBucket).
+		err = tx.Where("objects.object_id = ? AND DBBucket.name = ?", srcPath, srcBucket).
 			Joins("DBBucket").
 			Take(&srcObj).
 			Error
@@ -1251,6 +1251,24 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath
 			return fmt.Errorf("failed to fetch src object health: %w", err)
 		}
 
+		if srcBucket == dstBucket && srcPath == dstPath {
+			// No copying is happening. We just update the metadata on the src
+			// object.
+			srcObj.MimeType = mimeType
+			om = api.ObjectMetadata{
+				Health:   srcObjHealth,
+				MimeType: srcObj.MimeType,
+				ModTime:  srcObj.CreatedAt.UTC(),
+				Name:     srcObj.ObjectID,
+				Size:     srcObj.Size,
+			}
+			return tx.Save(&srcObj).Error
+		}
+		_, err = deleteObject(tx, dstBucket, dstPath)
+		if err != nil {
+			return fmt.Errorf("failed to delete object: %w", err)
+		}
+
 		var srcSlices []dbSlice
 		err = tx.Where("db_object_id = ?", srcObj.ID).
 			Find(&srcSlices).

stores/multipart.go

@@ -313,6 +313,13 @@ func (s *SQLStore) CompleteMultipartUpload(ctx context.Context, bucket, path str
 		if mu.DBBucket.Name != bucket {
 			return fmt.Errorf("bucket name mismatch: %v != %v: %w", mu.DBBucket.Name, bucket, api.ErrBucketNotFound)
 		}
+
+		// Delete potentially existing object.
+		_, err := deleteObject(tx, bucket, path)
+		if err != nil {
+			return fmt.Errorf("failed to delete object: %w", err)
+		}
+
 		// Sort the parts.
 		sort.Slice(mu.Parts, func(i, j int) bool {
 			return mu.Parts[i].PartNumber < mu.Parts[j].PartNumber