ETags

api/bus.go

@@ -213,6 +213,7 @@ type Object struct {
 
 // ObjectMetadata contains various metadata about an object.
 type ObjectMetadata struct {
+	ETag     string    `json:"eTag"`
 	Health   float64   `json:"health"`
 	MimeType string    `json:"mimeType"`
 	ModTime  time.Time `json:"modTime"`
@@ -222,14 +223,14 @@ type ObjectMetadata struct {
 
 // LastModified returns the object's ModTime formatted for use in the
 // 'Last-Modified' header
-func (o *Object) LastModified() string {
+func (o ObjectMetadata) LastModified() string {
 	return o.ModTime.UTC().Format(http.TimeFormat)
 }
 
 // ContentType returns the object's MimeType for use in the 'Content-Type'
 // header, if the object's mime type is empty we try and deduce it from the
 // extension in the object's name.
-func (o Object) ContentType() string {
+func (o ObjectMetadata) ContentType() string {
 	if o.MimeType != "" {
 		return o.MimeType
 	}
@@ -248,6 +249,7 @@ type ObjectAddRequest struct {
 	Object        object.Object                            `json:"object"`
 	UsedContracts map[types.PublicKey]types.FileContractID `json:"usedContracts"`
 	MimeType      string                                   `json:"mimeType"`
+	ETag          string                                   `json:"eTag"`
 }
 
 // ObjectsResponse is the response type for the /objects endpoint.
@@ -573,7 +575,7 @@ type (
 	}
 	MultipartAddPartRequest struct {
 		Bucket        string                                   `json:"bucket"`
-		Etag          string                                   `json:"eTag"`
+		ETag          string                                   `json:"eTag"`
 		Path          string                                   `json:"path"`
 		ContractSet   string                                   `json:"contractSet"`
 		UploadID      string                                   `json:"uploadID"`
@@ -712,6 +714,6 @@ type AddPartialSlabResponse struct {
 	Slabs                        []object.PartialSlab `json:"slabs"`
 }
 
-func FormatEtag(etag string) string {
-	return fmt.Sprintf("\"%s\"", etag)
+func FormatETag(ETag string) string {
+	return fmt.Sprintf("\"%s\"", ETag)
 }

bus/bus.go

@@ -117,14 +117,14 @@ type (
 		ObjectsBySlabKey(ctx context.Context, bucket string, slabKey object.EncryptionKey) ([]api.ObjectMetadata, error)
 		SearchObjects(ctx context.Context, bucket, substring string, offset, limit int) ([]api.ObjectMetadata, error)
 		CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath, dstPath, mimeType string) (api.ObjectMetadata, error)
-		UpdateObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) error
+		UpdateObject(ctx context.Context, bucket, path, contractSet, ETag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) error
 		RemoveObject(ctx context.Context, bucket, path string) error
 		RemoveObjects(ctx context.Context, bucket, prefix string) error
 		RenameObject(ctx context.Context, bucket, from, to string) error
 		RenameObjects(ctx context.Context, bucket, from, to string) error
 
 		AbortMultipartUpload(ctx context.Context, bucket, path string, uploadID string) (err error)
-		AddMultipartPart(ctx context.Context, bucket, path, contractSet, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, eTag string, usedContracts map[types.PublicKey]types.FileContractID) (err error)
+		AddMultipartPart(ctx context.Context, bucket, path, contractSet, eTag, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, usedContracts map[types.PublicKey]types.FileContractID) (err error)
 		CompleteMultipartUpload(ctx context.Context, bucket, path, uploadID string, parts []api.MultipartCompletedPart) (_ api.MultipartCompleteResponse, err error)
 		CreateMultipartUpload(ctx context.Context, bucket, path string, ec object.EncryptionKey, mimeType string) (api.MultipartCreateResponse, error)
 		MultipartUpload(ctx context.Context, uploadID string) (resp api.MultipartUpload, _ error)
@@ -1026,7 +1026,7 @@ func (b *bus) objectsHandlerPUT(jc jape.Context) {
 	} else if aor.Bucket == "" {
 		aor.Bucket = api.DefaultBucketName
 	}
-	jc.Check("couldn't store object", b.ms.UpdateObject(jc.Request.Context(), aor.Bucket, jc.PathParam("path"), aor.ContractSet, aor.Object, aor.UsedContracts, aor.MimeType))
+	jc.Check("couldn't store object", b.ms.UpdateObject(jc.Request.Context(), aor.Bucket, jc.PathParam("path"), aor.ContractSet, aor.ETag, aor.MimeType, aor.Object, aor.UsedContracts))
 }
 
 func (b *bus) objectsCopyHandlerPOST(jc jape.Context) {
@@ -1039,6 +1039,9 @@ func (b *bus) objectsCopyHandlerPOST(jc jape.Context) {
 	if jc.Check("couldn't copy object", err) != nil {
 		return
 	}
+
+	jc.ResponseWriter.Header().Set("Last-Modified", om.LastModified())
+	jc.ResponseWriter.Header().Set("ETag", api.FormatETag(om.ETag))
 	jc.Encode(om)
 }
 
@@ -1897,7 +1900,7 @@ func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) {
 	} else if req.ContractSet == "" {
 		jc.Error(errors.New("contract_set must be non-empty"), http.StatusBadRequest)
 		return
-	} else if req.Etag == "" {
+	} else if req.ETag == "" {
 		jc.Error(errors.New("etag must be non-empty"), http.StatusBadRequest)
 		return
 	} else if req.PartNumber <= 0 || req.PartNumber > gofakes3.MaxUploadPartNumber {
@@ -1907,7 +1910,7 @@ func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) {
 		jc.Error(errors.New("upload_id must be non-empty"), http.StatusBadRequest)
 		return
 	}
-	err := b.ms.AddMultipartPart(jc.Request.Context(), req.Bucket, req.Path, req.ContractSet, req.UploadID, req.PartNumber, req.Slices, req.PartialSlabs, req.Etag, req.UsedContracts)
+	err := b.ms.AddMultipartPart(jc.Request.Context(), req.Bucket, req.Path, req.ContractSet, req.ETag, req.UploadID, req.PartNumber, req.Slices, req.PartialSlabs, req.UsedContracts)
 	if jc.Check("failed to upload part", err) != nil {
 		return
 	}

bus/client.go

@@ -671,14 +671,15 @@ func (c *Client) Object(ctx context.Context, path string, opts ...api.ObjectsOpt
 }
 
 // AddObject stores the provided object under the given path.
-func (c *Client) AddObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) (err error) {
+func (c *Client) AddObject(ctx context.Context, bucket, path, contractSet, eTag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
 	path = strings.TrimPrefix(path, "/")
 	err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/objects/%s", path), api.ObjectAddRequest{
 		Bucket:        bucket,
 		ContractSet:   contractSet,
 		Object:        o,
 		UsedContracts: usedContracts,
 		MimeType:      mimeType,
+		ETag:          eTag,
 	})
 	return
 }
@@ -975,10 +976,10 @@ func (c *Client) CreateMultipartUpload(ctx context.Context, bucket, path string,
 	return
 }
 
-func (c *Client) AddMultipartPart(ctx context.Context, bucket, path, contractSet, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, etag string, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
+func (c *Client) AddMultipartPart(ctx context.Context, bucket, path, contractSet, eTag, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
 	err = c.c.WithContext(ctx).PUT("/multipart/part", api.MultipartAddPartRequest{
 		Bucket:        bucket,
-		Etag:          etag,
+		ETag:          eTag,
 		Path:          path,
 		ContractSet:   contractSet,
 		UploadID:      uploadID,

internal/testing/cluster_test.go

@@ -31,6 +31,11 @@ import (
 	"lukechampine.com/frand"
 )
 
+const (
+	testEtag     = "d34db33f"
+	testMimeType = "application/octet-stream"
+)
+
 // TestNewTestCluster is a test for creating a cluster of Nodes for testing,
 // making sure that it forms contracts, renews contracts and shuts down.
 func TestNewTestCluster(t *testing.T) {
@@ -60,7 +65,7 @@ func TestNewTestCluster(t *testing.T) {
 	}
 
 	// Try talking to the bus API by adding an object.
-	err = b.AddObject(context.Background(), api.DefaultBucketName, "foo", testAutopilotConfig.Contracts.Set, object.Object{
+	err = b.AddObject(context.Background(), api.DefaultBucketName, "foo", testAutopilotConfig.Contracts.Set, testEtag, testMimeType, object.Object{
 		Key: object.GenerateEncryptionKey(),
 		Slabs: []object.SlabSlice{
 			{
@@ -73,7 +78,7 @@ func TestNewTestCluster(t *testing.T) {
 				Length: 0,
 			},
 		},
-	}, map[types.PublicKey]types.FileContractID{}, "application/octet-stream")
+	}, map[types.PublicKey]types.FileContractID{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -300,7 +305,8 @@ func TestObjectEntries(t *testing.T) {
 		t.SkipNow()
 	}
 
-	// assert mod time & mime type and clear it afterwards so we can compare
+	// assertMetadata asserts ModTime, ETag and MimeType are set and then clears
+	// them afterwards so we can compare without having to specify the metadata
 	start := time.Now()
 	assertMetadata := func(entries []api.ObjectMetadata) {
 		for i := range entries {
@@ -315,6 +321,12 @@ func TestObjectEntries(t *testing.T) {
 				t.Fatal("mime type should be set", entries[i].MimeType, entries[i].Name)
 			}
 			entries[i].MimeType = ""
+
+			// assert etag
+			if entries[i].ETag == "" {
+				t.Fatal("ETag should be set")
+			}
+			entries[i].ETag = ""
 		}
 	}
 
@@ -398,8 +410,6 @@ func TestObjectEntries(t *testing.T) {
 		if err != nil {
 			t.Fatal(err, test.path)
 		}
-
-		// assert mod time & mime type and clear it afterwards so we can compare
 		assertMetadata(res.Entries)
 
 		if !(len(res.Entries) == 0 && len(test.want) == 0) && !reflect.DeepEqual(res.Entries, test.want) {
@@ -410,8 +420,6 @@ func TestObjectEntries(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-
-			// assert mod time & mime type and clear it afterwards so we can compare
 			assertMetadata(res.Entries)
 
 			if len(res.Entries) != 1 || res.Entries[0] != test.want[offset] {
@@ -431,8 +439,6 @@ func TestObjectEntries(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-
-			// assert mod time & mime type and clear it afterwards so we can compare
 			assertMetadata(res.Entries)
 
 			if len(res.Entries) != 1 || res.Entries[0] != test.want[offset+1] {
@@ -450,8 +456,6 @@ func TestObjectEntries(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-
-		// assert mod time & mime type and clear it afterwards so we can compare
 		assertMetadata(got)
 
 		if !(len(got) == 0 && len(test.want) == 0) && !reflect.DeepEqual(got, test.want) {

internal/testing/s3_test.go

@@ -137,6 +137,8 @@ func TestS3Basic(t *testing.T) {
 		t.Fatal("expected LastModified to be non-zero")
 	} else if !res.LastModified.After(start.UTC()) {
 		t.Fatal("expected LastModified to be after the start of our test")
+	} else if res.ETag == "" {
+		t.Fatal("expected ETag to be set")
 	}
 
 	// get copied object

s3/backend.go

@@ -372,7 +372,7 @@ func (s *s3) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKe
 		return gofakes3.CopyObjectResult{}, gofakes3.ErrorMessage(gofakes3.ErrInternal, err.Error())
 	}
 	return gofakes3.CopyObjectResult{
-		ETag:         "", // TODO: don't have that
+		ETag:         api.FormatETag(obj.ETag),
 		LastModified: gofakes3.NewContentTime(obj.ModTime.UTC()),
 	}, nil
 }
@@ -399,7 +399,7 @@ func (s *s3) UploadPart(ctx context.Context, bucket, object string, id gofakes3.
 	if err != nil {
 		return nil, gofakes3.ErrorMessage(gofakes3.ErrInternal, err.Error())
 	}
-	return &gofakes3.UploadPartResult{ETag: api.FormatEtag(etag)}, nil
+	return &gofakes3.UploadPartResult{ETag: api.FormatETag(etag)}, nil
 }
 
 func (s *s3) ListMultipartUploads(ctx context.Context, bucket string, marker *gofakes3.UploadListMarker, prefix gofakes3.Prefix, limit int64) (*gofakes3.ListMultipartUploadsResult, error) {
@@ -486,8 +486,7 @@ func (s *s3) CompleteMultipartUpload(ctx context.Context, bucket, object string,
 	if err != nil {
 		return nil, gofakes3.ErrorMessage(gofakes3.ErrInternal, err.Error())
 	}
-
 	return &gofakes3.CompleteMultipartUploadResult{
-		ETag: api.FormatEtag(resp.ETag),
+		ETag: api.FormatETag(resp.ETag),
 	}, nil
 }

s3/s3.go

@@ -29,7 +29,7 @@ type bus interface {
 	DeleteBucket(ctx context.Context, name string) error
 	ListBuckets(ctx context.Context) (buckets []api.Bucket, err error)
 
-	AddObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) (err error)
+	AddObject(ctx context.Context, bucket, path, contractSet, eTag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) (err error)
 	CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath, dstPath string, opts api.CopyObjectOptions) (om api.ObjectMetadata, err error)
 	DeleteObject(ctx context.Context, bucket, path string, batch bool) (err error)
 	Object(ctx context.Context, path string, opts ...api.ObjectsOption) (res api.ObjectsResponse, err error)
@@ -47,7 +47,7 @@ type bus interface {
 type worker interface {
 	UploadObject(ctx context.Context, r io.Reader, path string, opts ...api.UploadOption) (err error)
 	GetObject(ctx context.Context, path, bucket string, opts ...api.DownloadObjectOption) (api.GetObjectResponse, error)
-	UploadMultipartUploadPart(ctx context.Context, r io.Reader, path, uploadID string, partNumber int, opts ...api.UploadOption) (etag string, err error)
+	UploadMultipartUploadPart(ctx context.Context, r io.Reader, path, uploadID string, partNumber int, opts ...api.UploadOption) (eTag string, err error)
 }
 
 func (l *gofakes3Logger) Print(level gofakes3.LogLevel, v ...interface{}) {

stores/metadata.go

@@ -89,6 +89,7 @@ type (
 		Size  int64
 
 		MimeType string `json:"index"`
+		Etag     string `gorm:"index"`
 	}
 
 	dbBucket struct {
@@ -164,6 +165,7 @@ type (
 		ObjectModTime  time.Time
 		ObjectMimeType string
 		ObjectHealth   float64
+		ObjectETag     string
 
 		// slice
 		SliceOffset uint32
@@ -184,11 +186,12 @@ type (
 
 	// rawObjectMetadata is used for hydrating object metadata.
 	rawObjectMetadata struct {
+		ETag     string
 		Health   float64
-		Name     string
-		Size     int64
 		MimeType string
 		ModTime  datetime
+		Name     string
+		Size     int64
 	}
 )
 
@@ -304,11 +307,12 @@ func (s dbSlab) convert() (slab object.Slab, err error) {
 
 func (raw rawObjectMetadata) convert() api.ObjectMetadata {
 	return api.ObjectMetadata{
+		ETag:     raw.ETag,
 		Health:   raw.Health,
-		Name:     raw.Name,
-		Size:     raw.Size,
 		MimeType: raw.MimeType,
 		ModTime:  time.Time(raw.ModTime).UTC(),
+		Name:     raw.Name,
+		Size:     raw.Size,
 	}
 }
 
@@ -395,11 +399,12 @@ func (raw rawObject) convert() (api.Object, error) {
 	// return object
 	return api.Object{
 		ObjectMetadata: api.ObjectMetadata{
-			Name:     raw[0].ObjectName,
-			Size:     raw[0].ObjectSize,
+			ETag:     raw[0].ObjectETag,
 			Health:   minHealth,
 			MimeType: raw[0].ObjectMimeType,
 			ModTime:  raw[0].ObjectModTime.UTC(),
+			Name:     raw[0].ObjectName,
+			Size:     raw[0].ObjectSize,
 		},
 		Object: object.Object{
 			Key:          key,
@@ -1015,13 +1020,14 @@ func (s *SQLStore) ObjectEntries(ctx context.Context, bucket, path, prefix, mark
 	var rows []rawObjectMetadata
 	query := fmt.Sprintf(`
 	SELECT
+		MAX(etag) AS ETag,
 		MAX(created_at) AS ModTime,
 		CASE slashindex WHEN 0 THEN %s ELSE %s END AS name,
 		SUM(size) AS size,
 		MIN(health) as health,
 		MAX(mimeType) as MimeType
 	FROM (
-		SELECT MAX(objects.created_at) AS created_at, MAX(size) AS size, MIN(slabs.health) as health, MAX(objects.mime_type) as mimeType, SUBSTR(object_id, ?) AS trimmed , INSTR(SUBSTR(object_id, ?), "/") AS slashindex
+		SELECT MAX(etag) AS etag, MAX(objects.created_at) AS created_at, MAX(size) AS size, MIN(slabs.health) as health, MAX(objects.mime_type) as mimeType, SUBSTR(object_id, ?) AS trimmed , INSTR(SUBSTR(object_id, ?), "/") AS slashindex
 		FROM objects
 		INNER JOIN buckets b ON objects.db_bucket_id = b.id AND b.name = ?
 		LEFT JOIN slices ON objects.id = slices.db_object_id 
@@ -1269,8 +1275,9 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath
 		}
 
 		om = api.ObjectMetadata{
-			Health:   srcObjHealth,
 			MimeType: dstObj.MimeType,
+			ETag:     dstObj.Etag,
+			Health:   srcObjHealth,
 			ModTime:  dstObj.CreatedAt.UTC(),
 			Name:     dstObj.ObjectID,
 			Size:     dstObj.Size,
@@ -1280,7 +1287,7 @@ func (s *SQLStore) CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath
 	return
 }
 
-func (s *SQLStore) UpdateObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) error {
+func (s *SQLStore) UpdateObject(ctx context.Context, bucket, path, contractSet, ETag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) error {
 	s.objectsMu.Lock()
 	defer s.objectsMu.Unlock()
 
@@ -1333,6 +1340,7 @@ func (s *SQLStore) UpdateObject(ctx context.Context, bucket, path, contractSet s
 			Key:        objKey,
 			Size:       o.TotalSize(),
 			MimeType:   mimeType,
+			Etag:       ETag,
 		}
 		err = tx.Create(&obj).Error
 		if err != nil {
@@ -1697,7 +1705,7 @@ func (s *SQLStore) object(ctx context.Context, txn *gorm.DB, bucket string, path
 	// accordingly
 	var rows rawObject
 	tx := s.db.
-		Select("o.id as ObjectID, o.key as ObjectKey, o.object_id as ObjectName, o.size as ObjectSize, o.mime_type as ObjectMimeType, o.created_at as ObjectModTime, sli.id as SliceID, sli.offset as SliceOffset, sli.length as SliceLength, sla.id as SlabID, sla.health as SlabHealth, sla.key as SlabKey, sla.min_shards as SlabMinShards, bs.id IS NOT NULL AS SlabBuffered, sec.id as SectorID, sec.root as SectorRoot, sec.latest_host as SectorHost").
+		Select("o.id as ObjectID, o.key as ObjectKey, o.object_id as ObjectName, o.size as ObjectSize, o.mime_type as ObjectMimeType, o.created_at as ObjectModTime, o.etag as ObjectETag, sli.id as SliceID, sli.offset as SliceOffset, sli.length as SliceLength, sla.id as SlabID, sla.health as SlabHealth, sla.key as SlabKey, sla.min_shards as SlabMinShards, bs.id IS NOT NULL AS SlabBuffered, sec.id as SectorID, sec.root as SectorRoot, sec.latest_host as SectorHost").
 		Model(&dbObject{}).
 		Table("objects o").
 		Joins("INNER JOIN buckets b ON o.db_bucket_id = b.id AND b.name = ?", bucket).