Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ETags #625

Merged
merged 45 commits into from
Oct 3, 2023
Merged

ETags #625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
bd15bfd
worker,bus: add ObjectMetadata
peterjan Sep 21, 2023
f9bd141
stores: add MimeType to object
peterjan Sep 22, 2023
3096b84
stores: add CreateMultipartOptions
peterjan Sep 24, 2023
b471ac7
bus,worker: fix CompleteMultipartUpload, CreateMultipartUpload signature
peterjan Sep 25, 2023
0a9428e
worker: cleanup PR
peterjan Sep 25, 2023
795b36f
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 25, 2023
efac727
bus: fix MultipartAddPartRequest
peterjan Sep 25, 2023
1f43282
testing: fix TestUploadDownloadExtended,TestObjectEntries
peterjan Sep 25, 2023
7a4461e
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 25, 2023
669e76f
testing: fix TestObjectEntries
peterjan Sep 25, 2023
4f59a5e
testing: avoid NDF in TestSlabBufferStats
peterjan Sep 25, 2023
ed28864
bus: add ContentType method
peterjan Sep 25, 2023
6f1a8b2
api: add CopyObjectOptions
peterjan Sep 25, 2023
6b58856
worker,bus: add etag
peterjan Sep 25, 2023
7c72be1
testing: fix TestSQLMetadataStore
peterjan Sep 25, 2023
821a212
wip: tmp commit
peterjan Sep 25, 2023
484f2fe
testing: fix TestCopyObject
peterjan Sep 25, 2023
d2d8b6b
testing: fix TestObjectEntries
peterjan Sep 26, 2023
57c2846
modules: update gofakes3
peterjan Sep 26, 2023
70e4ad0
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 27, 2023
1b13773
stores: update migration label
peterjan Sep 27, 2023
e595555
fix lint
peterjan Sep 27, 2023
074aa64
testing,migrations: fix TestObjectEntries + avoid conflict
peterjan Sep 27, 2023
95ef7ea
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 27, 2023
dc59b40
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 27, 2023
769f42b
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 27, 2023
2cc2a59
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 27, 2023
fa53b11
worker: fix multiple write
peterjan Sep 27, 2023
67180da
Merge branch 'pj/mime-type' into pj/serve-content
peterjan Sep 28, 2023
09967ac
Merge branch 'pj/etags' into pj/serve-content
peterjan Sep 28, 2023
9e30adc
worker: add serveContent
peterjan Sep 28, 2023
d3dc995
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 28, 2023
2335248
fix: lint
peterjan Sep 28, 2023
67fa235
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 28, 2023
a9cc9ea
fix: TestMultipartUploadWithUploadPackingRegression
peterjan Sep 28, 2023
01827c5
bus: move ContentType method
peterjan Sep 28, 2023
2a3a3a2
worker: set etag on response header on upload
peterjan Sep 28, 2023
c9d897e
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Sep 29, 2023
423a41a
worker: implement MR remarks
peterjan Oct 2, 2023
bbe8054
api: add UploadObjectResponse,UploadMultipartUploadPartResponse
peterjan Oct 2, 2023
758da29
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Oct 2, 2023
12e06a4
worker: allow seek at offset
peterjan Oct 2, 2023
3059531
Merge branch 'master' of https://github.com/SiaFoundation/renterd int…
peterjan Oct 3, 2023
c65708d
Merge branch 'master' into pj/serve-content
ChrisSchinnerl Oct 3, 2023
3b15871
worker: etag -> eTag
ChrisSchinnerl Oct 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions api/bus.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ type Object struct {

// ObjectMetadata contains various metadata about an object.
type ObjectMetadata struct {
ETag string `json:"eTag"`
Health float64 `json:"health"`
MimeType string `json:"mimeType"`
ModTime time.Time `json:"modTime"`
Expand All @@ -223,14 +224,14 @@ type ObjectMetadata struct {

// LastModified returns the object's ModTime formatted for use in the
// 'Last-Modified' header
func (o *Object) LastModified() string {
func (o ObjectMetadata) LastModified() string {
return o.ModTime.UTC().Format(http.TimeFormat)
}

// ContentType returns the object's MimeType for use in the 'Content-Type'
// header, if the object's mime type is empty we try and deduce it from the
// extension in the object's name.
func (o Object) ContentType() string {
func (o ObjectMetadata) ContentType() string {
if o.MimeType != "" {
return o.MimeType
}
Expand All @@ -249,6 +250,7 @@ type ObjectAddRequest struct {
Object object.Object `json:"object"`
UsedContracts map[types.PublicKey]types.FileContractID `json:"usedContracts"`
MimeType string `json:"mimeType"`
ETag string `json:"eTag"`
}

// ObjectsResponse is the response type for the /objects endpoint.
Expand Down Expand Up @@ -578,7 +580,7 @@ type (
}
MultipartAddPartRequest struct {
Bucket string `json:"bucket"`
Etag string `json:"eTag"`
ETag string `json:"eTag"`
Path string `json:"path"`
ContractSet string `json:"contractSet"`
UploadID string `json:"uploadID"`
Expand Down Expand Up @@ -717,6 +719,6 @@ type AddPartialSlabResponse struct {
Slabs []object.PartialSlab `json:"slabs"`
}

func FormatEtag(etag string) string {
return fmt.Sprintf("\"%s\"", etag)
func FormatETag(ETag string) string {
return fmt.Sprintf("\"%s\"", ETag)
}
8 changes: 8 additions & 0 deletions api/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,14 @@ type DownloadRange struct {
Length int64
}

type UploadObjectResponse struct {
ETag string `json:"etag"`
}

type UploadMultipartUploadPartResponse struct {
ETag string `json:"etag"`
}

type GetObjectResponse struct {
Content io.ReadCloser `json:"content"`
ContentType string `json:"contentType"`
Expand Down
13 changes: 8 additions & 5 deletions bus/bus.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,14 +117,14 @@ type (
ObjectsBySlabKey(ctx context.Context, bucket string, slabKey object.EncryptionKey) ([]api.ObjectMetadata, error)
SearchObjects(ctx context.Context, bucket, substring string, offset, limit int) ([]api.ObjectMetadata, error)
CopyObject(ctx context.Context, srcBucket, dstBucket, srcPath, dstPath, mimeType string) (api.ObjectMetadata, error)
UpdateObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) error
UpdateObject(ctx context.Context, bucket, path, contractSet, ETag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) error
RemoveObject(ctx context.Context, bucket, path string) error
RemoveObjects(ctx context.Context, bucket, prefix string) error
RenameObject(ctx context.Context, bucket, from, to string) error
RenameObjects(ctx context.Context, bucket, from, to string) error

AbortMultipartUpload(ctx context.Context, bucket, path string, uploadID string) (err error)
AddMultipartPart(ctx context.Context, bucket, path, contractSet, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, eTag string, usedContracts map[types.PublicKey]types.FileContractID) (err error)
AddMultipartPart(ctx context.Context, bucket, path, contractSet, eTag, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, usedContracts map[types.PublicKey]types.FileContractID) (err error)
CompleteMultipartUpload(ctx context.Context, bucket, path, uploadID string, parts []api.MultipartCompletedPart) (_ api.MultipartCompleteResponse, err error)
CreateMultipartUpload(ctx context.Context, bucket, path string, ec object.EncryptionKey, mimeType string) (api.MultipartCreateResponse, error)
MultipartUpload(ctx context.Context, uploadID string) (resp api.MultipartUpload, _ error)
Expand Down Expand Up @@ -1027,7 +1027,7 @@ func (b *bus) objectsHandlerPUT(jc jape.Context) {
} else if aor.Bucket == "" {
aor.Bucket = api.DefaultBucketName
}
jc.Check("couldn't store object", b.ms.UpdateObject(jc.Request.Context(), aor.Bucket, jc.PathParam("path"), aor.ContractSet, aor.Object, aor.UsedContracts, aor.MimeType))
jc.Check("couldn't store object", b.ms.UpdateObject(jc.Request.Context(), aor.Bucket, jc.PathParam("path"), aor.ContractSet, aor.ETag, aor.MimeType, aor.Object, aor.UsedContracts))
}

func (b *bus) objectsCopyHandlerPOST(jc jape.Context) {
Expand All @@ -1040,6 +1040,9 @@ func (b *bus) objectsCopyHandlerPOST(jc jape.Context) {
if jc.Check("couldn't copy object", err) != nil {
return
}

jc.ResponseWriter.Header().Set("Last-Modified", om.LastModified())
jc.ResponseWriter.Header().Set("ETag", api.FormatETag(om.ETag))
jc.Encode(om)
}

Expand Down Expand Up @@ -1904,7 +1907,7 @@ func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) {
} else if req.ContractSet == "" {
jc.Error(errors.New("contract_set must be non-empty"), http.StatusBadRequest)
return
} else if req.Etag == "" {
} else if req.ETag == "" {
ChrisSchinnerl marked this conversation as resolved.
Show resolved Hide resolved
jc.Error(errors.New("etag must be non-empty"), http.StatusBadRequest)
return
} else if req.PartNumber <= 0 || req.PartNumber > gofakes3.MaxUploadPartNumber {
Expand All @@ -1914,7 +1917,7 @@ func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) {
jc.Error(errors.New("upload_id must be non-empty"), http.StatusBadRequest)
return
}
err := b.ms.AddMultipartPart(jc.Request.Context(), req.Bucket, req.Path, req.ContractSet, req.UploadID, req.PartNumber, req.Slices, req.PartialSlabs, req.Etag, req.UsedContracts)
err := b.ms.AddMultipartPart(jc.Request.Context(), req.Bucket, req.Path, req.ContractSet, req.ETag, req.UploadID, req.PartNumber, req.Slices, req.PartialSlabs, req.UsedContracts)
if jc.Check("failed to upload part", err) != nil {
return
}
Expand Down
7 changes: 4 additions & 3 deletions bus/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -677,14 +677,15 @@ func (c *Client) Object(ctx context.Context, path string, opts ...api.ObjectsOpt
}

// AddObject stores the provided object under the given path.
func (c *Client) AddObject(ctx context.Context, bucket, path, contractSet string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID, mimeType string) (err error) {
func (c *Client) AddObject(ctx context.Context, bucket, path, contractSet, eTag, mimeType string, o object.Object, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
path = strings.TrimPrefix(path, "/")
err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/objects/%s", path), api.ObjectAddRequest{
Bucket: bucket,
ContractSet: contractSet,
Object: o,
UsedContracts: usedContracts,
MimeType: mimeType,
ETag: eTag,
})
return
}
Expand Down Expand Up @@ -981,10 +982,10 @@ func (c *Client) CreateMultipartUpload(ctx context.Context, bucket, path string,
return
}

func (c *Client) AddMultipartPart(ctx context.Context, bucket, path, contractSet, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, etag string, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
func (c *Client) AddMultipartPart(ctx context.Context, bucket, path, contractSet, eTag, uploadID string, partNumber int, slices []object.SlabSlice, partialSlab []object.PartialSlab, usedContracts map[types.PublicKey]types.FileContractID) (err error) {
err = c.c.WithContext(ctx).PUT("/multipart/part", api.MultipartAddPartRequest{
Bucket: bucket,
Etag: etag,
ETag: eTag,
Path: path,
ContractSet: contractSet,
UploadID: uploadID,
Expand Down
Loading