Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove ObjectsBySlabKey #842

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion autopilot/autopilot.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ type Bus interface {
RecordContractPruneMetric(ctx context.Context, metrics ...api.ContractPruneMetric) error

// objects
ObjectsBySlabKey(ctx context.Context, bucket string, key object.EncryptionKey) (objects []api.ObjectMetadata, err error)
RefreshHealth(ctx context.Context) error
Slab(ctx context.Context, key object.EncryptionKey) (object.Slab, error)
SlabsForMigration(ctx context.Context, healthCutoff float64, set string, limit int) ([]api.UnhealthySlab, error)
Expand Down
18 changes: 0 additions & 18 deletions bus/bus.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,6 @@ type (
ListObjects(ctx context.Context, bucketName, prefix, sortBy, sortDir, marker string, limit int) (api.ObjectsListResponse, error)
Object(ctx context.Context, bucketName, path string) (api.Object, error)
ObjectEntries(ctx context.Context, bucketName, path, prefix, sortBy, sortDir, marker string, offset, limit int) ([]api.ObjectMetadata, bool, error)
ObjectsBySlabKey(ctx context.Context, bucketName string, slabKey object.EncryptionKey) ([]api.ObjectMetadata, error)
ObjectsStats(ctx context.Context) (api.ObjectsStatsResponse, error)
RemoveObject(ctx context.Context, bucketName, path string) error
RemoveObjects(ctx context.Context, bucketName, prefix string) error
Expand Down Expand Up @@ -339,7 +338,6 @@ func (b *bus) Handler() http.Handler {
"POST /slabs/partial": b.slabsPartialHandlerPOST,
"POST /slabs/refreshhealth": b.slabsRefreshHealthHandlerPOST,
"GET /slab/:key": b.slabHandlerGET,
"GET /slab/:key/objects": b.slabObjectsHandlerGET,
"PUT /slab": b.slabHandlerPUT,

"GET /state": b.stateHandlerGET,
Expand Down Expand Up @@ -1405,22 +1403,6 @@ func (b *bus) sectorsHostRootHandlerDELETE(jc jape.Context) {
}
}

func (b *bus) slabObjectsHandlerGET(jc jape.Context) {
var key object.EncryptionKey
if jc.DecodeParam("key", &key) != nil {
return
}
bucket := api.DefaultBucketName
if jc.DecodeForm("bucket", &bucket) != nil {
return
}
objects, err := b.ms.ObjectsBySlabKey(jc.Request.Context(), bucket, key)
if jc.Check("failed to retrieve objects by slab", err) != nil {
return
}
jc.Encode(objects)
}

func (b *bus) slabHandlerGET(jc jape.Context) {
var key object.EncryptionKey
if jc.DecodeParam("key", &key) != nil {
Expand Down
8 changes: 0 additions & 8 deletions bus/client/objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,6 @@ func (c *Client) Object(ctx context.Context, bucket, path string, opts api.GetOb
return
}

// ObjectsBySlabKey returns all objects that reference a given slab.
func (c *Client) ObjectsBySlabKey(ctx context.Context, bucket string, key object.EncryptionKey) (objects []api.ObjectMetadata, err error) {
values := url.Values{}
values.Set("bucket", bucket)
err = c.c.WithContext(ctx).GET(fmt.Sprintf("/slab/%v/objects?"+values.Encode(), key), &objects)
return
}

// ObjectsStats returns information about the number of objects and their size.
func (c *Client) ObjectsStats() (osr api.ObjectsStatsResponse, err error) {
err = c.c.GET("/stats/objects", &osr)
Expand Down
29 changes: 16 additions & 13 deletions internal/testing/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1637,22 +1637,25 @@ func TestUploadPacking(t *testing.T) {
return nil
})

// ObjectsBySlabKey should return 2 objects for the slab of file1 since file1
// and file2 share the same slab.
res, err := b.Object(context.Background(), api.DefaultBucketName, "file1", api.GetObjectOptions{})
// fetch both objects
res1, err := b.Object(context.Background(), api.DefaultBucketName, "file1", api.GetObjectOptions{})
tt.OK(err)
objs, err := b.ObjectsBySlabKey(context.Background(), api.DefaultBucketName, res.Object.Slabs[0].Key)
res2, err := b.Object(context.Background(), api.DefaultBucketName, "file2", api.GetObjectOptions{})
tt.OK(err)
if len(objs) != 2 {
t.Fatal("expected 2 objects", len(objs))

// assert they share a slab
var found bool
if len(res1.Object.Slabs) != 1 {
t.Fatalf("expected 1 slab, got %v", len(res1.Object.Slabs))
}
sort.Slice(objs, func(i, j int) bool {
return objs[i].Name < objs[j].Name // make result deterministic
})
if objs[0].Name != "/file1" {
t.Fatal("expected file1", objs[0].Name)
} else if objs[1].Name != "/file2" {
t.Fatal("expected file2", objs[1].Name)
for _, slab := range res2.Object.Slabs {
if slab.Key.Equals(res1.Object.Slabs[0].Key) {
found = true
break
}
}
if !found {
t.Fatal("object should share a slab")
}
}

Expand Down
5 changes: 5 additions & 0 deletions object/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ func (k *EncryptionKey) UnmarshalBinary(b []byte) error {
return nil
}

// Equals implements the Equaler interface.
func (k EncryptionKey) Equals(o EncryptionKey) bool {
return bytes.Equal(k.entropy[:], o.entropy[:])
}

// String implements fmt.Stringer.
func (k EncryptionKey) String() string {
return "key:" + hex.EncodeToString(k.entropy[:])
Expand Down
28 changes: 0 additions & 28 deletions stores/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -2171,34 +2171,6 @@ func (s *SQLStore) PackedSlabsForUpload(ctx context.Context, lockingDuration tim
return s.slabBufferMgr.SlabsForUpload(ctx, lockingDuration, minShards, totalShards, contractSetID, limit)
}

func (s *SQLStore) ObjectsBySlabKey(ctx context.Context, bucket string, slabKey object.EncryptionKey) (metadata []api.ObjectMetadata, err error) {
var rows []rawObjectMetadata
key, err := slabKey.MarshalBinary()
if err != nil {
return nil, err
}

err = s.db.Raw(`
SELECT DISTINCT obj.object_id as Name, obj.size as Size, obj.mime_type as MimeType, sla.health as Health
FROM slabs sla
INNER JOIN slices sli ON sli.db_slab_id = sla.id
INNER JOIN objects obj ON sli.db_object_id = obj.id
INNER JOIN buckets b ON obj.db_bucket_id = b.id AND b.name = ?
WHERE sla.key = ?
`, bucket, key).
Scan(&rows).
Error
if err != nil {
return nil, err
}

// convert rows
for _, row := range rows {
metadata = append(metadata, row.convert())
}
return
}

// MarkPackedSlabsUploaded marks the given slabs as uploaded and deletes them
// from the buffer.
func (s *SQLStore) MarkPackedSlabsUploaded(ctx context.Context, slabs []api.UploadedPackedSlab) error {
Expand Down
79 changes: 0 additions & 79 deletions stores/metadata_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2984,69 +2984,6 @@ func (s *SQLStore) dbSlab(key []byte) (dbSlab, error) {
return slab, nil
}

func TestObjectsBySlabKey(t *testing.T) {
ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
defer ss.Close()

// create a host
hks, err := ss.addTestHosts(1)
if err != nil {
t.Fatal(err)
}
hk1 := hks[0]

// create a contract
fcids, _, err := ss.addTestContracts(hks)
if err != nil {
t.Fatal(err)
}
fcid1 := fcids[0]

// create a slab.
slab := object.Slab{
Health: 1.0,
Key: object.GenerateEncryptionKey(),
MinShards: 1,
Shards: newTestShards(hk1, fcid1, types.Hash256{1}),
}

// Add 3 objects that all reference the slab.
obj := object.Object{
Key: object.GenerateEncryptionKey(),
Slabs: []object.SlabSlice{
{
Slab: slab,
Offset: 1,
Length: 0, // incremented later
},
},
}
for _, name := range []string{"obj1", "obj2", "obj3"} {
obj.Slabs[0].Length++
err = ss.UpdateObject(context.Background(), api.DefaultBucketName, name, testContractSet, testETag, testMimeType, obj)
if err != nil {
t.Fatal(err)
}
}

// Fetch the objects by slab.
objs, err := ss.ObjectsBySlabKey(context.Background(), api.DefaultBucketName, slab.Key)
if err != nil {
t.Fatal(err)
}
for i, name := range []string{"obj1", "obj2", "obj3"} {
if objs[i].Name != name {
t.Fatal("unexpected object name", objs[i].Name, name)
}
if objs[i].Size != int64(i)+1 {
t.Fatal("unexpected object size", objs[i].Size, i+1)
}
if objs[i].Health != 1.0 {
t.Fatal("unexpected object health", objs[i].Health)
}
}
}

func TestBuckets(t *testing.T) {
ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
defer ss.Close()
Expand Down Expand Up @@ -3241,22 +3178,6 @@ func TestBucketObjects(t *testing.T) {
} else if _, err := ss.Object(context.Background(), b2, "/bar"); !errors.Is(err, api.ErrObjectNotFound) {
t.Fatal(err)
}

// See if we can fetch the object by slab.
var ec object.EncryptionKey
if obj, err := ss.object(context.Background(), ss.db, b1, "/bar"); err != nil {
t.Fatal(err)
} else if err := ec.UnmarshalBinary(obj[0].SlabKey); err != nil {
t.Fatal(err)
} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b1, ec); err != nil {
t.Fatal(err)
} else if len(objects) != 1 {
t.Fatal("expected 1 object", len(objects))
} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b2, ec); err != nil {
t.Fatal(err)
} else if len(objects) != 0 {
t.Fatal("expected 0 objects", len(objects))
}
}

func TestCopyObject(t *testing.T) {
Expand Down