forked from cortexproject/cortex
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
update thanos and add block_ids_fetcher to bucketindex (cortexproject…
…#5681) * update thanos and add bucket-index-ids-fetcher to compactor Signed-off-by: Wen Xu <[email protected]> * make mod-check Signed-off-by: Wen Xu <[email protected]> * udpate docs Signed-off-by: Wen Xu <[email protected]> * add unit test for bucketindex block ids fetcher Signed-off-by: Wen Xu <[email protected]> * group imports Signed-off-by: Wen Xu <[email protected]> * initialize the baseBlockIDsFetcher in the constructor Signed-off-by: Wen Xu <[email protected]> * set the bucketindex block ids fetcher as default if bucketindex is enabled Signed-off-by: Wen Xu <[email protected]> * remove TODO Signed-off-by: Wen Xu <[email protected]> --------- Signed-off-by: Wen Xu <[email protected]>
- Loading branch information
Showing
600 changed files
with
11,978 additions
and
6,626 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package bucketindex | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/go-kit/log" | ||
| "github.com/go-kit/log/level" | ||
| "github.com/oklog/ulid" | ||
| "github.com/pkg/errors" | ||
| "github.com/thanos-io/objstore" | ||
| "github.com/thanos-io/thanos/pkg/block" | ||
|
|
||
| "github.com/cortexproject/cortex/pkg/storage/bucket" | ||
| ) | ||
|
|
||
| type BlockIDsFetcher struct { | ||
| logger log.Logger | ||
| bkt objstore.Bucket | ||
| userID string | ||
| cfgProvider bucket.TenantConfigProvider | ||
| baseBlockIDsFetcher block.BlockIDsFetcher | ||
| } | ||
|
|
||
| func NewBlockIDsFetcher(logger log.Logger, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider) *BlockIDsFetcher { | ||
| userBkt := bucket.NewUserBucketClient(userID, bkt, cfgProvider) | ||
| baseBlockIDsFetcher := block.NewBaseBlockIDsFetcher(logger, userBkt) | ||
| return &BlockIDsFetcher{ | ||
| logger: logger, | ||
| bkt: bkt, | ||
| userID: userID, | ||
| cfgProvider: cfgProvider, | ||
| baseBlockIDsFetcher: baseBlockIDsFetcher, | ||
| } | ||
| } | ||
|
|
||
| func (f *BlockIDsFetcher) GetActiveAndPartialBlockIDs(ctx context.Context, ch chan<- ulid.ULID) (partialBlocks map[ulid.ULID]bool, err error) { | ||
| // Fetch the bucket index. | ||
| idx, err := ReadIndex(ctx, f.bkt, f.userID, f.cfgProvider, f.logger) | ||
| if errors.Is(err, ErrIndexNotFound) { | ||
| // This is a legit case happening when the first blocks of a tenant have recently been uploaded by ingesters | ||
| // and their bucket index has not been created yet. | ||
| // Fallback to BaseBlockIDsFetcher. | ||
| return f.baseBlockIDsFetcher.GetActiveAndPartialBlockIDs(ctx, ch) | ||
| } | ||
| if errors.Is(err, ErrIndexCorrupted) { | ||
| // In case a single tenant bucket index is corrupted, we want to return empty active blocks and parital blocks, so skipping this compaction cycle | ||
| level.Error(f.logger).Log("msg", "corrupted bucket index found", "user", f.userID, "err", err) | ||
| // Fallback to BaseBlockIDsFetcher. | ||
| return f.baseBlockIDsFetcher.GetActiveAndPartialBlockIDs(ctx, ch) | ||
| } | ||
|
|
||
| if errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) { | ||
| // stop the job and return the error | ||
| // this error should be used to return Access Denied to the caller | ||
| level.Error(f.logger).Log("msg", "bucket index key permission revoked", "user", f.userID, "err", err) | ||
| return nil, err | ||
| } | ||
|
|
||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Sent the active block ids | ||
| for _, b := range idx.Blocks { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return nil, ctx.Err() | ||
| case ch <- b.ID: | ||
| } | ||
| } | ||
| return nil, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| package bucketindex | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "encoding/json" | ||
| "path" | ||
| "sync" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/go-kit/log" | ||
| "github.com/oklog/ulid" | ||
| "github.com/stretchr/testify/require" | ||
| "github.com/thanos-io/thanos/pkg/block/metadata" | ||
|
|
||
| cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil" | ||
| "github.com/cortexproject/cortex/pkg/util/concurrency" | ||
| ) | ||
|
|
||
| func TestBlockIDsFetcher_Fetch(t *testing.T) { | ||
| t.Parallel() | ||
| const userID = "user-1" | ||
|
|
||
| bkt, _ := cortex_testutil.PrepareFilesystemBucket(t) | ||
| ctx := context.Background() | ||
| now := time.Now() | ||
| logs := &concurrency.SyncBuffer{} | ||
| logger := log.NewLogfmtLogger(logs) | ||
|
|
||
| // Create a bucket index. | ||
| block1 := &Block{ID: ulid.MustNew(1, nil)} | ||
| block2 := &Block{ID: ulid.MustNew(2, nil)} | ||
| block3 := &Block{ID: ulid.MustNew(3, nil)} | ||
| mark1 := &BlockDeletionMark{ID: block1.ID, DeletionTime: now.Add(-time.Hour).Unix()} // Below the ignore delay threshold. | ||
| mark2 := &BlockDeletionMark{ID: block2.ID, DeletionTime: now.Add(-3 * time.Hour).Unix()} // Above the ignore delay threshold. | ||
|
|
||
| require.NoError(t, WriteIndex(ctx, bkt, userID, nil, &Index{ | ||
| Version: IndexVersion1, | ||
| Blocks: Blocks{block1, block2, block3}, | ||
| BlockDeletionMarks: BlockDeletionMarks{mark1, mark2}, | ||
| UpdatedAt: now.Unix(), | ||
| })) | ||
|
|
||
| blockIdsFetcher := NewBlockIDsFetcher(logger, bkt, userID, nil) | ||
| ch := make(chan ulid.ULID) | ||
| var wg sync.WaitGroup | ||
| var blockIds []ulid.ULID | ||
| wg.Add(1) | ||
| go func() { | ||
| defer wg.Done() | ||
| for id := range ch { | ||
| blockIds = append(blockIds, id) | ||
| } | ||
| }() | ||
| blockIdsFetcher.GetActiveAndPartialBlockIDs(ctx, ch) | ||
| close(ch) | ||
| wg.Wait() | ||
| require.Equal(t, []ulid.ULID{block1.ID, block2.ID, block3.ID}, blockIds) | ||
| } | ||
|
|
||
| func TestBlockIDsFetcherFetcher_Fetch_NoBucketIndex(t *testing.T) { | ||
| t.Parallel() | ||
| const userID = "user-1" | ||
|
|
||
| bkt, _ := cortex_testutil.PrepareFilesystemBucket(t) | ||
| ctx := context.Background() | ||
| now := time.Now() | ||
| logs := &concurrency.SyncBuffer{} | ||
| logger := log.NewLogfmtLogger(logs) | ||
|
|
||
| //prepare tenant bucket | ||
| var meta1, meta2, meta3 metadata.Meta | ||
| block1 := &Block{ID: ulid.MustNew(1, nil)} | ||
| meta1.Version = 1 | ||
| meta1.ULID = block1.ID | ||
| block2 := &Block{ID: ulid.MustNew(2, nil)} | ||
| meta2.Version = 1 | ||
| meta2.ULID = block2.ID | ||
| block3 := &Block{ID: ulid.MustNew(3, nil)} | ||
| meta3.Version = 1 | ||
| meta3.ULID = block3.ID | ||
| metas := []metadata.Meta{meta1, meta2, meta3} | ||
| mark1 := &BlockDeletionMark{ID: block1.ID, DeletionTime: now.Add(-time.Hour).Unix()} // Below the ignore delay threshold. | ||
| mark2 := &BlockDeletionMark{ID: block2.ID, DeletionTime: now.Add(-3 * time.Hour).Unix()} // Above the ignore delay threshold. | ||
| marks := []*BlockDeletionMark{mark1, mark2} | ||
| var buf bytes.Buffer | ||
| for _, meta := range metas { | ||
| require.NoError(t, json.NewEncoder(&buf).Encode(&meta)) | ||
| require.NoError(t, bkt.Upload(ctx, path.Join(userID, meta.ULID.String(), metadata.MetaFilename), &buf)) | ||
| } | ||
| for _, mark := range marks { | ||
| require.NoError(t, json.NewEncoder(&buf).Encode(mark)) | ||
| require.NoError(t, bkt.Upload(ctx, path.Join(userID, mark.ID.String(), metadata.DeletionMarkFilename), &buf)) | ||
| } | ||
| blockIdsFetcher := NewBlockIDsFetcher(logger, bkt, userID, nil) | ||
| ch := make(chan ulid.ULID) | ||
| var wg sync.WaitGroup | ||
| var blockIds []ulid.ULID | ||
| wg.Add(1) | ||
| go func() { | ||
| defer wg.Done() | ||
| for id := range ch { | ||
| blockIds = append(blockIds, id) | ||
| } | ||
| }() | ||
| blockIdsFetcher.GetActiveAndPartialBlockIDs(ctx, ch) | ||
| close(ch) | ||
| wg.Wait() | ||
| require.Equal(t, []ulid.ULID{block1.ID, block2.ID, block3.ID}, blockIds) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.