Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable the log recoverer #13704

Closed
wants to merge 5 commits into from
Closed

Disable the log recoverer #13704

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d23b83c
Reduce recoverer workload
ferglor Jun 26, 2024
89346c5
Recover filters in a single thread
ferglor Jun 26, 2024
fedb8cb
Have the recoverer do nothing on startup
ferglor Jun 27, 2024
56701a6
Return 0 payloads
ferglor Jun 27, 2024
601794c
Clean up unreachable code
ferglor Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/scripts/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
github.com/pelletier/go-toml/v2 v2.1.1
github.com/prometheus/client_golang v1.17.0
github.com/shopspring/decimal v1.3.1
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468
github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
Expand Down
4 changes: 2 additions & 2 deletions core/scripts/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1210,8 +1210,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg=
github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8=
github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8 h1:iVHJSS0TLcEFnq/VLc71+16aSFnj9or0gHGs/etPAjA=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468 h1:rhsUMdSrCerrUzzsOWaHkcb+qlB7knEAXYv/P29YUn8=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468/go.mod h1:L32xvCpk84Nglit64OhySPMP1tM3TTBK7Tw0qZl7Sd4=
github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240621143432-85370a54b141 h1:TMOoYaeSDkkI3jkCH7lKHOZaLkeDuxFTNC+XblD6M0M=
Expand Down
179 changes: 52 additions & 127 deletions core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,17 @@ var (
// GCInterval is the interval at which the recovery cache is cleaned up
GCInterval = RecoveryCacheTTL - time.Second
// MaxProposals is the maximum number of proposals that can be returned by GetRecoveryProposals
MaxProposals = 20
MaxProposals = 10
// recoveryBatchSize is the number of filters to recover in a single batch
recoveryBatchSize = 10
recoveryBatchSize = 5
// recoveryLogsBuffer is the number of blocks to be used as a safety buffer when reading logs
recoveryLogsBuffer = int64(200)
recoveryLogsBurst = int64(500)
// blockTimeUpdateCadence is the cadence at which the chain's blocktime is re-calculated
blockTimeUpdateCadence = 10 * time.Minute
// maxPendingPayloadsPerUpkeep is the number of logs we can have pending for a single upkeep
// at any given time
maxPendingPayloadsPerUpkeep = 500
maxPendingPayloadsPerUpkeep = 250
)

type LogRecoverer interface {
Expand Down Expand Up @@ -130,53 +130,53 @@ func (r *logRecoverer) Start(ctx context.Context) error {
return r.StartOnce(LogRecovererServiceName, func() error {
r.updateBlockTime(ctx)

r.lggr.Infow("starting log recoverer", "blockTime", r.blockTime.Load(), "lookbackBlocks", r.lookbackBlocks.Load(), "interval", r.interval)

r.threadCtrl.Go(func(ctx context.Context) {
recoveryTicker := time.NewTicker(r.interval)
defer recoveryTicker.Stop()

for {
select {
case <-recoveryTicker.C:
if err := r.recover(ctx); err != nil {
r.lggr.Warnw("failed to recover logs", "err", err)
}
case <-ctx.Done():
return
}
}
})

r.threadCtrl.Go(func(ctx context.Context) {
cleanupTicker := time.NewTicker(utils.WithJitter(GCInterval))
defer cleanupTicker.Stop()

for {
select {
case <-cleanupTicker.C:
r.clean(ctx)
cleanupTicker.Reset(utils.WithJitter(GCInterval))
case <-ctx.Done():
return
}
}
})

r.threadCtrl.Go(func(ctx context.Context) {
blockTimeTicker := time.NewTicker(blockTimeUpdateCadence)
defer blockTimeTicker.Stop()

for {
select {
case <-blockTimeTicker.C:
r.updateBlockTime(ctx)
blockTimeTicker.Reset(utils.WithJitter(blockTimeUpdateCadence))
case <-ctx.Done():
return
}
}
})
r.lggr.Infow("starting no op log recoverer", "blockTime", r.blockTime.Load(), "lookbackBlocks", r.lookbackBlocks.Load(), "interval", r.interval)

//r.threadCtrl.Go(func(ctx context.Context) {
// recoveryTicker := time.NewTicker(r.interval)
// defer recoveryTicker.Stop()
//
// for {
// select {
// case <-recoveryTicker.C:
// if err := r.recover(ctx); err != nil {
// r.lggr.Warnw("failed to recover logs", "err", err)
// }
// case <-ctx.Done():
// return
// }
// }
//})
//
//r.threadCtrl.Go(func(ctx context.Context) {
// cleanupTicker := time.NewTicker(utils.WithJitter(GCInterval))
// defer cleanupTicker.Stop()
//
// for {
// select {
// case <-cleanupTicker.C:
// r.clean(ctx)
// cleanupTicker.Reset(utils.WithJitter(GCInterval))
// case <-ctx.Done():
// return
// }
// }
//})
//
//r.threadCtrl.Go(func(ctx context.Context) {
// blockTimeTicker := time.NewTicker(blockTimeUpdateCadence)
// defer blockTimeTicker.Stop()
//
// for {
// select {
// case <-blockTimeTicker.C:
// r.updateBlockTime(ctx)
// blockTimeTicker.Reset(utils.WithJitter(blockTimeUpdateCadence))
// case <-ctx.Done():
// return
// }
// }
//})

return nil
})
Expand Down Expand Up @@ -285,84 +285,9 @@ func (r *logRecoverer) getLogTriggerCheckData(ctx context.Context, proposal ocr2
}

func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers.UpkeepPayload, error) {
latestBlock, err := r.poller.LatestBlock(ctx)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrHeadNotAvailable, err)
}

r.lock.Lock()
defer r.lock.Unlock()

if len(r.pending) == 0 {
return nil, nil
}

allLogsCounter := 0
logsCount := map[string]int{}

r.sortPending(uint64(latestBlock.BlockNumber))

var results, pending []ocr2keepers.UpkeepPayload
for _, payload := range r.pending {
if allLogsCounter >= MaxProposals {
// we have enough proposals, the rest are pushed back to pending
pending = append(pending, payload)
continue
}
uid := payload.UpkeepID.String()
if logsCount[uid] >= AllowedLogsPerUpkeep {
// we have enough proposals for this upkeep, the rest are pushed back to pending
pending = append(pending, payload)
continue
}
results = append(results, payload)
logsCount[uid]++
allLogsCounter++
}

r.pending = pending
prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending)))
r.lggr.Debugf("returning 0 recoverable payloads in no op recoverer")

r.lggr.Debugf("found %d recoverable payloads", len(results))

return results, nil
}

func (r *logRecoverer) recover(ctx context.Context) error {
latest, err := r.poller.LatestBlock(ctx)
if err != nil {
return fmt.Errorf("%w: %s", ErrHeadNotAvailable, err)
}

start, offsetBlock := r.getRecoveryWindow(latest.BlockNumber)
if offsetBlock < 0 {
// too soon to recover, we don't have enough blocks
return nil
}
if start < 0 {
start = 0
}

filters := r.getFilterBatch(offsetBlock)
if len(filters) == 0 {
return nil
}

r.lggr.Debugw("recovering logs", "filters", filters, "startBlock", start, "offsetBlock", offsetBlock, "latestBlock", latest)

var wg sync.WaitGroup
for _, f := range filters {
wg.Add(1)
go func(f upkeepFilter) {
defer wg.Done()
if err := r.recoverFilter(ctx, f, start, offsetBlock); err != nil {
r.lggr.Debugw("error recovering filter", "err", err.Error())
}
}(f)
}
wg.Wait()

return nil
return nil, nil
}

// recoverFilter recovers logs for a single upkeep filter.
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ require (
github.com/shirou/gopsutil/v3 v3.24.3
github.com/shopspring/decimal v1.3.1
github.com/smartcontractkit/chain-selectors v1.0.10
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468
github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240621143432-85370a54b141
github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1169,8 +1169,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg=
github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8=
github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8 h1:iVHJSS0TLcEFnq/VLc71+16aSFnj9or0gHGs/etPAjA=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468 h1:rhsUMdSrCerrUzzsOWaHkcb+qlB7knEAXYv/P29YUn8=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468/go.mod h1:L32xvCpk84Nglit64OhySPMP1tM3TTBK7Tw0qZl7Sd4=
github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240621143432-85370a54b141 h1:TMOoYaeSDkkI3jkCH7lKHOZaLkeDuxFTNC+XblD6M0M=
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ require (
github.com/segmentio/ksuid v1.0.4
github.com/shopspring/decimal v1.3.1
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468
github.com/smartcontractkit/chainlink-testing-framework v1.31.5
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1511,8 +1511,8 @@ github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ
github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg=
github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8=
github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8 h1:iVHJSS0TLcEFnq/VLc71+16aSFnj9or0gHGs/etPAjA=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468 h1:rhsUMdSrCerrUzzsOWaHkcb+qlB7knEAXYv/P29YUn8=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468/go.mod h1:L32xvCpk84Nglit64OhySPMP1tM3TTBK7Tw0qZl7Sd4=
github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240621143432-85370a54b141 h1:TMOoYaeSDkkI3jkCH7lKHOZaLkeDuxFTNC+XblD6M0M=
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/load/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.31.0
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468
github.com/smartcontractkit/chainlink-testing-framework v1.31.5
github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/load/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1501,8 +1501,8 @@ github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ
github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCqR1LNS7aI3jT0V+xGrg=
github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE=
github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8=
github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8 h1:iVHJSS0TLcEFnq/VLc71+16aSFnj9or0gHGs/etPAjA=
github.com/smartcontractkit/chainlink-automation v1.0.5-0.20240626230921-6ba72816a5b8/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468 h1:rhsUMdSrCerrUzzsOWaHkcb+qlB7knEAXYv/P29YUn8=
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240625145034-72dab520f468/go.mod h1:L32xvCpk84Nglit64OhySPMP1tM3TTBK7Tw0qZl7Sd4=
github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240621143432-85370a54b141 h1:TMOoYaeSDkkI3jkCH7lKHOZaLkeDuxFTNC+XblD6M0M=
Expand Down