diff --git a/.changeset/late-windows-clean.md b/.changeset/late-windows-clean.md new file mode 100644 index 00000000000..261747efa6c --- /dev/null +++ b/.changeset/late-windows-clean.md @@ -0,0 +1,11 @@ +--- +"chainlink": minor +--- + +#internal Updated the TXM confirmation logic to use the mined transaction count to identify re-org'd or confirmed transactions. + +- Confirmer uses the mined transaction count to determine if transactions have been re-org'd or confirmed. +- Confirmer no longer sets transaction states to `confirmed_missing_receipt`. This state is maintained in queries for backwards compatibility. +- Finalizer now responsible for fetching and storing receipts for confirmed transactions. +- Finalizer now responsible for resuming pending task runs. +- Finalizer now responsible for marking old transactions without receipts broadcasted before the finalized head as fatal. diff --git a/.changeset/sixty-queens-wait.md b/.changeset/sixty-queens-wait.md new file mode 100644 index 00000000000..cd9fc9ea65c --- /dev/null +++ b/.changeset/sixty-queens-wait.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#updated chain config: allow chain id and account address to be manually provided when no selections are available diff --git a/.github/actions/goreleaser-build-sign-publish/action.yml b/.github/actions/goreleaser-build-sign-publish/action.yml index c57bff91488..fa72216d70d 100644 --- a/.github/actions/goreleaser-build-sign-publish/action.yml +++ b/.github/actions/goreleaser-build-sign-publish/action.yml @@ -26,19 +26,32 @@ inputs: description: "The goreleaser configuration yaml" default: ".goreleaser.yaml" required: false + # other inputs + enable-debug: + description: | + Enable debug information for the run (true/false). This includes + buildkit debug information, and goreleaser debug, etc. + required: false + default: "${{ runner.debug == '1' }}" + runs: using: composite steps: - # We need QEMU to test the cross architecture builds after they're built. name: Set up QEMU uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + - name: Setup docker buildx uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.0 + with: + buildkitd-flags: ${{ inputs.enable-debug == 'true' && '--debug' || '' }} + - name: Set up Go uses: ./.github/actions/setup-go with: go-version-file: 'go.mod' only-modules: 'true' + - name: Setup goreleaser uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 with: @@ -65,6 +78,7 @@ runs: IMAGE_TAG: ${{ inputs.docker-image-tag }} GORELEASER_KEY: ${{ inputs.goreleaser-key }} GITHUB_TOKEN: ${{ github.token }} + DEBUG: ${{ inputs.enable-debug }} run: | # https://github.com/orgs/community/discussions/24950 ${GITHUB_ACTION_PATH}/release.js diff --git a/.github/actions/goreleaser-build-sign-publish/release.js b/.github/actions/goreleaser-build-sign-publish/release.js index 0dbd58ca6cf..cd0521c991e 100755 --- a/.github/actions/goreleaser-build-sign-publish/release.js +++ b/.github/actions/goreleaser-build-sign-publish/release.js @@ -168,6 +168,7 @@ function extractDockerImages(artifacts) { function constructGoreleaserCommand(releaseType, version, goreleaserConfig) { const flags = []; + const debugFlag = (process.env.DEBUG == 'true') ? '--verbose' : ''; checkReleaseType(releaseType); @@ -192,9 +193,9 @@ function constructGoreleaserCommand(releaseType, version, goreleaserConfig) { const flagsStr = flags.join(" "); if (releaseType === "merge") { - return `CHAINLINK_VERSION=${version} goreleaser ${subCmd} ${flagsStr}`; + return `CHAINLINK_VERSION=${version} goreleaser ${debugFlag} ${subCmd} ${flagsStr}`; } else { - return `CHAINLINK_VERSION=${version} goreleaser ${subCmd} --config ${goreleaserConfig} ${flagsStr}`; + return `CHAINLINK_VERSION=${version} goreleaser ${debugFlag} ${subCmd} --config ${goreleaserConfig} ${flagsStr}`; } } diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 1bbae3769a4..ec24c4c9706 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -947,7 +947,7 @@ runner-test-matrix: pyroscope_env: ci-smoke-ccipv1_6-evm-simulated test_env_vars: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 - E2E_JD_VERSION: 0.4.0 + E2E_JD_VERSION: 0.6.0 - id: smoke/ccip_messaging_test.go:* path: integration-tests/smoke/ccip_messaging_test.go diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 48977cee35e..5c931ed9870 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -526,10 +526,14 @@ jobs: make rm-mocked make generate - name: Ensure clean after generate - run: git diff --stat --exit-code + run: | + git add --all + git diff --stat --cached --exit-code - run: make gomodtidy - name: Ensure clean after tidy - run: git diff --minimal --exit-code + run: | + git add --all + git diff --minimal --cached --exit-code run-frequency: name: Scheduled Run Frequency diff --git a/.github/workflows/solidity.yml b/.github/workflows/solidity.yml index fb826b0f185..c76fbe6b671 100644 --- a/.github/workflows/solidity.yml +++ b/.github/workflows/solidity.yml @@ -97,7 +97,9 @@ jobs: run: ./tools/ci/check_solc_hashes - name: Check if Go solidity wrappers are updated if: ${{ needs.changes.outputs.changes == 'true' }} - run: git diff --minimal --color --exit-code | diff-so-fancy + run: | + git add --all + git diff --minimal --color --cached --exit-code | diff-so-fancy # The if statements for steps after checkout repo is a workaround for # passing required check for PRs that don't have filtered changes. diff --git a/common/txmgr/broadcaster.go b/common/txmgr/broadcaster.go index 2a234ab3340..14e959c39ae 100644 --- a/common/txmgr/broadcaster.go +++ b/common/txmgr/broadcaster.go @@ -769,7 +769,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) save } } } - return eb.txStore.UpdateTxFatalError(ctx, etx) + return eb.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, etx) } func observeTimeUntilBroadcast[CHAIN_ID types.ID](chainID CHAIN_ID, createdAt, broadcastAt time.Time) { diff --git a/common/txmgr/confirmer.go b/common/txmgr/confirmer.go index e6fd5b61f68..7c5ba798cf2 100644 --- a/common/txmgr/confirmer.go +++ b/common/txmgr/confirmer.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "sort" - "strconv" "sync" "time" @@ -34,14 +33,6 @@ const ( // processHeadTimeout represents a sanity limit on how long ProcessHead // should take to complete processHeadTimeout = 10 * time.Minute - - // logAfterNConsecutiveBlocksChainTooShort logs a warning if we go at least - // this many consecutive blocks with a re-org protection chain that is too - // short - // - // we don't log every time because on startup it can be lower, only if it - // persists does it indicate a serious problem - logAfterNConsecutiveBlocksChainTooShort = 10 ) var ( @@ -58,22 +49,6 @@ var ( Name: "tx_manager_num_confirmed_transactions", Help: "Total number of confirmed transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", }, []string{"chainID"}) - promNumSuccessfulTxs = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_successful_transactions", - Help: "Total number of successful transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", - }, []string{"chainID"}) - promRevertedTxCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_num_tx_reverted", - Help: "Number of times a transaction reverted on-chain. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", - }, []string{"chainID"}) - promFwdTxCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tx_manager_fwd_tx_count", - Help: "The number of forwarded transaction attempts labeled by status", - }, []string{"chainID", "successful"}) - promTxAttemptCount = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "tx_manager_tx_attempt_count", - Help: "The number of transaction attempts that are currently being processed by the transaction manager", - }, []string{"chainID"}) promTimeUntilTxConfirmed = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "tx_manager_time_until_tx_confirmed", Help: "The amount of time elapsed from a transaction being broadcast to being included in a block.", @@ -103,15 +78,11 @@ var ( }, []string{"chainID"}) ) -type confirmerHeadTracker[HEAD types.Head[BLOCK_HASH], BLOCK_HASH types.Hashable] interface { - LatestAndFinalizedBlock(ctx context.Context) (latest, finalized HEAD, err error) -} - // Confirmer is a broad service which performs four different tasks in sequence on every new longest chain // Step 1: Mark that all currently pending transaction attempts were broadcast before this block -// Step 2: Check pending transactions for receipts -// Step 3: See if any transactions have exceeded the gas bumping block threshold and, if so, bump them -// Step 4: Check confirmed transactions to make sure they are still in the longest chain (reorg protection) +// Step 2: Check pending transactions for confirmation and confirmed transactions for re-org +// Step 3: Check if any pending transaction is stuck in the mempool. If so, mark for purge. +// Step 4: See if any transactions have exceeded the gas bumping block threshold and, if so, bump them type Confirmer[ CHAIN_ID types.ID, HEAD types.Head[BLOCK_HASH], @@ -129,7 +100,6 @@ type Confirmer[ txmgrtypes.TxAttemptBuilder[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] resumeCallback ResumeCallback - chainConfig txmgrtypes.ConfirmerChainConfig feeConfig txmgrtypes.ConfirmerFeeConfig txConfig txmgrtypes.ConfirmerTransactionsConfig dbConfig txmgrtypes.ConfirmerDatabaseConfig @@ -138,15 +108,12 @@ type Confirmer[ ks txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] enabledAddresses []ADDR - mb *mailbox.Mailbox[HEAD] - stopCh services.StopChan - wg sync.WaitGroup - initSync sync.Mutex - isStarted bool - nConsecutiveBlocksChainTooShort int - isReceiptNil func(R) bool - - headTracker confirmerHeadTracker[HEAD, BLOCK_HASH] + mb *mailbox.Mailbox[HEAD] + stopCh services.StopChan + wg sync.WaitGroup + initSync sync.Mutex + isStarted bool + isReceiptNil func(R) bool } func NewConfirmer[ @@ -161,7 +128,6 @@ func NewConfirmer[ ]( txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], client txmgrtypes.TxmClient[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], - chainConfig txmgrtypes.ConfirmerChainConfig, feeConfig txmgrtypes.ConfirmerFeeConfig, txConfig txmgrtypes.ConfirmerTransactionsConfig, dbConfig txmgrtypes.ConfirmerDatabaseConfig, @@ -170,7 +136,6 @@ func NewConfirmer[ lggr logger.Logger, isReceiptNil func(R) bool, stuckTxDetector txmgrtypes.StuckTxDetector[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - headTracker confirmerHeadTracker[HEAD, BLOCK_HASH], ) *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { lggr = logger.Named(lggr, "Confirmer") return &Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ @@ -179,7 +144,6 @@ func NewConfirmer[ client: client, TxAttemptBuilder: txAttemptBuilder, resumeCallback: nil, - chainConfig: chainConfig, feeConfig: feeConfig, txConfig: txConfig, dbConfig: dbConfig, @@ -188,7 +152,6 @@ func NewConfirmer[ mb: mailbox.NewSingle[HEAD](), isReceiptNil: isReceiptNil, stuckTxDetector: stuckTxDetector, - headTracker: headTracker, } } @@ -294,178 +257,146 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pro // NOTE: This SHOULD NOT be run concurrently or it could behave badly func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) processHead(ctx context.Context, head types.Head[BLOCK_HASH]) error { - mark := time.Now() - ec.lggr.Debugw("processHead start", "headNum", head.BlockNumber(), "id", "confirmer") + mark := time.Now() if err := ec.txStore.SetBroadcastBeforeBlockNum(ctx, head.BlockNumber(), ec.chainID); err != nil { - return fmt.Errorf("SetBroadcastBeforeBlockNum failed: %w", err) - } - if err := ec.CheckConfirmedMissingReceipt(ctx); err != nil { - return fmt.Errorf("CheckConfirmedMissingReceipt failed: %w", err) - } - - _, latestFinalizedHead, err := ec.headTracker.LatestAndFinalizedBlock(ctx) - if err != nil { - return fmt.Errorf("failed to retrieve latest finalized head: %w", err) - } - - if !latestFinalizedHead.IsValid() { - return fmt.Errorf("latest finalized head is not valid") - } - - if latestFinalizedHead.BlockNumber() > head.BlockNumber() { - ec.lggr.Debugw("processHead received old block", "latestFinalizedHead", latestFinalizedHead.BlockNumber(), "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") + return err } + ec.lggr.Debugw("Finished SetBroadcastBeforeBlockNum", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - if err := ec.CheckForReceipts(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()); err != nil { - return fmt.Errorf("CheckForReceipts failed: %w", err) + mark = time.Now() + if err := ec.CheckForConfirmation(ctx, head); err != nil { + return err } + ec.lggr.Debugw("Finished CheckForConfirmation", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - ec.lggr.Debugw("Finished CheckForReceipts", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") mark = time.Now() - if err := ec.ProcessStuckTransactions(ctx, head.BlockNumber()); err != nil { - return fmt.Errorf("ProcessStuckTransactions failed: %w", err) + return err } - ec.lggr.Debugw("Finished ProcessStuckTransactions", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - mark = time.Now() + mark = time.Now() if err := ec.RebroadcastWhereNecessary(ctx, head.BlockNumber()); err != nil { - return fmt.Errorf("RebroadcastWhereNecessary failed: %w", err) + return err } - ec.lggr.Debugw("Finished RebroadcastWhereNecessary", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - mark = time.Now() - - if err := ec.EnsureConfirmedTransactionsInLongestChain(ctx, head); err != nil { - return fmt.Errorf("EnsureConfirmedTransactionsInLongestChain failed: %w", err) - } - - ec.lggr.Debugw("Finished EnsureConfirmedTransactionsInLongestChain", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - - if ec.resumeCallback != nil { - mark = time.Now() - if err := ec.ResumePendingTaskRuns(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()); err != nil { - return fmt.Errorf("ResumePendingTaskRuns failed: %w", err) - } - - ec.lggr.Debugw("Finished ResumePendingTaskRuns", "headNum", head.BlockNumber(), "time", time.Since(mark), "id", "confirmer") - } - ec.lggr.Debugw("processHead finish", "headNum", head.BlockNumber(), "id", "confirmer") return nil } -// CheckConfirmedMissingReceipt will attempt to re-send any transaction in the -// state of "confirmed_missing_receipt". If we get back any type of senderror -// other than "sequence too low" it means that this transaction isn't actually -// confirmed and needs to be put back into "unconfirmed" state, so that it can enter -// the gas bumping cycle. This is necessary in rare cases (e.g. Polygon) where -// network conditions are extremely hostile. -// -// For example, assume the following scenario: -// -// 0. We are connected to multiple primary nodes via load balancer -// 1. We send a transaction, it is confirmed and, we get a receipt -// 2. A new head comes in from RPC node 1 indicating that this transaction was re-org'd, so we put it back into unconfirmed state -// 3. We re-send that transaction to a RPC node 2 **which hasn't caught up to this re-org yet** -// 4. RPC node 2 still has an old view of the chain, so it returns us "sequence too low" indicating "no problem this transaction is already mined" -// 5. Now the transaction is marked "confirmed_missing_receipt" but the latest chain does not actually include it -// 6. Now we are reliant on the Resender to propagate it, and this transaction will not be gas bumped, so in the event of gas spikes it could languish or even be evicted from the mempool and hold up the queue -// 7. Even if/when RPC node 2 catches up, the transaction is still stuck in state "confirmed_missing_receipt" -// -// This scenario might sound unlikely but has been observed to happen multiple times in the wild on Polygon. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckConfirmedMissingReceipt(ctx context.Context) (err error) { - attempts, err := ec.txStore.FindTxAttemptsConfirmedMissingReceipt(ctx, ec.chainID) - if err != nil { - return err - } - if len(attempts) == 0 { - return nil - } - ec.lggr.Infow(fmt.Sprintf("Found %d transactions confirmed_missing_receipt. The RPC node did not give us a receipt for these transactions even though it should have been mined. This could be due to using the wallet with an external account, or if the primary node is not synced or not propagating transactions properly", len(attempts)), "attempts", attempts) - txCodes, txErrs, broadcastTime, txIDs, err := ec.client.BatchSendTransactions(ctx, attempts, int(ec.chainConfig.RPCDefaultBatchSize()), ec.lggr) - // update broadcast times before checking additional errors - if len(txIDs) > 0 { - if updateErr := ec.txStore.UpdateBroadcastAts(ctx, broadcastTime, txIDs); updateErr != nil { - err = fmt.Errorf("%w: failed to update broadcast time: %w", err, updateErr) +// CheckForConfirmation fetches the mined transaction count for each enabled address and marks transactions with a lower sequence as confirmed and ones with equal or higher sequence as unconfirmed +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckForConfirmation(ctx context.Context, head types.Head[BLOCK_HASH]) error { + var errorList []error + for _, fromAddress := range ec.enabledAddresses { + minedTxCount, err := ec.client.SequenceAt(ctx, fromAddress, nil) + if err != nil { + errorList = append(errorList, fmt.Errorf("unable to fetch mined transaction count for address %s: %w", fromAddress.String(), err)) + continue } - } - if err != nil { - ec.lggr.Debugw("Batch sending transactions failed", "err", err) - } - var txIDsToUnconfirm []int64 - for idx, txErr := range txErrs { - // Add to Unconfirm array, all tx where error wasn't TransactionAlreadyKnown. - if txErr != nil { - if txCodes[idx] == client.TransactionAlreadyKnown { - continue - } + reorgTxs, includedTxs, err := ec.txStore.FindReorgOrIncludedTxs(ctx, fromAddress, minedTxCount, ec.chainID) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to find re-org'd or included transactions based on the mined transaction count %d: %w", minedTxCount.Int64(), err)) + continue + } + // If re-org'd transactions are identified, process them and mark them for rebroadcast + err = ec.ProcessReorgTxs(ctx, reorgTxs, head) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to process re-org'd transactions: %w", err)) + continue + } + // If unconfirmed transactions are identified as included, process them and mark them as confirmed or terminally stuck (if purge attempt exists) + err = ec.ProcessIncludedTxs(ctx, includedTxs, head) + if err != nil { + errorList = append(errorList, fmt.Errorf("failed to process confirmed transactions: %w", err)) + continue } - - txIDsToUnconfirm = append(txIDsToUnconfirm, attempts[idx].TxID) } - err = ec.txStore.UpdateTxsUnconfirmed(ctx, txIDsToUnconfirm) - - if err != nil { - return err + if len(errorList) > 0 { + return errors.Join(errorList...) } - return + return nil } -// CheckForReceipts finds attempts that are still pending and checks to see if a receipt is present for the given block number. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckForReceipts(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64) error { - attempts, err := ec.txStore.FindTxAttemptsRequiringReceiptFetch(ctx, ec.chainID) - if err != nil { - return fmt.Errorf("FindTxAttemptsRequiringReceiptFetch failed: %w", err) - } - if len(attempts) == 0 { +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessReorgTxs(ctx context.Context, reorgTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head types.Head[BLOCK_HASH]) error { + if len(reorgTxs) == 0 { return nil } + etxIDs := make([]int64, 0, len(reorgTxs)) + attemptIDs := make([]int64, 0, len(reorgTxs)) + for _, etx := range reorgTxs { + if len(etx.TxAttempts) == 0 { + return fmt.Errorf("invariant violation: expected tx %v to have at least one attempt", etx.ID) + } - ec.lggr.Debugw(fmt.Sprintf("Fetching receipts for %v transaction attempts", len(attempts)), "blockNum", blockNum) + // Rebroadcast the one with the highest gas price + attempt := etx.TxAttempts[0] - attemptsByAddress := make(map[ADDR][]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) - for _, att := range attempts { - attemptsByAddress[att.Tx.FromAddress] = append(attemptsByAddress[att.Tx.FromAddress], att) - } + logValues := []interface{}{ + "txhash", attempt.Hash.String(), + "currentBlockNum", head.BlockNumber(), + "currentBlockHash", head.BlockHash().String(), + "txID", etx.ID, + "attemptID", attempt.ID, + "nReceipts", len(attempt.Receipts), + "attemptState", attempt.State, + "id", "confirmer", + } - for from, attempts := range attemptsByAddress { - minedSequence, err := ec.getMinedSequenceForAddress(ctx, from) - if err != nil { - return fmt.Errorf("unable to fetch pending sequence for address: %v: %w", from, err) + if len(attempt.Receipts) > 0 && attempt.Receipts[0] != nil { + receipt := attempt.Receipts[0] + logValues = append(logValues, + "replacementBlockHashAtConfirmedHeight", head.HashAtHeight(receipt.GetBlockNumber().Int64()), + "confirmedInBlockNum", receipt.GetBlockNumber(), + "confirmedInBlockHash", receipt.GetBlockHash(), + "confirmedInTxIndex", receipt.GetTransactionIndex(), + ) + } + + if etx.State == TxFinalized { + ec.lggr.AssumptionViolationw(fmt.Sprintf("Re-org detected for finalized transaction. This should never happen. Rebroadcasting transaction %s which may have been re-org'd out of the main chain", attempt.Hash.String()), logValues...) + } else { + ec.lggr.Infow(fmt.Sprintf("Re-org detected. Rebroadcasting transaction %s which may have been re-org'd out of the main chain", attempt.Hash.String()), logValues...) } - // separateLikelyConfirmedAttempts is used as an optimisation: there is - // no point trying to fetch receipts for attempts with a sequence higher - // than the highest sequence the RPC node thinks it has seen - likelyConfirmed := ec.separateLikelyConfirmedAttempts(from, attempts, minedSequence) - likelyConfirmedCount := len(likelyConfirmed) - if likelyConfirmedCount > 0 { - likelyUnconfirmedCount := len(attempts) - likelyConfirmedCount + etxIDs = append(etxIDs, etx.ID) + attemptIDs = append(attemptIDs, attempt.ID) + } - ec.lggr.Debugf("Fetching and saving %v likely confirmed receipts. Skipping checking the others (%v)", - likelyConfirmedCount, likelyUnconfirmedCount) + // Mark transactions as unconfirmed, mark attempts as in-progress, and delete receipts since they do not apply to the new chain + // This may revert some fatal error transactions to unconfirmed if terminally stuck transactions purge attempts get re-org'd + return ec.txStore.UpdateTxsForRebroadcast(ctx, etxIDs, attemptIDs) +} - start := time.Now() - err = ec.fetchAndSaveReceipts(ctx, likelyConfirmed, blockNum) - if err != nil { - return fmt.Errorf("unable to fetch and save receipts for likely confirmed txs, for address: %v: %w", from, err) - } - ec.lggr.Debugw(fmt.Sprintf("Fetching and saving %v likely confirmed receipts done", likelyConfirmedCount), - "time", time.Since(start)) +func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ProcessIncludedTxs(ctx context.Context, includedTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head types.Head[BLOCK_HASH]) error { + if len(includedTxs) == 0 { + return nil + } + // Add newly confirmed transactions to the prom metric + promNumConfirmedTxs.WithLabelValues(ec.chainID.String()).Add(float64(len(includedTxs))) + + purgeTxIDs := make([]int64, 0, len(includedTxs)) + confirmedTxIDs := make([]int64, 0, len(includedTxs)) + for _, tx := range includedTxs { + // If any attempt in the transaction is marked for purge, the transaction was terminally stuck and should be marked as fatal error + if tx.HasPurgeAttempt() { + // Setting the purged block num here is ok since we have confirmation the tx has been included + ec.stuckTxDetector.SetPurgeBlockNum(tx.FromAddress, head.BlockNumber()) + purgeTxIDs = append(purgeTxIDs, tx.ID) + continue } + confirmedTxIDs = append(confirmedTxIDs, tx.ID) + observeUntilTxConfirmed(ec.chainID, tx.TxAttempts, head) } - - if err := ec.txStore.MarkAllConfirmedMissingReceipt(ctx, ec.chainID); err != nil { - return fmt.Errorf("unable to mark txes as 'confirmed_missing_receipt': %w", err) + // Mark the transactions included on-chain with a purge attempt as fatal error with the terminally stuck error message + if err := ec.txStore.UpdateTxFatalError(ctx, purgeTxIDs, ec.stuckTxDetector.StuckTxFatalError()); err != nil { + return fmt.Errorf("failed to update terminally stuck transactions: %w", err) } - - if err := ec.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, latestFinalizedBlockNum, ec.chainID); err != nil { - return fmt.Errorf("unable to confirm buried unconfirmed txes': %w", err) + // Mark the transactions included on-chain as confirmed + if err := ec.txStore.UpdateTxConfirmed(ctx, confirmedTxIDs); err != nil { + return fmt.Errorf("failed to update confirmed transactions: %w", err) } return nil } @@ -528,103 +459,6 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pro return errors.Join(errorList...) } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) separateLikelyConfirmedAttempts(from ADDR, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], minedSequence SEQ) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - if len(attempts) == 0 { - return attempts - } - - firstAttemptSequence := *attempts[len(attempts)-1].Tx.Sequence - lastAttemptSequence := *attempts[0].Tx.Sequence - latestMinedSequence := minedSequence.Int64() - 1 // this can be -1 if a transaction has never been mined on this account - ec.lggr.Debugw(fmt.Sprintf("There are %d attempts from address %s, mined transaction count is %d (latest mined sequence is %d) and for the attempts' sequences: first = %d, last = %d", - len(attempts), from, minedSequence.Int64(), latestMinedSequence, firstAttemptSequence.Int64(), lastAttemptSequence.Int64()), "nAttempts", len(attempts), "fromAddress", from, "minedSequence", minedSequence, "latestMinedSequence", latestMinedSequence, "firstAttemptSequence", firstAttemptSequence, "lastAttemptSequence", lastAttemptSequence) - - likelyConfirmed := attempts - // attempts are ordered by sequence ASC - for i := 0; i < len(attempts); i++ { - // If the attempt sequence is lower or equal to the latestBlockSequence - // it must have been confirmed, we just didn't get a receipt yet - // - // Examples: - // 3 transactions confirmed, highest has sequence 2 - // 5 total attempts, highest has sequence 4 - // minedSequence=3 - // likelyConfirmed will be attempts[0:3] which gives the first 3 transactions, as expected - if (*attempts[i].Tx.Sequence).Int64() > minedSequence.Int64() { - ec.lggr.Debugf("Marking attempts as likely confirmed just before index %v, at sequence: %v", i, *attempts[i].Tx.Sequence) - likelyConfirmed = attempts[0:i] - break - } - } - - if len(likelyConfirmed) == 0 { - ec.lggr.Debug("There are no likely confirmed attempts - so will skip checking any") - } - - return likelyConfirmed -} - -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchAndSaveReceipts(ctx context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], blockNum int64) error { - promTxAttemptCount.WithLabelValues(ec.chainID.String()).Set(float64(len(attempts))) - - batchSize := int(ec.chainConfig.RPCDefaultBatchSize()) - if batchSize == 0 { - batchSize = len(attempts) - } - var allReceipts []R - for i := 0; i < len(attempts); i += batchSize { - j := i + batchSize - if j > len(attempts) { - j = len(attempts) - } - - ec.lggr.Debugw(fmt.Sprintf("Batch fetching receipts at indexes %d until (excluded) %d", i, j), "blockNum", blockNum) - - batch := attempts[i:j] - - receipts, err := ec.batchFetchReceipts(ctx, batch, blockNum) - if err != nil { - return fmt.Errorf("batchFetchReceipts failed: %w", err) - } - validReceipts, purgeReceipts := ec.separateValidAndPurgeAttemptReceipts(receipts, batch) - // Saves the receipts and mark the associated transactions as Confirmed - if err := ec.txStore.SaveFetchedReceipts(ctx, validReceipts, TxConfirmed, nil, ec.chainID); err != nil { - return fmt.Errorf("saveFetchedReceipts failed: %w", err) - } - // Save the receipts but mark the associated transactions as Fatal Error since the original transaction was purged - stuckTxFatalErrMsg := ec.stuckTxDetector.StuckTxFatalError() - if err := ec.txStore.SaveFetchedReceipts(ctx, purgeReceipts, TxFatalError, &stuckTxFatalErrMsg, ec.chainID); err != nil { - return fmt.Errorf("saveFetchedReceipts failed: %w", err) - } - promNumConfirmedTxs.WithLabelValues(ec.chainID.String()).Add(float64(len(receipts))) - - allReceipts = append(allReceipts, receipts...) - } - - observeUntilTxConfirmed(ec.chainID, attempts, allReceipts) - - return nil -} - -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) separateValidAndPurgeAttemptReceipts(receipts []R, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) (valid []R, purge []R) { - receiptMap := make(map[TX_HASH]R) - for _, receipt := range receipts { - receiptMap[receipt.GetTxHash()] = receipt - } - for _, attempt := range attempts { - if receipt, ok := receiptMap[attempt.Hash]; ok { - if attempt.IsPurgeAttempt { - // Setting the purged block num here is ok since we have confirmation the tx has been purged with the receipt - ec.stuckTxDetector.SetPurgeBlockNum(attempt.Tx.FromAddress, receipt.GetBlockNumber().Int64()) - purge = append(purge, receipt) - } else { - valid = append(valid, receipt) - } - } - } - return -} - func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) resumeFailedTaskRuns(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { if !etx.PipelineTaskRunID.Valid || ec.resumeCallback == nil || !etx.SignalCallback || etx.CallbackCompleted { return nil @@ -636,113 +470,13 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) res return fmt.Errorf("failed to resume pipeline: %w", err) } else { // Mark tx as having completed callback - if err = ec.txStore.UpdateTxCallbackCompleted(ctx, etx.PipelineTaskRunID.UUID, ec.chainID); err != nil { + if err := ec.txStore.UpdateTxCallbackCompleted(ctx, etx.PipelineTaskRunID.UUID, ec.chainID); err != nil { return err } } return nil } -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) getMinedSequenceForAddress(ctx context.Context, from ADDR) (SEQ, error) { - return ec.client.SequenceAt(ctx, from, nil) -} - -// Note this function will increment promRevertedTxCount upon receiving -// a reverted transaction receipt. Should only be called with unconfirmed attempts. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) batchFetchReceipts(ctx context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], blockNum int64) (receipts []R, err error) { - // Metadata is required to determine whether a tx is forwarded or not. - if ec.txConfig.ForwardersEnabled() { - err = ec.txStore.PreloadTxes(ctx, attempts) - if err != nil { - return nil, fmt.Errorf("Confirmer#batchFetchReceipts error loading txs for attempts: %w", err) - } - } - - lggr := ec.lggr.Named("BatchFetchReceipts").With("blockNum", blockNum) - - txReceipts, txErrs, err := ec.client.BatchGetReceipts(ctx, attempts) - if err != nil { - return nil, err - } - - for i := range txReceipts { - attempt := attempts[i] - receipt := txReceipts[i] - err := txErrs[i] - - l := attempt.Tx.GetLogger(lggr).With("txHash", attempt.Hash.String(), "txAttemptID", attempt.ID, - "txID", attempt.TxID, "err", err, "sequence", attempt.Tx.Sequence, - ) - - if err != nil { - l.Error("FetchReceipt failed") - continue - } - - if ec.isReceiptNil(receipt) { - // NOTE: This should never happen, but it seems safer to check - // regardless to avoid a potential panic - l.AssumptionViolation("got nil receipt") - continue - } - - if receipt.IsZero() { - l.Debug("Still waiting for receipt") - continue - } - - l = l.With("blockHash", receipt.GetBlockHash().String(), "status", receipt.GetStatus(), "transactionIndex", receipt.GetTransactionIndex()) - - if receipt.IsUnmined() { - l.Debug("Got receipt for transaction but it's still in the mempool and not included in a block yet") - continue - } - - l.Debugw("Got receipt for transaction", "blockNumber", receipt.GetBlockNumber(), "feeUsed", receipt.GetFeeUsed()) - - if receipt.GetTxHash().String() != attempt.Hash.String() { - l.Errorf("Invariant violation, expected receipt with hash %s to have same hash as attempt with hash %s", receipt.GetTxHash().String(), attempt.Hash.String()) - continue - } - - if receipt.GetBlockNumber() == nil { - l.Error("Invariant violation, receipt was missing block number") - continue - } - - if receipt.GetStatus() == 0 { - if receipt.GetRevertReason() != nil { - l.Warnw("transaction reverted on-chain", "hash", receipt.GetTxHash(), "revertReason", *receipt.GetRevertReason()) - } else { - rpcError, errExtract := ec.client.CallContract(ctx, attempt, receipt.GetBlockNumber()) - if errExtract == nil { - l.Warnw("transaction reverted on-chain", "hash", receipt.GetTxHash(), "rpcError", rpcError.String()) - } else { - l.Warnw("transaction reverted on-chain unable to extract revert reason", "hash", receipt.GetTxHash(), "err", err) - } - } - // This might increment more than once e.g. in case of re-orgs going back and forth we might re-fetch the same receipt - promRevertedTxCount.WithLabelValues(ec.chainID.String()).Add(1) - } else { - promNumSuccessfulTxs.WithLabelValues(ec.chainID.String()).Add(1) - } - - // This is only recording forwarded tx that were mined and have a status. - // Counters are prone to being inaccurate due to re-orgs. - if ec.txConfig.ForwardersEnabled() { - meta, metaErr := attempt.Tx.GetMeta() - if metaErr == nil && meta != nil && meta.FwdrDestAddress != nil { - // promFwdTxCount takes two labels, chainId and a boolean of whether a tx was successful or not. - promFwdTxCount.WithLabelValues(ec.chainID.String(), strconv.FormatBool(receipt.GetStatus() != 0)).Add(1) - } - } - - receipts = append(receipts, receipt) - } - - return -} - // RebroadcastWhereNecessary bumps gas or resends transactions that were previously out-of-funds func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RebroadcastWhereNecessary(ctx context.Context, blockHeight int64) error { var wg sync.WaitGroup @@ -1046,7 +780,7 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han // Mark confirmed_missing_receipt and wait for the next cycle to try to get a receipt lggr.Debugw("Sequence already used", "txAttemptID", attempt.ID, "txHash", attempt.Hash.String()) timeout := ec.dbConfig.DefaultQueryTimeout() - return ec.txStore.SaveConfirmedMissingReceiptAttempt(ctx, timeout, &attempt, now) + return ec.txStore.SaveConfirmedAttempt(ctx, timeout, &attempt, now) case client.InsufficientFunds: timeout := ec.dbConfig.DefaultQueryTimeout() return ec.txStore.SaveInsufficientFundsAttempt(ctx, timeout, &attempt, now) @@ -1066,139 +800,6 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) han } } -// EnsureConfirmedTransactionsInLongestChain finds all confirmed txes up to the earliest head -// of the given chain and ensures that every one has a receipt with a block hash that is -// in the given chain. -// -// If any of the confirmed transactions does not have a receipt in the chain, it has been -// re-org'd out and will be rebroadcast. -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) EnsureConfirmedTransactionsInLongestChain(ctx context.Context, head types.Head[BLOCK_HASH]) error { - logArgs := []interface{}{ - "chainLength", head.ChainLength(), - } - - // Here, we rely on the finalized block provided in the chain instead of the one - // provided via a dedicated method to avoid the false warning of the chain being - // too short. When `FinalityTagBypass = true,` HeadTracker tracks `finality depth - // + history depth` to prevent excessive CPU usage. Thus, the provided chain may - // be shorter than the chain from the latest to the latest finalized, marked with - // a tag. A proper fix of this issue and complete switch to finality tag support - // will be introduced in BCFR-620 - latestFinalized := head.LatestFinalizedHead() - if latestFinalized == nil || !latestFinalized.IsValid() { - if ec.nConsecutiveBlocksChainTooShort > logAfterNConsecutiveBlocksChainTooShort { - warnMsg := "Chain length supplied for re-org detection was shorter than the depth from the latest head to the finalized head. Re-org protection is not working properly. This could indicate a problem with the remote RPC endpoint, a compatibility issue with a particular blockchain, a bug with this particular blockchain, heads table being truncated too early, remote node out of sync, or something else. If this happens a lot please raise a bug with the Chainlink team including a log output sample and details of the chain and RPC endpoint you are using." - ec.lggr.Warnw(warnMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) - } else { - logMsg := "Chain length supplied for re-org detection was shorter than the depth from the latest head to the finalized head" - ec.lggr.Debugw(logMsg, append(logArgs, "nConsecutiveBlocksChainTooShort", ec.nConsecutiveBlocksChainTooShort)...) - } - ec.nConsecutiveBlocksChainTooShort++ - } else { - ec.nConsecutiveBlocksChainTooShort = 0 - } - etxs, err := ec.txStore.FindTransactionsConfirmedInBlockRange(ctx, head.BlockNumber(), head.EarliestHeadInChain().BlockNumber(), ec.chainID) - if err != nil { - return fmt.Errorf("findTransactionsConfirmedInBlockRange failed: %w", err) - } - - for _, etx := range etxs { - if !hasReceiptInLongestChain(*etx, head) { - if err := ec.markForRebroadcast(ctx, *etx, head); err != nil { - return fmt.Errorf("markForRebroadcast failed for etx %v: %w", etx.ID, err) - } - } - } - - // It is safe to process separate keys concurrently - // NOTE: This design will block one key if another takes a really long time to execute - var wg sync.WaitGroup - errors := []error{} - var errMu sync.Mutex - wg.Add(len(ec.enabledAddresses)) - for _, address := range ec.enabledAddresses { - go func(fromAddress ADDR) { - if err := ec.handleAnyInProgressAttempts(ctx, fromAddress, head.BlockNumber()); err != nil { - errMu.Lock() - errors = append(errors, err) - errMu.Unlock() - ec.lggr.Errorw("Error in handleAnyInProgressAttempts", "err", err, "fromAddress", fromAddress) - } - - wg.Done() - }(address) - } - - wg.Wait() - - return multierr.Combine(errors...) -} - -func hasReceiptInLongestChain[ - CHAIN_ID types.ID, - ADDR types.Hashable, - TX_HASH, BLOCK_HASH types.Hashable, - SEQ types.Sequence, - FEE feetypes.Fee, -](etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head types.Head[BLOCK_HASH]) bool { - for { - for _, attempt := range etx.TxAttempts { - for _, receipt := range attempt.Receipts { - if receipt.GetBlockHash().String() == head.BlockHash().String() && receipt.GetBlockNumber().Int64() == head.BlockNumber() { - return true - } - } - } - - head = head.GetParent() - if head == nil { - return false - } - } -} - -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) markForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head types.Head[BLOCK_HASH]) error { - if len(etx.TxAttempts) == 0 { - return fmt.Errorf("invariant violation: expected tx %v to have at least one attempt", etx.ID) - } - - // Rebroadcast the one with the highest gas price - attempt := etx.TxAttempts[0] - var receipt txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH] - if len(attempt.Receipts) > 0 { - receipt = attempt.Receipts[0] - } - - logValues := []interface{}{ - "txhash", attempt.Hash.String(), - "currentBlockNum", head.BlockNumber(), - "currentBlockHash", head.BlockHash().String(), - "txID", etx.ID, - "attemptID", attempt.ID, - "nReceipts", len(attempt.Receipts), - "id", "confirmer", - } - - // nil check on receipt interface - if receipt != nil { - logValues = append(logValues, - "replacementBlockHashAtConfirmedHeight", head.HashAtHeight(receipt.GetBlockNumber().Int64()), - "confirmedInBlockNum", receipt.GetBlockNumber(), - "confirmedInBlockHash", receipt.GetBlockHash(), - "confirmedInTxIndex", receipt.GetTransactionIndex(), - ) - } - - ec.lggr.Infow(fmt.Sprintf("Re-org detected. Rebroadcasting transaction %s which may have been re-org'd out of the main chain", attempt.Hash.String()), logValues...) - - // Put it back in progress and delete all receipts (they do not apply to the new chain) - if err := ec.txStore.UpdateTxForRebroadcast(ctx, etx, attempt); err != nil { - return fmt.Errorf("markForRebroadcast failed: %w", err) - } - - return nil -} - // ForceRebroadcast sends a transaction for every sequence in the given sequence range at the given gas price. // If an tx exists for this sequence, we re-send the existing tx with the supplied parameters. // If an tx doesn't exist for this sequence, we send a zero transaction. @@ -1259,80 +860,38 @@ func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sen return txhash, nil } -// ResumePendingTaskRuns issues callbacks to task runs that are pending waiting for receipts -func (ec *Confirmer[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ResumePendingTaskRuns(ctx context.Context, latest, finalized int64) error { - receiptsPlus, err := ec.txStore.FindTxesPendingCallback(ctx, latest, finalized, ec.chainID) - - if err != nil { - return err - } - - if len(receiptsPlus) > 0 { - ec.lggr.Debugf("Resuming %d task runs pending receipt", len(receiptsPlus)) - } else { - ec.lggr.Debug("No task runs to resume") - } - for _, data := range receiptsPlus { - var taskErr error - var output interface{} - if data.FailOnRevert && data.Receipt.GetStatus() == 0 { - taskErr = fmt.Errorf("transaction %s reverted on-chain", data.Receipt.GetTxHash()) - } else { - output = data.Receipt - } - - ec.lggr.Debugw("Callback: resuming tx with receipt", "output", output, "taskErr", taskErr, "pipelineTaskRunID", data.ID) - if err := ec.resumeCallback(ctx, data.ID, output, taskErr); err != nil { - return fmt.Errorf("failed to resume suspended pipeline run: %w", err) - } - // Mark tx as having completed callback - if err := ec.txStore.UpdateTxCallbackCompleted(ctx, data.ID, ec.chainID); err != nil { - return err - } - } - - return nil -} - // observeUntilTxConfirmed observes the promBlocksUntilTxConfirmed metric for each confirmed // transaction. func observeUntilTxConfirmed[ CHAIN_ID types.ID, ADDR types.Hashable, TX_HASH, BLOCK_HASH types.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee, -](chainID CHAIN_ID, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], receipts []R) { +](chainID CHAIN_ID, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], head types.Head[BLOCK_HASH]) { for _, attempt := range attempts { - for _, r := range receipts { - if attempt.Hash.String() != r.GetTxHash().String() { - continue + // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) + // was created. We want to measure the amount of time taken from when a transaction is created + // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts + // were needed to achieve this. + duration := time.Since(attempt.Tx.CreatedAt) + promTimeUntilTxConfirmed. + WithLabelValues(chainID.String()). + Observe(float64(duration)) + + // Since a tx can have many attempts, we take the number of blocks to confirm as the block number + // of the receipt minus the block number of the first ever broadcast for this transaction. + broadcastBefore := iutils.MinFunc(attempt.Tx.TxAttempts, func(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) int64 { + if attempt.BroadcastBeforeBlockNum != nil { + return *attempt.BroadcastBeforeBlockNum } - - // We estimate the time until confirmation by subtracting from the time the tx (not the attempt) - // was created. We want to measure the amount of time taken from when a transaction is created - // via e.g Txm.CreateTransaction to when it is confirmed on-chain, regardless of how many attempts - // were needed to achieve this. - duration := time.Since(attempt.Tx.CreatedAt) - promTimeUntilTxConfirmed. + return 0 + }) + if broadcastBefore > 0 { + blocksElapsed := head.BlockNumber() - broadcastBefore + promBlocksUntilTxConfirmed. WithLabelValues(chainID.String()). - Observe(float64(duration)) - - // Since a tx can have many attempts, we take the number of blocks to confirm as the block number - // of the receipt minus the block number of the first ever broadcast for this transaction. - broadcastBefore := iutils.MinFunc(attempt.Tx.TxAttempts, func(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) int64 { - if attempt.BroadcastBeforeBlockNum != nil { - return *attempt.BroadcastBeforeBlockNum - } - return 0 - }) - if broadcastBefore > 0 { - blocksElapsed := r.GetBlockNumber().Int64() - broadcastBefore - promBlocksUntilTxConfirmed. - WithLabelValues(chainID.String()). - Observe(float64(blocksElapsed)) - } + Observe(float64(blocksElapsed)) } } } diff --git a/common/txmgr/tracker.go b/common/txmgr/tracker.go index a7236472710..408ae62173a 100644 --- a/common/txmgr/tracker.go +++ b/common/txmgr/tracker.go @@ -304,7 +304,7 @@ func (tr *Tracker[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) markTxFatal // Set state to TxInProgress so the tracker can attempt to mark it as fatal tx.State = TxInProgress - if err := tr.txStore.UpdateTxFatalError(ctx, tx); err != nil { + if err := tr.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, tx); err != nil { return fmt.Errorf("failed to mark tx %v as abandoned: %w", tx.ID, err) } return nil diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index 28d505e5e05..3776f62254c 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -118,6 +118,7 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RegisterRe b.resumeCallback = fn b.broadcaster.SetResumeCallback(fn) b.confirmer.SetResumeCallback(fn) + b.finalizer.SetResumeCallback(fn) } // NewTxm creates a new Txm with the given configuration. diff --git a/common/txmgr/types/config.go b/common/txmgr/types/config.go index 8b11a45d11d..1ab334b3a48 100644 --- a/common/txmgr/types/config.go +++ b/common/txmgr/types/config.go @@ -4,7 +4,6 @@ import "time" type TransactionManagerChainConfig interface { BroadcasterChainConfig - ConfirmerChainConfig } type TransactionManagerFeeConfig interface { @@ -46,12 +45,6 @@ type ConfirmerFeeConfig interface { // from gas.Config BumpThreshold() uint64 MaxFeePrice() string // logging value - BumpPercent() uint16 -} - -type ConfirmerChainConfig interface { - RPCDefaultBatchSize() uint32 - FinalityDepth() uint32 } type ConfirmerDatabaseConfig interface { diff --git a/common/txmgr/types/finalizer.go b/common/txmgr/types/finalizer.go index be3c897d0e2..4ed25111faa 100644 --- a/common/txmgr/types/finalizer.go +++ b/common/txmgr/types/finalizer.go @@ -1,6 +1,10 @@ package types import ( + "context" + + "github.com/google/uuid" + "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/common/types" ) @@ -9,4 +13,5 @@ type Finalizer[BLOCK_HASH types.Hashable, HEAD types.Head[BLOCK_HASH]] interface // interfaces for running the underlying estimator services.Service DeliverLatestHead(head HEAD) bool + SetResumeCallback(callback func(ctx context.Context, id uuid.UUID, result interface{}, err error) error) } diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 8dc816e9bec..b75ee69302a 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -555,9 +555,9 @@ func (_c *TxStore_FindEarliestUnconfirmedTxAttemptBlock_Call[ADDR, CHAIN_ID, TX_ return _c } -// FindLatestSequence provides a mock function with given fields: ctx, fromAddress, chainId -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID) (SEQ, error) { - ret := _m.Called(ctx, fromAddress, chainId) +// FindLatestSequence provides a mock function with given fields: ctx, fromAddress, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) { + ret := _m.Called(ctx, fromAddress, chainID) if len(ret) == 0 { panic("no return value specified for FindLatestSequence") @@ -566,16 +566,16 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestS var r0 SEQ var r1 error if rf, ok := ret.Get(0).(func(context.Context, ADDR, CHAIN_ID) (SEQ, error)); ok { - return rf(ctx, fromAddress, chainId) + return rf(ctx, fromAddress, chainID) } if rf, ok := ret.Get(0).(func(context.Context, ADDR, CHAIN_ID) SEQ); ok { - r0 = rf(ctx, fromAddress, chainId) + r0 = rf(ctx, fromAddress, chainID) } else { r0 = ret.Get(0).(SEQ) } if rf, ok := ret.Get(1).(func(context.Context, ADDR, CHAIN_ID) error); ok { - r1 = rf(ctx, fromAddress, chainId) + r1 = rf(ctx, fromAddress, chainID) } else { r1 = ret.Error(1) } @@ -591,12 +591,12 @@ type TxStore_FindLatestSequence_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_ // FindLatestSequence is a helper method to define mock.On call // - ctx context.Context // - fromAddress ADDR -// - chainId CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx interface{}, fromAddress interface{}, chainId interface{}) *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("FindLatestSequence", ctx, fromAddress, chainId)} +// - chainID CHAIN_ID +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx interface{}, fromAddress interface{}, chainID interface{}) *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("FindLatestSequence", ctx, fromAddress, chainID)} } -func (_c *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID)) *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID)) *TxStore_FindLatestSequence_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(ADDR), args[2].(CHAIN_ID)) }) @@ -673,63 +673,72 @@ func (_c *TxStore_FindNextUnstartedTransactionFromAddress_Call[ADDR, CHAIN_ID, T return _c } -// FindTransactionsConfirmedInBlockRange provides a mock function with given fields: ctx, highBlockNumber, lowBlockNumber, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber int64, lowBlockNumber int64, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - ret := _m.Called(ctx, highBlockNumber, lowBlockNumber, chainID) +// FindReorgOrIncludedTxs provides a mock function with given fields: ctx, fromAddress, nonce, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindReorgOrIncludedTxs(ctx context.Context, fromAddress ADDR, nonce SEQ, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ret := _m.Called(ctx, fromAddress, nonce, chainID) if len(ret) == 0 { - panic("no return value specified for FindTransactionsConfirmedInBlockRange") + panic("no return value specified for FindReorgOrIncludedTxs") } var r0 []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)); ok { - return rf(ctx, highBlockNumber, lowBlockNumber, chainID) + var r1 []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, ADDR, SEQ, CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)); ok { + return rf(ctx, fromAddress, nonce, chainID) } - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, CHAIN_ID) []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]); ok { - r0 = rf(ctx, highBlockNumber, lowBlockNumber, chainID) + if rf, ok := ret.Get(0).(func(context.Context, ADDR, SEQ, CHAIN_ID) []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]); ok { + r0 = rf(ctx, fromAddress, nonce, chainID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) } } - if rf, ok := ret.Get(1).(func(context.Context, int64, int64, CHAIN_ID) error); ok { - r1 = rf(ctx, highBlockNumber, lowBlockNumber, chainID) + if rf, ok := ret.Get(1).(func(context.Context, ADDR, SEQ, CHAIN_ID) []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]); ok { + r1 = rf(ctx, fromAddress, nonce, chainID) } else { - r1 = ret.Error(1) + if ret.Get(1) != nil { + r1 = ret.Get(1).([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) + } } - return r0, r1 + if rf, ok := ret.Get(2).(func(context.Context, ADDR, SEQ, CHAIN_ID) error); ok { + r2 = rf(ctx, fromAddress, nonce, chainID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } -// TxStore_FindTransactionsConfirmedInBlockRange_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTransactionsConfirmedInBlockRange' -type TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { +// TxStore_FindReorgOrIncludedTxs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindReorgOrIncludedTxs' +type TxStore_FindReorgOrIncludedTxs_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { *mock.Call } -// FindTransactionsConfirmedInBlockRange is a helper method to define mock.On call +// FindReorgOrIncludedTxs is a helper method to define mock.On call // - ctx context.Context -// - highBlockNumber int64 -// - lowBlockNumber int64 +// - fromAddress ADDR +// - nonce SEQ // - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(ctx interface{}, highBlockNumber interface{}, lowBlockNumber interface{}, chainID interface{}) *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("FindTransactionsConfirmedInBlockRange", ctx, highBlockNumber, lowBlockNumber, chainID)} +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindReorgOrIncludedTxs(ctx interface{}, fromAddress interface{}, nonce interface{}, chainID interface{}) *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("FindReorgOrIncludedTxs", ctx, fromAddress, nonce, chainID)} } -func (_c *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, highBlockNumber int64, lowBlockNumber int64, chainID CHAIN_ID)) *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, fromAddress ADDR, nonce SEQ, chainID CHAIN_ID)) *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(CHAIN_ID)) + run(args[0].(context.Context), args[1].(ADDR), args[2].(SEQ), args[3].(CHAIN_ID)) }) return _c } -func (_c *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(etxs, err) +func (_c *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(reorgTx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], includedTxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Return(reorgTx, includedTxs, err) return _c } -func (_c *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, int64, int64, CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)) *TxStore_FindTransactionsConfirmedInBlockRange_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, ADDR, SEQ, CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)) *TxStore_FindReorgOrIncludedTxs_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } @@ -793,65 +802,6 @@ func (_c *TxStore_FindTxAttemptsConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_ return _c } -// FindTxAttemptsRequiringReceiptFetch provides a mock function with given fields: ctx, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - ret := _m.Called(ctx, chainID) - - if len(ret) == 0 { - panic("no return value specified for FindTxAttemptsRequiringReceiptFetch") - } - - var r0 []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)); ok { - return rf(ctx, chainID) - } - if rf, ok := ret.Get(0).(func(context.Context, CHAIN_ID) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]); ok { - r0 = rf(ctx, chainID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, CHAIN_ID) error); ok { - r1 = rf(ctx, chainID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// TxStore_FindTxAttemptsRequiringReceiptFetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTxAttemptsRequiringReceiptFetch' -type TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { - *mock.Call -} - -// FindTxAttemptsRequiringReceiptFetch is a helper method to define mock.On call -// - ctx context.Context -// - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx interface{}, chainID interface{}) *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("FindTxAttemptsRequiringReceiptFetch", ctx, chainID)} -} - -func (_c *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, chainID CHAIN_ID)) *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(CHAIN_ID)) - }) - return _c -} - -func (_c *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(attempts, err) - return _c -} - -func (_c *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)) *TxStore_FindTxAttemptsRequiringReceiptFetch_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(run) - return _c -} - // FindTxAttemptsRequiringResend provides a mock function with given fields: ctx, olderThan, maxInFlightTransactions, chainID, address func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { ret := _m.Called(ctx, olderThan, maxInFlightTransactions, chainID, address) @@ -1808,102 +1758,6 @@ func (_c *TxStore_LoadTxAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SE return _c } -// MarkAllConfirmedMissingReceipt provides a mock function with given fields: ctx, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) error { - ret := _m.Called(ctx, chainID) - - if len(ret) == 0 { - panic("no return value specified for MarkAllConfirmedMissingReceipt") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, CHAIN_ID) error); ok { - r0 = rf(ctx, chainID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// TxStore_MarkAllConfirmedMissingReceipt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkAllConfirmedMissingReceipt' -type TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { - *mock.Call -} - -// MarkAllConfirmedMissingReceipt is a helper method to define mock.On call -// - ctx context.Context -// - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx interface{}, chainID interface{}) *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("MarkAllConfirmedMissingReceipt", ctx, chainID)} -} - -func (_c *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, chainID CHAIN_ID)) *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(CHAIN_ID)) - }) - return _c -} - -func (_c *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(err error) *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(err) - return _c -} - -func (_c *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, CHAIN_ID) error) *TxStore_MarkAllConfirmedMissingReceipt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(run) - return _c -} - -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID) error { - ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) - - if len(ret) == 0 { - panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, CHAIN_ID) error); ok { - r0 = rf(ctx, blockNum, latestFinalizedBlockNum, chainID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// TxStore_MarkOldTxesMissingReceiptAsErrored_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkOldTxesMissingReceiptAsErrored' -type TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { - *mock.Call -} - -// MarkOldTxesMissingReceiptAsErrored is a helper method to define mock.On call -// - ctx context.Context -// - blockNum int64 -// - latestFinalizedBlockNum int64 -// - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, latestFinalizedBlockNum interface{}, chainID interface{}) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, latestFinalizedBlockNum, chainID)} -} - -func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID)) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(CHAIN_ID)) - }) - return _c -} - -func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(_a0) - return _c -} - -func (_c *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, int64, int64, CHAIN_ID) error) *TxStore_MarkOldTxesMissingReceiptAsErrored_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - _c.Call.Return(run) - return _c -} - // PreloadTxes provides a mock function with given fields: ctx, attempts func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PreloadTxes(ctx context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { ret := _m.Called(ctx, attempts) @@ -2059,12 +1913,12 @@ func (_c *TxStore_ReapTxHistory_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ return _c } -// SaveConfirmedMissingReceiptAttempt provides a mock function with given fields: ctx, timeout, attempt, broadcastAt -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { +// SaveConfirmedAttempt provides a mock function with given fields: ctx, timeout, attempt, broadcastAt +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { ret := _m.Called(ctx, timeout, attempt, broadcastAt) if len(ret) == 0 { - panic("no return value specified for SaveConfirmedMissingReceiptAttempt") + panic("no return value specified for SaveConfirmedAttempt") } var r0 error @@ -2077,48 +1931,48 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirm return r0 } -// TxStore_SaveConfirmedMissingReceiptAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveConfirmedMissingReceiptAttempt' -type TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { +// TxStore_SaveConfirmedAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveConfirmedAttempt' +type TxStore_SaveConfirmedAttempt_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { *mock.Call } -// SaveConfirmedMissingReceiptAttempt is a helper method to define mock.On call +// SaveConfirmedAttempt is a helper method to define mock.On call // - ctx context.Context // - timeout time.Duration // - attempt *txmgrtypes.TxAttempt[CHAIN_ID,ADDR,TX_HASH,BLOCK_HASH,SEQ,FEE] // - broadcastAt time.Time -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx interface{}, timeout interface{}, attempt interface{}, broadcastAt interface{}) *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("SaveConfirmedMissingReceiptAttempt", ctx, timeout, attempt, broadcastAt)} +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedAttempt(ctx interface{}, timeout interface{}, attempt interface{}, broadcastAt interface{}) *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("SaveConfirmedAttempt", ctx, timeout, attempt, broadcastAt)} } -func (_c *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time)) *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time)) *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(time.Duration), args[2].(*txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), args[3].(time.Time)) }) return _c } -func (_c *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(_a0) return _c } -func (_c *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, time.Duration, *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], time.Time) error) *TxStore_SaveConfirmedMissingReceiptAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, time.Duration, *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], time.Time) error) *TxStore_SaveConfirmedAttempt_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } -// SaveFetchedReceipts provides a mock function with given fields: ctx, r, state, errorMsg, chainID -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, r []R, state txmgrtypes.TxState, errorMsg *string, chainID CHAIN_ID) error { - ret := _m.Called(ctx, r, state, errorMsg, chainID) +// SaveFetchedReceipts provides a mock function with given fields: ctx, r +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, r []R) error { + ret := _m.Called(ctx, r) if len(ret) == 0 { panic("no return value specified for SaveFetchedReceipts") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []R, txmgrtypes.TxState, *string, CHAIN_ID) error); ok { - r0 = rf(ctx, r, state, errorMsg, chainID) + if rf, ok := ret.Get(0).(func(context.Context, []R) error); ok { + r0 = rf(ctx, r) } else { r0 = ret.Error(0) } @@ -2134,16 +1988,13 @@ type TxStore_SaveFetchedReceipts_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX // SaveFetchedReceipts is a helper method to define mock.On call // - ctx context.Context // - r []R -// - state txmgrtypes.TxState -// - errorMsg *string -// - chainID CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx interface{}, r interface{}, state interface{}, errorMsg interface{}, chainID interface{}) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("SaveFetchedReceipts", ctx, r, state, errorMsg, chainID)} +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx interface{}, r interface{}) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("SaveFetchedReceipts", ctx, r)} } -func (_c *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, r []R, state txmgrtypes.TxState, errorMsg *string, chainID CHAIN_ID)) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, r []R)) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]R), args[2].(txmgrtypes.TxState), args[3].(*string), args[4].(CHAIN_ID)) + run(args[0].(context.Context), args[1].([]R)) }) return _c } @@ -2153,7 +2004,7 @@ func (_c *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, return _c } -func (_c *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, []R, txmgrtypes.TxState, *string, CHAIN_ID) error) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, []R) error) *TxStore_SaveFetchedReceipts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } @@ -2496,9 +2347,9 @@ func (_c *TxStore_UpdateTxAttemptInProgressToBroadcast_Call[ADDR, CHAIN_ID, TX_H return _c } -// UpdateTxCallbackCompleted provides a mock function with given fields: ctx, pipelineTaskRunRid, chainId -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error { - ret := _m.Called(ctx, pipelineTaskRunRid, chainId) +// UpdateTxCallbackCompleted provides a mock function with given fields: ctx, pipelineTaskRunRid, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID) error { + ret := _m.Called(ctx, pipelineTaskRunRid, chainID) if len(ret) == 0 { panic("no return value specified for UpdateTxCallbackCompleted") @@ -2506,7 +2357,7 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCal var r0 error if rf, ok := ret.Get(0).(func(context.Context, uuid.UUID, CHAIN_ID) error); ok { - r0 = rf(ctx, pipelineTaskRunRid, chainId) + r0 = rf(ctx, pipelineTaskRunRid, chainID) } else { r0 = ret.Error(0) } @@ -2522,12 +2373,12 @@ type TxStore_UpdateTxCallbackCompleted_Call[ADDR types.Hashable, CHAIN_ID types. // UpdateTxCallbackCompleted is a helper method to define mock.On call // - ctx context.Context // - pipelineTaskRunRid uuid.UUID -// - chainId CHAIN_ID -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx interface{}, pipelineTaskRunRid interface{}, chainId interface{}) *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxCallbackCompleted", ctx, pipelineTaskRunRid, chainId)} +// - chainID CHAIN_ID +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx interface{}, pipelineTaskRunRid interface{}, chainID interface{}) *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxCallbackCompleted", ctx, pipelineTaskRunRid, chainID)} } -func (_c *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID)) *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID)) *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(uuid.UUID), args[2].(CHAIN_ID)) }) @@ -2544,17 +2395,64 @@ func (_c *TxStore_UpdateTxCallbackCompleted_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_ return _c } -// UpdateTxFatalError provides a mock function with given fields: ctx, etx -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - ret := _m.Called(ctx, etx) +// UpdateTxConfirmed provides a mock function with given fields: ctx, etxIDs +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxConfirmed(ctx context.Context, etxIDs []int64) error { + ret := _m.Called(ctx, etxIDs) + + if len(ret) == 0 { + panic("no return value specified for UpdateTxConfirmed") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []int64) error); ok { + r0 = rf(ctx, etxIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_UpdateTxConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxConfirmed' +type TxStore_UpdateTxConfirmed_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { + *mock.Call +} + +// UpdateTxConfirmed is a helper method to define mock.On call +// - ctx context.Context +// - etxIDs []int64 +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxConfirmed(ctx interface{}, etxIDs interface{}) *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxConfirmed", ctx, etxIDs)} +} + +func (_c *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etxIDs []int64)) *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64)) + }) + return _c +} + +func (_c *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, []int64) error) *TxStore_UpdateTxConfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Return(run) + return _c +} + +// UpdateTxFatalError provides a mock function with given fields: ctx, etxIDs, errMsg +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx context.Context, etxIDs []int64, errMsg string) error { + ret := _m.Called(ctx, etxIDs, errMsg) if len(ret) == 0 { panic("no return value specified for UpdateTxFatalError") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error); ok { - r0 = rf(ctx, etx) + if rf, ok := ret.Get(0).(func(context.Context, []int64, string) error); ok { + r0 = rf(ctx, etxIDs, errMsg) } else { r0 = ret.Error(0) } @@ -2569,14 +2467,15 @@ type TxStore_UpdateTxFatalError_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_ // UpdateTxFatalError is a helper method to define mock.On call // - ctx context.Context -// - etx *txmgrtypes.Tx[CHAIN_ID,ADDR,TX_HASH,BLOCK_HASH,SEQ,FEE] -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx interface{}, etx interface{}) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxFatalError", ctx, etx)} +// - etxIDs []int64 +// - errMsg string +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx interface{}, etxIDs interface{}, errMsg interface{}) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxFatalError", ctx, etxIDs, errMsg)} } -func (_c *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etxIDs []int64, errMsg string)) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) + run(args[0].(context.Context), args[1].([]int64), args[2].(string)) }) return _c } @@ -2586,22 +2485,22 @@ func (_c *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R return _c } -func (_c *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, []int64, string) error) *TxStore_UpdateTxFatalError_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } -// UpdateTxForRebroadcast provides a mock function with given fields: ctx, etx, etxAttempt -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - ret := _m.Called(ctx, etx, etxAttempt) +// UpdateTxFatalErrorAndDeleteAttempts provides a mock function with given fields: ctx, etx +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + ret := _m.Called(ctx, etx) if len(ret) == 0 { - panic("no return value specified for UpdateTxForRebroadcast") + panic("no return value specified for UpdateTxFatalErrorAndDeleteAttempts") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error); ok { - r0 = rf(ctx, etx, etxAttempt) + if rf, ok := ret.Get(0).(func(context.Context, *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error); ok { + r0 = rf(ctx, etx) } else { r0 = ret.Error(0) } @@ -2609,32 +2508,31 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFor return r0 } -// TxStore_UpdateTxForRebroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxForRebroadcast' -type TxStore_UpdateTxForRebroadcast_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { +// TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxFatalErrorAndDeleteAttempts' +type TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { *mock.Call } -// UpdateTxForRebroadcast is a helper method to define mock.On call +// UpdateTxFatalErrorAndDeleteAttempts is a helper method to define mock.On call // - ctx context.Context -// - etx txmgrtypes.Tx[CHAIN_ID,ADDR,TX_HASH,BLOCK_HASH,SEQ,FEE] -// - etxAttempt txmgrtypes.TxAttempt[CHAIN_ID,ADDR,TX_HASH,BLOCK_HASH,SEQ,FEE] -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx interface{}, etx interface{}, etxAttempt interface{}) *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxForRebroadcast", ctx, etx, etxAttempt)} +// - etx *txmgrtypes.Tx[CHAIN_ID,ADDR,TX_HASH,BLOCK_HASH,SEQ,FEE] +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalErrorAndDeleteAttempts(ctx interface{}, etx interface{}) *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxFatalErrorAndDeleteAttempts", ctx, etx)} } -func (_c *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), args[2].(txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) + run(args[0].(context.Context), args[1].(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) }) return _c } -func (_c *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(_a0) return _c } -func (_c *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error) *TxStore_UpdateTxForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error) *TxStore_UpdateTxFatalErrorAndDeleteAttempts_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Return(run) return _c } @@ -2687,9 +2585,57 @@ func (_c *TxStore_UpdateTxUnstartedToInProgress_Call[ADDR, CHAIN_ID, TX_HASH, BL return _c } -// UpdateTxsUnconfirmed provides a mock function with given fields: ctx, ids -func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) error { - ret := _m.Called(ctx, ids) +// UpdateTxsForRebroadcast provides a mock function with given fields: ctx, etxIDs, attemptIDs +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsForRebroadcast(ctx context.Context, etxIDs []int64, attemptIDs []int64) error { + ret := _m.Called(ctx, etxIDs, attemptIDs) + + if len(ret) == 0 { + panic("no return value specified for UpdateTxsForRebroadcast") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []int64, []int64) error); ok { + r0 = rf(ctx, etxIDs, attemptIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TxStore_UpdateTxsForRebroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxsForRebroadcast' +type TxStore_UpdateTxsForRebroadcast_Call[ADDR types.Hashable, CHAIN_ID types.ID, TX_HASH types.Hashable, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee] struct { + *mock.Call +} + +// UpdateTxsForRebroadcast is a helper method to define mock.On call +// - ctx context.Context +// - etxIDs []int64 +// - attemptIDs []int64 +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsForRebroadcast(ctx interface{}, etxIDs interface{}, attemptIDs interface{}) *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxsForRebroadcast", ctx, etxIDs, attemptIDs)} +} + +func (_c *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etxIDs []int64, attemptIDs []int64)) *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64), args[2].([]int64)) + }) + return _c +} + +func (_c *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Return(_a0 error) *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Return(_a0) + return _c +} + +func (_c *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RunAndReturn(run func(context.Context, []int64, []int64) error) *TxStore_UpdateTxsForRebroadcast_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + _c.Call.Return(run) + return _c +} + +// UpdateTxsUnconfirmed provides a mock function with given fields: ctx, etxIDs +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx context.Context, etxIDs []int64) error { + ret := _m.Called(ctx, etxIDs) if len(ret) == 0 { panic("no return value specified for UpdateTxsUnconfirmed") @@ -2697,7 +2643,7 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUn var r0 error if rf, ok := ret.Get(0).(func(context.Context, []int64) error); ok { - r0 = rf(ctx, ids) + r0 = rf(ctx, etxIDs) } else { r0 = ret.Error(0) } @@ -2712,12 +2658,12 @@ type TxStore_UpdateTxsUnconfirmed_Call[ADDR types.Hashable, CHAIN_ID types.ID, T // UpdateTxsUnconfirmed is a helper method to define mock.On call // - ctx context.Context -// - ids []int64 -func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx interface{}, ids interface{}) *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - return &TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxsUnconfirmed", ctx, ids)} +// - etxIDs []int64 +func (_e *TxStore_Expecter[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx interface{}, etxIDs interface{}) *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + return &TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{Call: _e.mock.On("UpdateTxsUnconfirmed", ctx, etxIDs)} } -func (_c *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, ids []int64)) *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +func (_c *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Run(run func(ctx context.Context, etxIDs []int64)) *TxStore_UpdateTxsUnconfirmed_Call[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].([]int64)) }) diff --git a/common/txmgr/types/tx.go b/common/txmgr/types/tx.go index f04047a36c1..b0bc2ca7025 100644 --- a/common/txmgr/types/tx.go +++ b/common/txmgr/types/tx.go @@ -342,6 +342,15 @@ func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) GetChecker() (Transm return t, nil } +func (e *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) HasPurgeAttempt() bool { + for _, attempt := range e.TxAttempts { + if attempt.IsPurgeAttempt { + return true + } + } + return false +} + // Provides error classification to external components in a chain agnostic way // Only exposes the error types that could be set in the transaction error field type ErrorClassifier interface { diff --git a/common/txmgr/types/tx_store.go b/common/txmgr/types/tx_store.go index 668b8db2049..d685a6c5ce7 100644 --- a/common/txmgr/types/tx_store.go +++ b/common/txmgr/types/tx_store.go @@ -36,8 +36,8 @@ type TxStore[ // Find confirmed txes beyond the minConfirmations param that require callback but have not yet been signaled FindTxesPendingCallback(ctx context.Context, latest, finalized int64, chainID CHAIN_ID) (receiptsPlus []ReceiptPlus[R], err error) // Update tx to mark that its callback has been signaled - UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error - SaveFetchedReceipts(ctx context.Context, r []R, state TxState, errorMsg *string, chainID CHAIN_ID) error + UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID) error + SaveFetchedReceipts(ctx context.Context, r []R) error // additional methods for tx store management CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) (err error) @@ -68,11 +68,12 @@ type TransactionStore[ CountUnstartedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) CreateTransaction(ctx context.Context, txRequest TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) DeleteInProgressAttempt(ctx context.Context, attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - FindLatestSequence(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID) (SEQ, error) + FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) + // FindReorgOrIncludedTxs returns either a list of re-org'd transactions or included transactions based on the provided sequence + FindReorgOrIncludedTxs(ctx context.Context, fromAddress ADDR, nonce SEQ, chainID CHAIN_ID) (reorgTx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], includedTxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindTxsRequiringResubmissionDueToInsufficientFunds(ctx context.Context, address ADDR, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) - FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) // Search for Tx using the idempotencyKey and chainID FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) @@ -80,8 +81,6 @@ type TransactionStore[ FindTxWithSequence(ctx context.Context, fromAddress ADDR, seq SEQ) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindNextUnstartedTransactionFromAddress(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) - // FindTransactionsConfirmedInBlockRange retrieves tx with attempts and partial receipt values for optimization purpose - FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) (etxs []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, chainID CHAIN_ID) (null.Time, error) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context, chainID CHAIN_ID) (null.Int, error) GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) @@ -90,10 +89,8 @@ type TransactionStore[ GetTxByID(ctx context.Context, id int64) (tx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) HasInProgressTransaction(ctx context.Context, account ADDR, chainID CHAIN_ID) (exists bool, err error) LoadTxAttempts(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (err error) - MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID CHAIN_ID) error PreloadTxes(ctx context.Context, attempts []TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error + SaveConfirmedAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error SaveInProgressAttempt(ctx context.Context, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error @@ -101,12 +98,17 @@ type TransactionStore[ SetBroadcastBeforeBlockNum(ctx context.Context, blockNum int64, chainID CHAIN_ID) error UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState TxAttemptState) error - // Update tx to mark that its callback has been signaled - UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error - UpdateTxsUnconfirmed(ctx context.Context, ids []int64) error + // UpdateTxCallbackCompleted updates tx to mark that its callback has been signaled + UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID CHAIN_ID) error + // UpdateTxConfirmed updates transaction states to confirmed + UpdateTxConfirmed(ctx context.Context, etxIDs []int64) error + // UpdateTxFatalErrorAndDeleteAttempts updates transaction states to fatal error, deletes attempts, and clears broadcast info and sequence + UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + // UpdateTxFatalError updates transaction states to fatal error with error message + UpdateTxFatalError(ctx context.Context, etxIDs []int64, errMsg string) error + UpdateTxsForRebroadcast(ctx context.Context, etxIDs []int64, attemptIDs []int64) error + UpdateTxsUnconfirmed(ctx context.Context, etxIDs []int64) error UpdateTxUnstartedToInProgress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - UpdateTxFatalError(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - UpdateTxForRebroadcast(ctx context.Context, etx Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error } type TxHistoryReaper[CHAIN_ID types.ID] interface { diff --git a/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go b/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go index d9979a02e40..16546b26999 100644 --- a/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go +++ b/core/capabilities/ccip/ocrimpls/contract_transmitter_test.go @@ -159,12 +159,12 @@ func testTransmitter( // wait for receipt to be written to the db require.Eventually(t, func() bool { - rows, err := uni.db.QueryContext(testutils.Context(t), `SELECT count(*) as cnt FROM evm.receipts LIMIT 1`) - require.NoError(t, err, "failed to query receipts") - defer rows.Close() - var count int - for rows.Next() { - require.NoError(t, rows.Scan(&count), "failed to scan") + uni.backend.Commit() + var count uint32 + err := uni.db.GetContext(testutils.Context(t), &count, `SELECT count(*) as cnt FROM evm.receipts LIMIT 1`) + require.NoError(t, err) + if count == 1 { + t.Log("tx receipt found in db") } return count == 1 }, testutils.WaitTimeout(t), 2*time.Second) diff --git a/core/chains/evm/txmgr/builder.go b/core/chains/evm/txmgr/builder.go index cbfb8775cfb..73c5614aba3 100644 --- a/core/chains/evm/txmgr/builder.go +++ b/core/chains/evm/txmgr/builder.go @@ -5,6 +5,8 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" "github.com/smartcontractkit/chainlink/v2/common/txmgr" @@ -59,8 +61,8 @@ func NewTxm( evmBroadcaster := NewEvmBroadcaster(txStore, txmClient, txmCfg, feeCfg, txConfig, listenerConfig, keyStore, txAttemptBuilder, lggr, checker, chainConfig.NonceAutoSync(), chainConfig.ChainType()) evmTracker := NewEvmTracker(txStore, keyStore, chainID, lggr) stuckTxDetector := NewStuckTxDetector(lggr, client.ConfiguredChainID(), chainConfig.ChainType(), fCfg.PriceMax(), txConfig.AutoPurge(), estimator, txStore, client) - evmConfirmer := NewEvmConfirmer(txStore, txmClient, txmCfg, feeCfg, txConfig, dbConfig, keyStore, txAttemptBuilder, lggr, stuckTxDetector, headTracker) - evmFinalizer := NewEvmFinalizer(lggr, client.ConfiguredChainID(), chainConfig.RPCDefaultBatchSize(), txStore, client, headTracker) + evmConfirmer := NewEvmConfirmer(txStore, txmClient, feeCfg, txConfig, dbConfig, keyStore, txAttemptBuilder, lggr, stuckTxDetector, headTracker) + evmFinalizer := NewEvmFinalizer(lggr, client.ConfiguredChainID(), chainConfig.RPCDefaultBatchSize(), txConfig.ForwardersEnabled(), txStore, txmClient, headTracker) var evmResender *Resender if txConfig.ResendAfterThreshold() > 0 { evmResender = NewEvmResender(lggr, txStore, txmClient, evmTracker, keyStore, txmgr.DefaultResenderPollInterval, chainConfig, txConfig) @@ -112,7 +114,6 @@ func NewEvmReaper(lggr logger.Logger, store txmgrtypes.TxHistoryReaper[*big.Int] func NewEvmConfirmer( txStore TxStore, client TxmClient, - chainConfig txmgrtypes.ConfirmerChainConfig, feeConfig txmgrtypes.ConfirmerFeeConfig, txConfig txmgrtypes.ConfirmerTransactionsConfig, dbConfig txmgrtypes.ConfirmerDatabaseConfig, @@ -122,7 +123,7 @@ func NewEvmConfirmer( stuckTxDetector StuckTxDetector, headTracker latestAndFinalizedBlockHeadTracker, ) *Confirmer { - return txmgr.NewConfirmer(txStore, client, chainConfig, feeConfig, txConfig, dbConfig, keystore, txAttemptBuilder, lggr, func(r *evmtypes.Receipt) bool { return r == nil }, stuckTxDetector, headTracker) + return txmgr.NewConfirmer(txStore, client, feeConfig, txConfig, dbConfig, keystore, txAttemptBuilder, lggr, func(r *evmtypes.Receipt) bool { return r == nil }, stuckTxDetector) } // NewEvmTracker instantiates a new EVM tracker for abandoned transactions @@ -132,7 +133,7 @@ func NewEvmTracker( chainID *big.Int, lggr logger.Logger, ) *Tracker { - return txmgr.NewTracker(txStore, keyStore, chainID, lggr) + return txmgr.NewTracker[*big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt](txStore, keyStore, chainID, lggr) } // NewEvmBroadcaster returns a new concrete EvmBroadcaster diff --git a/core/chains/evm/txmgr/client.go b/core/chains/evm/txmgr/client.go index dfaa4e6bfd8..9b2bcab6ebc 100644 --- a/core/chains/evm/txmgr/client.go +++ b/core/chains/evm/txmgr/client.go @@ -139,7 +139,7 @@ func (c *evmTxmClient) BatchGetReceipts(ctx context.Context, attempts []TxAttemp } if err := c.client.BatchCallContext(ctx, reqs); err != nil { - return nil, nil, fmt.Errorf("EthConfirmer#batchFetchReceipts error fetching receipts with BatchCallContext: %w", err) + return nil, nil, fmt.Errorf("error fetching receipts with BatchCallContext: %w", err) } for _, req := range reqs { @@ -192,3 +192,7 @@ func (c *evmTxmClient) CallContract(ctx context.Context, a TxAttempt, blockNumbe func (c *evmTxmClient) HeadByHash(ctx context.Context, hash common.Hash) (*evmtypes.Head, error) { return c.client.HeadByHash(ctx, hash) } + +func (c *evmTxmClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { + return c.client.BatchCallContext(ctx, b) +} diff --git a/core/chains/evm/txmgr/config.go b/core/chains/evm/txmgr/config.go index af20c9a5901..d79a4e0d8af 100644 --- a/core/chains/evm/txmgr/config.go +++ b/core/chains/evm/txmgr/config.go @@ -46,7 +46,6 @@ type ( EvmTxmConfig txmgrtypes.TransactionManagerChainConfig EvmTxmFeeConfig txmgrtypes.TransactionManagerFeeConfig EvmBroadcasterConfig txmgrtypes.BroadcasterChainConfig - EvmConfirmerConfig txmgrtypes.ConfirmerChainConfig EvmResenderConfig txmgrtypes.ResenderChainConfig ) diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index d468d1b4c10..ea251971860 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -2,7 +2,6 @@ package txmgr_test import ( "context" - "encoding/json" "errors" "fmt" "math/big" @@ -14,9 +13,7 @@ import ( pkgerrors "github.com/pkg/errors" gethCommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rpc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -63,12 +60,6 @@ func newBroadcastLegacyEthTxAttempt(t *testing.T, etxID int64, gasPrice ...int64 return attempt } -func mustTxBeInState(t *testing.T, txStore txmgr.TestEvmTxStore, tx txmgr.Tx, expectedState txmgrtypes.TxState) { - etx, err := txStore.FindTxWithAttempts(tests.Context(t), tx.ID) - require.NoError(t, err) - require.Equal(t, expectedState, etx.State) -} - func newTxReceipt(hash gethCommon.Hash, blockNumber int, txIndex uint) evmtypes.Receipt { return evmtypes.Receipt{ TxHash: hash, @@ -133,7 +124,7 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ethKeyStore, feeEstimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), config.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) ctx := tests.Context(t) // Can't close unstarted instance @@ -166,8 +157,7 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { } head.Parent.Store(h9) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once() - ethClient.On("LatestFinalizedBlock", mock.Anything).Return(latestFinalizedHead, nil).Once() + ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), nil) err = ec.ProcessHead(ctx, head) require.NoError(t, err) @@ -193,1155 +183,233 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { require.NoError(t, ec.XXXTestCloseInternal()) } -func TestEthConfirmer_CheckForReceipts(t *testing.T) { +func TestEthConfirmer_CheckForConfirmation(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) - gconfig, config := newTestChainScopedConfig(t) + cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.EVM[0].GasEstimator.PriceMax = assets.GWei(500) + }) txStore := cltest.NewTestTxStore(t, db) - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) - - ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) + evmcfg := evmtest.NewChainScopedConfig(t, cfg) - nonce := int64(0) ctx := tests.Context(t) - blockNum := int64(0) - latestFinalizedBlockNum := int64(0) - - t.Run("only finds eth_txes in unconfirmed state with at least one broadcast attempt", func(t *testing.T) { - mustInsertFatalErrorEthTx(t, txStore, fromAddress) - mustInsertInProgressEthTx(t, txStore, nonce, fromAddress) - nonce++ - cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, nonce, 1, fromAddress) - nonce++ - mustInsertUnconfirmedEthTxWithInsufficientEthAttempt(t, txStore, nonce, fromAddress) - nonce++ - mustCreateUnstartedGeneratedTx(t, txStore, fromAddress, config.EVM().ChainID()) - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - }) + blockNum := int64(100) + head := evmtypes.Head{ + Hash: testutils.NewHash(), + Number: blockNum, + } + head.IsFinalized.Store(true) - etx1 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - nonce++ - require.Len(t, etx1.TxAttempts, 1) - attempt1_1 := etx1.TxAttempts[0] - hashAttempt1_1 := attempt1_1.Hash - require.Len(t, attempt1_1.Receipts, 0) - - t.Run("fetches receipt for one unconfirmed eth_tx", func(t *testing.T) { - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - // Transaction not confirmed yet, receipt is nil - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], hashAttempt1_1, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &evmtypes.Receipt{} - }).Once() + t.Run("does nothing if no re-org'd or included transactions found", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx1 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + etx2 := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 4, fromAddress, 1, blockNum, assets.NewWeiI(1)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(1), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) var err error etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - assert.NoError(t, err) - require.Len(t, etx1.TxAttempts, 1) - attempt1_1 = etx1.TxAttempts[0] - require.NoError(t, err) - require.Len(t, attempt1_1.Receipts, 0) - }) - - t.Run("saves nothing if returned receipt does not match the attempt", func(t *testing.T) { - txmReceipt := evmtypes.Receipt{ - TxHash: testutils.NewHash(), - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - // First transaction confirmed - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], hashAttempt1_1, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - }).Once() - - // No error because it is merely logged - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) require.NoError(t, err) - require.Len(t, etx.TxAttempts, 1) - - require.Len(t, etx.TxAttempts[0].Receipts, 0) - }) - - t.Run("saves nothing if query returns error", func(t *testing.T) { - txmReceipt := evmtypes.Receipt{ - TxHash: attempt1_1.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - } + require.Equal(t, txmgrcommon.TxConfirmed, etx1.State) - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - // First transaction confirmed - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], hashAttempt1_1, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - elems[0].Error = errors.New("foo") - }).Once() - - // No error because it is merely logged - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) + etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) require.NoError(t, err) - require.Len(t, etx.TxAttempts, 1) - require.Len(t, etx.TxAttempts[0].Receipts, 0) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) }) - etx2 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - nonce++ - require.Len(t, etx2.TxAttempts, 1) - attempt2_1 := etx2.TxAttempts[0] - require.Len(t, attempt2_1.Receipts, 0) - - t.Run("saves eth_receipt and marks eth_tx as confirmed when geth client returns valid receipt", func(t *testing.T) { - txmReceipt := evmtypes.Receipt{ - TxHash: attempt1_1.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempt1_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt2_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction confirmed - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - // Second transaction still unconfirmed - elems[1].Result = &evmtypes.Receipt{} - }).Once() - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - // Check that the receipt was saved - etx, err := txStore.FindTxWithAttempts(ctx, etx1.ID) - require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) - assert.Len(t, etx.TxAttempts, 1) - attempt1_1 = etx.TxAttempts[0] - require.Len(t, attempt1_1.Receipts, 1) - - ethReceipt := attempt1_1.Receipts[0] - - assert.Equal(t, txmReceipt.TxHash, ethReceipt.GetTxHash()) - assert.Equal(t, txmReceipt.BlockHash, ethReceipt.GetBlockHash()) - assert.Equal(t, txmReceipt.BlockNumber.Int64(), ethReceipt.GetBlockNumber().Int64()) - assert.Equal(t, txmReceipt.TransactionIndex, ethReceipt.GetTransactionIndex()) + t.Run("marks re-org'd confirmed transaction as unconfirmed, marks latest attempt as in-progress, deletes receipt", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + // Insert confirmed transaction that stays confirmed + etx := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - receiptJSON, err := json.Marshal(txmReceipt) - require.NoError(t, err) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(0), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) - j, err := json.Marshal(ethReceipt) + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - assert.JSONEq(t, string(receiptJSON), string(j)) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) + attempt := etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptInProgress, attempt.State) + require.Empty(t, attempt.Receipts) }) - t.Run("fetches and saves receipts for several attempts in gas price order", func(t *testing.T) { - attempt2_2 := newBroadcastLegacyEthTxAttempt(t, etx2.ID) - attempt2_2.TxFee = gas.EvmFee{GasPrice: assets.NewWeiI(10)} - - attempt2_3 := newBroadcastLegacyEthTxAttempt(t, etx2.ID) - attempt2_3.TxFee = gas.EvmFee{GasPrice: assets.NewWeiI(20)} - - // Insert order deliberately reversed to test sorting by gas price - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2_3)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2_2)) - - txmReceipt := evmtypes.Receipt{ - TxHash: attempt2_2.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 3 && - cltest.BatchElemMatchesParams(b[2], attempt2_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt2_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[0], attempt2_3.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // Most expensive attempt still unconfirmed - elems[2].Result = &evmtypes.Receipt{} - // Second most expensive attempt is confirmed - *(elems[1].Result.(*evmtypes.Receipt)) = txmReceipt - // Cheapest attempt still unconfirmed - elems[0].Result = &evmtypes.Receipt{} - }).Once() + t.Run("marks re-org'd terminally stuck transaction as unconfirmed, marks latest attempt as in-progress, deletes receipt, removed error", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + // Insert terminally stuck transaction that stays fatal error + etx := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 0, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), etx.TxAttempts[0].Hash) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(0), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) - // Check that the state was updated - etx, err := txStore.FindTxWithAttempts(ctx, etx2.ID) + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - require.Equal(t, txmgrcommon.TxConfirmed, etx.State) - require.Len(t, etx.TxAttempts, 3) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) + require.Equal(t, "", etx.Error.String) + attempt := etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptInProgress, attempt.State) + require.Empty(t, attempt.Receipts) }) - etx3 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - attempt3_1 := etx3.TxAttempts[0] - nonce++ - - t.Run("ignores receipt missing BlockHash that comes from querying parity too early", func(t *testing.T) { - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - receipt := evmtypes.Receipt{ - TxHash: attempt3_1.Hash, - Status: uint64(1), - } - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt3_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = receipt - }).Once() + t.Run("handles multiple re-org transactions at a time", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + // Insert confirmed transaction that stays confirmed + etx1 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + // Insert terminally stuck transaction that stays fatal error + etx2 := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), etx2.TxAttempts[0].Hash) + // Insert confirmed transaction that gets re-org'd + etx3 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 2, blockNum) + // Insert terminally stuck transaction that gets re-org'd + etx4 := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 3, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), etx4.TxAttempts[0].Hash) + // Insert unconfirmed transaction that is untouched + etx5 := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 4, fromAddress, 1, blockNum, assets.NewWeiI(1)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(2), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) - // No receipt, but no error either - etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) + var err error + etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxConfirmed, etx1.State) + attempt1 := etx1.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1.State) + require.Len(t, attempt1.Receipts, 1) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - assert.Len(t, etx.TxAttempts, 1) - attempt3_1 = etx.TxAttempts[0] - require.Len(t, attempt3_1.Receipts, 0) - }) - - t.Run("does not panic if receipt has BlockHash but is missing some other fields somehow", func(t *testing.T) { - // NOTE: This should never happen, but we shouldn't panic regardless - receipt := evmtypes.Receipt{ - TxHash: attempt3_1.Hash, - BlockHash: testutils.NewHash(), - Status: uint64(1), - } - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt3_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = receipt - }).Once() - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - // No receipt, but no error either - etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) + etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx2.State) + require.Equal(t, client.TerminallyStuckMsg, etx2.Error.String) + attempt2 := etx2.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt2.State) + require.Len(t, attempt2.Receipts, 1) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - assert.Len(t, etx.TxAttempts, 1) - attempt3_1 = etx.TxAttempts[0] - require.Len(t, attempt3_1.Receipts, 0) - }) - t.Run("handles case where eth_receipt already exists somehow", func(t *testing.T) { - ethReceipt := mustInsertEthReceipt(t, txStore, 42, testutils.NewHash(), attempt3_1.Hash) - txmReceipt := evmtypes.Receipt{ - TxHash: attempt3_1.Hash, - BlockHash: ethReceipt.BlockHash, - BlockNumber: big.NewInt(ethReceipt.BlockNumber), - TransactionIndex: ethReceipt.TransactionIndex, - Status: uint64(1), - } - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt3_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - }).Once() - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - // Check that the receipt was unchanged - etx, err := txStore.FindTxWithAttempts(ctx, etx3.ID) + etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) + attempt3 := etx3.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptInProgress, attempt3.State) + require.Empty(t, attempt3.Receipts) - assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) - assert.Len(t, etx.TxAttempts, 1) - attempt3_1 = etx.TxAttempts[0] - require.Len(t, attempt3_1.Receipts, 1) - - ethReceipt3_1 := attempt3_1.Receipts[0] - - assert.Equal(t, txmReceipt.TxHash, ethReceipt3_1.GetTxHash()) - assert.Equal(t, txmReceipt.BlockHash, ethReceipt3_1.GetBlockHash()) - assert.Equal(t, txmReceipt.BlockNumber.Int64(), ethReceipt3_1.GetBlockNumber().Int64()) - assert.Equal(t, txmReceipt.TransactionIndex, ethReceipt3_1.GetTransactionIndex()) - }) - - etx4 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - attempt4_1 := etx4.TxAttempts[0] - nonce++ - - t.Run("on receipt fetch marks in_progress eth_tx_attempt as broadcast", func(t *testing.T) { - attempt4_2 := newInProgressLegacyEthTxAttempt(t, etx4.ID) - attempt4_2.TxFee = gas.EvmFee{GasPrice: assets.NewWeiI(10)} - - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt4_2)) - - txmReceipt := evmtypes.Receipt{ - TxHash: attempt4_2.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - // Second attempt is confirmed - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempt4_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt4_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First attempt still unconfirmed - elems[1].Result = &evmtypes.Receipt{} - // Second attempt is confirmed - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - }).Once() - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - // Check that the state was updated - var err error etx4, err = txStore.FindTxWithAttempts(ctx, etx4.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx4.State) + require.Equal(t, "", etx4.Error.String) + attempt4 := etx4.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptInProgress, attempt4.State) + require.True(t, attempt4.IsPurgeAttempt) + require.Empty(t, attempt4.Receipts) - attempt4_1 = etx4.TxAttempts[1] - attempt4_2 = etx4.TxAttempts[0] - - // And the attempts - require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt4_1.State) - require.Nil(t, attempt4_1.BroadcastBeforeBlockNum) - require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt4_2.State) - require.Equal(t, int64(42), *attempt4_2.BroadcastBeforeBlockNum) - - // Check receipts - require.Len(t, attempt4_1.Receipts, 0) - require.Len(t, attempt4_2.Receipts, 1) - }) - - etx5 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - attempt5_1 := etx5.TxAttempts[0] - nonce++ - - t.Run("simulate on revert", func(t *testing.T) { - txmReceipt := evmtypes.Receipt{ - TxHash: attempt5_1.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(0), - } - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - // First attempt is confirmed and reverted - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && - cltest.BatchElemMatchesParams(b[0], attempt5_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First attempt still unconfirmed - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt - }).Once() - data, err := utils.ABIEncode(`[{"type":"uint256"}]`, big.NewInt(10)) - require.NoError(t, err) - sig := utils.Keccak256Fixed([]byte(`MyError(uint256)`)) - ethClient.On("CallContract", mock.Anything, mock.Anything, mock.Anything).Return(nil, &client.JsonError{ - Code: 1, - Message: "reverted", - Data: utils.ConcatBytes(sig[:4], data), - }).Once() - - // Do the thing - require.NoError(t, ec.CheckForReceipts(ctx, blockNum, latestFinalizedBlockNum)) - - // Check that the state was updated etx5, err = txStore.FindTxWithAttempts(ctx, etx5.ID) require.NoError(t, err) - - attempt5_1 = etx5.TxAttempts[0] - - // And the attempts - require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt5_1.State) - require.NotNil(t, attempt5_1.BroadcastBeforeBlockNum) - // Check receipts - require.Len(t, attempt5_1.Receipts, 1) - }) -} - -func TestEthConfirmer_CheckForReceipts_batching(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].RPCDefaultBatchSize = ptr[uint32](2) - }) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - - etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) - var attempts []txmgr.TxAttempt - latestFinalizedBlockNum := int64(0) - - // Total of 5 attempts should lead to 3 batched fetches (2, 2, 1) - for i := 0; i < 5; i++ { - attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, int64(i+2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) - attempts = append(attempts, attempt) - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempts[4].Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempts[3].Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - }).Once() - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempts[2].Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempts[1].Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - }).Once() - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && - cltest.BatchElemMatchesParams(b[0], attempts[0].Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - elems[0].Result = &evmtypes.Receipt{} - }).Once() - - require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) -} - -func TestEthConfirmer_CheckForReceipts_HandlesNonFwdTxsWithForwardingEnabled(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].RPCDefaultBatchSize = ptr[uint32](1) - c.EVM[0].Transactions.ForwardersEnabled = ptr(true) - }) - - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - latestFinalizedBlockNum := int64(0) - - // tx is not forwarded and doesn't have meta set. EthConfirmer should handle nil meta values - etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) - attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 2) - attempt.Tx.Meta = nil - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) - dbtx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - require.Equal(t, 0, len(dbtx.TxAttempts[0].Receipts)) - - txmReceipt := evmtypes.Receipt{ - TxHash: attempt.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && - cltest.BatchElemMatchesParams(b[0], attempt.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt // confirmed - }).Once() - - require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) - - // Check receipt is inserted correctly. - dbtx, err = txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - require.Equal(t, 1, len(dbtx.TxAttempts[0].Receipts)) -} - -func TestEthConfirmer_CheckForReceipts_only_likely_confirmed(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].RPCDefaultBatchSize = ptr[uint32](6) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx5.State) + attempt5 := etx5.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt5.State) }) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - latestFinalizedBlockNum := int64(0) - - var attempts []txmgr.TxAttempt - // inserting in DESC nonce order to test DB ASC ordering - etx2 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 1, fromAddress) - for i := 0; i < 4; i++ { - attempt := newBroadcastLegacyEthTxAttempt(t, etx2.ID, int64(100-i)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) - } - etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) - for i := 0; i < 4; i++ { - attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, int64(100-i)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) - - // only adding these because a batch for only those attempts should be sent - attempts = append(attempts, attempt) - } - - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), nil) - - var captured []rpc.BatchElem - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 4 - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - captured = append(captured, elems...) - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - elems[2].Result = &evmtypes.Receipt{} - elems[3].Result = &evmtypes.Receipt{} - }).Once() - - require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) - - cltest.BatchElemMustMatchParams(t, captured[0], attempts[0].Hash, "eth_getTransactionReceipt") - cltest.BatchElemMustMatchParams(t, captured[1], attempts[1].Hash, "eth_getTransactionReceipt") - cltest.BatchElemMustMatchParams(t, captured[2], attempts[2].Hash, "eth_getTransactionReceipt") - cltest.BatchElemMustMatchParams(t, captured[3], attempts[3].Hash, "eth_getTransactionReceipt") -} - -func TestEthConfirmer_CheckForReceipts_should_not_check_for_likely_unconfirmed(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - gconfig, config := newTestChainScopedConfig(t) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) - ctx := tests.Context(t) - latestFinalizedBlockNum := int64(0) - - etx := cltest.MustInsertUnconfirmedEthTx(t, txStore, 1, fromAddress) - for i := 0; i < 4; i++ { - attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, int64(100-i)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) - } - - // latest nonce is lower that all attempts' nonces - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), nil) - - require.NoError(t, ec.CheckForReceipts(ctx, 42, latestFinalizedBlockNum)) -} - -func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt_scoped_to_key(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewTestGeneralConfig(t) - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - _, fromAddress1_1 := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - _, fromAddress1_2 := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - _, fromAddress2_1 := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(20), nil) - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - latestFinalizedBlockNum := int64(0) - - // STATE - // key 1, tx with nonce 0 is unconfirmed - // key 1, tx with nonce 1 is unconfirmed - // key 2, tx with nonce 9 is unconfirmed and gets a receipt in block 10 - etx1_0 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress1_1) - etx1_1 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 1, fromAddress1_1) - etx2_9 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 3, fromAddress1_2) - // there also happens to be a confirmed tx with a higher nonce from a different chain in the DB - etx_other_chain := cltest.MustInsertUnconfirmedEthTx(t, txStore, 8, fromAddress2_1) - pgtest.MustExec(t, db, `UPDATE evm.txes SET state='confirmed' WHERE id = $1`, etx_other_chain.ID) - - attempt2_9 := newBroadcastLegacyEthTxAttempt(t, etx2_9.ID, int64(1)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2_9)) - txmReceipt2_9 := newTxReceipt(attempt2_9.Hash, 10, 1) - - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt2_9.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt2_9 - }).Once() - - require.NoError(t, ec.CheckForReceipts(ctx, 10, latestFinalizedBlockNum)) - - mustTxBeInState(t, txStore, etx1_0, txmgrcommon.TxUnconfirmed) - mustTxBeInState(t, txStore, etx1_1, txmgrcommon.TxUnconfirmed) - mustTxBeInState(t, txStore, etx2_9, txmgrcommon.TxConfirmed) - - // Now etx1_1 gets a receipt in block 11, which should mark etx1_0 as confirmed_missing_receipt - attempt1_1 := newBroadcastLegacyEthTxAttempt(t, etx1_1.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_1)) - txmReceipt1_1 := newTxReceipt(attempt1_1.Hash, 11, 1) - - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt1_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt1_1 - }).Once() - - require.NoError(t, ec.CheckForReceipts(ctx, 11, latestFinalizedBlockNum)) - - mustTxBeInState(t, txStore, etx1_0, txmgrcommon.TxConfirmedMissingReceipt) - mustTxBeInState(t, txStore, etx1_1, txmgrcommon.TxConfirmed) - mustTxBeInState(t, txStore, etx2_9, txmgrcommon.TxConfirmed) -} - -func TestEthConfirmer_CheckForReceipts_confirmed_missing_receipt(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - latestFinalizedBlockNum := int64(0) - - // STATE - // eth_txes with nonce 0 has two attempts (broadcast before block 21 and 41) the first of which will get a receipt - // eth_txes with nonce 1 has two attempts (broadcast before block 21 and 41) neither of which will ever get a receipt - // eth_txes with nonce 2 has an attempt (broadcast before block 41) that will not get a receipt on the first try but will get one later - // eth_txes with nonce 3 has an attempt (broadcast before block 41) that has been confirmed in block 42 - // All other attempts were broadcast before block 41 - b := int64(21) - - etx0 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) - attempt0_1 := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(1)) - attempt0_2 := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(2)) - attempt0_2.BroadcastBeforeBlockNum = &b - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt0_1)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt0_2)) - - etx1 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 1, fromAddress) - attempt1_1 := newBroadcastLegacyEthTxAttempt(t, etx1.ID, int64(1)) - attempt1_2 := newBroadcastLegacyEthTxAttempt(t, etx1.ID, int64(2)) - attempt1_2.BroadcastBeforeBlockNum = &b - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_1)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_2)) - - etx2 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 2, fromAddress) - attempt2_1 := newBroadcastLegacyEthTxAttempt(t, etx2.ID, int64(1)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2_1)) - - etx3 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 3, fromAddress) - attempt3_1 := newBroadcastLegacyEthTxAttempt(t, etx3.ID, int64(1)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt3_1)) + t.Run("marks valid transaction as confirmed if nonce less than mined tx count", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, blockNum, assets.NewWeiI(1)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - pgtest.MustExec(t, db, `UPDATE evm.tx_attempts SET broadcast_before_block_num = 41 WHERE broadcast_before_block_num IS NULL`) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(1), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) - t.Run("marks buried eth_txes as 'confirmed_missing_receipt'", func(t *testing.T) { - txmReceipt0 := evmtypes.Receipt{ - TxHash: attempt0_2.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - txmReceipt3 := evmtypes.Receipt{ - TxHash: attempt3_1.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(42), - TransactionIndex: uint(1), - Status: uint64(1), - } - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(4), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 6 && - cltest.BatchElemMatchesParams(b[0], attempt0_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt0_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[2], attempt1_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[3], attempt1_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[4], attempt2_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[5], attempt3_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction confirmed - *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt0 - elems[1].Result = &evmtypes.Receipt{} - // Second transaction stil unconfirmed - elems[2].Result = &evmtypes.Receipt{} - elems[3].Result = &evmtypes.Receipt{} - // Third transaction still unconfirmed - elems[4].Result = &evmtypes.Receipt{} - // Fourth transaction is confirmed - *(elems[5].Result.(*evmtypes.Receipt)) = txmReceipt3 - }).Once() - - // PERFORM - // Block num of 43 is one higher than the receipt (as would generally be expected) - require.NoError(t, ec.CheckForReceipts(ctx, 43, latestFinalizedBlockNum)) - - // Expected state is that the "top" eth_tx is now confirmed, with the - // two below it "confirmed_missing_receipt" and the "bottom" eth_tx also confirmed var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx3.State) + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + }) - ethReceipt := etx3.TxAttempts[0].Receipts[0] - require.Equal(t, txmReceipt3.BlockHash, ethReceipt.GetBlockHash()) + t.Run("marks purge transaction as terminally stuck if nonce less than mined tx count", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedEthTxWithBroadcastPurgeAttempt(t, txStore, 0, fromAddress) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx2.State) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx1.State) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(1), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx0.State) - - require.Len(t, etx0.TxAttempts, 2) - require.Len(t, etx0.TxAttempts[0].Receipts, 1) - ethReceipt = etx0.TxAttempts[0].Receipts[0] - require.Equal(t, txmReceipt0.BlockHash, ethReceipt.GetBlockHash()) + require.Equal(t, txmgrcommon.TxFatalError, etx.State) + require.Equal(t, client.TerminallyStuckMsg, etx.Error.String) }) - // STATE - // eth_txes with nonce 0 is confirmed - // eth_txes with nonce 1 is confirmed_missing_receipt - // eth_txes with nonce 2 is confirmed_missing_receipt - // eth_txes with nonce 3 is confirmed - - t.Run("marks eth_txes with state 'confirmed_missing_receipt' as 'confirmed' if a receipt finally shows up", func(t *testing.T) { - txmReceipt := evmtypes.Receipt{ - TxHash: attempt2_1.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(43), - TransactionIndex: uint(1), - Status: uint64(1), - } - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 3 && - cltest.BatchElemMatchesParams(b[0], attempt1_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt1_1.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[2], attempt2_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction still unconfirmed - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - // Second transaction confirmed - *(elems[2].Result.(*evmtypes.Receipt)) = txmReceipt - }).Once() - - // PERFORM - // Block num of 44 is one higher than the receipt (as would generally be expected) - require.NoError(t, ec.CheckForReceipts(ctx, 44, latestFinalizedBlockNum)) - - // Expected state is that the "top" two eth_txes are now confirmed, with the - // one below it still "confirmed_missing_receipt" and the bottom one remains confirmed - var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx3.State) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx2.State) + t.Run("handles multiple confirmed transactions at a time", func(t *testing.T) { + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + // Insert valid confirmed transaction that is untouched + etx1 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + // Insert terminally stuck transaction that is untouched + etx2 := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), etx2.TxAttempts[0].Hash) + // Insert valid unconfirmed transaction that is confirmed + etx3 := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 2, fromAddress, 1, blockNum, assets.NewWeiI(1)) + // Insert unconfirmed purge transaction that is confirmed and marked as terminally stuck + etx4 := mustInsertUnconfirmedEthTxWithBroadcastPurgeAttempt(t, txStore, 3, fromAddress) + // Insert unconfirmed transact that is not confirmed and left untouched + etx5 := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 4, fromAddress, 1, blockNum, assets.NewWeiI(1)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethReceipt := etx2.TxAttempts[0].Receipts[0] - require.Equal(t, txmReceipt.BlockHash, ethReceipt.GetBlockHash()) + ethClient.On("NonceAt", mock.Anything, fromAddress, mock.Anything).Return(uint64(4), nil).Maybe() + require.NoError(t, ec.CheckForConfirmation(ctx, &head)) + var err error etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx1.State) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx0.State) - }) + require.Equal(t, txmgrcommon.TxConfirmed, etx1.State) + attempt1 := etx1.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1.State) + require.Len(t, attempt1.Receipts, 1) - // STATE - // eth_txes with nonce 0 is confirmed - // eth_txes with nonce 1 is confirmed_missing_receipt - // eth_txes with nonce 2 is confirmed - // eth_txes with nonce 3 is confirmed - - t.Run("continues to leave eth_txes with state 'confirmed_missing_receipt' unchanged if at least one attempt is above LatestFinalizedBlockNum", func(t *testing.T) { - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempt1_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt1_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // Both attempts still unconfirmed - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - }).Once() - - latestFinalizedBlockNum = 30 - - // PERFORM - // Block num of 80 puts the first attempt (21) below threshold but second attempt (41) still above - require.NoError(t, ec.CheckForReceipts(ctx, 80, latestFinalizedBlockNum)) - - // Expected state is that the "top" two eth_txes are now confirmed, with the - // one below it still "confirmed_missing_receipt" and the bottom one remains confirmed - var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx3.State) etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx2.State) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx1.State) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx0.State) - }) + require.Equal(t, txmgrcommon.TxFatalError, etx2.State) + require.Equal(t, client.TerminallyStuckMsg, etx2.Error.String) + attempt2 := etx2.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt2.State) + require.Len(t, attempt2.Receipts, 1) - // STATE - // eth_txes with nonce 0 is confirmed - // eth_txes with nonce 1 is confirmed_missing_receipt - // eth_txes with nonce 2 is confirmed - // eth_txes with nonce 3 is confirmed - - t.Run("marks eth_Txes with state 'confirmed_missing_receipt' as 'errored' if a receipt fails to show up and all attempts are buried deeper than LatestFinalizedBlockNum", func(t *testing.T) { - ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(10), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 2 && - cltest.BatchElemMatchesParams(b[0], attempt1_2.Hash, "eth_getTransactionReceipt") && - cltest.BatchElemMatchesParams(b[1], attempt1_1.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // Both attempts still unconfirmed - elems[0].Result = &evmtypes.Receipt{} - elems[1].Result = &evmtypes.Receipt{} - }).Once() - - latestFinalizedBlockNum = 50 - - // PERFORM - // Block num of 100 puts the first attempt (21) and second attempt (41) below threshold - require.NoError(t, ec.CheckForReceipts(ctx, 100, latestFinalizedBlockNum)) - - // Expected state is that the "top" two eth_txes are now confirmed, with the - // one below it marked as "fatal_error" and the bottom one remains confirmed - var err error etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) require.NoError(t, err) require.Equal(t, txmgrcommon.TxConfirmed, etx3.State) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx2.State) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxFatalError, etx1.State) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - require.NoError(t, err) - require.Equal(t, txmgrcommon.TxConfirmed, etx0.State) - }) -} - -func TestEthConfirmer_CheckConfirmedMissingReceipt(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() + attempt3 := etx3.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt3.State) + require.Empty(t, attempt3.Receipts) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethClient.On("IsL2").Return(false).Maybe() - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - - // STATE - // eth_txes with nonce 0 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 1 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 2 has one attempt - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - attempt0_2 := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt0_2)) - etx1 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 1, 1, originalBroadcastAt, fromAddress) - attempt1_2 := newBroadcastLegacyEthTxAttempt(t, etx1.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_2)) - etx2 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 2, 1, originalBroadcastAt, fromAddress) - attempt2_1 := etx2.TxAttempts[0] - etx3 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 3, 1, originalBroadcastAt, fromAddress) - attempt3_1 := etx3.TxAttempts[0] - - ethClient.On("BatchCallContextAll", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 4 && - cltest.BatchElemMatchesParams(b[0], hexutil.Encode(attempt0_2.SignedRawTx), "eth_sendRawTransaction") && - cltest.BatchElemMatchesParams(b[1], hexutil.Encode(attempt1_2.SignedRawTx), "eth_sendRawTransaction") && - cltest.BatchElemMatchesParams(b[2], hexutil.Encode(attempt2_1.SignedRawTx), "eth_sendRawTransaction") && - cltest.BatchElemMatchesParams(b[3], hexutil.Encode(attempt3_1.SignedRawTx), "eth_sendRawTransaction") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction confirmed - elems[0].Error = errors.New("nonce too low") - elems[1].Error = errors.New("transaction underpriced") - elems[2].Error = nil - elems[3].Error = errors.New("transaction already finalized") - }).Once() - - // PERFORM - require.NoError(t, ec.CheckConfirmedMissingReceipt(ctx)) - - // Expected state is that the "top" eth_tx is untouched but the other two - // are marked as unconfirmed - var err error - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx0.State) - assert.Greater(t, etx0.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx1.State) - assert.Greater(t, etx1.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) - assert.Greater(t, etx2.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx3.State) - assert.Greater(t, etx3.BroadcastAt.Unix(), originalBroadcastAt.Unix()) -} - -func TestEthConfirmer_CheckConfirmedMissingReceipt_batchSendTransactions_fails(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) {}) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethClient.On("IsL2").Return(false).Maybe() - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) - - // STATE - // eth_txes with nonce 0 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 1 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 2 has one attempt - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - attempt0_2 := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt0_2)) - etx1 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 1, 1, originalBroadcastAt, fromAddress) - attempt1_2 := newBroadcastLegacyEthTxAttempt(t, etx1.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_2)) - etx2 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 2, 1, originalBroadcastAt, fromAddress) - attempt2_1 := etx2.TxAttempts[0] - - ethClient.On("BatchCallContextAll", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 3 && - cltest.BatchElemMatchesParams(b[0], hexutil.Encode(attempt0_2.SignedRawTx), "eth_sendRawTransaction") && - cltest.BatchElemMatchesParams(b[1], hexutil.Encode(attempt1_2.SignedRawTx), "eth_sendRawTransaction") && - cltest.BatchElemMatchesParams(b[2], hexutil.Encode(attempt2_1.SignedRawTx), "eth_sendRawTransaction") - })).Return(errors.New("Timed out")).Once() - - // PERFORM - require.NoError(t, ec.CheckConfirmedMissingReceipt(ctx)) - - // Expected state is that all txes are marked as unconfirmed, since the batch call had failed - var err error - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx0.State) - assert.Equal(t, etx0.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx1.State) - assert.Equal(t, etx1.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) - assert.Equal(t, etx2.BroadcastAt.Unix(), originalBroadcastAt.Unix()) -} - -func TestEthConfirmer_CheckConfirmedMissingReceipt_smallEvmRPCBatchSize_middleBatchSendTransactionFails(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].RPCDefaultBatchSize = ptr[uint32](1) - }) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethClient.On("IsL2").Return(false).Maybe() - - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ctx := tests.Context(t) + etx4, err = txStore.FindTxWithAttempts(ctx, etx4.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx4.State) + require.Equal(t, client.TerminallyStuckMsg, etx4.Error.String) + attempt4 := etx4.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt4.State) + require.True(t, attempt4.IsPurgeAttempt) + require.Empty(t, attempt4.Receipts) - // STATE - // eth_txes with nonce 0 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 1 has two attempts, the later attempt with higher gas fees - // eth_txes with nonce 2 has one attempt - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - attempt0_2 := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt0_2)) - etx1 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 1, 1, originalBroadcastAt, fromAddress) - attempt1_2 := newBroadcastLegacyEthTxAttempt(t, etx1.ID, int64(2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt1_2)) - etx2 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 2, 1, originalBroadcastAt, fromAddress) - - // Expect eth_sendRawTransaction in 3 batches. First batch will pass, 2nd will fail, 3rd never attempted. - ethClient.On("BatchCallContextAll", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && - cltest.BatchElemMatchesParams(b[0], hexutil.Encode(attempt0_2.SignedRawTx), "eth_sendRawTransaction") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction confirmed - elems[0].Error = errors.New("nonce too low") - }).Once() - ethClient.On("BatchCallContextAll", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 1 && - cltest.BatchElemMatchesParams(b[0], hexutil.Encode(attempt1_2.SignedRawTx), "eth_sendRawTransaction") - })).Return(errors.New("Timed out")).Once() - - // PERFORM - require.NoError(t, ec.CheckConfirmedMissingReceipt(ctx)) - - // Expected state is that all transactions since failed batch will be unconfirmed - var err error - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx0.State) - assert.Greater(t, etx0.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx1.State) - assert.Equal(t, etx1.BroadcastAt.Unix(), originalBroadcastAt.Unix()) - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - assert.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) - assert.Equal(t, etx2.BroadcastAt.Unix(), originalBroadcastAt.Unix()) + etx5, err = txStore.FindTxWithAttempts(ctx, etx5.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx5.State) + attempt5 := etx5.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt5.State) + require.Empty(t, attempt3.Receipts) + }) } func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { @@ -1381,7 +449,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + require.Empty(t, etxs) }) mustInsertInProgressEthTx(t, txStore, nonce, fromAddress) @@ -1391,7 +459,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + require.Empty(t, etxs) }) // This one has BroadcastBeforeBlockNum set as nil... which can happen, but it should be ignored @@ -1402,7 +470,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + require.Empty(t, etxs) }) etx1 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) @@ -1420,7 +488,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + assert.Empty(t, etxs) }) etx2 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) @@ -1434,7 +502,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + assert.Empty(t, etxs) }) etxWithoutAttempts := cltest.NewEthTx(fromAddress) @@ -1453,7 +521,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmOtherAddress, currentHead, gasBumpThreshold, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - assert.Len(t, etxs, 0) + assert.Empty(t, etxs) }) t.Run("returns the transaction if it is unconfirmed and has no attempts (note that this is an invariant violation, but we handle it anyway)", func(t *testing.T) { @@ -1468,7 +536,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, gasBumpThreshold, 10, 0, big.NewInt(42)) require.NoError(t, err) - require.Len(t, etxs, 0) + require.Empty(t, etxs) }) etx3 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) @@ -1498,7 +566,7 @@ func TestEthConfirmer_FindTxsRequiringRebroadcast(t *testing.T) { etxs, err := ec.FindTxsRequiringRebroadcast(tests.Context(t), lggr, evmFromAddress, currentHead, 0, 10, 0, &cltest.FixtureChainID) require.NoError(t, err) - require.Len(t, etxs, 0) + require.Empty(t, etxs) }) t.Run("does not return more transactions for gas bumping than gasBumpThreshold", func(t *testing.T) { @@ -1669,7 +737,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), ccfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) // Create confirmer with necessary state - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) servicetest.Run(t, ec) currentHead := int64(30) oldEnough := int64(15) @@ -1718,7 +786,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing kst.On("EnabledAddressesForChain", mock.Anything, &cltest.FixtureChainID).Return(addresses, nil).Maybe() stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), ccfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), ccfg.EVM(), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmFeeConfig(ccfg.EVM().GasEstimator()), ccfg.EVM().Transactions(), cfg.Database(), kst, txBuilder, lggr, stuckTxDetector, ht) servicetest.Run(t, ec) currentHead := int64(30) oldEnough := int64(15) @@ -1793,7 +861,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_MaxFeeScenario(t *testing.T) { // Once for the bumped attempt which exceeds limit ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Nonce() == uint64(*etx.Sequence) && tx.GasPrice().Int64() == int64(20000000000) + return tx.GasPrice().Int64() == int64(20000000000) && tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.ExceedsMaxFee, errors.New("tx fee (1.10 ether) exceeds the configured cap (1.00 ether)")).Once() // Do the thing @@ -1814,51 +882,44 @@ func TestEthConfirmer_RebroadcastWhereNecessary_MaxFeeScenario(t *testing.T) { func TestEthConfirmer_RebroadcastWhereNecessary(t *testing.T) { t.Parallel() - db := pgtest.NewSqlxDB(t) cfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.EVM[0].GasEstimator.PriceMax = assets.GWei(500) + c.EVM[0].GasEstimator.BumpMin = assets.NewWeiI(0) }) - txStore := cltest.NewTestTxStore(t, db) ctx := tests.Context(t) - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - - _, _ = cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - kst := ksmocks.NewEth(t) - addresses := []gethCommon.Address{fromAddress} - kst.On("EnabledAddressesForChain", mock.Anything, &cltest.FixtureChainID).Return(addresses, nil).Maybe() - // Use a mock keystore for this test - ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, kst, nil) currentHead := int64(30) - oldEnough := int64(19) - nonce := int64(0) t.Run("does nothing if no transactions require bumping", func(t *testing.T) { - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - }) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - originalBroadcastAt := time.Unix(1616509100, 0) - etx := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress, originalBroadcastAt) - nonce++ - attempt1_1 := etx.TxAttempts[0] - var dbAttempt txmgr.DbEthTxAttempt - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt1_1.ID)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) + }) t.Run("re-sends previous transaction on keystore error", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, 25, assets.NewWeiI(100)) + kst := ksmocks.NewEth(t) + addresses := []gethCommon.Address{fromAddress} + kst.On("EnabledAddressesForChain", mock.Anything, &cltest.FixtureChainID).Return(addresses, nil).Maybe() // simulate bumped transaction that is somehow impossible to sign kst.On("SignTx", mock.Anything, fromAddress, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Nonce() == uint64(*etx.Sequence) + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), mock.Anything).Return(nil, errors.New("signing error")).Once() + // Use a mock keystore for this test + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, kst, nil) - // Do the thing - err := ec.RebroadcastWhereNecessary(tests.Context(t), currentHead) + err := ec.RebroadcastWhereNecessary(ctx, currentHead) require.Error(t, err) require.Contains(t, err.Error(), "signing error") @@ -1870,557 +931,364 @@ func TestEthConfirmer_RebroadcastWhereNecessary(t *testing.T) { }) t.Run("does nothing and continues on fatal error", func(t *testing.T) { - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if tx.Nonce() != uint64(*etx.Sequence) { - return false - } - ethTx = *tx - return true - }), - mock.MatchedBy(func(chainID *big.Int) bool { - return chainID.Cmp(evmcfg.EVM().ChainID()) == 0 - })).Return(ðTx, nil).Once() + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, 25, assets.NewWeiI(100)) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) + ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Nonce() == uint64(*etx.Sequence) + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Fatal, errors.New("exceeds block gas limit")).Once() - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - require.Len(t, etx.TxAttempts, 1) }) - var attempt1_2 txmgr.TxAttempt - ethClient = testutils.NewEthClientMockWithDefaultChain(t) - ec.XXXTestSetClient(txmgr.NewEvmTxmClient(ethClient, nil)) - t.Run("creates new attempt with higher gas price if transaction has an attempt older than threshold", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(20000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt1_1.TxFee.GasPrice.ToInt().Int64()) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestGasPrice := assets.GWei(20) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, 25, latestGasPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.MatchedBy(func(chainID *big.Int) bool { - return chainID.Cmp(evmcfg.EVM().ChainID()) == 0 - })).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Successful, nil).Once() - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) require.Len(t, etx.TxAttempts, 2) - require.Equal(t, attempt1_1.ID, etx.TxAttempts[1].ID) // Got the new attempt - attempt1_2 = etx.TxAttempts[0] - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt1_2.TxFee.GasPrice.ToInt().Int64()) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1_2.State) + bumpAttempt := etx.TxAttempts[0] + expectedBumpedGas := latestGasPrice.AddPercentage(evmcfg.EVM().GasEstimator().BumpPercent()) + require.Equal(t, expectedBumpedGas.Int64(), bumpAttempt.TxFee.GasPrice.Int64()) + require.Equal(t, txmgrtypes.TxAttemptBroadcast, bumpAttempt.State) }) t.Run("does nothing if there is an attempt without BroadcastBeforeBlockNum set", func(t *testing.T) { - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 0, fromAddress, txmgrtypes.TxAttemptBroadcast) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) + + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - require.Len(t, etx.TxAttempts, 2) + require.Len(t, etx.TxAttempts, 1) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt1_2.ID)) - var attempt1_3 txmgr.TxAttempt t.Run("creates new attempt with higher gas price if transaction is already in mempool (e.g. due to previous crash before we could save the new attempt)", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(25000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt1_2.TxFee.GasPrice.ToInt().Int64()) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestGasPrice := assets.GWei(20) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, 25, latestGasPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx.Sequence || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, fmt.Errorf("known transaction: %s", ethTx.Hash().Hex())).Once() + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Successful, fmt.Errorf("known transaction: %s", etx.TxAttempts[0].Hash.Hex())).Once() - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - require.Len(t, etx.TxAttempts, 3) - require.Equal(t, attempt1_1.ID, etx.TxAttempts[2].ID) - require.Equal(t, attempt1_2.ID, etx.TxAttempts[1].ID) + require.Len(t, etx.TxAttempts, 2) // Got the new attempt - attempt1_3 = etx.TxAttempts[0] - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt1_3.TxFee.GasPrice.ToInt().Int64()) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1_3.State) + bumpAttempt := etx.TxAttempts[0] + expectedBumpedGas := latestGasPrice.AddPercentage(evmcfg.EVM().GasEstimator().BumpPercent()) + require.Equal(t, expectedBumpedGas.Int64(), bumpAttempt.TxFee.GasPrice.Int64()) + require.Equal(t, txmgrtypes.TxAttemptBroadcast, bumpAttempt.State) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt1_3.ID)) - var attempt1_4 txmgr.TxAttempt - t.Run("saves new attempt even for transaction that has already been confirmed (nonce already used)", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(30000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt1_2.TxFee.GasPrice.ToInt().Int64()) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestGasPrice := assets.GWei(20) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, 25, latestGasPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - receipt := evmtypes.Receipt{BlockNumber: big.NewInt(40)} - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx.Sequence || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - receipt.TxHash = tx.Hash() - return true - }), - mock.Anything).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.TransactionAlreadyKnown, errors.New("nonce too low")).Once() - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx.State) - // Got the new attempt - attempt1_4 = etx.TxAttempts[0] - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt1_4.TxFee.GasPrice.ToInt().Int64()) - - require.Len(t, etx.TxAttempts, 4) - require.Equal(t, attempt1_1.ID, etx.TxAttempts[3].ID) - require.Equal(t, attempt1_2.ID, etx.TxAttempts[2].ID) - require.Equal(t, attempt1_3.ID, etx.TxAttempts[1].ID) - require.Equal(t, attempt1_4.ID, etx.TxAttempts[0].ID) + // Got the new attempt + bumpedAttempt := etx.TxAttempts[0] + expectedBumpedGas := latestGasPrice.AddPercentage(evmcfg.EVM().GasEstimator().BumpPercent()) + require.Equal(t, expectedBumpedGas.Int64(), bumpedAttempt.TxFee.GasPrice.Int64()) + + require.Len(t, etx.TxAttempts, 2) require.Equal(t, txmgrtypes.TxAttemptBroadcast, etx.TxAttempts[0].State) require.Equal(t, txmgrtypes.TxAttemptBroadcast, etx.TxAttempts[1].State) - require.Equal(t, txmgrtypes.TxAttemptBroadcast, etx.TxAttempts[2].State) - require.Equal(t, txmgrtypes.TxAttemptBroadcast, etx.TxAttempts[3].State) }) - // Mark original tx as confirmed, so we won't pick it up anymore - pgtest.MustExec(t, db, `UPDATE evm.txes SET state = 'confirmed'`) - - etx2 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - nonce++ - attempt2_1 := etx2.TxAttempts[0] - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt2_1.ID)) - var attempt2_2 txmgr.TxAttempt - t.Run("saves in-progress attempt on temporary error and returns error", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(20000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt2_1.TxFee.GasPrice.ToInt().Int64()) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestGasPrice := assets.GWei(20) + broadcastBlockNum := int64(25) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, broadcastBlockNum, latestGasPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - n := *etx2.Sequence - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != n || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == n && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Unknown, errors.New("some network error")).Once() - // Do the thing - err := ec.RebroadcastWhereNecessary(tests.Context(t), currentHead) + err := ec.RebroadcastWhereNecessary(ctx, currentHead) require.Error(t, err) require.Contains(t, err.Error(), "some network error") - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) // Old attempt is untouched - require.Len(t, etx2.TxAttempts, 2) - require.Equal(t, attempt2_1.ID, etx2.TxAttempts[1].ID) - attempt2_1 = etx2.TxAttempts[1] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt2_1.State) - assert.Equal(t, oldEnough, *attempt2_1.BroadcastBeforeBlockNum) + require.Len(t, etx.TxAttempts, 2) + originalAttempt := etx.TxAttempts[1] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, originalAttempt.State) + require.Equal(t, broadcastBlockNum, *originalAttempt.BroadcastBeforeBlockNum) // New in_progress attempt saved - attempt2_2 = etx2.TxAttempts[0] - assert.Equal(t, txmgrtypes.TxAttemptInProgress, attempt2_2.State) - assert.Nil(t, attempt2_2.BroadcastBeforeBlockNum) + bumpedAttempt := etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptInProgress, bumpedAttempt.State) + require.Nil(t, bumpedAttempt.BroadcastBeforeBlockNum) - // Do it again and move the attempt into "broadcast" - n = *etx2.Sequence + // Try again and move the attempt into "broadcast" ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == n && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Successful, nil).Once() - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - - // Attempt marked "broadcast" - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx2.State) - - // New in_progress attempt saved - require.Len(t, etx2.TxAttempts, 2) - require.Equal(t, attempt2_2.ID, etx2.TxAttempts[0].ID) - attempt2_2 = etx2.TxAttempts[0] - require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt2_2.State) - assert.Nil(t, attempt2_2.BroadcastBeforeBlockNum) - }) - - // Set BroadcastBeforeBlockNum again so the next test will pick it up - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt2_2.ID)) - - t.Run("assumes that 'nonce too low' error means confirmed_missing_receipt", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(25000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt2_1.TxFee.GasPrice.ToInt().Int64()) - - ethTx := *types.NewTx(&types.LegacyTx{}) - n := *etx2.Sequence - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != n || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() - ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == n && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.TransactionAlreadyKnown, errors.New("nonce too low")).Once() - - // Creates new attempt as normal if currentHead is not high enough - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx2.State) - - // One new attempt saved - require.Len(t, etx2.TxAttempts, 3) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx2.TxAttempts[0].State) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx2.TxAttempts[1].State) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx2.TxAttempts[2].State) - }) - - // Original tx is confirmed, so we won't pick it up anymore - etx3 := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, nonce, fromAddress) - nonce++ - attempt3_1 := etx3.TxAttempts[0] - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1, gas_price=$2 WHERE id=$3 RETURNING *`, oldEnough, assets.NewWeiI(35000000000), attempt3_1.ID)) - - var attempt3_2 txmgr.TxAttempt - - t.Run("saves attempt anyway if replacement transaction is underpriced because the bumped gas price is insufficiently higher than the previous one", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(42000000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt3_1.TxFee.GasPrice.ToInt().Int64()) - - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx3.Sequence || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() - ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx3.Sequence && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, errors.New("replacement transaction underpriced")).Once() + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) - - require.Len(t, etx3.TxAttempts, 2) - require.Equal(t, attempt3_1.ID, etx3.TxAttempts[1].ID) - attempt3_2 = etx3.TxAttempts[0] - - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt3_2.TxFee.GasPrice.ToInt().Int64()) + // New in_progress attempt saved and marked "broadcast" + require.Len(t, etx.TxAttempts, 2) + bumpedAttempt = etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, bumpedAttempt.State) + require.Nil(t, bumpedAttempt.BroadcastBeforeBlockNum) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt3_2.ID)) - var attempt3_3 txmgr.TxAttempt - - t.Run("handles case where transaction is already known somehow", func(t *testing.T) { - expectedBumpedGasPrice := big.NewInt(50400000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt3_1.TxFee.GasPrice.ToInt().Int64()) + t.Run("re-bumps attempt if initial bump is underpriced because the bumped gas price is insufficiently higher than the previous one", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + latestGasPrice := assets.GWei(20) + broadcastBlockNum := int64(25) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, broadcastBlockNum, latestGasPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, evmcfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx3.Sequence || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx3.Sequence && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, fmt.Errorf("known transaction: %s", ethTx.Hash().Hex())).Once() - - // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) - require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) - - require.Len(t, etx3.TxAttempts, 3) - attempt3_3 = etx3.TxAttempts[0] - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt3_3.TxFee.GasPrice.ToInt().Int64()) - }) - - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt3_3.ID)) - var attempt3_4 txmgr.TxAttempt - - t.Run("pretends it was accepted and continues the cycle if rejected for being temporarily underpriced", func(t *testing.T) { - // This happens if parity is rejecting transactions that are not priced high enough to even get into the mempool at all - // It should pretend it was accepted into the mempool and hand off to the next cycle to continue bumping gas as normal - temporarilyUnderpricedError := "There are too many transactions in the queue. Your transaction was dropped due to limit. Try increasing the fee." - - expectedBumpedGasPrice := big.NewInt(60480000000) - require.Greater(t, expectedBumpedGasPrice.Int64(), attempt3_2.TxFee.GasPrice.ToInt().Int64()) - - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx3.Sequence || expectedBumpedGasPrice.Cmp(tx.GasPrice()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Underpriced, errors.New("replacement transaction underpriced")).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx3.Sequence && expectedBumpedGasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, errors.New(temporarilyUnderpricedError)).Once() + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Successful, nil).Once() // Do the thing - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) - - require.Len(t, etx3.TxAttempts, 4) - attempt3_4 = etx3.TxAttempts[0] - assert.Equal(t, expectedBumpedGasPrice.Int64(), attempt3_4.TxFee.GasPrice.ToInt().Int64()) + require.Len(t, etx.TxAttempts, 2) + bumpedAttempt := etx.TxAttempts[0] + expectedBumpedGas := latestGasPrice.AddPercentage(evmcfg.EVM().GasEstimator().BumpPercent()) + expectedBumpedGas = expectedBumpedGas.AddPercentage(evmcfg.EVM().GasEstimator().BumpPercent()) + require.Equal(t, expectedBumpedGas.Int64(), bumpedAttempt.TxFee.GasPrice.Int64()) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt3_4.ID)) - t.Run("resubmits at the old price and does not create a new attempt if one of the bumped transactions would exceed EVM.GasEstimator.PriceMax", func(t *testing.T) { - // Set price such that the next bump will exceed EVM.GasEstimator.PriceMax - // Existing gas price is: 60480000000 - gasPrice := attempt3_4.TxFee.GasPrice.ToInt() + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + priceMax := assets.GWei(30) gcfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].GasEstimator.PriceMax = assets.NewWeiI(60500000000) + c.EVM[0].GasEstimator.PriceMax = priceMax }) newCfg := evmtest.NewChainScopedConfig(t, gcfg) - ec2 := newEthConfirmer(t, txStore, ethClient, gcfg, newCfg, ethKeyStore, nil) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + broadcastBlockNum := int64(25) + currentAttemptPrice := priceMax.Sub(assets.GWei(1)) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, broadcastBlockNum, currentAttemptPrice) + ec := newEthConfirmer(t, txStore, ethClient, cfg, newCfg, ethKeyStore, nil) ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx3.Sequence && gasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, errors.New("already known")).Once() // we already submitted at this price, now it's time to bump and submit again but since we simply resubmitted rather than increasing gas price, geth already knows about this tx + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Underpriced, errors.New("underpriced")).Once() // we already submitted at this price, now it's time to bump and submit again but since we simply resubmitted rather than increasing gas price, geth already knows about this tx // Do the thing - require.NoError(t, ec2.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.Error(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) // No new tx attempts - require.Len(t, etx3.TxAttempts, 4) - attempt3_4 = etx3.TxAttempts[0] - assert.Equal(t, gasPrice.Int64(), attempt3_4.TxFee.GasPrice.ToInt().Int64()) + require.Len(t, etx.TxAttempts, 1) + bumpedAttempt := etx.TxAttempts[0] + require.Equal(t, currentAttemptPrice.Int64(), bumpedAttempt.TxFee.GasPrice.Int64()) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1 WHERE id=$2 RETURNING *`, oldEnough, attempt3_4.ID)) - t.Run("resubmits at the old price and does not create a new attempt if the current price is exactly EVM.GasEstimator.PriceMax", func(t *testing.T) { - // Set price such that the current price is already at EVM.GasEstimator.PriceMax - // Existing gas price is: 60480000000 - gasPrice := attempt3_4.TxFee.GasPrice.ToInt() + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + priceMax := assets.GWei(30) gcfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].GasEstimator.PriceMax = assets.NewWeiI(60480000000) + c.EVM[0].GasEstimator.PriceMax = priceMax }) newCfg := evmtest.NewChainScopedConfig(t, gcfg) - ec2 := newEthConfirmer(t, txStore, ethClient, gcfg, newCfg, ethKeyStore, nil) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + broadcastBlockNum := int64(25) + etx := mustInsertUnconfirmedTxWithBroadcastAttempts(t, txStore, 0, fromAddress, 1, broadcastBlockNum, priceMax) + ec := newEthConfirmer(t, txStore, ethClient, cfg, newCfg, ethKeyStore, nil) ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx3.Sequence && gasPrice.Cmp(tx.GasPrice()) == 0 - }), fromAddress).Return(commonclient.Successful, errors.New("already known")).Once() // we already submitted at this price, now it's time to bump and submit again but since we simply resubmitted rather than increasing gas price, geth already knows about this tx + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Underpriced, errors.New("underpriced")).Once() // we already submitted at this price, now it's time to bump and submit again but since we simply resubmitted rather than increasing gas price, geth already knows about this tx // Do the thing - require.NoError(t, ec2.RebroadcastWhereNecessary(tests.Context(t), currentHead)) + require.Error(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) var err error - etx3, err = txStore.FindTxWithAttempts(ctx, etx3.ID) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx3.State) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) // No new tx attempts - require.Len(t, etx3.TxAttempts, 4) - attempt3_4 = etx3.TxAttempts[0] - assert.Equal(t, gasPrice.Int64(), attempt3_4.TxFee.GasPrice.ToInt().Int64()) + require.Len(t, etx.TxAttempts, 1) + bumpedAttempt := etx.TxAttempts[0] + require.Equal(t, priceMax.Int64(), bumpedAttempt.TxFee.GasPrice.Int64()) }) - // The EIP-1559 etx and attempt - etx4 := mustInsertUnconfirmedEthTxWithBroadcastDynamicFeeAttempt(t, txStore, nonce, fromAddress) - attempt4_1 := etx4.TxAttempts[0] - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1, gas_tip_cap=$2, gas_fee_cap=$3 WHERE id=$4 RETURNING *`, - oldEnough, assets.GWei(35), assets.GWei(100), attempt4_1.ID)) - var attempt4_2 txmgr.TxAttempt - t.Run("EIP-1559: bumps using EIP-1559 rules when existing attempts are of type 0x2", func(t *testing.T) { - ethTx := *types.NewTx(&types.DynamicFeeTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx4.Sequence { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() - // This is the new, EIP-1559 attempt - gasTipCap := assets.GWei(42) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + gcfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.EVM[0].GasEstimator.BumpMin = assets.GWei(1) + }) + newCfg := evmtest.NewChainScopedConfig(t, gcfg) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedEthTxWithBroadcastDynamicFeeAttempt(t, txStore, 0, fromAddress) + err := txStore.UpdateTxAttemptBroadcastBeforeBlockNum(ctx, etx.ID, uint(25)) + require.NoError(t, err) + ec := newEthConfirmer(t, txStore, ethClient, cfg, newCfg, ethKeyStore, nil) + ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx4.Sequence && gasTipCap.ToInt().Cmp(tx.GasTipCap()) == 0 + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Successful, nil).Once() - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx4, err = txStore.FindTxWithAttempts(ctx, etx4.ID) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx4.State) + require.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) // A new, bumped attempt - require.Len(t, etx4.TxAttempts, 2) - attempt4_2 = etx4.TxAttempts[0] - assert.Nil(t, attempt4_2.TxFee.GasPrice) - assert.Equal(t, assets.GWei(42).String(), attempt4_2.TxFee.GasTipCap.String()) - assert.Equal(t, assets.GWei(120).String(), attempt4_2.TxFee.GasFeeCap.String()) - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1_2.State) + require.Len(t, etx.TxAttempts, 2) + bumpAttempt := etx.TxAttempts[0] + require.Nil(t, bumpAttempt.TxFee.GasPrice) + bumpedGas := assets.NewWeiI(1).Add(newCfg.EVM().GasEstimator().BumpMin()) + require.Equal(t, bumpedGas.Int64(), bumpAttempt.TxFee.GasTipCap.Int64()) + require.Equal(t, bumpedGas.Int64(), bumpAttempt.TxFee.GasFeeCap.Int64()) + require.Equal(t, txmgrtypes.TxAttemptBroadcast, bumpAttempt.State) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1, gas_tip_cap=$2, gas_fee_cap=$3 WHERE id=$4 RETURNING *`, - oldEnough, assets.GWei(999), assets.GWei(1000), attempt4_2.ID)) - t.Run("EIP-1559: resubmits at the old price and does not create a new attempt if one of the bumped EIP-1559 transactions would have its tip cap exceed EVM.GasEstimator.PriceMax", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) gcfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.EVM[0].GasEstimator.PriceMax = assets.GWei(1000) + c.EVM[0].GasEstimator.PriceMax = assets.NewWeiI(1) }) newCfg := evmtest.NewChainScopedConfig(t, gcfg) - ec2 := newEthConfirmer(t, txStore, ethClient, gcfg, newCfg, ethKeyStore, nil) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedEthTxWithBroadcastDynamicFeeAttempt(t, txStore, 0, fromAddress) + err := txStore.UpdateTxAttemptBroadcastBeforeBlockNum(ctx, etx.ID, uint(25)) + require.NoError(t, err) + ec := newEthConfirmer(t, txStore, ethClient, cfg, newCfg, ethKeyStore, nil) - // Third attempt failed to bump, resubmits old one instead ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx4.Sequence && attempt4_2.Hash.String() == tx.Hash().String() - }), fromAddress).Return(commonclient.Successful, nil).Once() + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Underpriced, errors.New("underpriced")).Once() - require.NoError(t, ec2.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx4, err = txStore.FindTxWithAttempts(ctx, etx4.ID) + require.Error(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx4.State) + assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) // No new tx attempts - require.Len(t, etx4.TxAttempts, 2) - assert.Equal(t, assets.GWei(999).Int64(), etx4.TxAttempts[0].TxFee.GasTipCap.ToInt().Int64()) - assert.Equal(t, assets.GWei(1000).Int64(), etx4.TxAttempts[0].TxFee.GasFeeCap.ToInt().Int64()) + require.Len(t, etx.TxAttempts, 1) + bumpedAttempt := etx.TxAttempts[0] + assert.Equal(t, assets.NewWeiI(1).Int64(), bumpedAttempt.TxFee.GasTipCap.Int64()) + assert.Equal(t, assets.NewWeiI(1).Int64(), bumpedAttempt.TxFee.GasFeeCap.Int64()) }) - require.NoError(t, db.Get(&dbAttempt, `UPDATE evm.tx_attempts SET broadcast_before_block_num=$1, gas_tip_cap=$2, gas_fee_cap=$3 WHERE id=$4 RETURNING *`, - oldEnough, assets.GWei(45), assets.GWei(100), attempt4_2.ID)) - - t.Run("EIP-1559: saves attempt anyway if replacement transaction is underpriced because the bumped gas price is insufficiently higher than the previous one", func(t *testing.T) { + t.Run("EIP-1559: re-bumps attempt if initial bump is underpriced because the bumped gas price is insufficiently higher than the previous one", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + gcfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.EVM[0].GasEstimator.BumpMin = assets.GWei(1) + }) + newCfg := evmtest.NewChainScopedConfig(t, gcfg) // NOTE: This test case was empirically impossible when I tried it on eth mainnet (any EIP1559 transaction with a higher tip cap is accepted even if it's only 1 wei more) but appears to be possible on Polygon/Matic, probably due to poor design that applies the 10% minimum to the overall value (base fee + tip cap) - expectedBumpedTipCap := assets.GWei(54) - require.Greater(t, expectedBumpedTipCap.Int64(), attempt4_2.TxFee.GasTipCap.ToInt().Int64()) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + etx := mustInsertUnconfirmedEthTxWithBroadcastDynamicFeeAttempt(t, txStore, 0, fromAddress) + err := txStore.UpdateTxAttemptBroadcastBeforeBlockNum(ctx, etx.ID, uint(25)) + require.NoError(t, err) + ec := newEthConfirmer(t, txStore, ethClient, cfg, newCfg, ethKeyStore, nil) - ethTx := *types.NewTx(&types.LegacyTx{}) - kst.On("SignTx", mock.Anything, - fromAddress, - mock.MatchedBy(func(tx *types.Transaction) bool { - if evmtypes.Nonce(tx.Nonce()) != *etx4.Sequence || expectedBumpedTipCap.ToInt().Cmp(tx.GasTipCap()) != 0 { - return false - } - ethTx = *tx - return true - }), - mock.Anything).Return(ðTx, nil).Once() ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return evmtypes.Nonce(tx.Nonce()) == *etx4.Sequence && expectedBumpedTipCap.ToInt().Cmp(tx.GasTipCap()) == 0 - }), fromAddress).Return(commonclient.Successful, errors.New("replacement transaction underpriced")).Once() + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Underpriced, errors.New("replacement transaction underpriced")).Once() + ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 + }), fromAddress).Return(commonclient.Successful, nil).Once() // Do it - require.NoError(t, ec.RebroadcastWhereNecessary(tests.Context(t), currentHead)) - var err error - etx4, err = txStore.FindTxWithAttempts(ctx, etx4.ID) + require.NoError(t, ec.RebroadcastWhereNecessary(ctx, currentHead)) + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) + assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx4.State) - - require.Len(t, etx4.TxAttempts, 3) - require.Equal(t, attempt4_1.ID, etx4.TxAttempts[2].ID) - require.Equal(t, attempt4_2.ID, etx4.TxAttempts[1].ID) - attempt4_3 := etx4.TxAttempts[0] - - assert.Equal(t, expectedBumpedTipCap.Int64(), attempt4_3.TxFee.GasTipCap.ToInt().Int64()) + require.Len(t, etx.TxAttempts, 2) + bumpAttempt := etx.TxAttempts[0] + bumpedGas := assets.NewWeiI(1).Add(newCfg.EVM().GasEstimator().BumpMin()) + bumpedGas = bumpedGas.Add(newCfg.EVM().GasEstimator().BumpMin()) + assert.Equal(t, bumpedGas.Int64(), bumpAttempt.TxFee.GasTipCap.Int64()) }) } @@ -2491,7 +1359,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_TerminallyUnderpriced_ThenGoesTh commonclient.Successful, nil).Once() signedLegacyTx := new(types.Transaction) kst.On("SignTx", mock.Anything, mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Type() == 0x0 && tx.Nonce() == uint64(*etx.Sequence) + return tx.Type() == 0x0 && tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), mock.Anything).Return( signedLegacyTx, nil, ).Run(func(args mock.Arguments) { @@ -2523,7 +1391,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_TerminallyUnderpriced_ThenGoesTh commonclient.Successful, nil).Once() signedDxFeeTx := new(types.Transaction) kst.On("SignTx", mock.Anything, mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Type() == 0x2 && tx.Nonce() == uint64(*etx.Sequence) + return tx.Type() == 0x2 && tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), mock.Anything).Return( signedDxFeeTx, nil, ).Run(func(args mock.Arguments) { @@ -2670,7 +1538,7 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WhenOutOfEth(t *testing.T) { var dbAttempts []txmgr.DbEthTxAttempt require.NoError(t, db.Select(&dbAttempts, "SELECT * FROM evm.tx_attempts WHERE state = 'insufficient_eth'")) - require.Len(t, dbAttempts, 0) + require.Empty(t, dbAttempts) }) } @@ -2707,11 +1575,11 @@ func TestEthConfirmer_RebroadcastWhereNecessary_TerminallyStuckError(t *testing. // Return terminally stuck error on first rebroadcast ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Nonce() == uint64(*etx.Sequence) + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.TerminallyStuck, errors.New(terminallyStuckError)).Once() // Return successful for purge attempt ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - return tx.Nonce() == uint64(*etx.Sequence) + return tx.Nonce() == uint64(*etx.Sequence) //nolint:gosec // disable G115 }), fromAddress).Return(commonclient.Successful, nil).Once() // Start processing transactions for rebroadcast @@ -2726,180 +1594,6 @@ func TestEthConfirmer_RebroadcastWhereNecessary_TerminallyStuckError(t *testing. }) } -func TestEthConfirmer_EnsureConfirmedTransactionsInLongestChain(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ctx := tests.Context(t) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - gconfig, config := newTestChainScopedConfig(t) - ec := newEthConfirmer(t, txStore, ethClient, gconfig, config, ethKeyStore, nil) - - h8 := &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - } - h9 := &evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 9, - } - h9.Parent.Store(h8) - head := &evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 10, - } - head.Parent.Store(h9) - t.Run("does nothing if there aren't any transactions", func(t *testing.T) { - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - }) - - t.Run("does nothing to unconfirmed transactions", func(t *testing.T) { - etx := cltest.MustInsertUnconfirmedEthTxWithBroadcastLegacyAttempt(t, txStore, 0, fromAddress) - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - }) - - t.Run("does nothing to confirmed transactions with receipts within head height of the chain and included in the chain", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 2, 1, fromAddress) - mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) - }) - - t.Run("does nothing to confirmed transactions that only have receipts older than the start of the chain", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 3, 1, fromAddress) - // Add receipt that is older than the lowest block of the chain - mustInsertEthReceipt(t, txStore, h8.Number-1, testutils.NewHash(), etx.TxAttempts[0].Hash) - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) - }) - - t.Run("unconfirms and rebroadcasts transactions that have receipts within head height of the chain but not included in the chain", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 4, 1, fromAddress) - attempt := etx.TxAttempts[0] - // Include one within head height but a different block hash - mustInsertEthReceipt(t, txStore, head.Parent.Load().Number, testutils.NewHash(), attempt.Hash) - - ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - atx, err := txmgr.GetGethSignedTx(attempt.SignedRawTx) - require.NoError(t, err) - // Keeps gas price and nonce the same - return atx.GasPrice().Cmp(tx.GasPrice()) == 0 && atx.Nonce() == tx.Nonce() - }), fromAddress).Return(commonclient.Successful, nil).Once() - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - require.Len(t, etx.TxAttempts, 1) - attempt = etx.TxAttempts[0] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt.State) - }) - - t.Run("unconfirms and rebroadcasts transactions that have receipts within head height of chain but not included in the chain even if a receipt exists older than the start of the chain", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 5, 1, fromAddress) - attempt := etx.TxAttempts[0] - attemptHash := attempt.Hash - // Add receipt that is older than the lowest block of the chain - mustInsertEthReceipt(t, txStore, h8.Number-1, testutils.NewHash(), attemptHash) - // Include one within head height but a different block hash - mustInsertEthReceipt(t, txStore, head.Parent.Load().Number, testutils.NewHash(), attemptHash) - - ethClient.On("SendTransactionReturnCode", mock.Anything, mock.Anything, fromAddress).Return( - commonclient.Successful, nil).Once() - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - require.Len(t, etx.TxAttempts, 1) - attempt = etx.TxAttempts[0] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt.State) - }) - - t.Run("if more than one attempt has a receipt (should not be possible but isn't prevented by database constraints) unconfirms and rebroadcasts only the attempt with the highest gas price", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 6, 1, fromAddress) - require.Len(t, etx.TxAttempts, 1) - // Sanity check to assert the included attempt has the lowest gas price - require.Less(t, etx.TxAttempts[0].TxFee.GasPrice.ToInt().Int64(), int64(30000)) - - attempt2 := newBroadcastLegacyEthTxAttempt(t, etx.ID, 30000) - attempt2.SignedRawTx = hexutil.MustDecode("0xf88c8301f3a98503b9aca000832ab98094f5fff180082d6017036b771ba883025c654bc93580a4daa6d556000000000000000000000000000000000000000000000000000000000000000026a0f25601065ee369b6470c0399a2334afcfbeb0b5c8f3d9a9042e448ed29b5bcbda05b676e00248b85faf4dd889f0e2dcf91eb867e23ac9eeb14a73f9e4c14972cdf") - attempt3 := newBroadcastLegacyEthTxAttempt(t, etx.ID, 40000) - attempt3.SignedRawTx = hexutil.MustDecode("0xf88c8301f3a88503b9aca0008316e36094151445852b0cfdf6a4cc81440f2af99176e8ad0880a4daa6d556000000000000000000000000000000000000000000000000000000000000000026a0dcb5a7ad52b96a866257134429f944c505820716567f070e64abb74899803855a04c13eff2a22c218e68da80111e1bb6dc665d3dea7104ab40ff8a0275a99f630d") - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2)) - require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt3)) - - // Receipt is within head height but a different block hash - mustInsertEthReceipt(t, txStore, head.Parent.Load().Number, testutils.NewHash(), attempt2.Hash) - // Receipt is within head height but a different block hash - mustInsertEthReceipt(t, txStore, head.Parent.Load().Number, testutils.NewHash(), attempt3.Hash) - - ethClient.On("SendTransactionReturnCode", mock.Anything, mock.MatchedBy(func(tx *types.Transaction) bool { - s, err := txmgr.GetGethSignedTx(attempt3.SignedRawTx) - require.NoError(t, err) - return tx.Hash() == s.Hash() - }), fromAddress).Return(commonclient.Successful, nil).Once() - - // Do the thing - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx.State) - require.Len(t, etx.TxAttempts, 3) - attempt1 := etx.TxAttempts[0] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt1.State) - attempt2 = etx.TxAttempts[1] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt2.State) - attempt3 = etx.TxAttempts[2] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt3.State) - }) - - t.Run("if receipt has a block number that is in the future, does not mark for rebroadcast (the safe thing to do is simply wait until heads catches up)", func(t *testing.T) { - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 7, 1, fromAddress) - attempt := etx.TxAttempts[0] - // Add receipt that is higher than head - mustInsertEthReceipt(t, txStore, head.Number+1, testutils.NewHash(), attempt.Hash) - - require.NoError(t, ec.EnsureConfirmedTransactionsInLongestChain(tests.Context(t), head)) - - etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) - require.Len(t, etx.TxAttempts, 1) - attempt = etx.TxAttempts[0] - assert.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt.State) - assert.Len(t, attempt.Receipts, 1) - }) -} - func TestEthConfirmer_ForceRebroadcast(t *testing.T) { t.Parallel() @@ -3000,203 +1694,6 @@ func TestEthConfirmer_ForceRebroadcast(t *testing.T) { }) } -func TestEthConfirmer_ResumePendingRuns(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - config := configtest.NewTestGeneralConfig(t) - txStore := cltest.NewTestTxStore(t, db) - - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - - _, fromAddress := cltest.MustInsertRandomKey(t, ethKeyStore) - - ethClient := testutils.NewEthClientMockWithDefaultChain(t) - - evmcfg := evmtest.NewChainScopedConfig(t, config) - - h8 := &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - } - h9 := &evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 9, - } - h9.Parent.Store(h8) - head := evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 10, - } - head.Parent.Store(h9) - - minConfirmations := int64(2) - - pgtest.MustExec(t, db, `SET CONSTRAINTS fk_pipeline_runs_pruning_key DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS pipeline_runs_pipeline_spec_id_fkey DEFERRED`) - - t.Run("doesn't process task runs that are not suspended (possibly already previously resumed)", func(t *testing.T) { - ec := newEthConfirmer(t, txStore, ethClient, config, evmcfg, ethKeyStore, func(context.Context, uuid.UUID, interface{}, error) error { - t.Fatal("No value expected") - return nil - }) - - run := cltest.MustInsertPipelineRun(t, db) - tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) - - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 1, 1, fromAddress) - mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) - // Setting both signal_callback and callback_completed to TRUE to simulate a completed pipeline task - // It would only be in a state past suspended if the resume callback was called and callback_completed was set to TRUE - pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE, callback_completed = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) - - err := ec.ResumePendingTaskRuns(tests.Context(t), head.Number, 0) - require.NoError(t, err) - }) - - t.Run("doesn't process task runs where the receipt is younger than minConfirmations", func(t *testing.T) { - ec := newEthConfirmer(t, txStore, ethClient, config, evmcfg, ethKeyStore, func(context.Context, uuid.UUID, interface{}, error) error { - t.Fatal("No value expected") - return nil - }) - - run := cltest.MustInsertPipelineRun(t, db) - tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) - - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 2, 1, fromAddress) - mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) - - pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) - - err := ec.ResumePendingTaskRuns(tests.Context(t), head.Number, 0) - require.NoError(t, err) - }) - - t.Run("processes eth_txes with receipts older than minConfirmations", func(t *testing.T) { - ch := make(chan interface{}) - nonce := evmtypes.Nonce(3) - var err error - ec := newEthConfirmer(t, txStore, ethClient, config, evmcfg, ethKeyStore, func(ctx context.Context, id uuid.UUID, value interface{}, thisErr error) error { - err = thisErr - ch <- value - return nil - }) - - run := cltest.MustInsertPipelineRun(t, db) - tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) - pgtest.MustExec(t, db, `UPDATE pipeline_runs SET state = 'suspended' WHERE id = $1`, run.ID) - - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) - pgtest.MustExec(t, db, `UPDATE evm.txes SET meta='{"FailOnRevert": true}'`) - receipt := mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) - - pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) - - done := make(chan struct{}) - t.Cleanup(func() { <-done }) - go func() { - defer close(done) - err2 := ec.ResumePendingTaskRuns(tests.Context(t), head.Number, 0) - if !assert.NoError(t, err2) { - return - } - // Retrieve Tx to check if callback completed flag was set to true - updateTx, err3 := txStore.FindTxWithSequence(tests.Context(t), fromAddress, nonce) - if assert.NoError(t, err3) { - assert.Equal(t, true, updateTx.CallbackCompleted) - } - }() - - select { - case data := <-ch: - assert.NoError(t, err) - - require.IsType(t, &evmtypes.Receipt{}, data) - r := data.(*evmtypes.Receipt) - require.Equal(t, receipt.TxHash, r.TxHash) - - case <-time.After(time.Second): - t.Fatal("no value received") - } - }) - - pgtest.MustExec(t, db, `DELETE FROM pipeline_runs`) - - t.Run("processes eth_txes with receipt older than minConfirmations that reverted", func(t *testing.T) { - type data struct { - value any - error - } - ch := make(chan data) - nonce := evmtypes.Nonce(4) - ec := newEthConfirmer(t, txStore, ethClient, config, evmcfg, ethKeyStore, func(ctx context.Context, id uuid.UUID, value interface{}, err error) error { - ch <- data{value, err} - return nil - }) - - run := cltest.MustInsertPipelineRun(t, db) - tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) - pgtest.MustExec(t, db, `UPDATE pipeline_runs SET state = 'suspended' WHERE id = $1`, run.ID) - - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) - pgtest.MustExec(t, db, `UPDATE evm.txes SET meta='{"FailOnRevert": true}'`) - - // receipt is not passed through as a value since it reverted and caused an error - mustInsertRevertedEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) - - pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) - - done := make(chan struct{}) - t.Cleanup(func() { <-done }) - go func() { - defer close(done) - err2 := ec.ResumePendingTaskRuns(tests.Context(t), head.Number, 0) - if !assert.NoError(t, err2) { - return - } - // Retrieve Tx to check if callback completed flag was set to true - updateTx, err3 := txStore.FindTxWithSequence(tests.Context(t), fromAddress, nonce) - if assert.NoError(t, err3) { - assert.Equal(t, true, updateTx.CallbackCompleted) - } - }() - - select { - case data := <-ch: - assert.Error(t, data.error) - - assert.EqualError(t, data.error, fmt.Sprintf("transaction %s reverted on-chain", etx.TxAttempts[0].Hash.String())) - - assert.Nil(t, data.value) - - case <-time.After(tests.WaitTimeout(t)): - t.Fatal("no value received") - } - }) - - t.Run("does not mark callback complete if callback fails", func(t *testing.T) { - nonce := evmtypes.Nonce(5) - ec := newEthConfirmer(t, txStore, ethClient, config, evmcfg, ethKeyStore, func(context.Context, uuid.UUID, interface{}, error) error { - return errors.New("error") - }) - - run := cltest.MustInsertPipelineRun(t, db) - tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) - - etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) - mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) - pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) - - err := ec.ResumePendingTaskRuns(tests.Context(t), head.Number, 0) - require.Error(t, err) - - // Retrieve Tx to check if callback completed flag was left unchanged - updateTx, err := txStore.FindTxWithSequence(tests.Context(t), fromAddress, nonce) - require.NoError(t, err) - require.Equal(t, false, updateTx.CallbackCompleted) - }) -} - func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { t.Parallel() @@ -3230,7 +1727,7 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ethKeyStore, feeEstimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), evmcfg.EVM().Transactions().AutoPurge(), feeEstimator, txStore, ethClient) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(evmcfg.EVM()), txmgr.NewEvmTxmFeeConfig(ge), evmcfg.EVM().Transactions(), cfg.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmFeeConfig(ge), evmcfg.EVM().Transactions(), cfg.Database(), ethKeyStore, txBuilder, lggr, stuckTxDetector, ht) fn := func(ctx context.Context, id uuid.UUID, result interface{}, err error) error { require.ErrorContains(t, err, client.TerminallyStuckMsg) return nil @@ -3255,10 +1752,8 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { } head.IsFinalized.Store(true) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(&head, nil).Once() - ethClient.On("LatestFinalizedBlock", mock.Anything).Return(&head, nil).Once() + // Mined tx count does not increment due to terminally stuck transaction ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(0), nil).Once() - ethClient.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil).Once() // First call to ProcessHead should: // 1. Detect a stuck transaction @@ -3273,7 +1768,7 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { require.NoError(t, err) require.NotNil(t, dbTx) latestAttempt := dbTx.TxAttempts[0] - require.Equal(t, true, latestAttempt.IsPurgeAttempt) + require.True(t, latestAttempt.IsPurgeAttempt) require.Equal(t, limitDefault, latestAttempt.ChainSpecificFeeLimit) require.Equal(t, bumpedFee.GasPrice, latestAttempt.TxFee.GasPrice) @@ -3281,23 +1776,8 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { Hash: testutils.NewHash(), Number: blockNum + 1, } - head.IsFinalized.Store(true) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(&head, nil).Once() - ethClient.On("LatestFinalizedBlock", mock.Anything).Return(&head, nil).Once() + // Mined tx count incremented because of purge attempt ethClient.On("NonceAt", mock.Anything, mock.Anything, mock.Anything).Return(uint64(1), nil) - ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { - return len(b) == 4 && cltest.BatchElemMatchesParams(b[0], latestAttempt.Hash, "eth_getTransactionReceipt") - })).Return(nil).Run(func(args mock.Arguments) { - elems := args.Get(1).([]rpc.BatchElem) - // First transaction confirmed - *(elems[0].Result.(*evmtypes.Receipt)) = evmtypes.Receipt{ - TxHash: latestAttempt.Hash, - BlockHash: testutils.NewHash(), - BlockNumber: big.NewInt(blockNum + 1), - TransactionIndex: uint(1), - Status: uint64(1), - } - }).Once() // Second call to ProcessHead on next head should: // 1. Check for receipts for purged transaction @@ -3309,7 +1789,7 @@ func TestEthConfirmer_ProcessStuckTransactions(t *testing.T) { require.NotNil(t, dbTx) require.Equal(t, txmgrcommon.TxFatalError, dbTx.State) require.Equal(t, client.TerminallyStuckMsg, dbTx.Error.String) - require.Equal(t, true, dbTx.CallbackCompleted) + require.True(t, dbTx.CallbackCompleted) }) } @@ -3324,7 +1804,7 @@ func newEthConfirmer(t testing.TB, txStore txmgr.EvmTxStore, ethClient client.Cl txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ks, estimator) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, testutils.FixtureChainID, "", assets.NewWei(assets.NewEth(100).ToInt()), config.EVM().Transactions().AutoPurge(), estimator, txStore, ethClient) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) - ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ks, txBuilder, lggr, stuckTxDetector, ht) + ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient, nil), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), gconfig.Database(), ks, txBuilder, lggr, stuckTxDetector, ht) ec.SetResumeCallback(fn) servicetest.Run(t, ec) return ec diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index b75533e8d05..d76580907b3 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -34,9 +34,6 @@ import ( var ( ErrKeyNotUpdated = errors.New("evmTxStore: Key not updated") - // ErrCouldNotGetReceipt is the error string we save if we reach our LatestFinalizedBlockNum for a confirmed transaction - // without ever getting a receipt. This most likely happened because an external wallet used the account for this nonce - ErrCouldNotGetReceipt = "could not get receipt" ) // EvmTxStore combines the txmgr tx store interface and the interface needed for the API to read from the tx DB @@ -46,8 +43,13 @@ type EvmTxStore interface { TxStoreWebApi // methods used solely in EVM components - FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) (receipts []Receipt, err error) - UpdateTxStatesToFinalizedUsingReceiptIds(ctx context.Context, etxIDs []int64, chainId *big.Int) error + DeleteReceiptByTxHash(ctx context.Context, txHash common.Hash) error + FindAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) (hashes []TxAttempt, err error) + FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) (receipts []*evmtypes.Receipt, err error) + FindTxesPendingCallback(ctx context.Context, latest, finalized int64, chainID *big.Int) (receiptsPlus []ReceiptPlus, err error) + FindTxesByIDs(ctx context.Context, etxIDs []int64, chainID *big.Int) (etxs []*Tx, err error) + SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt) (err error) + UpdateTxStatesToFinalizedUsingTxHashes(ctx context.Context, txHashes []common.Hash, chainID *big.Int) error } // TxStoreWebApi encapsulates the methods that are not used by the txmgr and only used by the various web controllers, readers, or evm specific components @@ -844,28 +846,6 @@ func (o *evmTxStore) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) erro return nil } -func (o *evmTxStore) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) (attempts []TxAttempt, err error) { - var cancel context.CancelFunc - ctx, cancel = o.stopCh.Ctx(ctx) - defer cancel() - err = o.Transact(ctx, true, func(orm *evmTxStore) error { - var dbAttempts []DbEthTxAttempt - err = orm.q.SelectContext(ctx, &dbAttempts, ` -SELECT evm.tx_attempts.* FROM evm.tx_attempts -JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id AND evm.txes.state IN ('unconfirmed', 'confirmed_missing_receipt') AND evm.txes.evm_chain_id = $1 -WHERE evm.tx_attempts.state != 'insufficient_eth' -ORDER BY evm.txes.nonce ASC, evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC -`, chainID.String()) - if err != nil { - return pkgerrors.Wrap(err, "FindEthTxAttemptsRequiringReceiptFetch failed to load evm.tx_attempts") - } - attempts = dbEthTxAttemptsToEthTxAttempts(dbAttempts) - err = orm.preloadTxesAtomic(ctx, attempts) - return pkgerrors.Wrap(err, "FindEthTxAttemptsRequiringReceiptFetch failed to load evm.txes") - }) - return -} - // Returns the transaction by state and from addresses // Loads attempt and receipts in the transactions func (o *evmTxStore) FindTxsByStateAndFromAddresses(ctx context.Context, addresses []common.Address, state txmgrtypes.TxState, chainID *big.Int) (txs []*Tx, err error) { @@ -898,7 +878,7 @@ func (o *evmTxStore) FindTxsByStateAndFromAddresses(ctx context.Context, address return } -func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt, state txmgrtypes.TxState, errorMsg *string, chainID *big.Int) (err error) { +func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt) (err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() @@ -945,7 +925,6 @@ func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Rece valueStrs = append(valueStrs, "(?,?,?,?,?,NOW())") valueArgs = append(valueArgs, r.TxHash, r.BlockHash, r.BlockNumber.Int64(), r.TransactionIndex, receiptJSON) } - valueArgs = append(valueArgs, state, errorMsg, chainID.String()) /* #nosec G201 */ sql := ` @@ -957,21 +936,13 @@ func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Rece transaction_index = EXCLUDED.transaction_index, receipt = EXCLUDED.receipt RETURNING evm.receipts.tx_hash, evm.receipts.block_number - ), - updated_eth_tx_attempts AS ( - UPDATE evm.tx_attempts - SET - state = 'broadcast', - broadcast_before_block_num = COALESCE(evm.tx_attempts.broadcast_before_block_num, inserted_receipts.block_number) - FROM inserted_receipts - WHERE inserted_receipts.tx_hash = evm.tx_attempts.hash - RETURNING evm.tx_attempts.eth_tx_id ) - UPDATE evm.txes - SET state = ?, error = ? - FROM updated_eth_tx_attempts - WHERE updated_eth_tx_attempts.eth_tx_id = evm.txes.id - AND evm_chain_id = ? + UPDATE evm.tx_attempts + SET + state = 'broadcast', + broadcast_before_block_num = COALESCE(evm.tx_attempts.broadcast_before_block_num, inserted_receipts.block_number) + FROM inserted_receipts + WHERE inserted_receipts.tx_hash = evm.tx_attempts.hash ` stmt := fmt.Sprintf(sql, strings.Join(valueStrs, ",")) @@ -982,57 +953,6 @@ func (o *evmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Rece return pkgerrors.Wrap(err, "SaveFetchedReceipts failed to save receipts") } -// MarkAllConfirmedMissingReceipt -// It is possible that we can fail to get a receipt for all evm.tx_attempts -// even though a transaction with this nonce has long since been confirmed (we -// know this because transactions with higher nonces HAVE returned a receipt). -// -// This can probably only happen if an external wallet used the account (or -// conceivably because of some bug in the remote eth node that prevents it -// from returning a receipt for a valid transaction). -// -// In this case we mark these transactions as 'confirmed_missing_receipt' to -// prevent gas bumping. -// -// NOTE: We continue to attempt to resend evm.txes in this state on -// every head to guard against the extremely rare scenario of nonce gap due to -// reorg that excludes the transaction (from another wallet) that had this -// nonce (until LatestFinalizedBlockNum is reached, after which we make the explicit -// decision to give up). This is done in the EthResender. -// -// We will continue to try to fetch a receipt for these attempts until all -// attempts are equal to or below the LatestFinalizedBlockNum from current head. -func (o *evmTxStore) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID *big.Int) (err error) { - var cancel context.CancelFunc - ctx, cancel = o.stopCh.Ctx(ctx) - defer cancel() - res, err := o.q.ExecContext(ctx, ` -UPDATE evm.txes -SET state = 'confirmed_missing_receipt' -FROM ( - SELECT from_address, MAX(nonce) as max_nonce - FROM evm.txes - WHERE state = 'confirmed' AND evm_chain_id = $1 - GROUP BY from_address -) AS max_table -WHERE state = 'unconfirmed' - AND evm_chain_id = $1 - AND nonce < max_table.max_nonce - AND evm.txes.from_address = max_table.from_address - `, chainID.String()) - if err != nil { - return pkgerrors.Wrap(err, "markAllConfirmedMissingReceipt failed") - } - rowsAffected, err := res.RowsAffected() - if err != nil { - return pkgerrors.Wrap(err, "markAllConfirmedMissingReceipt RowsAffected failed") - } - if rowsAffected > 0 { - o.logger.Infow(fmt.Sprintf("%d transactions missing receipt", rowsAffected), "n", rowsAffected) - } - return -} - func (o *evmTxStore) GetInProgressTxAttempts(ctx context.Context, address common.Address, chainID *big.Int) (attempts []TxAttempt, err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) @@ -1080,23 +1000,23 @@ func (o *evmTxStore) FindTxesPendingCallback(ctx context.Context, latest, finali } // Update tx to mark that its callback has been signaled -func (o *evmTxStore) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunId uuid.UUID, chainId *big.Int) error { +func (o *evmTxStore) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunID uuid.UUID, chainID *big.Int) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - _, err := o.q.ExecContext(ctx, `UPDATE evm.txes SET callback_completed = TRUE WHERE pipeline_task_run_id = $1 AND evm_chain_id = $2`, pipelineTaskRunId, chainId.String()) + _, err := o.q.ExecContext(ctx, `UPDATE evm.txes SET callback_completed = TRUE WHERE pipeline_task_run_id = $1 AND evm_chain_id = $2`, pipelineTaskRunID, chainID.String()) if err != nil { return fmt.Errorf("failed to mark callback completed for transaction: %w", err) } return nil } -func (o *evmTxStore) FindLatestSequence(ctx context.Context, fromAddress common.Address, chainId *big.Int) (nonce evmtypes.Nonce, err error) { +func (o *evmTxStore) FindLatestSequence(ctx context.Context, fromAddress common.Address, chainID *big.Int) (nonce evmtypes.Nonce, err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() sql := `SELECT nonce FROM evm.txes WHERE from_address = $1 AND evm_chain_id = $2 AND nonce IS NOT NULL ORDER BY nonce DESC LIMIT 1` - err = o.q.GetContext(ctx, &nonce, sql, fromAddress, chainId.String()) + err = o.q.GetContext(ctx, &nonce, sql, fromAddress, chainID.String()) return } @@ -1152,74 +1072,47 @@ SELECT * FROM evm.txes WHERE from_address = $1 AND nonce = $2 AND state IN ('con return } -func updateEthTxAttemptUnbroadcast(ctx context.Context, orm *evmTxStore, attempt TxAttempt) error { - if attempt.State != txmgrtypes.TxAttemptBroadcast { - return errors.New("expected eth_tx_attempt to be broadcast") - } - _, err := orm.q.ExecContext(ctx, `UPDATE evm.tx_attempts SET broadcast_before_block_num = NULL, state = 'in_progress' WHERE id = $1`, attempt.ID) - return pkgerrors.Wrap(err, "updateEthTxAttemptUnbroadcast failed") +func updateEthTxAttemptsUnbroadcast(ctx context.Context, orm *evmTxStore, attemptIDs []int64) error { + _, err := orm.q.ExecContext(ctx, `UPDATE evm.tx_attempts SET broadcast_before_block_num = NULL, state = 'in_progress' WHERE id = ANY($1)`, pq.Array(attemptIDs)) + return err } -func updateEthTxUnconfirm(ctx context.Context, orm *evmTxStore, etx Tx) error { - if etx.State != txmgr.TxConfirmed { - return errors.New("expected tx state to be confirmed") - } - _, err := orm.q.ExecContext(ctx, `UPDATE evm.txes SET state = 'unconfirmed' WHERE id = $1`, etx.ID) - return pkgerrors.Wrap(err, "updateEthTxUnconfirm failed") +func updateEthTxsUnconfirm(ctx context.Context, orm *evmTxStore, etxIDs []int64) error { + _, err := orm.q.ExecContext(ctx, `UPDATE evm.txes SET state = 'unconfirmed', error = NULL WHERE id = ANY($1)`, pq.Array(etxIDs)) + return err } -func deleteEthReceipts(ctx context.Context, orm *evmTxStore, etxID int64) (err error) { +func deleteEthReceipts(ctx context.Context, orm *evmTxStore, etxIDs []int64) (err error) { _, err = orm.q.ExecContext(ctx, ` DELETE FROM evm.receipts USING evm.tx_attempts WHERE evm.receipts.tx_hash = evm.tx_attempts.hash -AND evm.tx_attempts.eth_tx_id = $1 - `, etxID) +AND evm.tx_attempts.eth_tx_id = ANY($1) + `, pq.Array(etxIDs)) return pkgerrors.Wrap(err, "deleteEthReceipts failed") } -func (o *evmTxStore) UpdateTxForRebroadcast(ctx context.Context, etx Tx, etxAttempt TxAttempt) error { +func (o *evmTxStore) DeleteReceiptByTxHash(ctx context.Context, txHash common.Hash) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - return o.Transact(ctx, false, func(orm *evmTxStore) error { - if err := deleteEthReceipts(ctx, orm, etx.ID); err != nil { - return pkgerrors.Wrapf(err, "deleteEthReceipts failed for etx %v", etx.ID) - } - if err := updateEthTxUnconfirm(ctx, orm, etx); err != nil { - return pkgerrors.Wrapf(err, "updateEthTxUnconfirm failed for etx %v", etx.ID) - } - return updateEthTxAttemptUnbroadcast(ctx, orm, etxAttempt) - }) + _, err := o.q.ExecContext(ctx, `DELETE FROM evm.receipts WHERE tx_hash = $1`, txHash) + return err } -func (o *evmTxStore) FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber, lowBlockNumber int64, chainID *big.Int) (etxs []*Tx, err error) { +func (o *evmTxStore) UpdateTxsForRebroadcast(ctx context.Context, etxIDs []int64, attemptIDs []int64) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - err = o.Transact(ctx, true, func(orm *evmTxStore) error { - var dbEtxs []DbEthTx - err = orm.q.SelectContext(ctx, &dbEtxs, ` -SELECT DISTINCT evm.txes.* FROM evm.txes -INNER JOIN evm.tx_attempts ON evm.txes.id = evm.tx_attempts.eth_tx_id AND evm.tx_attempts.state = 'broadcast' -INNER JOIN evm.receipts ON evm.receipts.tx_hash = evm.tx_attempts.hash -WHERE evm.txes.state IN ('confirmed', 'confirmed_missing_receipt') AND block_number BETWEEN $1 AND $2 AND evm_chain_id = $3 -ORDER BY nonce ASC -`, lowBlockNumber, highBlockNumber, chainID.String()) - if err != nil { - return pkgerrors.Wrap(err, "FindTransactionsConfirmedInBlockRange failed to load evm.txes") + return o.Transact(ctx, false, func(orm *evmTxStore) error { + if err := deleteEthReceipts(ctx, orm, etxIDs); err != nil { + return pkgerrors.Wrapf(err, "deleteEthReceipts failed for etx %v", etxIDs) } - etxs = make([]*Tx, len(dbEtxs)) - dbEthTxsToEvmEthTxPtrs(dbEtxs, etxs) - if err = orm.LoadTxesAttempts(ctx, etxs); err != nil { - return pkgerrors.Wrap(err, "FindTransactionsConfirmedInBlockRange failed to load evm.tx_attempts") + if err := updateEthTxsUnconfirm(ctx, orm, etxIDs); err != nil { + return pkgerrors.Wrapf(err, "updateEthTxUnconfirm failed for etx %v", etxIDs) } - - // retrieve tx with attempts and partial receipt values for optimization purpose - err = orm.loadEthTxesAttemptsWithPartialReceipts(ctx, etxs) - return pkgerrors.Wrap(err, "FindTransactionsConfirmedInBlockRange failed to load evm.receipts") + return updateEthTxAttemptsUnbroadcast(ctx, orm, attemptIDs) }) - return etxs, pkgerrors.Wrap(err, "FindTransactionsConfirmedInBlockRange failed") } func (o *evmTxStore) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, chainID *big.Int) (broadcastAt nullv4.Time, err error) { @@ -1298,7 +1191,7 @@ func (o *evmTxStore) SaveSentAttempt(ctx context.Context, timeout time.Duration, return o.saveSentAttempt(ctx, timeout, attempt, broadcastAt) } -func (o *evmTxStore) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt, broadcastAt time.Time) error { +func (o *evmTxStore) SaveConfirmedAttempt(ctx context.Context, timeout time.Duration, attempt *TxAttempt, broadcastAt time.Time) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() @@ -1306,12 +1199,12 @@ func (o *evmTxStore) SaveConfirmedMissingReceiptAttempt(ctx context.Context, tim if err := orm.saveSentAttempt(ctx, timeout, attempt, broadcastAt); err != nil { return err } - if _, err := orm.q.ExecContext(ctx, `UPDATE evm.txes SET state = 'confirmed_missing_receipt' WHERE id = $1`, attempt.TxID); err != nil { + if _, err := orm.q.ExecContext(ctx, `UPDATE evm.txes SET state = 'confirmed' WHERE id = $1`, attempt.TxID); err != nil { return pkgerrors.Wrap(err, "failed to update evm.txes") } return nil }) - return pkgerrors.Wrap(err, "SaveConfirmedMissingReceiptAttempt failed") + return pkgerrors.Wrap(err, "SaveConfirmedAttempt failed") } func (o *evmTxStore) DeleteInProgressAttempt(ctx context.Context, attempt TxAttempt) error { @@ -1472,101 +1365,6 @@ ORDER BY nonce ASC return } -// markOldTxesMissingReceiptAsErrored -// -// Once eth_tx has all of its attempts broadcast equal to or before latestFinalizedBlockNum -// without receiving any receipts, we mark it as fatally errored (never sent). -// -// The job run will also be marked as errored in this case since we never got a -// receipt and thus cannot pass on any transaction hash -func (o *evmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int) error { - var cancel context.CancelFunc - ctx, cancel = o.stopCh.Ctx(ctx) - defer cancel() - // Any 'confirmed_missing_receipt' eth_tx with all attempts equal to or older than latestFinalizedBlockNum will be marked as errored - // We will not try to query for receipts for this transaction anymore - if latestFinalizedBlockNum <= 0 { - return nil - } - // note: if QOpt passes in a sql.Tx this will reuse it - return o.Transact(ctx, false, func(orm *evmTxStore) error { - type etx struct { - ID int64 - Nonce int64 - } - var data []etx - err := orm.q.SelectContext(ctx, &data, ` -UPDATE evm.txes -SET state='fatal_error', nonce=NULL, error=$1, broadcast_at=NULL, initial_broadcast_at=NULL -FROM ( - SELECT e1.id, e1.nonce, e1.from_address FROM evm.txes AS e1 WHERE id IN ( - SELECT e2.id FROM evm.txes AS e2 - INNER JOIN evm.tx_attempts ON e2.id = evm.tx_attempts.eth_tx_id - WHERE e2.state = 'confirmed_missing_receipt' - AND e2.evm_chain_id = $3 - GROUP BY e2.id - HAVING max(evm.tx_attempts.broadcast_before_block_num) <= $2 - ) - FOR UPDATE OF e1 -) e0 -WHERE e0.id = evm.txes.id -RETURNING e0.id, e0.nonce`, ErrCouldNotGetReceipt, latestFinalizedBlockNum, chainID.String()) - - if err != nil { - return pkgerrors.Wrap(err, "markOldTxesMissingReceiptAsErrored failed to query") - } - - // We need this little lookup table because we have to have the nonce - // from the first query, BEFORE it was updated/nullified - lookup := make(map[int64]etx) - for _, d := range data { - lookup[d.ID] = d - } - etxIDs := make([]int64, len(data)) - for i := 0; i < len(data); i++ { - etxIDs[i] = data[i].ID - } - - type result struct { - ID int64 - FromAddress common.Address - MaxBroadcastBeforeBlockNum int64 - TxHashes pq.ByteaArray - } - - var results []result - err = orm.q.SelectContext(ctx, &results, ` -SELECT e.id, e.from_address, max(a.broadcast_before_block_num) AS max_broadcast_before_block_num, array_agg(a.hash) AS tx_hashes -FROM evm.txes e -INNER JOIN evm.tx_attempts a ON e.id = a.eth_tx_id -WHERE e.id = ANY($1) -GROUP BY e.id -`, etxIDs) - - if err != nil { - return pkgerrors.Wrap(err, "markOldTxesMissingReceiptAsErrored failed to load additional data") - } - - for _, r := range results { - nonce := lookup[r.ID].Nonce - txHashesHex := make([]common.Address, len(r.TxHashes)) - for i := 0; i < len(r.TxHashes); i++ { - txHashesHex[i] = common.BytesToAddress(r.TxHashes[i]) - } - - orm.logger.Criticalw(fmt.Sprintf("eth_tx with ID %v expired without ever getting a receipt for any of our attempts. "+ - "Current block height is %v, transaction was broadcast before block height %v. This transaction may not have not been sent and will be marked as fatally errored. "+ - "This can happen if there is another instance of chainlink running that is using the same private key, or if "+ - "an external wallet has been used to send a transaction from account %s with nonce %v."+ - " Please note that Chainlink requires exclusive ownership of it's private keys and sharing keys across multiple"+ - " chainlink instances, or using the chainlink keys with an external wallet is NOT SUPPORTED and WILL lead to missed transactions", - r.ID, blockNum, r.MaxBroadcastBeforeBlockNum, r.FromAddress, nonce), "ethTxID", r.ID, "nonce", nonce, "fromAddress", r.FromAddress, "txHashes", txHashesHex) - } - - return nil - }) -} - func (o *evmTxStore) SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt TxAttempt, replacementAttempt *TxAttempt) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) @@ -1609,12 +1407,12 @@ func (o *evmTxStore) FindNextUnstartedTransactionFromAddress(ctx context.Context return etx, nil } -func (o *evmTxStore) UpdateTxFatalError(ctx context.Context, etx *Tx) error { +func (o *evmTxStore) UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *Tx) error { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() - if etx.State != txmgr.TxInProgress && etx.State != txmgr.TxUnstarted { - return pkgerrors.Errorf("can only transition to fatal_error from in_progress or unstarted, transaction is currently %s", etx.State) + if etx.State != txmgr.TxInProgress && etx.State != txmgr.TxUnstarted && etx.State != txmgr.TxConfirmed { + return pkgerrors.Errorf("can only transition to fatal_error from in_progress, unstarted, or confirmed, transaction is currently %s", etx.State) } if !etx.Error.Valid { return errors.New("expected error field to be set") @@ -2112,35 +1910,147 @@ func (o *evmTxStore) UpdateTxAttemptBroadcastBeforeBlockNum(ctx context.Context, return err } -// FindConfirmedTxesReceipts Returns all confirmed transactions with receipt block nums older than or equal to the finalized block number -func (o *evmTxStore) FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) (receipts []Receipt, err error) { +// FindAttemptsRequiringReceiptFetch returns all broadcasted attempts for confirmed or terminally stuck transactions that do not have receipts stored in the DB +func (o *evmTxStore) FindAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) (attempts []TxAttempt, err error) { + var cancel context.CancelFunc + ctx, cancel = o.stopCh.Ctx(ctx) + defer cancel() + var dbTxAttempts []DbEthTxAttempt + query := ` + SELECT evm.tx_attempts.* FROM evm.tx_attempts + JOIN evm.txes ON evm.txes.ID = evm.tx_attempts.eth_tx_id + WHERE evm.tx_attempts.state = 'broadcast' AND evm.txes.state IN ('confirmed', 'confirmed_missing_receipt', 'fatal_error') AND evm.txes.evm_chain_id = $1 AND evm.txes.ID NOT IN ( + SELECT DISTINCT evm.txes.ID FROM evm.txes + JOIN evm.tx_attempts ON evm.tx_attempts.eth_tx_id = evm.txes.ID + JOIN evm.receipts ON evm.receipts.tx_hash = evm.tx_attempts.hash + WHERE evm.txes.evm_chain_id = $1 AND evm.txes.state IN ('confirmed', 'confirmed_missing_receipt', 'fatal_error') AND evm.receipts.ID IS NOT NULL + ) + ORDER BY evm.txes.nonce ASC, evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC + ` + err = o.q.SelectContext(ctx, &dbTxAttempts, query, chainID.String()) + attempts = dbEthTxAttemptsToEthTxAttempts(dbTxAttempts) + return attempts, err +} + +// FindConfirmedTxesReceipts returns all confirmed transactions with receipt block nums older than or equal to the finalized block number +func (o *evmTxStore) FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) (receipts []*evmtypes.Receipt, err error) { var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() + var dbReceipts []Receipt // note the receipts are partially loaded for performance reason - query := `SELECT evm.receipts.id, evm.receipts.tx_hash, evm.receipts.block_hash, evm.receipts.block_number FROM evm.receipts + query := `SELECT evm.receipts.tx_hash, evm.receipts.block_hash, evm.receipts.block_number FROM evm.receipts INNER JOIN evm.tx_attempts ON evm.tx_attempts.hash = evm.receipts.tx_hash INNER JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id WHERE evm.txes.state = 'confirmed' AND evm.receipts.block_number <= $1 AND evm.txes.evm_chain_id = $2` - err = o.q.SelectContext(ctx, &receipts, query, finalizedBlockNum, chainID.String()) + err = o.q.SelectContext(ctx, &dbReceipts, query, finalizedBlockNum, chainID.String()) + for _, dbReceipt := range dbReceipts { + receipts = append(receipts, &evmtypes.Receipt{ + TxHash: dbReceipt.TxHash, + BlockHash: dbReceipt.BlockHash, + BlockNumber: big.NewInt(dbReceipt.BlockNumber), + }) + } return receipts, err } -// Mark transactions corresponding to receipt IDs as finalized -func (o *evmTxStore) UpdateTxStatesToFinalizedUsingReceiptIds(ctx context.Context, receiptIDs []int64, chainId *big.Int) error { - if len(receiptIDs) == 0 { +// Mark transactions corresponding to attempt hashes as finalized +func (o *evmTxStore) UpdateTxStatesToFinalizedUsingTxHashes(ctx context.Context, txHashes []common.Hash, chainID *big.Int) error { + if len(txHashes) == 0 { return nil } + txHashBytea := make([][]byte, len(txHashes)) + for i, hash := range txHashes { + txHashBytea[i] = hash.Bytes() + } var cancel context.CancelFunc ctx, cancel = o.stopCh.Ctx(ctx) defer cancel() sql := ` UPDATE evm.txes SET state = 'finalized' WHERE evm.txes.evm_chain_id = $1 AND evm.txes.id IN (SELECT evm.txes.id FROM evm.txes INNER JOIN evm.tx_attempts ON evm.tx_attempts.eth_tx_id = evm.txes.id - INNER JOIN evm.receipts ON evm.receipts.tx_hash = evm.tx_attempts.hash - WHERE evm.receipts.id = ANY($2)) + WHERE evm.tx_attempts.hash = ANY($2)) ` - _, err := o.q.ExecContext(ctx, sql, chainId.String(), pq.Array(receiptIDs)) + _, err := o.q.ExecContext(ctx, sql, chainID.String(), txHashBytea) return err } + +// FindReorgOrIncludedTxs finds transactions that have either been re-org'd or included on-chain based on the mined transaction count +// If the mined transaction count receeds, transactions could have beeen re-org'd +// If it proceeds, transactions could have been included +// This check assumes transactions are broadcasted in ascending order and not out of order +func (o *evmTxStore) FindReorgOrIncludedTxs(ctx context.Context, fromAddress common.Address, minedTxCount evmtypes.Nonce, chainID *big.Int) (reorgTxs []*Tx, includedTxs []*Tx, err error) { + var cancel context.CancelFunc + ctx, cancel = o.stopCh.Ctx(ctx) + defer cancel() + err = o.Transact(ctx, true, func(orm *evmTxStore) error { + var dbReOrgEtxs []DbEthTx + query := `SELECT * FROM evm.txes WHERE from_address = $1 AND state IN ('confirmed', 'confirmed_missing_receipt', 'fatal_error', 'finalized') AND nonce >= $2 AND evm_chain_id = $3` + err = o.q.SelectContext(ctx, &dbReOrgEtxs, query, fromAddress, minedTxCount.Int64(), chainID.String()) + // If re-org'd transactions found, populate them with attempts and partial receipts, then return since new transactions could not have been included + if len(dbReOrgEtxs) > 0 { + reorgTxs = make([]*Tx, len(dbReOrgEtxs)) + dbEthTxsToEvmEthTxPtrs(dbReOrgEtxs, reorgTxs) + if err = orm.LoadTxesAttempts(ctx, reorgTxs); err != nil { + return fmt.Errorf("failed to load evm.tx_attempts: %w", err) + } + // retrieve tx with attempts and partial receipt values for optimization purpose + if err = orm.loadEthTxesAttemptsWithPartialReceipts(ctx, reorgTxs); err != nil { + return fmt.Errorf("failed to load partial evm.receipts: %w", err) + } + return nil + } + // If re-org'd transactions not found, find unconfirmed transactions could have been included and populate with attempts + var dbIncludedEtxs []DbEthTx + query = `SELECT * FROM evm.txes WHERE state = 'unconfirmed' AND from_address = $1 AND nonce < $2 AND evm_chain_id = $3` + err = o.q.SelectContext(ctx, &dbIncludedEtxs, query, fromAddress, minedTxCount.Int64(), chainID.String()) + includedTxs = make([]*Tx, len(dbIncludedEtxs)) + dbEthTxsToEvmEthTxPtrs(dbIncludedEtxs, includedTxs) + if err = orm.LoadTxesAttempts(ctx, includedTxs); err != nil { + return fmt.Errorf("failed to load evm.tx_attempts: %w", err) + } + return nil + }) + return +} + +func (o *evmTxStore) UpdateTxConfirmed(ctx context.Context, etxIDs []int64) error { + var cancel context.CancelFunc + ctx, cancel = o.stopCh.Ctx(ctx) + defer cancel() + err := o.Transact(ctx, true, func(orm *evmTxStore) error { + sql := `UPDATE evm.txes SET state = 'confirmed' WHERE id = ANY($1)` + _, err := o.q.ExecContext(ctx, sql, pq.Array(etxIDs)) + if err != nil { + return err + } + sql = `UPDATE evm.tx_attempts SET state = 'broadcast' WHERE state = 'in_progress' AND eth_tx_id = ANY($1)` + _, err = o.q.ExecContext(ctx, sql, pq.Array(etxIDs)) + return err + }) + return err +} + +func (o *evmTxStore) UpdateTxFatalError(ctx context.Context, etxIDs []int64, errMsg string) error { + var cancel context.CancelFunc + ctx, cancel = o.stopCh.Ctx(ctx) + defer cancel() + sql := `UPDATE evm.txes SET state = 'fatal_error', error = $1 WHERE id = ANY($2)` + _, err := o.q.ExecContext(ctx, sql, errMsg, pq.Array(etxIDs)) + return err +} + +func (o *evmTxStore) FindTxesByIDs(ctx context.Context, etxIDs []int64, chainID *big.Int) (etxs []*Tx, err error) { + var cancel context.CancelFunc + ctx, cancel = o.stopCh.Ctx(ctx) + defer cancel() + var dbEtxs []DbEthTx + sql := `SELECT * FROM evm.txes WHERE id = ANY($1) AND evm_chain_id = $2 ORDER BY created_at ASC, id ASC` + if err = o.q.SelectContext(ctx, &dbEtxs, sql, pq.Array(etxIDs), chainID.String()); err != nil { + return nil, fmt.Errorf("failed to find evm.tx: %w", err) + } + etxs = make([]*Tx, len(dbEtxs)) + dbEthTxsToEvmEthTxPtrs(dbEtxs, etxs) + return +} diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index 9e1f135e0b2..a05cf3f9010 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -21,6 +21,7 @@ import ( txmgrcommon "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" @@ -112,8 +113,8 @@ func TestORM_Transactions(t *testing.T) { assert.Len(t, txs, 2) assert.Equal(t, evmtypes.Nonce(1), *txs[0].Sequence, "transactions should be sorted by nonce") assert.Equal(t, evmtypes.Nonce(0), *txs[1].Sequence, "transactions should be sorted by nonce") - assert.Len(t, txs[0].TxAttempts, 0, "eth tx attempts should not be preloaded") - assert.Len(t, txs[1].TxAttempts, 0) + assert.Empty(t, txs[0].TxAttempts, "eth tx attempts should not be preloaded") + assert.Empty(t, txs[1].TxAttempts) } func TestORM(t *testing.T) { @@ -164,7 +165,7 @@ func TestORM(t *testing.T) { assert.Equal(t, etx.TxAttempts[0].ID, attemptD.ID) assert.Equal(t, etx.TxAttempts[1].ID, attemptL.ID) require.Len(t, etx.TxAttempts[0].Receipts, 1) - require.Len(t, etx.TxAttempts[1].Receipts, 0) + require.Empty(t, etx.TxAttempts[1].Receipts) assert.Equal(t, r.BlockHash, etx.TxAttempts[0].Receipts[0].GetBlockHash()) }) t.Run("FindTxByHash", func(t *testing.T) { @@ -180,7 +181,7 @@ func TestORM(t *testing.T) { assert.Equal(t, etx.TxAttempts[0].ID, attemptD.ID) assert.Equal(t, etx.TxAttempts[1].ID, attemptL.ID) require.Len(t, etx.TxAttempts[0].Receipts, 1) - require.Len(t, etx.TxAttempts[1].Receipts, 0) + require.Empty(t, etx.TxAttempts[1].Receipts) assert.Equal(t, r.BlockHash, etx.TxAttempts[0].Receipts[0].GetBlockHash()) }) } @@ -248,7 +249,7 @@ func TestORM_FindTxAttemptsRequiringResend(t *testing.T) { olderThan := time.Now() attempts, err := txStore.FindTxAttemptsRequiringResend(tests.Context(t), olderThan, 10, testutils.FixtureChainID, fromAddress) require.NoError(t, err) - assert.Len(t, attempts, 0) + assert.Empty(t, attempts) }) // Mix up the insert order to assure that they come out sorted by nonce not implicitly or by ID @@ -291,7 +292,7 @@ func TestORM_FindTxAttemptsRequiringResend(t *testing.T) { olderThan := time.Now() attempts, err := txStore.FindTxAttemptsRequiringResend(tests.Context(t), olderThan, 10, testutils.FixtureChainID, utils.RandomAddress()) require.NoError(t, err) - assert.Len(t, attempts, 0) + assert.Empty(t, attempts) }) t.Run("returns the highest price attempt for each transaction that was last broadcast before or on the given time", func(t *testing.T) { @@ -437,29 +438,7 @@ func TestORM_SetBroadcastBeforeBlockNum(t *testing.T) { }) } -func TestORM_FindTxAttemptsConfirmedMissingReceipt(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - - attempts, err := txStore.FindTxAttemptsConfirmedMissingReceipt(tests.Context(t), ethClient.ConfiguredChainID()) - - require.NoError(t, err) - - assert.Len(t, attempts, 1) - assert.Len(t, etx0.TxAttempts, 1) - assert.Equal(t, etx0.TxAttempts[0].ID, attempts[0].ID) -} - -func TestORM_UpdateTxsUnconfirmed(t *testing.T) { +func TestORM_UpdateTxConfirmed(t *testing.T) { t.Parallel() ctx := tests.Context(t) @@ -468,35 +447,23 @@ func TestORM_UpdateTxsUnconfirmed(t *testing.T) { ethKeyStore := cltest.NewKeyStore(t, db).Eth() _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - assert.Equal(t, etx0.State, txmgrcommon.TxConfirmedMissingReceipt) - require.NoError(t, txStore.UpdateTxsUnconfirmed(tests.Context(t), []int64{etx0.ID})) - - etx0, err := txStore.FindTxWithAttempts(ctx, etx0.ID) - require.NoError(t, err) - assert.Equal(t, etx0.State, txmgrcommon.TxUnconfirmed) -} - -func TestORM_FindTxAttemptsRequiringReceiptFetch(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) + etx0 := mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 0, fromAddress, txmgrtypes.TxAttemptBroadcast) + etx1 := mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 1, fromAddress, txmgrtypes.TxAttemptInProgress) + assert.Equal(t, txmgrcommon.TxUnconfirmed, etx0.State) + assert.Equal(t, txmgrcommon.TxUnconfirmed, etx1.State) + require.NoError(t, txStore.UpdateTxConfirmed(tests.Context(t), []int64{etx0.ID, etx1.ID})) - attempts, err := txStore.FindTxAttemptsRequiringReceiptFetch(tests.Context(t), ethClient.ConfiguredChainID()) + var err error + etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) require.NoError(t, err) - assert.Len(t, attempts, 1) + assert.Equal(t, txmgrcommon.TxConfirmed, etx0.State) assert.Len(t, etx0.TxAttempts, 1) - assert.Equal(t, etx0.TxAttempts[0].ID, attempts[0].ID) + assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx0.TxAttempts[0].State) + etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) + require.NoError(t, err) + assert.Equal(t, txmgrcommon.TxConfirmed, etx1.State) + assert.Len(t, etx1.TxAttempts, 1) + assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx1.TxAttempts[0].State) } func TestORM_SaveFetchedReceipts(t *testing.T) { @@ -505,62 +472,45 @@ func TestORM_SaveFetchedReceipts(t *testing.T) { db := pgtest.NewSqlxDB(t) txStore := cltest.NewTestTxStore(t, db) ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) ctx := tests.Context(t) - originalBroadcastAt := time.Unix(1616509100, 0) - etx0 := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt( - t, txStore, 0, 1, originalBroadcastAt, fromAddress) - require.Len(t, etx0.TxAttempts, 1) + tx1 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, 100, fromAddress) + require.Len(t, tx1.TxAttempts, 1) + + tx2 := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, 100) + require.Len(t, tx2.TxAttempts, 1) - // create receipt associated with transaction - txmReceipt := evmtypes.Receipt{ - TxHash: etx0.TxAttempts[0].Hash, + // create receipts associated with transactions + txmReceipt1 := evmtypes.Receipt{ + TxHash: tx1.TxAttempts[0].Hash, + BlockHash: utils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + } + txmReceipt2 := evmtypes.Receipt{ + TxHash: tx2.TxAttempts[0].Hash, BlockHash: utils.NewHash(), BlockNumber: big.NewInt(42), TransactionIndex: uint(1), } - err := txStore.SaveFetchedReceipts(tests.Context(t), []*evmtypes.Receipt{&txmReceipt}, txmgrcommon.TxConfirmed, nil, ethClient.ConfiguredChainID()) - - require.NoError(t, err) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) + err := txStore.SaveFetchedReceipts(tests.Context(t), []*evmtypes.Receipt{&txmReceipt1, &txmReceipt2}) require.NoError(t, err) - require.Len(t, etx0.TxAttempts, 1) - require.Len(t, etx0.TxAttempts[0].Receipts, 1) - require.Equal(t, txmReceipt.BlockHash, etx0.TxAttempts[0].Receipts[0].GetBlockHash()) - require.Equal(t, txmgrcommon.TxConfirmed, etx0.State) -} - -func TestORM_MarkAllConfirmedMissingReceipt(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - ctx := tests.Context(t) - - // create transaction 0 (nonce 0) that is unconfirmed (block 7) - etx0_blocknum := int64(7) - etx0 := cltest.MustInsertUnconfirmedEthTx(t, txStore, 0, fromAddress) - etx0_attempt := newBroadcastLegacyEthTxAttempt(t, etx0.ID, int64(1)) - etx0_attempt.BroadcastBeforeBlockNum = &etx0_blocknum - require.NoError(t, txStore.InsertTxAttempt(ctx, &etx0_attempt)) - assert.Equal(t, txmgrcommon.TxUnconfirmed, etx0.State) - - // create transaction 1 (nonce 1) that is confirmed (block 77) - etx1 := mustInsertConfirmedEthTxBySaveFetchedReceipts(t, txStore, fromAddress, int64(1), int64(77), *ethClient.ConfiguredChainID()) - assert.Equal(t, etx1.State, txmgrcommon.TxConfirmed) - // mark transaction 0 confirmed_missing_receipt - err := txStore.MarkAllConfirmedMissingReceipt(tests.Context(t), ethClient.ConfiguredChainID()) + tx1, err = txStore.FindTxWithAttempts(ctx, tx1.ID) require.NoError(t, err) - etx0, err = txStore.FindTxWithAttempts(ctx, etx0.ID) + require.Len(t, tx1.TxAttempts, 1) + require.Len(t, tx1.TxAttempts[0].Receipts, 1) + require.Equal(t, txmReceipt1.BlockHash, tx1.TxAttempts[0].Receipts[0].GetBlockHash()) + require.Equal(t, txmgrcommon.TxConfirmed, tx1.State) + + tx2, err = txStore.FindTxWithAttempts(ctx, tx2.ID) require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx0.State) + require.Len(t, tx2.TxAttempts, 1) + require.Len(t, tx2.TxAttempts[0].Receipts, 1) + require.Equal(t, txmReceipt2.BlockHash, tx2.TxAttempts[0].Receipts[0].GetBlockHash()) + require.Equal(t, txmgrcommon.TxFatalError, tx2.State) } func TestORM_PreloadTxes(t *testing.T) { @@ -641,7 +591,6 @@ func TestORM_FindTxesPendingCallback(t *testing.T) { Hash: testutils.NewHash(), Number: 10, } - head.Parent.Store(h9) minConfirmations := int64(2) @@ -769,7 +718,7 @@ func TestORM_UpdateTxForRebroadcast(t *testing.T) { _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) ctx := tests.Context(t) - t.Run("delete all receipts for eth transaction", func(t *testing.T) { + t.Run("marks confirmed tx as unconfirmed, marks latest attempt as in-progress, deletes receipt", func(t *testing.T) { etx := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 777, 1) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) assert.NoError(t, err) @@ -782,7 +731,7 @@ func TestORM_UpdateTxForRebroadcast(t *testing.T) { assert.Len(t, etx.TxAttempts[0].Receipts, 1) // use exported method - err = txStore.UpdateTxForRebroadcast(tests.Context(t), etx, attempt) + err = txStore.UpdateTxsForRebroadcast(tests.Context(t), []int64{etx.ID}, []int64{attempt.ID}) require.NoError(t, err) resultTx, err := txStore.FindTxWithAttempts(ctx, etx.ID) @@ -796,49 +745,39 @@ func TestORM_UpdateTxForRebroadcast(t *testing.T) { // assert tx state assert.Equal(t, txmgrcommon.TxUnconfirmed, resultTx.State) // assert receipt - assert.Len(t, resultTxAttempt.Receipts, 0) + assert.Empty(t, resultTxAttempt.Receipts) }) -} - -func TestORM_FindTransactionsConfirmedInBlockRange(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - h8 := &evmtypes.Head{ - Number: 8, - Hash: testutils.NewHash(), - } - h9 := &evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 9, - } - h9.Parent.Store(h8) - head := evmtypes.Head{ - Hash: testutils.NewHash(), - Number: 10, - } - head.Parent.Store(h9) - - t.Run("find all transactions confirmed in range", func(t *testing.T) { - etx_8 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 700, 8) - etx_9 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 777, 9) + t.Run("marks confirmed tx as unconfirmed, clears error, marks latest attempt as in-progress, deletes receipt", func(t *testing.T) { + blockNum := int64(100) + etx := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), etx.TxAttempts[0].Hash) + etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + // assert attempt state + attempt := etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt.State) + // assert tx state + assert.Equal(t, txmgrcommon.TxFatalError, etx.State) + // assert receipt + assert.Len(t, etx.TxAttempts[0].Receipts, 1) - etxes, err := txStore.FindTransactionsConfirmedInBlockRange(tests.Context(t), head.Number, 8, ethClient.ConfiguredChainID()) + // use exported method + err = txStore.UpdateTxsForRebroadcast(tests.Context(t), []int64{etx.ID}, []int64{attempt.ID}) require.NoError(t, err) - assert.Len(t, etxes, 2) - assert.Equal(t, etxes[0].Sequence, etx_8.Sequence) - assert.Equal(t, etxes[1].Sequence, etx_9.Sequence) - }) - t.Run("return empty txes when no transactions in range found", func(t *testing.T) { - etxes, err := txStore.FindTransactionsConfirmedInBlockRange(tests.Context(t), 0, 0, ethClient.ConfiguredChainID()) + resultTx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - assert.Len(t, etxes, 0) + require.Len(t, resultTx.TxAttempts, 1) + resultTxAttempt := resultTx.TxAttempts[0] + + // assert attempt state + assert.Equal(t, txmgrtypes.TxAttemptInProgress, resultTxAttempt.State) + assert.Nil(t, resultTxAttempt.BroadcastBeforeBlockNum) + // assert tx state + assert.Equal(t, txmgrcommon.TxUnconfirmed, resultTx.State) + // assert receipt + assert.Empty(t, resultTxAttempt.Receipts) }) } @@ -944,7 +883,7 @@ func TestORM_SaveSentAttempt(t *testing.T) { }) } -func TestORM_SaveConfirmedMissingReceiptAttempt(t *testing.T) { +func TestORM_SaveConfirmedAttempt(t *testing.T) { t.Parallel() ctx := tests.Context(t) @@ -959,12 +898,12 @@ func TestORM_SaveConfirmedMissingReceiptAttempt(t *testing.T) { etx := mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 1, fromAddress, txmgrtypes.TxAttemptInProgress) now := time.Now() - err = txStore.SaveConfirmedMissingReceiptAttempt(tests.Context(t), defaultDuration, &etx.TxAttempts[0], now) + err = txStore.SaveConfirmedAttempt(tests.Context(t), defaultDuration, &etx.TxAttempts[0], now) require.NoError(t, err) etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxConfirmedMissingReceipt, etx.State) + assert.Equal(t, txmgrcommon.TxConfirmed, etx.State) assert.Equal(t, txmgrtypes.TxAttemptBroadcast, etx.TxAttempts[0].State) }) } @@ -1115,7 +1054,7 @@ func TestEthConfirmer_FindTxsRequiringResubmissionDueToInsufficientEth(t *testin etxs, err := txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(tests.Context(t), fromAddress, big.NewInt(42)) require.NoError(t, err) - assert.Len(t, etxs, 0) + assert.Empty(t, etxs) }) t.Run("does not return confirmed or fatally errored eth_txes", func(t *testing.T) { @@ -1132,42 +1071,6 @@ func TestEthConfirmer_FindTxsRequiringResubmissionDueToInsufficientEth(t *testin }) } -func TestORM_MarkOldTxesMissingReceiptAsErrored(t *testing.T) { - t.Parallel() - - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - ctx := tests.Context(t) - ethKeyStore := cltest.NewKeyStore(t, db).Eth() - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) - latestFinalizedBlockNum := int64(8) - - // tx state should be confirmed missing receipt - // attempt should be before latestFinalizedBlockNum - t.Run("successfully mark errored transactions", func(t *testing.T) { - etx := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt(t, txStore, 1, 7, time.Now(), fromAddress) - - err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, latestFinalizedBlockNum, ethClient.ConfiguredChainID()) - require.NoError(t, err) - - etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxFatalError, etx.State) - }) - - t.Run("successfully mark errored transactions w/ qopt passing in sql.Tx", func(t *testing.T) { - etx := mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt(t, txStore, 1, 7, time.Now(), fromAddress) - err := txStore.MarkOldTxesMissingReceiptAsErrored(tests.Context(t), 10, latestFinalizedBlockNum, ethClient.ConfiguredChainID()) - require.NoError(t, err) - - // must run other query outside of postgres transaction so changes are committed - etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) - require.NoError(t, err) - assert.Equal(t, txmgrcommon.TxFatalError, etx.State) - }) -} - func TestORM_LoadEthTxesAttempts(t *testing.T) { t.Parallel() @@ -1271,7 +1174,7 @@ func TestORM_FindNextUnstartedTransactionFromAddress(t *testing.T) { }) } -func TestORM_UpdateTxFatalError(t *testing.T) { +func TestORM_UpdateTxFatalErrorAndDeleteAttempts(t *testing.T) { t.Parallel() ctx := tests.Context(t) @@ -1286,11 +1189,11 @@ func TestORM_UpdateTxFatalError(t *testing.T) { etxPretendError := null.StringFrom("no more toilet paper") etx.Error = etxPretendError - err := txStore.UpdateTxFatalError(tests.Context(t), &etx) + err := txStore.UpdateTxFatalErrorAndDeleteAttempts(tests.Context(t), &etx) require.NoError(t, err) etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) require.NoError(t, err) - assert.Len(t, etx.TxAttempts, 0) + assert.Empty(t, etx.TxAttempts) assert.Equal(t, txmgrcommon.TxFatalError, etx.State) }) } @@ -1804,7 +1707,7 @@ func TestORM_CreateTransaction(t *testing.T) { assert.Greater(t, etx.ID, int64(0)) assert.Equal(t, fromAddress, etx.FromAddress) - assert.Equal(t, true, etx.SignalCallback) + assert.True(t, etx.SignalCallback) cltest.AssertCount(t, db, "evm.txes", 3) @@ -1812,7 +1715,7 @@ func TestORM_CreateTransaction(t *testing.T) { require.NoError(t, db.Get(&dbEthTx, `SELECT * FROM evm.txes ORDER BY id DESC LIMIT 1`)) assert.Equal(t, fromAddress, dbEthTx.FromAddress) - assert.Equal(t, true, dbEthTx.SignalCallback) + assert.True(t, dbEthTx.SignalCallback) }) } @@ -1873,30 +1776,64 @@ func AssertCountPerSubject(t *testing.T, txStore txmgr.TestEvmTxStore, expected require.Equal(t, int(expected), count) } -func TestORM_FindTransactionsByState(t *testing.T) { +func TestORM_FindAttemptsRequiringReceiptFetch(t *testing.T) { t.Parallel() ctx := tests.Context(t) - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db) - kst := cltest.NewKeyStore(t, db) - _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) - finalizedBlockNum := int64(100) - - mustInsertUnstartedTx(t, txStore, fromAddress) - mustInsertInProgressEthTxWithAttempt(t, txStore, 0, fromAddress) - mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 1, fromAddress, txmgrtypes.TxAttemptBroadcast) - mustInsertConfirmedMissingReceiptEthTxWithLegacyAttempt(t, txStore, 2, finalizedBlockNum, time.Now(), fromAddress) - mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 3, finalizedBlockNum+1) - mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 4, finalizedBlockNum) - mustInsertFatalErrorEthTx(t, txStore, fromAddress) - - receipts, err := txStore.FindConfirmedTxesReceipts(ctx, finalizedBlockNum, testutils.FixtureChainID) - require.NoError(t, err) - require.Len(t, receipts, 1) + blockNum := int64(100) + + t.Run("finds confirmed transaction requiring receipt fetch", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + kst := cltest.NewKeyStore(t, db) + _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + // Transactions whose attempts should not be picked up for receipt fetch + mustInsertFatalErrorEthTx(t, txStore, fromAddress) + mustInsertUnstartedTx(t, txStore, fromAddress) + mustInsertInProgressEthTxWithAttempt(t, txStore, 4, fromAddress) + mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 3, fromAddress, txmgrtypes.TxAttemptBroadcast) + mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 2, blockNum) + // Terminally stuck transaction with receipt should NOT be picked up for receipt fetch + stuckTx := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), stuckTx.TxAttempts[0].Hash) + + // Confirmed transaction without receipt should be picked up for receipt fetch + confirmedTx := mustInsertConfirmedEthTx(t, txStore, 0, fromAddress) + attempt := newBroadcastLegacyEthTxAttempt(t, confirmedTx.ID) + err := txStore.InsertTxAttempt(ctx, &attempt) + require.NoError(t, err) + + attempts, err := txStore.FindAttemptsRequiringReceiptFetch(ctx, testutils.FixtureChainID) + require.NoError(t, err) + require.Len(t, attempts, 1) + require.Equal(t, attempt.Hash.String(), attempts[0].Hash.String()) + }) + + t.Run("finds terminally stuck transaction requiring receipt fetch", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + kst := cltest.NewKeyStore(t, db) + _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + // Transactions whose attempts should not be picked up for receipt fetch + mustInsertFatalErrorEthTx(t, txStore, fromAddress) + mustInsertUnstartedTx(t, txStore, fromAddress) + mustInsertInProgressEthTxWithAttempt(t, txStore, 4, fromAddress) + mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 3, fromAddress, txmgrtypes.TxAttemptBroadcast) + mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 2, blockNum) + // Terminally stuck transaction with receipt should NOT be picked up for receipt fetch + stuckTxWithReceipt := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + mustInsertEthReceipt(t, txStore, blockNum, utils.NewHash(), stuckTxWithReceipt.TxAttempts[0].Hash) + // Terminally stuck transaction without receipt should be picked up for receipt fetch + stuckTxWoutReceipt := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 0, blockNum) + + attempts, err := txStore.FindAttemptsRequiringReceiptFetch(ctx, testutils.FixtureChainID) + require.NoError(t, err) + require.Len(t, attempts, 1) + require.Equal(t, stuckTxWoutReceipt.TxAttempts[0].Hash.String(), attempts[0].Hash.String()) + }) } -func TestORM_UpdateTxesFinalized(t *testing.T) { +func TestORM_UpdateTxStatesToFinalizedUsingTxHashes(t *testing.T) { t.Parallel() ctx := tests.Context(t) @@ -1921,11 +1858,176 @@ func TestORM_UpdateTxesFinalized(t *testing.T) { attempt := newBroadcastLegacyEthTxAttempt(t, tx.ID) err = txStore.InsertTxAttempt(ctx, &attempt) require.NoError(t, err) - receipt := mustInsertEthReceipt(t, txStore, 100, testutils.NewHash(), attempt.Hash) - err = txStore.UpdateTxStatesToFinalizedUsingReceiptIds(ctx, []int64{receipt.ID}, testutils.FixtureChainID) + mustInsertEthReceipt(t, txStore, 100, testutils.NewHash(), attempt.Hash) + err = txStore.UpdateTxStatesToFinalizedUsingTxHashes(ctx, []common.Hash{attempt.Hash}, testutils.FixtureChainID) require.NoError(t, err) etx, err := txStore.FindTxWithAttempts(ctx, tx.ID) require.NoError(t, err) require.Equal(t, txmgrcommon.TxFinalized, etx.State) }) } + +func TestORM_FindReorgOrIncludedTxs(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + kst := cltest.NewKeyStore(t, db) + blockNum := int64(100) + t.Run("finds re-org'd transactions using the mined tx count", func(t *testing.T) { + _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + _, otherAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + // Unstarted can't be re-org'd + mustInsertUnstartedTx(t, txStore, fromAddress) + // In-Progress can't be re-org'd + mustInsertInProgressEthTxWithAttempt(t, txStore, 4, fromAddress) + // Unconfirmed can't be re-org'd + mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 3, fromAddress, txmgrtypes.TxAttemptBroadcast) + // Confirmed and nonce greater than mined tx count so has been re-org'd + mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 2, blockNum) + // Fatal error and nonce equal to mined tx count so has been re-org'd + mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + // Nonce lower than mined tx count so has not been re-org + mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + + // Tx for another from address should not be returned + mustInsertConfirmedEthTxWithReceipt(t, txStore, otherAddress, 1, blockNum) + mustInsertConfirmedEthTxWithReceipt(t, txStore, otherAddress, 0, blockNum) + + reorgTxs, includedTxs, err := txStore.FindReorgOrIncludedTxs(ctx, fromAddress, evmtypes.Nonce(1), testutils.FixtureChainID) + require.NoError(t, err) + require.Len(t, reorgTxs, 2) + require.Empty(t, includedTxs) + }) + + t.Run("finds transactions included on-chain using the mined tx count", func(t *testing.T) { + _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + _, otherAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + // Unstarted can't be included + mustInsertUnstartedTx(t, txStore, fromAddress) + // In-Progress can't be included + mustInsertInProgressEthTxWithAttempt(t, txStore, 5, fromAddress) + // Unconfirmed with higher nonce can't be included + mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 4, fromAddress, txmgrtypes.TxAttemptBroadcast) + // Unconfirmed with nonce less than mined tx count is newly included + mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 3, fromAddress, txmgrtypes.TxAttemptBroadcast) + // Unconfirmed with purge attempt with nonce less than mined tx cound is newly included + mustInsertUnconfirmedEthTxWithBroadcastPurgeAttempt(t, txStore, 2, fromAddress) + // Fatal error so already included + mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, blockNum) + // Confirmed so already included + mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, blockNum) + + // Tx for another from address should not be returned + mustInsertConfirmedEthTxWithReceipt(t, txStore, otherAddress, 1, blockNum) + mustInsertConfirmedEthTxWithReceipt(t, txStore, otherAddress, 0, blockNum) + + reorgTxs, includedTxs, err := txStore.FindReorgOrIncludedTxs(ctx, fromAddress, evmtypes.Nonce(4), testutils.FixtureChainID) + require.NoError(t, err) + require.Len(t, includedTxs, 2) + require.Empty(t, reorgTxs) + }) +} + +func TestORM_UpdateTxFatalError(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + kst := cltest.NewKeyStore(t, db) + _, fromAddress := cltest.MustInsertRandomKey(t, kst.Eth()) + t.Run("successfully marks transaction as fatal with error message", func(t *testing.T) { + // Unconfirmed with purge attempt with nonce less than mined tx cound is newly included + tx1 := mustInsertUnconfirmedEthTxWithBroadcastPurgeAttempt(t, txStore, 0, fromAddress) + tx2 := mustInsertUnconfirmedEthTxWithBroadcastPurgeAttempt(t, txStore, 1, fromAddress) + + err := txStore.UpdateTxFatalError(ctx, []int64{tx1.ID, tx2.ID}, client.TerminallyStuckMsg) + require.NoError(t, err) + + tx1, err = txStore.FindTxWithAttempts(ctx, tx1.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, tx1.State) + require.Equal(t, client.TerminallyStuckMsg, tx1.Error.String) + tx2, err = txStore.FindTxWithAttempts(ctx, tx2.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, tx2.State) + require.Equal(t, client.TerminallyStuckMsg, tx2.Error.String) + }) +} + +func TestORM_FindTxesByIDs(t *testing.T) { + t.Parallel() + + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ctx := tests.Context(t) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + + // tx state should be confirmed missing receipt + // attempt should be before latestFinalizedBlockNum + t.Run("successfully finds transactions with IDs", func(t *testing.T) { + etx1 := mustInsertInProgressEthTxWithAttempt(t, txStore, 3, fromAddress) + etx2 := mustInsertUnconfirmedEthTxWithAttemptState(t, txStore, 2, fromAddress, txmgrtypes.TxAttemptBroadcast) + etx3 := mustInsertTerminallyStuckTxWithAttempt(t, txStore, fromAddress, 1, 100) + etx4 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, 100) + + etxIDs := []int64{etx1.ID, etx2.ID, etx3.ID, etx4.ID} + oldTxs, err := txStore.FindTxesByIDs(ctx, etxIDs, testutils.FixtureChainID) + require.NoError(t, err) + require.Len(t, oldTxs, 4) + }) +} + +func TestORM_DeleteReceiptsByTxHash(t *testing.T) { + t.Parallel() + + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ctx := tests.Context(t) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + + etx1 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 0, 100) + etx2 := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 2, 100) + + // Delete one transaction's receipt + err := txStore.DeleteReceiptByTxHash(ctx, etx1.TxAttempts[0].Hash) + require.NoError(t, err) + + // receipt has been deleted + etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) + require.NoError(t, err) + require.Empty(t, etx1.TxAttempts[0].Receipts) + + // receipt still exists for other tx + etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) + require.NoError(t, err) + require.Len(t, etx2.TxAttempts[0].Receipts, 1) +} + +func mustInsertTerminallyStuckTxWithAttempt(t *testing.T, txStore txmgr.TestEvmTxStore, fromAddress common.Address, nonceInt int64, broadcastBeforeBlockNum int64) txmgr.Tx { + ctx := tests.Context(t) + broadcast := time.Now() + nonce := evmtypes.Nonce(nonceInt) + tx := txmgr.Tx{ + Sequence: &nonce, + FromAddress: fromAddress, + EncodedPayload: []byte{1, 2, 3}, + State: txmgrcommon.TxFatalError, + BroadcastAt: &broadcast, + InitialBroadcastAt: &broadcast, + Error: null.StringFrom(client.TerminallyStuckMsg), + } + require.NoError(t, txStore.InsertTx(ctx, &tx)) + attempt := cltest.NewLegacyEthTxAttempt(t, tx.ID) + attempt.BroadcastBeforeBlockNum = &broadcastBeforeBlockNum + attempt.State = txmgrtypes.TxAttemptBroadcast + attempt.IsPurgeAttempt = true + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + tx, err := txStore.FindTxWithAttempts(ctx, tx.ID) + require.NoError(t, err) + return tx +} diff --git a/core/chains/evm/txmgr/finalizer.go b/core/chains/evm/txmgr/finalizer.go index 60744636159..b5fe5ae37e2 100644 --- a/core/chains/evm/txmgr/finalizer.go +++ b/core/chains/evm/txmgr/finalizer.go @@ -2,13 +2,21 @@ package txmgr import ( "context" + "database/sql" + "errors" "fmt" "math/big" + "strconv" "sync" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "gopkg.in/guregu/null.v4" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -20,28 +28,70 @@ import ( var _ Finalizer = (*evmFinalizer)(nil) +var ( + promNumSuccessfulTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_successful_transactions", + Help: "Total number of successful transactions. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promRevertedTxCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_tx_reverted", + Help: "Number of times a transaction reverted on-chain. Note that this can err to be too high since transactions are counted on each confirmation, which can happen multiple times per transaction in the case of re-orgs", + }, []string{"chainID"}) + promFwdTxCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_fwd_tx_count", + Help: "The number of forwarded transaction attempts labeled by status", + }, []string{"chainID", "successful"}) + promTxAttemptCount = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tx_manager_tx_attempt_count", + Help: "The number of transaction attempts that are currently being processed by the transaction manager", + }, []string{"chainID"}) + promNumFinalizedTxs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tx_manager_num_finalized_transactions", + Help: "Total number of finalized transactions", + }, []string{"chainID"}) +) + +var ( + // ErrCouldNotGetReceipt is the error string we save if we reach our LatestFinalizedBlockNum for a confirmed transaction + // without ever getting a receipt. This most likely happened because an external wallet used the account for this nonce + ErrCouldNotGetReceipt = "could not get receipt" +) + // processHeadTimeout represents a sanity limit on how long ProcessHead should take to complete const processHeadTimeout = 10 * time.Minute type finalizerTxStore interface { - FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) ([]Receipt, error) - UpdateTxStatesToFinalizedUsingReceiptIds(ctx context.Context, txs []int64, chainId *big.Int) error + DeleteReceiptByTxHash(ctx context.Context, txHash common.Hash) error + FindAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) (hashes []TxAttempt, err error) + FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) (receipts []*evmtypes.Receipt, err error) + FindTxesPendingCallback(ctx context.Context, latest, finalized int64, chainID *big.Int) (receiptsPlus []ReceiptPlus, err error) + FindTxesByIDs(ctx context.Context, etxIDs []int64, chainID *big.Int) (etxs []*Tx, err error) + PreloadTxes(ctx context.Context, attempts []TxAttempt) error + SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt) (err error) + UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunID uuid.UUID, chainID *big.Int) error + UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *Tx) error + UpdateTxStatesToFinalizedUsingTxHashes(ctx context.Context, txHashes []common.Hash, chainID *big.Int) error } type finalizerChainClient interface { BatchCallContext(ctx context.Context, elems []rpc.BatchElem) error + BatchGetReceipts(ctx context.Context, attempts []TxAttempt) (txReceipt []*evmtypes.Receipt, txErr []error, funcErr error) + CallContract(ctx context.Context, a TxAttempt, blockNumber *big.Int) (rpcErr fmt.Stringer, extractErr error) } type finalizerHeadTracker interface { LatestAndFinalizedBlock(ctx context.Context) (latest, finalized *evmtypes.Head, err error) } +type resumeCallback = func(context.Context, uuid.UUID, interface{}, error) error + // Finalizer handles processing new finalized blocks and marking transactions as finalized accordingly in the TXM DB type evmFinalizer struct { services.StateMachine - lggr logger.SugaredLogger - chainId *big.Int - rpcBatchSize int + lggr logger.SugaredLogger + chainID *big.Int + rpcBatchSize int + forwardersEnabled bool txStore finalizerTxStore client finalizerChainClient @@ -52,32 +102,40 @@ type evmFinalizer struct { wg sync.WaitGroup lastProcessedFinalizedBlockNum int64 + resumeCallback resumeCallback } func NewEvmFinalizer( lggr logger.Logger, - chainId *big.Int, + chainID *big.Int, rpcBatchSize uint32, + forwardersEnabled bool, txStore finalizerTxStore, client finalizerChainClient, headTracker finalizerHeadTracker, ) *evmFinalizer { lggr = logger.Named(lggr, "Finalizer") return &evmFinalizer{ - lggr: logger.Sugared(lggr), - chainId: chainId, - rpcBatchSize: int(rpcBatchSize), - txStore: txStore, - client: client, - headTracker: headTracker, - mb: mailbox.NewSingle[*evmtypes.Head](), + lggr: logger.Sugared(lggr), + chainID: chainID, + rpcBatchSize: int(rpcBatchSize), + forwardersEnabled: forwardersEnabled, + txStore: txStore, + client: client, + headTracker: headTracker, + mb: mailbox.NewSingle[*evmtypes.Head](), + resumeCallback: nil, } } +func (f *evmFinalizer) SetResumeCallback(callback resumeCallback) { + f.resumeCallback = callback +} + // Start the finalizer func (f *evmFinalizer) Start(ctx context.Context) error { return f.StartOnce("Finalizer", func() error { - f.lggr.Debugf("started Finalizer with RPC batch size limit: %d", f.rpcBatchSize) + f.lggr.Debugw("started Finalizer", "rpcBatchSize", f.rpcBatchSize, "forwardersEnabled", f.forwardersEnabled) f.stopCh = make(chan struct{}) f.wg.Add(1) go f.runLoop() @@ -141,10 +199,23 @@ func (f *evmFinalizer) ProcessHead(ctx context.Context, head *evmtypes.Head) err if err != nil { return fmt.Errorf("failed to retrieve latest finalized head: %w", err) } + // Fetch and store receipts for confirmed transactions that do not have locally stored receipts + err = f.FetchAndStoreReceipts(ctx, head, latestFinalizedHead) + // Do not return on error since other functions are not dependent on results + if err != nil { + f.lggr.Errorf("failed to fetch and store receipts for confirmed transactions: %s", err.Error()) + } + // Resume pending task runs if any receipts match the min confirmation criteria + err = f.ResumePendingTaskRuns(ctx, head.BlockNumber(), latestFinalizedHead.BlockNumber()) + // Do not return on error since other functions are not dependent on results + if err != nil { + f.lggr.Errorf("failed to resume pending task runs: %s", err.Error()) + } return f.processFinalizedHead(ctx, latestFinalizedHead) } -// Determines if any confirmed transactions can be marked as finalized by comparing their receipts against the latest finalized block +// processFinalizedHead determines if any confirmed transactions can be marked as finalized by comparing their receipts against the latest finalized block +// Fetches receipts directly from on-chain so re-org detection is not needed during finalization func (f *evmFinalizer) processFinalizedHead(ctx context.Context, latestFinalizedHead *evmtypes.Head) error { // Cannot determine finality without a finalized head for comparison if latestFinalizedHead == nil || !latestFinalizedHead.IsValid() { @@ -156,23 +227,28 @@ func (f *evmFinalizer) processFinalizedHead(ctx context.Context, latestFinalized return nil } if latestFinalizedHead.BlockNumber() < f.lastProcessedFinalizedBlockNum { - f.lggr.Errorw("Received finalized block older than one already processed. This should never happen and could be an issue with RPCs.", "lastProcessedFinalizedBlockNum", f.lastProcessedFinalizedBlockNum, "retrievedFinalizedBlockNum", latestFinalizedHead.BlockNumber()) + f.lggr.Errorw("Received finalized block older than one already processed", "lastProcessedFinalizedBlockNum", f.lastProcessedFinalizedBlockNum, "retrievedFinalizedBlockNum", latestFinalizedHead.BlockNumber()) return nil } earliestBlockNumInChain := latestFinalizedHead.EarliestHeadInChain().BlockNumber() f.lggr.Debugw("processing latest finalized head", "blockNum", latestFinalizedHead.BlockNumber(), "blockHash", latestFinalizedHead.BlockHash(), "earliestBlockNumInChain", earliestBlockNumInChain) - // Retrieve all confirmed transactions with receipts older than or equal to the finalized block, loaded with attempts and receipts - unfinalizedReceipts, err := f.txStore.FindConfirmedTxesReceipts(ctx, latestFinalizedHead.BlockNumber(), f.chainId) + mark := time.Now() + // Retrieve all confirmed transactions with receipts older than or equal to the finalized block + unfinalizedReceipts, err := f.txStore.FindConfirmedTxesReceipts(ctx, latestFinalizedHead.BlockNumber(), f.chainID) if err != nil { return fmt.Errorf("failed to retrieve receipts for confirmed, unfinalized transactions: %w", err) } + if len(unfinalizedReceipts) > 0 { + f.lggr.Debugw(fmt.Sprintf("found %d receipts for potential finalized transactions", len(unfinalizedReceipts)), "timeElapsed", time.Since(mark)) + } + mark = time.Now() - var finalizedReceipts []Receipt + finalizedReceipts := make([]*evmtypes.Receipt, 0, len(unfinalizedReceipts)) // Group by block hash transactions whose receipts cannot be validated using the cached heads - blockNumToReceiptsMap := make(map[int64][]Receipt) - // Find transactions with receipt block nums older than the latest finalized block num and block hashes still in chain + blockNumToReceiptsMap := make(map[int64][]*evmtypes.Receipt) + // Find receipts with block nums older than or equal to the latest finalized block num for _, receipt := range unfinalizedReceipts { // The tx store query ensures transactions have receipts but leaving this check here for a belts and braces approach if receipt.TxHash == utils.EmptyHash || receipt.BlockHash == utils.EmptyHash { @@ -180,49 +256,64 @@ func (f *evmFinalizer) processFinalizedHead(ctx context.Context, latestFinalized continue } // The tx store query only returns transactions with receipts older than or equal to the finalized block but leaving this check here for a belts and braces approach - if receipt.BlockNumber > latestFinalizedHead.BlockNumber() { + if receipt.BlockNumber.Int64() > latestFinalizedHead.BlockNumber() { continue } // Receipt block num older than earliest head in chain. Validate hash using RPC call later - if receipt.BlockNumber < earliestBlockNumInChain { - blockNumToReceiptsMap[receipt.BlockNumber] = append(blockNumToReceiptsMap[receipt.BlockNumber], receipt) + if receipt.BlockNumber.Int64() < earliestBlockNumInChain { + blockNumToReceiptsMap[receipt.BlockNumber.Int64()] = append(blockNumToReceiptsMap[receipt.BlockNumber.Int64()], receipt) continue } - blockHashInChain := latestFinalizedHead.HashAtHeight(receipt.BlockNumber) + blockHashInChain := latestFinalizedHead.HashAtHeight(receipt.BlockNumber.Int64()) // Receipt block hash does not match the block hash in chain. Transaction has been re-org'd out but DB state has not been updated yet if blockHashInChain.String() != receipt.BlockHash.String() { // Log error if a transaction is marked as confirmed with a receipt older than the finalized block - // This scenario could potentially point to a re-org'd transaction the Confirmer has lost track of - f.lggr.Errorw("found confirmed transaction with re-org'd receipt older than finalized block", "receipt", receipt, "onchainBlockHash", blockHashInChain.String()) + // This scenario could potentially be caused by a stale receipt stored for a re-org'd transaction + f.lggr.Debugw("found confirmed transaction with re-org'd receipt", "receipt", receipt, "onchainBlockHash", blockHashInChain.String()) + err = f.txStore.DeleteReceiptByTxHash(ctx, receipt.GetTxHash()) + // Log error but allow process to continue so other transactions can still be marked as finalized + if err != nil { + f.lggr.Errorw("failed to delete receipt", "receipt", receipt) + } continue } finalizedReceipts = append(finalizedReceipts, receipt) } + if len(finalizedReceipts) > 0 { + f.lggr.Debugw(fmt.Sprintf("found %d finalized transactions using local block history", len(finalizedReceipts)), "latestFinalizedBlockNum", latestFinalizedHead.BlockNumber(), "timeElapsed", time.Since(mark)) + } + mark = time.Now() // Check if block hashes exist for receipts on-chain older than the earliest cached head // Transactions are grouped by their receipt block hash to avoid repeat requests on the same hash in case transactions were confirmed in the same block validatedReceipts := f.batchCheckReceiptHashesOnchain(ctx, blockNumToReceiptsMap) finalizedReceipts = append(finalizedReceipts, validatedReceipts...) + if len(validatedReceipts) > 0 { + f.lggr.Debugw(fmt.Sprintf("found %d finalized transactions validated against RPC", len(validatedReceipts)), "latestFinalizedBlockNum", latestFinalizedHead.BlockNumber(), "timeElapsed", time.Since(mark)) + } + txHashes := f.buildTxHashList(finalizedReceipts) - receiptIDs := f.buildReceiptIdList(finalizedReceipts) - - err = f.txStore.UpdateTxStatesToFinalizedUsingReceiptIds(ctx, receiptIDs, f.chainId) + err = f.txStore.UpdateTxStatesToFinalizedUsingTxHashes(ctx, txHashes, f.chainID) if err != nil { return fmt.Errorf("failed to update transactions as finalized: %w", err) } // Update lastProcessedFinalizedBlockNum after processing has completed to allow failed processing to retry on subsequent heads // Does not need to be protected with mutex lock because the Finalizer only runs in a single loop f.lastProcessedFinalizedBlockNum = latestFinalizedHead.BlockNumber() + + // add newly finalized transactions to the prom metric + promNumFinalizedTxs.WithLabelValues(f.chainID.String()).Add(float64(len(txHashes))) + return nil } -func (f *evmFinalizer) batchCheckReceiptHashesOnchain(ctx context.Context, blockNumToReceiptsMap map[int64][]Receipt) []Receipt { +func (f *evmFinalizer) batchCheckReceiptHashesOnchain(ctx context.Context, blockNumToReceiptsMap map[int64][]*evmtypes.Receipt) []*evmtypes.Receipt { if len(blockNumToReceiptsMap) == 0 { return nil } // Group the RPC batch calls in groups of rpcBatchSize - var rpcBatchGroups [][]rpc.BatchElem - var rpcBatch []rpc.BatchElem + rpcBatchGroups := make([][]rpc.BatchElem, 0, len(blockNumToReceiptsMap)) + rpcBatch := make([]rpc.BatchElem, 0, f.rpcBatchSize) for blockNum := range blockNumToReceiptsMap { elem := rpc.BatchElem{ Method: "eth_getBlockByNumber", @@ -235,14 +326,14 @@ func (f *evmFinalizer) batchCheckReceiptHashesOnchain(ctx context.Context, block rpcBatch = append(rpcBatch, elem) if len(rpcBatch) >= f.rpcBatchSize { rpcBatchGroups = append(rpcBatchGroups, rpcBatch) - rpcBatch = []rpc.BatchElem{} + rpcBatch = make([]rpc.BatchElem, 0, f.rpcBatchSize) } } if len(rpcBatch) > 0 { rpcBatchGroups = append(rpcBatchGroups, rpcBatch) } - var finalizedReceipts []Receipt + finalizedReceipts := make([]*evmtypes.Receipt, 0, len(blockNumToReceiptsMap)) for _, rpcBatch := range rpcBatchGroups { err := f.client.BatchCallContext(ctx, rpcBatch) if err != nil { @@ -271,8 +362,13 @@ func (f *evmFinalizer) batchCheckReceiptHashesOnchain(ctx context.Context, block finalizedReceipts = append(finalizedReceipts, receipt) } else { // Log error if a transaction is marked as confirmed with a receipt older than the finalized block - // This scenario could potentially point to a re-org'd transaction the Confirmer has lost track of - f.lggr.Errorw("found confirmed transaction with re-org'd receipt older than finalized block", "receipt", receipt, "onchainBlockHash", head.BlockHash().String()) + // This scenario could potentially be caused by a stale receipt stored for a re-org'd transaction + f.lggr.Debugw("found confirmed transaction with re-org'd receipt", "receipt", receipt, "onchainBlockHash", head.BlockHash().String()) + err = f.txStore.DeleteReceiptByTxHash(ctx, receipt.GetTxHash()) + // Log error but allow process to continue so other transactions can still be marked as finalized + if err != nil { + f.lggr.Errorw("failed to delete receipt", "receipt", receipt) + } } } } @@ -280,16 +376,293 @@ func (f *evmFinalizer) batchCheckReceiptHashesOnchain(ctx context.Context, block return finalizedReceipts } -// Build list of transaction IDs -func (f *evmFinalizer) buildReceiptIdList(finalizedReceipts []Receipt) []int64 { - receiptIds := make([]int64, len(finalizedReceipts)) +func (f *evmFinalizer) FetchAndStoreReceipts(ctx context.Context, head, latestFinalizedHead *evmtypes.Head) error { + attempts, err := f.txStore.FindAttemptsRequiringReceiptFetch(ctx, f.chainID) + if err != nil { + return fmt.Errorf("failed to fetch broadcasted attempts for confirmed transactions: %w", err) + } + if len(attempts) == 0 { + return nil + } + promTxAttemptCount.WithLabelValues(f.chainID.String()).Set(float64(len(attempts))) + + f.lggr.Debugw(fmt.Sprintf("Fetching receipts for %v transaction attempts", len(attempts))) + + batchSize := f.rpcBatchSize + if batchSize == 0 { + batchSize = len(attempts) + } + allReceipts := make([]*evmtypes.Receipt, 0, len(attempts)) + errorList := make([]error, 0, len(attempts)) + for i := 0; i < len(attempts); i += batchSize { + j := i + batchSize + if j > len(attempts) { + j = len(attempts) + } + batch := attempts[i:j] + + receipts, fetchErr := f.batchFetchReceipts(ctx, batch) + if fetchErr != nil { + errorList = append(errorList, fetchErr) + continue + } + + allReceipts = append(allReceipts, receipts...) + + if err = f.txStore.SaveFetchedReceipts(ctx, receipts); err != nil { + errorList = append(errorList, err) + continue + } + } + if len(errorList) > 0 { + return errors.Join(errorList...) + } + + oldTxIDs := findOldTxIDsWithoutReceipts(attempts, allReceipts, latestFinalizedHead) + // Process old transactions that never received receipts and need to be marked as fatal + err = f.ProcessOldTxsWithoutReceipts(ctx, oldTxIDs, head, latestFinalizedHead) + if err != nil { + return err + } + + return nil +} + +func (f *evmFinalizer) batchFetchReceipts(ctx context.Context, attempts []TxAttempt) (receipts []*evmtypes.Receipt, err error) { + // Metadata is required to determine whether a tx is forwarded or not. + if f.forwardersEnabled { + err = f.txStore.PreloadTxes(ctx, attempts) + if err != nil { + return nil, fmt.Errorf("Confirmer#batchFetchReceipts error loading txs for attempts: %w", err) + } + } + + txReceipts, txErrs, err := f.client.BatchGetReceipts(ctx, attempts) + if err != nil { + return nil, err + } + + for i, receipt := range txReceipts { + attempt := attempts[i] + err := txErrs[i] + if err != nil { + f.lggr.Error("FetchReceipts failed") + continue + } + ok := f.validateReceipt(ctx, receipt, attempt) + if !ok { + continue + } + receipts = append(receipts, receipt) + } + + return +} + +// Note this function will increment promRevertedTxCount upon receiving a reverted transaction receipt +func (f *evmFinalizer) validateReceipt(ctx context.Context, receipt *evmtypes.Receipt, attempt TxAttempt) bool { + l := attempt.Tx.GetLogger(f.lggr).With("txHash", attempt.Hash.String(), "txAttemptID", attempt.ID, + "txID", attempt.TxID, "nonce", attempt.Tx.Sequence, + ) + + if receipt == nil { + // NOTE: This should never happen, but it seems safer to check + // regardless to avoid a potential panic + l.AssumptionViolation("got nil receipt") + return false + } + + if receipt.IsZero() { + l.Debug("Still waiting for receipt") + return false + } + + l = l.With("blockHash", receipt.GetBlockHash().String(), "status", receipt.GetStatus(), "transactionIndex", receipt.GetTransactionIndex()) + + if receipt.IsUnmined() { + l.Debug("Got receipt for transaction but it's still in the mempool and not included in a block yet") + return false + } + + l.Debugw("Got receipt for transaction", "blockNumber", receipt.GetBlockNumber(), "feeUsed", receipt.GetFeeUsed()) + + if receipt.GetTxHash().String() != attempt.Hash.String() { + l.Errorf("Invariant violation, expected receipt with hash %s to have same hash as attempt with hash %s", receipt.GetTxHash().String(), attempt.Hash.String()) + return false + } + + if receipt.GetBlockNumber() == nil { + l.Error("Invariant violation, receipt was missing block number") + return false + } + + if receipt.GetStatus() == 0 { + if receipt.GetRevertReason() != nil { + l.Warnw("transaction reverted on-chain", "hash", receipt.GetTxHash(), "revertReason", *receipt.GetRevertReason()) + } else { + rpcError, errExtract := f.client.CallContract(ctx, attempt, receipt.GetBlockNumber()) + if errExtract == nil { + l.Warnw("transaction reverted on-chain", "hash", receipt.GetTxHash(), "rpcError", rpcError.String()) + } else { + l.Warnw("transaction reverted on-chain unable to extract revert reason", "hash", receipt.GetTxHash(), "err", errExtract) + } + } + // This might increment more than once e.g. in case of re-orgs going back and forth we might re-fetch the same receipt + promRevertedTxCount.WithLabelValues(f.chainID.String()).Add(1) + } else { + promNumSuccessfulTxs.WithLabelValues(f.chainID.String()).Add(1) + } + + // This is only recording forwarded tx that were mined and have a status. + // Counters are prone to being inaccurate due to re-orgs. + if f.forwardersEnabled { + meta, metaErr := attempt.Tx.GetMeta() + if metaErr == nil && meta != nil && meta.FwdrDestAddress != nil { + // promFwdTxCount takes two labels, chainID and a boolean of whether a tx was successful or not. + promFwdTxCount.WithLabelValues(f.chainID.String(), strconv.FormatBool(receipt.GetStatus() != 0)).Add(1) + } + } + return true +} + +// ResumePendingTaskRuns issues callbacks to task runs that are pending waiting for receipts +func (f *evmFinalizer) ResumePendingTaskRuns(ctx context.Context, latest, finalized int64) error { + if f.resumeCallback == nil { + return nil + } + receiptsPlus, err := f.txStore.FindTxesPendingCallback(ctx, latest, finalized, f.chainID) + + if err != nil { + return err + } + + if len(receiptsPlus) > 0 { + f.lggr.Debugf("Resuming %d task runs pending receipt", len(receiptsPlus)) + } else { + f.lggr.Debug("No task runs to resume") + } + for _, data := range receiptsPlus { + var taskErr error + var output interface{} + if data.FailOnRevert && data.Receipt.GetStatus() == 0 { + taskErr = fmt.Errorf("transaction %s reverted on-chain", data.Receipt.GetTxHash()) + } else { + output = data.Receipt + } + + f.lggr.Debugw("Callback: resuming tx with receipt", "output", output, "taskErr", taskErr, "pipelineTaskRunID", data.ID) + if err := f.resumeCallback(ctx, data.ID, output, taskErr); err != nil { + return fmt.Errorf("failed to resume suspended pipeline run: %w", err) + } + // Mark tx as having completed callback + if err := f.txStore.UpdateTxCallbackCompleted(ctx, data.ID, f.chainID); err != nil { + return err + } + } + + return nil +} + +func (f *evmFinalizer) ProcessOldTxsWithoutReceipts(ctx context.Context, oldTxIDs []int64, head, latestFinalizedHead *evmtypes.Head) error { + if len(oldTxIDs) == 0 { + return nil + } + oldTxs, err := f.txStore.FindTxesByIDs(ctx, oldTxIDs, f.chainID) + if err != nil { + return fmt.Errorf("failed to find transactions with IDs: %w", err) + } + + errorList := make([]error, 0, len(oldTxs)) + for _, oldTx := range oldTxs { + f.lggr.Criticalw(fmt.Sprintf("transaction with ID %v expired without ever getting a receipt for any of our attempts. "+ + "Current block height is %d, transaction was broadcast before finalized block %d. This transaction may not have not been sent and will be marked as fatally errored. "+ + "This can happen if there is another instance of chainlink running that is using the same private key, or if "+ + "an external wallet has been used to send a transaction from account %s with nonce %s."+ + " Please note that Chainlink requires exclusive ownership of it's private keys and sharing keys across multiple"+ + " chainlink instances, or using the chainlink keys with an external wallet is NOT SUPPORTED and WILL lead to missed transactions", + oldTx.ID, head.BlockNumber(), latestFinalizedHead.BlockNumber(), oldTx.FromAddress, oldTx.Sequence.String()), "txID", oldTx.ID, "sequence", oldTx.Sequence.String(), "fromAddress", oldTx.FromAddress) + + // Signal pending tasks for these transactions as failed + // Store errors and continue to allow all transactions a chance to be signaled + if f.resumeCallback != nil && oldTx.PipelineTaskRunID.Valid && oldTx.SignalCallback && !oldTx.CallbackCompleted { + err = f.resumeCallback(ctx, oldTx.PipelineTaskRunID.UUID, nil, errors.New(ErrCouldNotGetReceipt)) + switch { + case errors.Is(err, sql.ErrNoRows): + f.lggr.Debugw("callback missing or already resumed", "etxID", oldTx.ID) + case err != nil: + errorList = append(errorList, fmt.Errorf("failed to resume pipeline for ID %s: %w", oldTx.PipelineTaskRunID.UUID.String(), err)) + continue + default: + // Mark tx as having completed callback + if err = f.txStore.UpdateTxCallbackCompleted(ctx, oldTx.PipelineTaskRunID.UUID, f.chainID); err != nil { + errorList = append(errorList, fmt.Errorf("failed to update callback as complete for tx ID %d: %w", oldTx.ID, err)) + continue + } + } + } + + // Mark transaction as fatal error and delete attempts to prevent further receipt fetching + oldTx.Error = null.StringFrom(ErrCouldNotGetReceipt) + if err = f.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, oldTx); err != nil { + errorList = append(errorList, fmt.Errorf("failed to mark tx with ID %d as fatal: %w", oldTx.ID, err)) + } + } + if len(errorList) > 0 { + return errors.Join(errorList...) + } + + return nil +} + +// findOldTxIDsWithoutReceipts finds IDs for transactions without receipts and attempts broadcasted at or before the finalized head +func findOldTxIDsWithoutReceipts(attempts []TxAttempt, receipts []*evmtypes.Receipt, latestFinalizedHead *evmtypes.Head) []int64 { + if len(attempts) == 0 { + return nil + } + txIDToAttemptsMap := make(map[int64][]TxAttempt) + hashToReceiptMap := make(map[common.Hash]bool) + // Store all receipts hashes in a map to easily access which attempt hash has a receipt + for _, receipt := range receipts { + hashToReceiptMap[receipt.TxHash] = true + } + // Store all attempts in a map of tx ID to attempts + for _, attempt := range attempts { + txIDToAttemptsMap[attempt.TxID] = append(txIDToAttemptsMap[attempt.TxID], attempt) + } + + // Determine which transactions still do not have a receipt and if all of their attempts are older or equal to the latest finalized head + oldTxIDs := make([]int64, 0, len(txIDToAttemptsMap)) + for txID, attempts := range txIDToAttemptsMap { + hasReceipt := false + hasAttemptAfterFinalizedHead := false + for _, attempt := range attempts { + if _, exists := hashToReceiptMap[attempt.Hash]; exists { + hasReceipt = true + break + } + if attempt.BroadcastBeforeBlockNum != nil && *attempt.BroadcastBeforeBlockNum > latestFinalizedHead.BlockNumber() { + hasAttemptAfterFinalizedHead = true + break + } + } + if hasReceipt || hasAttemptAfterFinalizedHead { + continue + } + oldTxIDs = append(oldTxIDs, txID) + } + return oldTxIDs +} + +// buildTxHashList builds list of transaction hashes from receipts considered finalized +func (f *evmFinalizer) buildTxHashList(finalizedReceipts []*evmtypes.Receipt) []common.Hash { + txHashes := make([]common.Hash, len(finalizedReceipts)) for i, receipt := range finalizedReceipts { f.lggr.Debugw("transaction considered finalized", "txHash", receipt.TxHash.String(), - "receiptBlockNum", receipt.BlockNumber, + "receiptBlockNum", receipt.BlockNumber.Int64(), "receiptBlockHash", receipt.BlockHash.String(), ) - receiptIds[i] = receipt.ID + txHashes[i] = receipt.TxHash } - return receiptIds + return txHashes } diff --git a/core/chains/evm/txmgr/finalizer_test.go b/core/chains/evm/txmgr/finalizer_test.go index b91121d773f..76338d31836 100644 --- a/core/chains/evm/txmgr/finalizer_test.go +++ b/core/chains/evm/txmgr/finalizer_test.go @@ -1,7 +1,10 @@ package txmgr_test import ( + "context" + "encoding/json" "errors" + "fmt" "math/big" "testing" "time" @@ -10,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -18,13 +22,17 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" txmgrcommon "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/testutils" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" ) @@ -36,6 +44,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { ethKeyStore := cltest.NewKeyStore(t, db).Eth() feeLimit := uint64(10_000) ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) rpcBatchSize := uint32(1) ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) @@ -51,7 +60,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { head.Parent.Store(h99) t.Run("returns not finalized for tx with receipt newer than finalized block", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) idempotencyKey := uuid.New().String() @@ -80,8 +89,8 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { require.Equal(t, txmgrcommon.TxConfirmed, tx.State) }) - t.Run("returns not finalized for tx with receipt re-org'd out", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + t.Run("returns not finalized for tx with receipt re-org'd out and deletes stale receipt", func(t *testing.T) { + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) idempotencyKey := uuid.New().String() @@ -108,10 +117,12 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { tx, err = txStore.FindTxWithIdempotencyKey(ctx, idempotencyKey, testutils.FixtureChainID) require.NoError(t, err) require.Equal(t, txmgrcommon.TxConfirmed, tx.State) + require.Len(t, tx.TxAttempts, 1) + require.Empty(t, tx.TxAttempts[0].Receipts) }) t.Run("returns finalized for tx with receipt in a finalized block", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) idempotencyKey := uuid.New().String() @@ -141,7 +152,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { }) t.Run("returns finalized for tx with receipt older than block history depth", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) idempotencyKey := uuid.New().String() @@ -181,7 +192,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { // Separate batch calls will be made for each tx due to RPC batch size set to 1 when finalizer initialized above ethClient.On("BatchCallContext", mock.Anything, mock.IsType([]rpc.BatchElem{})).Run(func(args mock.Arguments) { rpcElements := args.Get(1).([]rpc.BatchElem) - require.Equal(t, 1, len(rpcElements)) + require.Len(t, rpcElements, 1) require.Equal(t, "eth_getBlockByNumber", rpcElements[0].Method) require.Equal(t, false, rpcElements[0].Args[1]) @@ -209,7 +220,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { }) t.Run("returns error if failed to retrieve latest head in headtracker", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(nil, errors.New("failed to get latest head")).Once() @@ -218,7 +229,7 @@ func TestFinalizer_MarkTxFinalized(t *testing.T) { }) t.Run("returns error if failed to calculate latest finalized head in headtracker", func(t *testing.T) { - finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, txStore, ethClient, ht) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) servicetest.Run(t, finalizer) ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(head, nil).Once() @@ -239,3 +250,917 @@ func insertTxAndAttemptWithIdempotencyKey(t *testing.T, txStore txmgr.TestEvmTxS require.NoError(t, err) return attempt.Hash } + +func TestFinalizer_ResumePendingRuns(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) + rpcBatchSize := uint32(1) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + + grandParentHead := &evmtypes.Head{ + Number: 8, + Hash: testutils.NewHash(), + } + parentHead := &evmtypes.Head{ + Hash: testutils.NewHash(), + Number: 9, + } + parentHead.Parent.Store(grandParentHead) + head := evmtypes.Head{ + Hash: testutils.NewHash(), + Number: 10, + } + head.Parent.Store(parentHead) + + minConfirmations := int64(2) + + pgtest.MustExec(t, db, `SET CONSTRAINTS fk_pipeline_runs_pruning_key DEFERRED`) + pgtest.MustExec(t, db, `SET CONSTRAINTS pipeline_runs_pipeline_spec_id_fkey DEFERRED`) + + t.Run("doesn't process task runs that are not suspended (possibly already previously resumed)", func(t *testing.T) { + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(context.Context, uuid.UUID, interface{}, error) error { + t.Fatal("No value expected") + return nil + }) + servicetest.Run(t, finalizer) + + run := cltest.MustInsertPipelineRun(t, db) + tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) + + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 1, 1, fromAddress) + mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) + // Setting both signal_callback and callback_completed to TRUE to simulate a completed pipeline task + // It would only be in a state past suspended if the resume callback was called and callback_completed was set to TRUE + pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE, callback_completed = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) + + err := finalizer.ResumePendingTaskRuns(ctx, head.BlockNumber(), 0) + require.NoError(t, err) + }) + + t.Run("doesn't process task runs where the receipt is younger than minConfirmations", func(t *testing.T) { + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(context.Context, uuid.UUID, interface{}, error) error { + t.Fatal("No value expected") + return nil + }) + servicetest.Run(t, finalizer) + + run := cltest.MustInsertPipelineRun(t, db) + tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) + + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 2, 1, fromAddress) + mustInsertEthReceipt(t, txStore, head.Number, head.Hash, etx.TxAttempts[0].Hash) + + pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) + + err := finalizer.ResumePendingTaskRuns(ctx, head.BlockNumber(), 0) + require.NoError(t, err) + }) + + t.Run("processes transactions with receipts older than minConfirmations", func(t *testing.T) { + ch := make(chan interface{}) + nonce := evmtypes.Nonce(3) + var err error + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(ctx context.Context, id uuid.UUID, value interface{}, thisErr error) error { + err = thisErr + ch <- value + return nil + }) + servicetest.Run(t, finalizer) + + run := cltest.MustInsertPipelineRun(t, db) + tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) + pgtest.MustExec(t, db, `UPDATE pipeline_runs SET state = 'suspended' WHERE id = $1`, run.ID) + + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) + pgtest.MustExec(t, db, `UPDATE evm.txes SET meta='{"FailOnRevert": true}'`) + receipt := mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) + + pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) + + done := make(chan struct{}) + t.Cleanup(func() { <-done }) + go func() { + defer close(done) + err2 := finalizer.ResumePendingTaskRuns(ctx, head.BlockNumber(), 0) + assert.NoError(t, err2) + + // Retrieve Tx to check if callback completed flag was set to true + updateTx, err3 := txStore.FindTxWithSequence(ctx, fromAddress, nonce) + assert.NoError(t, err3) + assert.True(t, updateTx.CallbackCompleted) + }() + + select { + case data := <-ch: + require.NoError(t, err) + + require.IsType(t, &evmtypes.Receipt{}, data) + r := data.(*evmtypes.Receipt) + require.Equal(t, receipt.TxHash, r.TxHash) + + case <-time.After(time.Second): + t.Fatal("no value received") + } + }) + + pgtest.MustExec(t, db, `DELETE FROM pipeline_runs`) + + t.Run("processes transactions with receipt older than minConfirmations that reverted", func(t *testing.T) { + type data struct { + value any + error + } + ch := make(chan data) + nonce := evmtypes.Nonce(4) + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(ctx context.Context, id uuid.UUID, value interface{}, err error) error { + ch <- data{value, err} + return nil + }) + servicetest.Run(t, finalizer) + + run := cltest.MustInsertPipelineRun(t, db) + tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) + pgtest.MustExec(t, db, `UPDATE pipeline_runs SET state = 'suspended' WHERE id = $1`, run.ID) + + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) + pgtest.MustExec(t, db, `UPDATE evm.txes SET meta='{"FailOnRevert": true}'`) + + // receipt is not passed through as a value since it reverted and caused an error + mustInsertRevertedEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) + + pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) + + done := make(chan struct{}) + t.Cleanup(func() { <-done }) + go func() { + defer close(done) + err2 := finalizer.ResumePendingTaskRuns(ctx, head.BlockNumber(), 0) + assert.NoError(t, err2) + + // Retrieve Tx to check if callback completed flag was set to true + updateTx, err3 := txStore.FindTxWithSequence(ctx, fromAddress, nonce) + assert.NoError(t, err3) + assert.True(t, updateTx.CallbackCompleted) + }() + + select { + case data := <-ch: + require.Error(t, data.error) + + require.EqualError(t, data.error, fmt.Sprintf("transaction %s reverted on-chain", etx.TxAttempts[0].Hash.String())) + + require.Nil(t, data.value) + + case <-time.After(tests.WaitTimeout(t)): + t.Fatal("no value received") + } + }) + + t.Run("does not mark callback complete if callback fails", func(t *testing.T) { + nonce := evmtypes.Nonce(5) + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(ctx context.Context, id uuid.UUID, value interface{}, err error) error { + return errors.New("error") + }) + servicetest.Run(t, finalizer) + + run := cltest.MustInsertPipelineRun(t, db) + tr := cltest.MustInsertUnfinishedPipelineTaskRun(t, db, run.ID) + + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, int64(nonce), 1, fromAddress) + mustInsertEthReceipt(t, txStore, head.Number-minConfirmations, head.Hash, etx.TxAttempts[0].Hash) + pgtest.MustExec(t, db, `UPDATE evm.txes SET pipeline_task_run_id = $1, min_confirmations = $2, signal_callback = TRUE WHERE id = $3`, &tr.ID, minConfirmations, etx.ID) + + err := finalizer.ResumePendingTaskRuns(ctx, head.BlockNumber(), 0) + require.Error(t, err) + + // Retrieve Tx to check if callback completed flag was left unchanged + updateTx, err := txStore.FindTxWithSequence(ctx, fromAddress, nonce) + require.NoError(t, err) + require.False(t, updateTx.CallbackCompleted) + }) +} + +func TestFinalizer_FetchAndStoreReceipts(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + cfg := configtest.NewTestGeneralConfig(t) + config := evmtest.NewChainScopedConfig(t, cfg) + ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) + rpcBatchSize := config.EVM().RPCDefaultBatchSize() + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + + latestFinalizedHead := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 99, + } + latestFinalizedHead.IsFinalized.Store(true) + head := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 100, + } + head.Parent.Store(latestFinalizedHead) + + t.Run("does nothing if no confirmed transactions without receipts found", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, config.EVM().RPCDefaultBatchSize(), false, txStore, txmClient, ht) + + mustInsertFatalErrorEthTx(t, txStore, fromAddress) + mustInsertInProgressEthTx(t, txStore, 0, fromAddress) + mustInsertUnconfirmedEthTxWithInsufficientEthAttempt(t, txStore, 2, fromAddress) + mustCreateUnstartedGeneratedTx(t, txStore, fromAddress, config.EVM().ChainID()) + // Insert confirmed transactions with receipt and multiple attempts to ensure none of the attempts are picked up + etx := mustInsertConfirmedEthTxWithReceipt(t, txStore, fromAddress, 3, head.Number) + attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 2) + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + }) + + t.Run("fetches receipt for confirmed transaction without a receipt", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + // Transaction not confirmed yet, receipt is nil + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &evmtypes.Receipt{} + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Len(t, etx.TxAttempts, 1) + attempt := etx.TxAttempts[0] + require.NoError(t, err) + require.Empty(t, attempt.Receipts) + }) + + t.Run("saves nothing if returned receipt does not match the attempt", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + txmReceipt := evmtypes.Receipt{ + TxHash: testutils.NewHash(), + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + } + + // First transaction confirmed + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt + }).Once() + + // No error because it is merely logged + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Len(t, etx.TxAttempts, 1) + require.Empty(t, etx.TxAttempts[0].Receipts) + }) + + t.Run("saves nothing if query returns error", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + txmReceipt := evmtypes.Receipt{ + TxHash: etx.TxAttempts[0].Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + } + + // Batch receipt call fails + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt + elems[0].Error = errors.New("foo") + }).Once() + + // No error because it is merely logged + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Len(t, etx.TxAttempts, 1) + require.Empty(t, etx.TxAttempts[0].Receipts) + }) + + t.Run("saves valid receipt returned by client", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx1 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + // Insert confirmed transaction without receipt + etx2 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 1, head.Number, fromAddress) + txmReceipt := evmtypes.Receipt{ + TxHash: etx1.TxAttempts[0].Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(1), + } + + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 2 && + cltest.BatchElemMatchesParams(b[0], etx1.TxAttempts[0].Hash, "eth_getTransactionReceipt") && + cltest.BatchElemMatchesParams(b[1], etx2.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + // First transaction confirmed + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt + // Second transaction still unconfirmed + elems[1].Result = &evmtypes.Receipt{} + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check that the receipt was saved + var err error + etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) + require.NoError(t, err) + + require.Equal(t, txmgrcommon.TxConfirmed, etx1.State) + require.Len(t, etx1.TxAttempts, 1) + attempt := etx1.TxAttempts[0] + require.Len(t, attempt.Receipts, 1) + receipt := attempt.Receipts[0] + require.Equal(t, txmReceipt.TxHash, receipt.GetTxHash()) + require.Equal(t, txmReceipt.BlockHash, receipt.GetBlockHash()) + require.Equal(t, txmReceipt.BlockNumber.Int64(), receipt.GetBlockNumber().Int64()) + require.Equal(t, txmReceipt.TransactionIndex, receipt.GetTransactionIndex()) + + receiptJSON, err := json.Marshal(txmReceipt) + require.NoError(t, err) + + storedReceiptJSON, err := json.Marshal(receipt) + require.NoError(t, err) + require.JSONEq(t, string(receiptJSON), string(storedReceiptJSON)) + }) + + t.Run("fetches and saves receipts for several attempts in gas price order", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + attempt1 := etx.TxAttempts[0] + attempt2 := newBroadcastLegacyEthTxAttempt(t, etx.ID, 2) + attempt3 := newBroadcastLegacyEthTxAttempt(t, etx.ID, 3) + + // Insert order deliberately reversed to test sorting by gas price + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt3)) + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt2)) + + txmReceipt := evmtypes.Receipt{ + TxHash: attempt2.Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(1), + } + + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 3 && + cltest.BatchElemMatchesParams(b[2], attempt1.Hash, "eth_getTransactionReceipt") && + cltest.BatchElemMatchesParams(b[1], attempt2.Hash, "eth_getTransactionReceipt") && + cltest.BatchElemMatchesParams(b[0], attempt3.Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + // Most expensive attempt still unconfirmed + elems[2].Result = &evmtypes.Receipt{} + // Second most expensive attempt is confirmed + *(elems[1].Result.(*evmtypes.Receipt)) = txmReceipt + // Cheapest attempt still unconfirmed + elems[0].Result = &evmtypes.Receipt{} + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check that the receipt was stored + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + require.Len(t, etx.TxAttempts, 3) + require.Empty(t, etx.TxAttempts[0].Receipts) + require.Len(t, etx.TxAttempts[1].Receipts, 1) + require.Empty(t, etx.TxAttempts[2].Receipts) + }) + + t.Run("ignores receipt missing BlockHash that comes from querying parity too early", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + receipt := evmtypes.Receipt{ + TxHash: etx.TxAttempts[0].Hash, + Status: uint64(1), + } + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = receipt + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // No receipt, but no error either + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + require.Len(t, etx.TxAttempts, 1) + attempt := etx.TxAttempts[0] + require.Empty(t, attempt.Receipts) + }) + + t.Run("does not panic if receipt has BlockHash but is missing some other fields somehow", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + // NOTE: This should never happen, but we shouldn't panic regardless + receipt := evmtypes.Receipt{ + TxHash: etx.TxAttempts[0].Hash, + BlockHash: testutils.NewHash(), + Status: uint64(1), + } + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = receipt + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // No receipt, but no error either + etx, err := txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + require.Len(t, etx.TxAttempts, 1) + attempt := etx.TxAttempts[0] + require.Empty(t, attempt.Receipts) + }) + + t.Run("simulate on revert", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + attempt := etx.TxAttempts[0] + txmReceipt := evmtypes.Receipt{ + TxHash: attempt.Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(0), + } + + // First attempt is confirmed and reverted + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], attempt.Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + // First attempt still unconfirmed + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt + }).Once() + data, err := utils.ABIEncode(`[{"type":"uint256"}]`, big.NewInt(10)) + require.NoError(t, err) + sig := utils.Keccak256Fixed([]byte(`MyError(uint256)`)) + ethClient.On("CallContract", mock.Anything, mock.Anything, mock.Anything).Return(nil, &client.JsonError{ + Code: 1, + Message: "reverted", + Data: utils.ConcatBytes(sig[:4], data), + }).Once() + + // Do the thing + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check that the state was updated + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + attempt = etx.TxAttempts[0] + require.Equal(t, txmgrtypes.TxAttemptBroadcast, attempt.State) + require.NotNil(t, attempt.BroadcastBeforeBlockNum) + // Check receipts + require.Len(t, attempt.Receipts, 1) + }) + + t.Run("find receipt for old transaction, avoid marking as fatal", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, true, txStore, txmClient, ht) + + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, latestFinalizedHead.Number, fromAddress) + + txmReceipt := evmtypes.Receipt{ + TxHash: etx.TxAttempts[0].Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(1), + } + + // Transaction receipt is nil + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check that transaction was picked up as old and marked as fatal + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + }) + + t.Run("old transaction failed to find receipt, marked as fatal", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, true, txStore, txmClient, ht) + + // Insert confirmed transaction without receipt + etx := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, latestFinalizedHead.Number, fromAddress) + + // Transaction receipt is nil + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && cltest.BatchElemMatchesParams(b[0], etx.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &evmtypes.Receipt{} + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check that transaction was picked up as old and marked as fatal + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx.State) + require.Equal(t, txmgr.ErrCouldNotGetReceipt, etx.Error.String) + }) +} + +func TestFinalizer_FetchAndStoreReceipts_batching(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + + latestFinalizedHead := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 99, + } + latestFinalizedHead.IsFinalized.Store(true) + head := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 100, + } + head.Parent.Store(latestFinalizedHead) + + t.Run("fetch and store receipts from multiple batch calls", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + rpcBatchSize := uint32(2) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + + // Insert confirmed transaction without receipt + etx := mustInsertConfirmedEthTx(t, txStore, 0, fromAddress) + + var attempts []txmgr.TxAttempt + // Total of 5 attempts should lead to 3 batched fetches (2, 2, 1)v + for i := 0; i < 5; i++ { + attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, int64(i+2)) + attempt.BroadcastBeforeBlockNum = &head.Number + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + attempts = append(attempts, attempt) + } + + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 2 && + cltest.BatchElemMatchesParams(b[0], attempts[4].Hash, "eth_getTransactionReceipt") && + cltest.BatchElemMatchesParams(b[1], attempts[3].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &evmtypes.Receipt{} + elems[1].Result = &evmtypes.Receipt{} + }).Once() + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 2 && + cltest.BatchElemMatchesParams(b[0], attempts[2].Hash, "eth_getTransactionReceipt") && + cltest.BatchElemMatchesParams(b[1], attempts[1].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &evmtypes.Receipt{} + elems[1].Result = &evmtypes.Receipt{} + }).Once() + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && + cltest.BatchElemMatchesParams(b[0], attempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + elems[0].Result = &evmtypes.Receipt{} + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + }) + + t.Run("continue to fetch and store receipts after batch call error", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + rpcBatchSize := uint32(1) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, rpcBatchSize, false, txStore, txmClient, ht) + + // Insert confirmed transactions without receipts + etx1 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, head.Number, fromAddress) + etx2 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 1, head.Number, fromAddress) + + txmReceipt := evmtypes.Receipt{ + TxHash: etx2.TxAttempts[0].Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(1), + } + + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && + cltest.BatchElemMatchesParams(b[0], etx1.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(errors.New("batch call failed")).Once() + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && + cltest.BatchElemMatchesParams(b[0], etx2.TxAttempts[0].Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt // confirmed + }).Once() + + // Returns error due to batch call failure + require.Error(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Still fetches and stores receipt for later batch call that succeeds + var err error + etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) + require.NoError(t, err) + require.Len(t, etx2.TxAttempts, 1) + attempt := etx2.TxAttempts[0] + require.Len(t, attempt.Receipts, 1) + }) +} + +func TestFinalizer_FetchAndStoreReceipts_HandlesNonFwdTxsWithForwardingEnabled(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + + latestFinalizedHead := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 99, + } + latestFinalizedHead.IsFinalized.Store(true) + head := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 100, + } + head.Parent.Store(latestFinalizedHead) + + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, 1, true, txStore, txmClient, ht) + + // tx is not forwarded and doesn't have meta set. Confirmer should handle nil meta values + etx := mustInsertConfirmedEthTx(t, txStore, 0, fromAddress) + attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 2) + attempt.Tx.Meta = nil + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + dbtx, err := txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Empty(t, dbtx.TxAttempts[0].Receipts) + + txmReceipt := evmtypes.Receipt{ + TxHash: attempt.Hash, + BlockHash: testutils.NewHash(), + BlockNumber: big.NewInt(42), + TransactionIndex: uint(1), + Status: uint64(1), + } + + ethClient.On("BatchCallContext", mock.Anything, mock.MatchedBy(func(b []rpc.BatchElem) bool { + return len(b) == 1 && + cltest.BatchElemMatchesParams(b[0], attempt.Hash, "eth_getTransactionReceipt") + })).Return(nil).Run(func(args mock.Arguments) { + elems := args.Get(1).([]rpc.BatchElem) + *(elems[0].Result.(*evmtypes.Receipt)) = txmReceipt // confirmed + }).Once() + + require.NoError(t, finalizer.FetchAndStoreReceipts(ctx, head, latestFinalizedHead)) + + // Check receipt is inserted correctly. + dbtx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Len(t, dbtx.TxAttempts[0].Receipts, 1) +} + +func TestFinalizer_ProcessOldTxsWithoutReceipts(t *testing.T) { + t.Parallel() + ctx := tests.Context(t) + ethClient := testutils.NewEthClientMockWithDefaultChain(t) + txmClient := txmgr.NewEvmTxmClient(ethClient, nil) + ht := headtracker.NewSimulatedHeadTracker(ethClient, true, 0) + + latestFinalizedHead := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 99, + } + latestFinalizedHead.IsFinalized.Store(true) + head := &evmtypes.Head{ + Hash: utils.NewHash(), + Number: 100, + } + head.Parent.Store(latestFinalizedHead) + + t.Run("does nothing if no old transactions found", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, 1, true, txStore, txmClient, ht) + require.NoError(t, finalizer.ProcessOldTxsWithoutReceipts(ctx, []int64{}, head, latestFinalizedHead)) + }) + + t.Run("marks multiple old transactions as fatal", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, 1, true, txStore, txmClient, ht) + + // Insert confirmed transaction without receipt + etx1 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 0, latestFinalizedHead.Number, fromAddress) + etx2 := cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, 1, latestFinalizedHead.Number, fromAddress) + + etxIDs := []int64{etx1.ID, etx2.ID} + require.NoError(t, finalizer.ProcessOldTxsWithoutReceipts(ctx, etxIDs, head, latestFinalizedHead)) + + // Check transactions marked as fatal + var err error + etx1, err = txStore.FindTxWithAttempts(ctx, etx1.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx1.State) + require.Equal(t, txmgr.ErrCouldNotGetReceipt, etx1.Error.String) + + etx2, err = txStore.FindTxWithAttempts(ctx, etx2.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx2.State) + require.Equal(t, txmgr.ErrCouldNotGetReceipt, etx2.Error.String) + }) + + t.Run("marks old transaction as fatal, resumes pending task as failed", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, 1, true, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(context.Context, uuid.UUID, interface{}, error) error { return nil }) + + // Insert confirmed transaction with pending task run + etx := cltest.NewEthTx(fromAddress) + etx.State = txmgrcommon.TxConfirmed + n := evmtypes.Nonce(0) + etx.Sequence = &n + now := time.Now() + etx.BroadcastAt = &now + etx.InitialBroadcastAt = &now + etx.SignalCallback = true + etx.PipelineTaskRunID = uuid.NullUUID{UUID: uuid.New(), Valid: true} + require.NoError(t, txStore.InsertTx(tests.Context(t), &etx)) + + attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 0) + attempt.BroadcastBeforeBlockNum = &latestFinalizedHead.Number // set broadcast time to finalized block num + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + + require.NoError(t, finalizer.ProcessOldTxsWithoutReceipts(ctx, []int64{etx.ID}, head, latestFinalizedHead)) + + // Check transaction marked as fatal + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxFatalError, etx.State) + require.Equal(t, txmgr.ErrCouldNotGetReceipt, etx.Error.String) + require.True(t, etx.CallbackCompleted) + }) + + t.Run("transaction stays confirmed if failure to resume pending task", func(t *testing.T) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTestTxStore(t, db) + ethKeyStore := cltest.NewKeyStore(t, db).Eth() + _, fromAddress := cltest.MustInsertRandomKeyReturningState(t, ethKeyStore) + finalizer := txmgr.NewEvmFinalizer(logger.Test(t), testutils.FixtureChainID, 1, true, txStore, txmClient, ht) + finalizer.SetResumeCallback(func(context.Context, uuid.UUID, interface{}, error) error { return errors.New("failure") }) + + // Insert confirmed transaction with pending task run + etx := cltest.NewEthTx(fromAddress) + etx.State = txmgrcommon.TxConfirmed + n := evmtypes.Nonce(0) + etx.Sequence = &n + now := time.Now() + etx.BroadcastAt = &now + etx.InitialBroadcastAt = &now + etx.SignalCallback = true + etx.PipelineTaskRunID = uuid.NullUUID{UUID: uuid.New(), Valid: true} + require.NoError(t, txStore.InsertTx(tests.Context(t), &etx)) + + attempt := newBroadcastLegacyEthTxAttempt(t, etx.ID, 0) + attempt.BroadcastBeforeBlockNum = &latestFinalizedHead.Number // set broadcast time to finalized block num + require.NoError(t, txStore.InsertTxAttempt(ctx, &attempt)) + + // Expect error since resuming pending task failed + require.Error(t, finalizer.ProcessOldTxsWithoutReceipts(ctx, []int64{etx.ID}, head, latestFinalizedHead)) + + // Check transaction marked as fatal + var err error + etx, err = txStore.FindTxWithAttempts(ctx, etx.ID) + require.NoError(t, err) + require.Equal(t, txmgrcommon.TxConfirmed, etx.State) + require.False(t, etx.CallbackCompleted) + }) +} diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index fa324d84fb5..ca98ad6ceb8 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -446,24 +446,130 @@ func (_c *EvmTxStore_DeleteInProgressAttempt_Call) RunAndReturn(run func(context return _c } +// DeleteReceiptByTxHash provides a mock function with given fields: ctx, txHash +func (_m *EvmTxStore) DeleteReceiptByTxHash(ctx context.Context, txHash common.Hash) error { + ret := _m.Called(ctx, txHash) + + if len(ret) == 0 { + panic("no return value specified for DeleteReceiptByTxHash") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, common.Hash) error); ok { + r0 = rf(ctx, txHash) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EvmTxStore_DeleteReceiptByTxHash_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteReceiptByTxHash' +type EvmTxStore_DeleteReceiptByTxHash_Call struct { + *mock.Call +} + +// DeleteReceiptByTxHash is a helper method to define mock.On call +// - ctx context.Context +// - txHash common.Hash +func (_e *EvmTxStore_Expecter) DeleteReceiptByTxHash(ctx interface{}, txHash interface{}) *EvmTxStore_DeleteReceiptByTxHash_Call { + return &EvmTxStore_DeleteReceiptByTxHash_Call{Call: _e.mock.On("DeleteReceiptByTxHash", ctx, txHash)} +} + +func (_c *EvmTxStore_DeleteReceiptByTxHash_Call) Run(run func(ctx context.Context, txHash common.Hash)) *EvmTxStore_DeleteReceiptByTxHash_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(common.Hash)) + }) + return _c +} + +func (_c *EvmTxStore_DeleteReceiptByTxHash_Call) Return(_a0 error) *EvmTxStore_DeleteReceiptByTxHash_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EvmTxStore_DeleteReceiptByTxHash_Call) RunAndReturn(run func(context.Context, common.Hash) error) *EvmTxStore_DeleteReceiptByTxHash_Call { + _c.Call.Return(run) + return _c +} + +// FindAttemptsRequiringReceiptFetch provides a mock function with given fields: ctx, chainID +func (_m *EvmTxStore) FindAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) ([]txmgr.TxAttempt, error) { + ret := _m.Called(ctx, chainID) + + if len(ret) == 0 { + panic("no return value specified for FindAttemptsRequiringReceiptFetch") + } + + var r0 []txmgr.TxAttempt + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) ([]txmgr.TxAttempt, error)); ok { + return rf(ctx, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, *big.Int) []txmgr.TxAttempt); ok { + r0 = rf(ctx, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]txmgr.TxAttempt) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *big.Int) error); ok { + r1 = rf(ctx, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EvmTxStore_FindAttemptsRequiringReceiptFetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindAttemptsRequiringReceiptFetch' +type EvmTxStore_FindAttemptsRequiringReceiptFetch_Call struct { + *mock.Call +} + +// FindAttemptsRequiringReceiptFetch is a helper method to define mock.On call +// - ctx context.Context +// - chainID *big.Int +func (_e *EvmTxStore_Expecter) FindAttemptsRequiringReceiptFetch(ctx interface{}, chainID interface{}) *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call { + return &EvmTxStore_FindAttemptsRequiringReceiptFetch_Call{Call: _e.mock.On("FindAttemptsRequiringReceiptFetch", ctx, chainID)} +} + +func (_c *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call) Run(run func(ctx context.Context, chainID *big.Int)) *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*big.Int)) + }) + return _c +} + +func (_c *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call) Return(hashes []txmgr.TxAttempt, err error) *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call { + _c.Call.Return(hashes, err) + return _c +} + +func (_c *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call) RunAndReturn(run func(context.Context, *big.Int) ([]txmgr.TxAttempt, error)) *EvmTxStore_FindAttemptsRequiringReceiptFetch_Call { + _c.Call.Return(run) + return _c +} + // FindConfirmedTxesReceipts provides a mock function with given fields: ctx, finalizedBlockNum, chainID -func (_m *EvmTxStore) FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) ([]txmgr.Receipt, error) { +func (_m *EvmTxStore) FindConfirmedTxesReceipts(ctx context.Context, finalizedBlockNum int64, chainID *big.Int) ([]*evmtypes.Receipt, error) { ret := _m.Called(ctx, finalizedBlockNum, chainID) if len(ret) == 0 { panic("no return value specified for FindConfirmedTxesReceipts") } - var r0 []txmgr.Receipt + var r0 []*evmtypes.Receipt var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64, *big.Int) ([]txmgr.Receipt, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64, *big.Int) ([]*evmtypes.Receipt, error)); ok { return rf(ctx, finalizedBlockNum, chainID) } - if rf, ok := ret.Get(0).(func(context.Context, int64, *big.Int) []txmgr.Receipt); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64, *big.Int) []*evmtypes.Receipt); ok { r0 = rf(ctx, finalizedBlockNum, chainID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]txmgr.Receipt) + r0 = ret.Get(0).([]*evmtypes.Receipt) } } @@ -496,12 +602,12 @@ func (_c *EvmTxStore_FindConfirmedTxesReceipts_Call) Run(run func(ctx context.Co return _c } -func (_c *EvmTxStore_FindConfirmedTxesReceipts_Call) Return(receipts []txmgr.Receipt, err error) *EvmTxStore_FindConfirmedTxesReceipts_Call { +func (_c *EvmTxStore_FindConfirmedTxesReceipts_Call) Return(receipts []*evmtypes.Receipt, err error) *EvmTxStore_FindConfirmedTxesReceipts_Call { _c.Call.Return(receipts, err) return _c } -func (_c *EvmTxStore_FindConfirmedTxesReceipts_Call) RunAndReturn(run func(context.Context, int64, *big.Int) ([]txmgr.Receipt, error)) *EvmTxStore_FindConfirmedTxesReceipts_Call { +func (_c *EvmTxStore_FindConfirmedTxesReceipts_Call) RunAndReturn(run func(context.Context, int64, *big.Int) ([]*evmtypes.Receipt, error)) *EvmTxStore_FindConfirmedTxesReceipts_Call { _c.Call.Return(run) return _c } @@ -620,9 +726,9 @@ func (_c *EvmTxStore_FindEarliestUnconfirmedTxAttemptBlock_Call) RunAndReturn(ru return _c } -// FindLatestSequence provides a mock function with given fields: ctx, fromAddress, chainId -func (_m *EvmTxStore) FindLatestSequence(ctx context.Context, fromAddress common.Address, chainId *big.Int) (evmtypes.Nonce, error) { - ret := _m.Called(ctx, fromAddress, chainId) +// FindLatestSequence provides a mock function with given fields: ctx, fromAddress, chainID +func (_m *EvmTxStore) FindLatestSequence(ctx context.Context, fromAddress common.Address, chainID *big.Int) (evmtypes.Nonce, error) { + ret := _m.Called(ctx, fromAddress, chainID) if len(ret) == 0 { panic("no return value specified for FindLatestSequence") @@ -631,16 +737,16 @@ func (_m *EvmTxStore) FindLatestSequence(ctx context.Context, fromAddress common var r0 evmtypes.Nonce var r1 error if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) (evmtypes.Nonce, error)); ok { - return rf(ctx, fromAddress, chainId) + return rf(ctx, fromAddress, chainID) } if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) evmtypes.Nonce); ok { - r0 = rf(ctx, fromAddress, chainId) + r0 = rf(ctx, fromAddress, chainID) } else { r0 = ret.Get(0).(evmtypes.Nonce) } if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { - r1 = rf(ctx, fromAddress, chainId) + r1 = rf(ctx, fromAddress, chainID) } else { r1 = ret.Error(1) } @@ -656,12 +762,12 @@ type EvmTxStore_FindLatestSequence_Call struct { // FindLatestSequence is a helper method to define mock.On call // - ctx context.Context // - fromAddress common.Address -// - chainId *big.Int -func (_e *EvmTxStore_Expecter) FindLatestSequence(ctx interface{}, fromAddress interface{}, chainId interface{}) *EvmTxStore_FindLatestSequence_Call { - return &EvmTxStore_FindLatestSequence_Call{Call: _e.mock.On("FindLatestSequence", ctx, fromAddress, chainId)} +// - chainID *big.Int +func (_e *EvmTxStore_Expecter) FindLatestSequence(ctx interface{}, fromAddress interface{}, chainID interface{}) *EvmTxStore_FindLatestSequence_Call { + return &EvmTxStore_FindLatestSequence_Call{Call: _e.mock.On("FindLatestSequence", ctx, fromAddress, chainID)} } -func (_c *EvmTxStore_FindLatestSequence_Call) Run(run func(ctx context.Context, fromAddress common.Address, chainId *big.Int)) *EvmTxStore_FindLatestSequence_Call { +func (_c *EvmTxStore_FindLatestSequence_Call) Run(run func(ctx context.Context, fromAddress common.Address, chainID *big.Int)) *EvmTxStore_FindLatestSequence_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(common.Address), args[2].(*big.Int)) }) @@ -738,63 +844,72 @@ func (_c *EvmTxStore_FindNextUnstartedTransactionFromAddress_Call) RunAndReturn( return _c } -// FindTransactionsConfirmedInBlockRange provides a mock function with given fields: ctx, highBlockNumber, lowBlockNumber, chainID -func (_m *EvmTxStore) FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber int64, lowBlockNumber int64, chainID *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { - ret := _m.Called(ctx, highBlockNumber, lowBlockNumber, chainID) +// FindReorgOrIncludedTxs provides a mock function with given fields: ctx, fromAddress, nonce, chainID +func (_m *EvmTxStore) FindReorgOrIncludedTxs(ctx context.Context, fromAddress common.Address, nonce evmtypes.Nonce, chainID *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { + ret := _m.Called(ctx, fromAddress, nonce, chainID) if len(ret) == 0 { - panic("no return value specified for FindTransactionsConfirmedInBlockRange") + panic("no return value specified for FindReorgOrIncludedTxs") } var r0 []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)); ok { - return rf(ctx, highBlockNumber, lowBlockNumber, chainID) + var r1 []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, evmtypes.Nonce, *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)); ok { + return rf(ctx, fromAddress, nonce, chainID) } - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { - r0 = rf(ctx, highBlockNumber, lowBlockNumber, chainID) + if rf, ok := ret.Get(0).(func(context.Context, common.Address, evmtypes.Nonce, *big.Int) []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { + r0 = rf(ctx, fromAddress, nonce, chainID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) } } - if rf, ok := ret.Get(1).(func(context.Context, int64, int64, *big.Int) error); ok { - r1 = rf(ctx, highBlockNumber, lowBlockNumber, chainID) + if rf, ok := ret.Get(1).(func(context.Context, common.Address, evmtypes.Nonce, *big.Int) []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { + r1 = rf(ctx, fromAddress, nonce, chainID) } else { - r1 = ret.Error(1) + if ret.Get(1) != nil { + r1 = ret.Get(1).([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) + } } - return r0, r1 + if rf, ok := ret.Get(2).(func(context.Context, common.Address, evmtypes.Nonce, *big.Int) error); ok { + r2 = rf(ctx, fromAddress, nonce, chainID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } -// EvmTxStore_FindTransactionsConfirmedInBlockRange_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTransactionsConfirmedInBlockRange' -type EvmTxStore_FindTransactionsConfirmedInBlockRange_Call struct { +// EvmTxStore_FindReorgOrIncludedTxs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindReorgOrIncludedTxs' +type EvmTxStore_FindReorgOrIncludedTxs_Call struct { *mock.Call } -// FindTransactionsConfirmedInBlockRange is a helper method to define mock.On call +// FindReorgOrIncludedTxs is a helper method to define mock.On call // - ctx context.Context -// - highBlockNumber int64 -// - lowBlockNumber int64 +// - fromAddress common.Address +// - nonce evmtypes.Nonce // - chainID *big.Int -func (_e *EvmTxStore_Expecter) FindTransactionsConfirmedInBlockRange(ctx interface{}, highBlockNumber interface{}, lowBlockNumber interface{}, chainID interface{}) *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call { - return &EvmTxStore_FindTransactionsConfirmedInBlockRange_Call{Call: _e.mock.On("FindTransactionsConfirmedInBlockRange", ctx, highBlockNumber, lowBlockNumber, chainID)} +func (_e *EvmTxStore_Expecter) FindReorgOrIncludedTxs(ctx interface{}, fromAddress interface{}, nonce interface{}, chainID interface{}) *EvmTxStore_FindReorgOrIncludedTxs_Call { + return &EvmTxStore_FindReorgOrIncludedTxs_Call{Call: _e.mock.On("FindReorgOrIncludedTxs", ctx, fromAddress, nonce, chainID)} } -func (_c *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call) Run(run func(ctx context.Context, highBlockNumber int64, lowBlockNumber int64, chainID *big.Int)) *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call { +func (_c *EvmTxStore_FindReorgOrIncludedTxs_Call) Run(run func(ctx context.Context, fromAddress common.Address, nonce evmtypes.Nonce, chainID *big.Int)) *EvmTxStore_FindReorgOrIncludedTxs_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(*big.Int)) + run(args[0].(context.Context), args[1].(common.Address), args[2].(evmtypes.Nonce), args[3].(*big.Int)) }) return _c } -func (_c *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call) Return(etxs []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call { - _c.Call.Return(etxs, err) +func (_c *EvmTxStore_FindReorgOrIncludedTxs_Call) Return(reorgTx []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], includedTxs []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) *EvmTxStore_FindReorgOrIncludedTxs_Call { + _c.Call.Return(reorgTx, includedTxs, err) return _c } -func (_c *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call) RunAndReturn(run func(context.Context, int64, int64, *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)) *EvmTxStore_FindTransactionsConfirmedInBlockRange_Call { +func (_c *EvmTxStore_FindReorgOrIncludedTxs_Call) RunAndReturn(run func(context.Context, common.Address, evmtypes.Nonce, *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], []*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)) *EvmTxStore_FindReorgOrIncludedTxs_Call { _c.Call.Return(run) return _c } @@ -976,65 +1091,6 @@ func (_c *EvmTxStore_FindTxAttemptsConfirmedMissingReceipt_Call) RunAndReturn(ru return _c } -// FindTxAttemptsRequiringReceiptFetch provides a mock function with given fields: ctx, chainID -func (_m *EvmTxStore) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID *big.Int) ([]types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { - ret := _m.Called(ctx, chainID) - - if len(ret) == 0 { - panic("no return value specified for FindTxAttemptsRequiringReceiptFetch") - } - - var r0 []types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *big.Int) ([]types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)); ok { - return rf(ctx, chainID) - } - if rf, ok := ret.Get(0).(func(context.Context, *big.Int) []types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { - r0 = rf(ctx, chainID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *big.Int) error); ok { - r1 = rf(ctx, chainID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTxAttemptsRequiringReceiptFetch' -type EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call struct { - *mock.Call -} - -// FindTxAttemptsRequiringReceiptFetch is a helper method to define mock.On call -// - ctx context.Context -// - chainID *big.Int -func (_e *EvmTxStore_Expecter) FindTxAttemptsRequiringReceiptFetch(ctx interface{}, chainID interface{}) *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call { - return &EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call{Call: _e.mock.On("FindTxAttemptsRequiringReceiptFetch", ctx, chainID)} -} - -func (_c *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call) Run(run func(ctx context.Context, chainID *big.Int)) *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*big.Int)) - }) - return _c -} - -func (_c *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call) Return(attempts []types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call { - _c.Call.Return(attempts, err) - return _c -} - -func (_c *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call) RunAndReturn(run func(context.Context, *big.Int) ([]types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)) *EvmTxStore_FindTxAttemptsRequiringReceiptFetch_Call { - _c.Call.Return(run) - return _c -} - // FindTxAttemptsRequiringResend provides a mock function with given fields: ctx, olderThan, maxInFlightTransactions, chainID, address func (_m *EvmTxStore) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID *big.Int, address common.Address) ([]types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { ret := _m.Called(ctx, olderThan, maxInFlightTransactions, chainID, address) @@ -1333,6 +1389,66 @@ func (_c *EvmTxStore_FindTxWithSequence_Call) RunAndReturn(run func(context.Cont return _c } +// FindTxesByIDs provides a mock function with given fields: ctx, etxIDs, chainID +func (_m *EvmTxStore) FindTxesByIDs(ctx context.Context, etxIDs []int64, chainID *big.Int) ([]*txmgr.Tx, error) { + ret := _m.Called(ctx, etxIDs, chainID) + + if len(ret) == 0 { + panic("no return value specified for FindTxesByIDs") + } + + var r0 []*txmgr.Tx + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []int64, *big.Int) ([]*txmgr.Tx, error)); ok { + return rf(ctx, etxIDs, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, []int64, *big.Int) []*txmgr.Tx); ok { + r0 = rf(ctx, etxIDs, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*txmgr.Tx) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []int64, *big.Int) error); ok { + r1 = rf(ctx, etxIDs, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EvmTxStore_FindTxesByIDs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTxesByIDs' +type EvmTxStore_FindTxesByIDs_Call struct { + *mock.Call +} + +// FindTxesByIDs is a helper method to define mock.On call +// - ctx context.Context +// - etxIDs []int64 +// - chainID *big.Int +func (_e *EvmTxStore_Expecter) FindTxesByIDs(ctx interface{}, etxIDs interface{}, chainID interface{}) *EvmTxStore_FindTxesByIDs_Call { + return &EvmTxStore_FindTxesByIDs_Call{Call: _e.mock.On("FindTxesByIDs", ctx, etxIDs, chainID)} +} + +func (_c *EvmTxStore_FindTxesByIDs_Call) Run(run func(ctx context.Context, etxIDs []int64, chainID *big.Int)) *EvmTxStore_FindTxesByIDs_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64), args[2].(*big.Int)) + }) + return _c +} + +func (_c *EvmTxStore_FindTxesByIDs_Call) Return(etxs []*txmgr.Tx, err error) *EvmTxStore_FindTxesByIDs_Call { + _c.Call.Return(etxs, err) + return _c +} + +func (_c *EvmTxStore_FindTxesByIDs_Call) RunAndReturn(run func(context.Context, []int64, *big.Int) ([]*txmgr.Tx, error)) *EvmTxStore_FindTxesByIDs_Call { + _c.Call.Return(run) + return _c +} + // FindTxesByMetaFieldAndStates provides a mock function with given fields: ctx, metaField, metaValue, states, chainID func (_m *EvmTxStore) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []types.TxState, chainID *big.Int) ([]*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { ret := _m.Called(ctx, metaField, metaValue, states, chainID) @@ -1396,23 +1512,23 @@ func (_c *EvmTxStore_FindTxesByMetaFieldAndStates_Call) RunAndReturn(run func(co } // FindTxesPendingCallback provides a mock function with given fields: ctx, latest, finalized, chainID -func (_m *EvmTxStore) FindTxesPendingCallback(ctx context.Context, latest int64, finalized int64, chainID *big.Int) ([]types.ReceiptPlus[*evmtypes.Receipt], error) { +func (_m *EvmTxStore) FindTxesPendingCallback(ctx context.Context, latest int64, finalized int64, chainID *big.Int) ([]txmgr.ReceiptPlus, error) { ret := _m.Called(ctx, latest, finalized, chainID) if len(ret) == 0 { panic("no return value specified for FindTxesPendingCallback") } - var r0 []types.ReceiptPlus[*evmtypes.Receipt] + var r0 []txmgr.ReceiptPlus var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) ([]types.ReceiptPlus[*evmtypes.Receipt], error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) ([]txmgr.ReceiptPlus, error)); ok { return rf(ctx, latest, finalized, chainID) } - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) []types.ReceiptPlus[*evmtypes.Receipt]); ok { + if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) []txmgr.ReceiptPlus); ok { r0 = rf(ctx, latest, finalized, chainID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]types.ReceiptPlus[*evmtypes.Receipt]) + r0 = ret.Get(0).([]txmgr.ReceiptPlus) } } @@ -1446,12 +1562,12 @@ func (_c *EvmTxStore_FindTxesPendingCallback_Call) Run(run func(ctx context.Cont return _c } -func (_c *EvmTxStore_FindTxesPendingCallback_Call) Return(receiptsPlus []types.ReceiptPlus[*evmtypes.Receipt], err error) *EvmTxStore_FindTxesPendingCallback_Call { +func (_c *EvmTxStore_FindTxesPendingCallback_Call) Return(receiptsPlus []txmgr.ReceiptPlus, err error) *EvmTxStore_FindTxesPendingCallback_Call { _c.Call.Return(receiptsPlus, err) return _c } -func (_c *EvmTxStore_FindTxesPendingCallback_Call) RunAndReturn(run func(context.Context, int64, int64, *big.Int) ([]types.ReceiptPlus[*evmtypes.Receipt], error)) *EvmTxStore_FindTxesPendingCallback_Call { +func (_c *EvmTxStore_FindTxesPendingCallback_Call) RunAndReturn(run func(context.Context, int64, int64, *big.Int) ([]txmgr.ReceiptPlus, error)) *EvmTxStore_FindTxesPendingCallback_Call { _c.Call.Return(run) return _c } @@ -2168,102 +2284,6 @@ func (_c *EvmTxStore_LoadTxAttempts_Call) RunAndReturn(run func(context.Context, return _c } -// MarkAllConfirmedMissingReceipt provides a mock function with given fields: ctx, chainID -func (_m *EvmTxStore) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID *big.Int) error { - ret := _m.Called(ctx, chainID) - - if len(ret) == 0 { - panic("no return value specified for MarkAllConfirmedMissingReceipt") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *big.Int) error); ok { - r0 = rf(ctx, chainID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EvmTxStore_MarkAllConfirmedMissingReceipt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkAllConfirmedMissingReceipt' -type EvmTxStore_MarkAllConfirmedMissingReceipt_Call struct { - *mock.Call -} - -// MarkAllConfirmedMissingReceipt is a helper method to define mock.On call -// - ctx context.Context -// - chainID *big.Int -func (_e *EvmTxStore_Expecter) MarkAllConfirmedMissingReceipt(ctx interface{}, chainID interface{}) *EvmTxStore_MarkAllConfirmedMissingReceipt_Call { - return &EvmTxStore_MarkAllConfirmedMissingReceipt_Call{Call: _e.mock.On("MarkAllConfirmedMissingReceipt", ctx, chainID)} -} - -func (_c *EvmTxStore_MarkAllConfirmedMissingReceipt_Call) Run(run func(ctx context.Context, chainID *big.Int)) *EvmTxStore_MarkAllConfirmedMissingReceipt_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*big.Int)) - }) - return _c -} - -func (_c *EvmTxStore_MarkAllConfirmedMissingReceipt_Call) Return(err error) *EvmTxStore_MarkAllConfirmedMissingReceipt_Call { - _c.Call.Return(err) - return _c -} - -func (_c *EvmTxStore_MarkAllConfirmedMissingReceipt_Call) RunAndReturn(run func(context.Context, *big.Int) error) *EvmTxStore_MarkAllConfirmedMissingReceipt_Call { - _c.Call.Return(run) - return _c -} - -// MarkOldTxesMissingReceiptAsErrored provides a mock function with given fields: ctx, blockNum, latestFinalizedBlockNum, chainID -func (_m *EvmTxStore) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int) error { - ret := _m.Called(ctx, blockNum, latestFinalizedBlockNum, chainID) - - if len(ret) == 0 { - panic("no return value specified for MarkOldTxesMissingReceiptAsErrored") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, int64, *big.Int) error); ok { - r0 = rf(ctx, blockNum, latestFinalizedBlockNum, chainID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkOldTxesMissingReceiptAsErrored' -type EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call struct { - *mock.Call -} - -// MarkOldTxesMissingReceiptAsErrored is a helper method to define mock.On call -// - ctx context.Context -// - blockNum int64 -// - latestFinalizedBlockNum int64 -// - chainID *big.Int -func (_e *EvmTxStore_Expecter) MarkOldTxesMissingReceiptAsErrored(ctx interface{}, blockNum interface{}, latestFinalizedBlockNum interface{}, chainID interface{}) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { - return &EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call{Call: _e.mock.On("MarkOldTxesMissingReceiptAsErrored", ctx, blockNum, latestFinalizedBlockNum, chainID)} -} - -func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) Run(run func(ctx context.Context, blockNum int64, latestFinalizedBlockNum int64, chainID *big.Int)) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int64), args[2].(int64), args[3].(*big.Int)) - }) - return _c -} - -func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) Return(_a0 error) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call) RunAndReturn(run func(context.Context, int64, int64, *big.Int) error) *EvmTxStore_MarkOldTxesMissingReceiptAsErrored_Call { - _c.Call.Return(run) - return _c -} - // PreloadTxes provides a mock function with given fields: ctx, attempts func (_m *EvmTxStore) PreloadTxes(ctx context.Context, attempts []types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error { ret := _m.Called(ctx, attempts) @@ -2419,12 +2439,12 @@ func (_c *EvmTxStore_ReapTxHistory_Call) RunAndReturn(run func(context.Context, return _c } -// SaveConfirmedMissingReceiptAttempt provides a mock function with given fields: ctx, timeout, attempt, broadcastAt -func (_m *EvmTxStore) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], broadcastAt time.Time) error { +// SaveConfirmedAttempt provides a mock function with given fields: ctx, timeout, attempt, broadcastAt +func (_m *EvmTxStore) SaveConfirmedAttempt(ctx context.Context, timeout time.Duration, attempt *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], broadcastAt time.Time) error { ret := _m.Called(ctx, timeout, attempt, broadcastAt) if len(ret) == 0 { - panic("no return value specified for SaveConfirmedMissingReceiptAttempt") + panic("no return value specified for SaveConfirmedAttempt") } var r0 error @@ -2437,48 +2457,48 @@ func (_m *EvmTxStore) SaveConfirmedMissingReceiptAttempt(ctx context.Context, ti return r0 } -// EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveConfirmedMissingReceiptAttempt' -type EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call struct { +// EvmTxStore_SaveConfirmedAttempt_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveConfirmedAttempt' +type EvmTxStore_SaveConfirmedAttempt_Call struct { *mock.Call } -// SaveConfirmedMissingReceiptAttempt is a helper method to define mock.On call +// SaveConfirmedAttempt is a helper method to define mock.On call // - ctx context.Context // - timeout time.Duration // - attempt *types.TxAttempt[*big.Int,common.Address,common.Hash,common.Hash,evmtypes.Nonce,gas.EvmFee] // - broadcastAt time.Time -func (_e *EvmTxStore_Expecter) SaveConfirmedMissingReceiptAttempt(ctx interface{}, timeout interface{}, attempt interface{}, broadcastAt interface{}) *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call { - return &EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call{Call: _e.mock.On("SaveConfirmedMissingReceiptAttempt", ctx, timeout, attempt, broadcastAt)} +func (_e *EvmTxStore_Expecter) SaveConfirmedAttempt(ctx interface{}, timeout interface{}, attempt interface{}, broadcastAt interface{}) *EvmTxStore_SaveConfirmedAttempt_Call { + return &EvmTxStore_SaveConfirmedAttempt_Call{Call: _e.mock.On("SaveConfirmedAttempt", ctx, timeout, attempt, broadcastAt)} } -func (_c *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call) Run(run func(ctx context.Context, timeout time.Duration, attempt *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], broadcastAt time.Time)) *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call { +func (_c *EvmTxStore_SaveConfirmedAttempt_Call) Run(run func(ctx context.Context, timeout time.Duration, attempt *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], broadcastAt time.Time)) *EvmTxStore_SaveConfirmedAttempt_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(time.Duration), args[2].(*types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]), args[3].(time.Time)) }) return _c } -func (_c *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call) Return(_a0 error) *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call { +func (_c *EvmTxStore_SaveConfirmedAttempt_Call) Return(_a0 error) *EvmTxStore_SaveConfirmedAttempt_Call { _c.Call.Return(_a0) return _c } -func (_c *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call) RunAndReturn(run func(context.Context, time.Duration, *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], time.Time) error) *EvmTxStore_SaveConfirmedMissingReceiptAttempt_Call { +func (_c *EvmTxStore_SaveConfirmedAttempt_Call) RunAndReturn(run func(context.Context, time.Duration, *types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], time.Time) error) *EvmTxStore_SaveConfirmedAttempt_Call { _c.Call.Return(run) return _c } -// SaveFetchedReceipts provides a mock function with given fields: ctx, r, state, errorMsg, chainID -func (_m *EvmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt, state types.TxState, errorMsg *string, chainID *big.Int) error { - ret := _m.Called(ctx, r, state, errorMsg, chainID) +// SaveFetchedReceipts provides a mock function with given fields: ctx, r +func (_m *EvmTxStore) SaveFetchedReceipts(ctx context.Context, r []*evmtypes.Receipt) error { + ret := _m.Called(ctx, r) if len(ret) == 0 { panic("no return value specified for SaveFetchedReceipts") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []*evmtypes.Receipt, types.TxState, *string, *big.Int) error); ok { - r0 = rf(ctx, r, state, errorMsg, chainID) + if rf, ok := ret.Get(0).(func(context.Context, []*evmtypes.Receipt) error); ok { + r0 = rf(ctx, r) } else { r0 = ret.Error(0) } @@ -2494,26 +2514,23 @@ type EvmTxStore_SaveFetchedReceipts_Call struct { // SaveFetchedReceipts is a helper method to define mock.On call // - ctx context.Context // - r []*evmtypes.Receipt -// - state types.TxState -// - errorMsg *string -// - chainID *big.Int -func (_e *EvmTxStore_Expecter) SaveFetchedReceipts(ctx interface{}, r interface{}, state interface{}, errorMsg interface{}, chainID interface{}) *EvmTxStore_SaveFetchedReceipts_Call { - return &EvmTxStore_SaveFetchedReceipts_Call{Call: _e.mock.On("SaveFetchedReceipts", ctx, r, state, errorMsg, chainID)} +func (_e *EvmTxStore_Expecter) SaveFetchedReceipts(ctx interface{}, r interface{}) *EvmTxStore_SaveFetchedReceipts_Call { + return &EvmTxStore_SaveFetchedReceipts_Call{Call: _e.mock.On("SaveFetchedReceipts", ctx, r)} } -func (_c *EvmTxStore_SaveFetchedReceipts_Call) Run(run func(ctx context.Context, r []*evmtypes.Receipt, state types.TxState, errorMsg *string, chainID *big.Int)) *EvmTxStore_SaveFetchedReceipts_Call { +func (_c *EvmTxStore_SaveFetchedReceipts_Call) Run(run func(ctx context.Context, r []*evmtypes.Receipt)) *EvmTxStore_SaveFetchedReceipts_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]*evmtypes.Receipt), args[2].(types.TxState), args[3].(*string), args[4].(*big.Int)) + run(args[0].(context.Context), args[1].([]*evmtypes.Receipt)) }) return _c } -func (_c *EvmTxStore_SaveFetchedReceipts_Call) Return(_a0 error) *EvmTxStore_SaveFetchedReceipts_Call { - _c.Call.Return(_a0) +func (_c *EvmTxStore_SaveFetchedReceipts_Call) Return(err error) *EvmTxStore_SaveFetchedReceipts_Call { + _c.Call.Return(err) return _c } -func (_c *EvmTxStore_SaveFetchedReceipts_Call) RunAndReturn(run func(context.Context, []*evmtypes.Receipt, types.TxState, *string, *big.Int) error) *EvmTxStore_SaveFetchedReceipts_Call { +func (_c *EvmTxStore_SaveFetchedReceipts_Call) RunAndReturn(run func(context.Context, []*evmtypes.Receipt) error) *EvmTxStore_SaveFetchedReceipts_Call { _c.Call.Return(run) return _c } @@ -3057,9 +3074,9 @@ func (_c *EvmTxStore_UpdateTxAttemptInProgressToBroadcast_Call) RunAndReturn(run return _c } -// UpdateTxCallbackCompleted provides a mock function with given fields: ctx, pipelineTaskRunRid, chainId -func (_m *EvmTxStore) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId *big.Int) error { - ret := _m.Called(ctx, pipelineTaskRunRid, chainId) +// UpdateTxCallbackCompleted provides a mock function with given fields: ctx, pipelineTaskRunRid, chainID +func (_m *EvmTxStore) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID *big.Int) error { + ret := _m.Called(ctx, pipelineTaskRunRid, chainID) if len(ret) == 0 { panic("no return value specified for UpdateTxCallbackCompleted") @@ -3067,7 +3084,7 @@ func (_m *EvmTxStore) UpdateTxCallbackCompleted(ctx context.Context, pipelineTas var r0 error if rf, ok := ret.Get(0).(func(context.Context, uuid.UUID, *big.Int) error); ok { - r0 = rf(ctx, pipelineTaskRunRid, chainId) + r0 = rf(ctx, pipelineTaskRunRid, chainID) } else { r0 = ret.Error(0) } @@ -3083,12 +3100,12 @@ type EvmTxStore_UpdateTxCallbackCompleted_Call struct { // UpdateTxCallbackCompleted is a helper method to define mock.On call // - ctx context.Context // - pipelineTaskRunRid uuid.UUID -// - chainId *big.Int -func (_e *EvmTxStore_Expecter) UpdateTxCallbackCompleted(ctx interface{}, pipelineTaskRunRid interface{}, chainId interface{}) *EvmTxStore_UpdateTxCallbackCompleted_Call { - return &EvmTxStore_UpdateTxCallbackCompleted_Call{Call: _e.mock.On("UpdateTxCallbackCompleted", ctx, pipelineTaskRunRid, chainId)} +// - chainID *big.Int +func (_e *EvmTxStore_Expecter) UpdateTxCallbackCompleted(ctx interface{}, pipelineTaskRunRid interface{}, chainID interface{}) *EvmTxStore_UpdateTxCallbackCompleted_Call { + return &EvmTxStore_UpdateTxCallbackCompleted_Call{Call: _e.mock.On("UpdateTxCallbackCompleted", ctx, pipelineTaskRunRid, chainID)} } -func (_c *EvmTxStore_UpdateTxCallbackCompleted_Call) Run(run func(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId *big.Int)) *EvmTxStore_UpdateTxCallbackCompleted_Call { +func (_c *EvmTxStore_UpdateTxCallbackCompleted_Call) Run(run func(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainID *big.Int)) *EvmTxStore_UpdateTxCallbackCompleted_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(uuid.UUID), args[2].(*big.Int)) }) @@ -3105,17 +3122,64 @@ func (_c *EvmTxStore_UpdateTxCallbackCompleted_Call) RunAndReturn(run func(conte return _c } -// UpdateTxFatalError provides a mock function with given fields: ctx, etx -func (_m *EvmTxStore) UpdateTxFatalError(ctx context.Context, etx *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error { - ret := _m.Called(ctx, etx) +// UpdateTxConfirmed provides a mock function with given fields: ctx, etxIDs +func (_m *EvmTxStore) UpdateTxConfirmed(ctx context.Context, etxIDs []int64) error { + ret := _m.Called(ctx, etxIDs) + + if len(ret) == 0 { + panic("no return value specified for UpdateTxConfirmed") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []int64) error); ok { + r0 = rf(ctx, etxIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EvmTxStore_UpdateTxConfirmed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxConfirmed' +type EvmTxStore_UpdateTxConfirmed_Call struct { + *mock.Call +} + +// UpdateTxConfirmed is a helper method to define mock.On call +// - ctx context.Context +// - etxIDs []int64 +func (_e *EvmTxStore_Expecter) UpdateTxConfirmed(ctx interface{}, etxIDs interface{}) *EvmTxStore_UpdateTxConfirmed_Call { + return &EvmTxStore_UpdateTxConfirmed_Call{Call: _e.mock.On("UpdateTxConfirmed", ctx, etxIDs)} +} + +func (_c *EvmTxStore_UpdateTxConfirmed_Call) Run(run func(ctx context.Context, etxIDs []int64)) *EvmTxStore_UpdateTxConfirmed_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64)) + }) + return _c +} + +func (_c *EvmTxStore_UpdateTxConfirmed_Call) Return(_a0 error) *EvmTxStore_UpdateTxConfirmed_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EvmTxStore_UpdateTxConfirmed_Call) RunAndReturn(run func(context.Context, []int64) error) *EvmTxStore_UpdateTxConfirmed_Call { + _c.Call.Return(run) + return _c +} + +// UpdateTxFatalError provides a mock function with given fields: ctx, etxIDs, errMsg +func (_m *EvmTxStore) UpdateTxFatalError(ctx context.Context, etxIDs []int64, errMsg string) error { + ret := _m.Called(ctx, etxIDs, errMsg) if len(ret) == 0 { panic("no return value specified for UpdateTxFatalError") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error); ok { - r0 = rf(ctx, etx) + if rf, ok := ret.Get(0).(func(context.Context, []int64, string) error); ok { + r0 = rf(ctx, etxIDs, errMsg) } else { r0 = ret.Error(0) } @@ -3130,14 +3194,15 @@ type EvmTxStore_UpdateTxFatalError_Call struct { // UpdateTxFatalError is a helper method to define mock.On call // - ctx context.Context -// - etx *types.Tx[*big.Int,common.Address,common.Hash,common.Hash,evmtypes.Nonce,gas.EvmFee] -func (_e *EvmTxStore_Expecter) UpdateTxFatalError(ctx interface{}, etx interface{}) *EvmTxStore_UpdateTxFatalError_Call { - return &EvmTxStore_UpdateTxFatalError_Call{Call: _e.mock.On("UpdateTxFatalError", ctx, etx)} +// - etxIDs []int64 +// - errMsg string +func (_e *EvmTxStore_Expecter) UpdateTxFatalError(ctx interface{}, etxIDs interface{}, errMsg interface{}) *EvmTxStore_UpdateTxFatalError_Call { + return &EvmTxStore_UpdateTxFatalError_Call{Call: _e.mock.On("UpdateTxFatalError", ctx, etxIDs, errMsg)} } -func (_c *EvmTxStore_UpdateTxFatalError_Call) Run(run func(ctx context.Context, etx *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) *EvmTxStore_UpdateTxFatalError_Call { +func (_c *EvmTxStore_UpdateTxFatalError_Call) Run(run func(ctx context.Context, etxIDs []int64, errMsg string)) *EvmTxStore_UpdateTxFatalError_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) + run(args[0].(context.Context), args[1].([]int64), args[2].(string)) }) return _c } @@ -3147,22 +3212,22 @@ func (_c *EvmTxStore_UpdateTxFatalError_Call) Return(_a0 error) *EvmTxStore_Upda return _c } -func (_c *EvmTxStore_UpdateTxFatalError_Call) RunAndReturn(run func(context.Context, *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error) *EvmTxStore_UpdateTxFatalError_Call { +func (_c *EvmTxStore_UpdateTxFatalError_Call) RunAndReturn(run func(context.Context, []int64, string) error) *EvmTxStore_UpdateTxFatalError_Call { _c.Call.Return(run) return _c } -// UpdateTxForRebroadcast provides a mock function with given fields: ctx, etx, etxAttempt -func (_m *EvmTxStore) UpdateTxForRebroadcast(ctx context.Context, etx types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], etxAttempt types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error { - ret := _m.Called(ctx, etx, etxAttempt) +// UpdateTxFatalErrorAndDeleteAttempts provides a mock function with given fields: ctx, etx +func (_m *EvmTxStore) UpdateTxFatalErrorAndDeleteAttempts(ctx context.Context, etx *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error { + ret := _m.Called(ctx, etx) if len(ret) == 0 { - panic("no return value specified for UpdateTxForRebroadcast") + panic("no return value specified for UpdateTxFatalErrorAndDeleteAttempts") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error); ok { - r0 = rf(ctx, etx, etxAttempt) + if rf, ok := ret.Get(0).(func(context.Context, *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error); ok { + r0 = rf(ctx, etx) } else { r0 = ret.Error(0) } @@ -3170,47 +3235,46 @@ func (_m *EvmTxStore) UpdateTxForRebroadcast(ctx context.Context, etx types.Tx[* return r0 } -// EvmTxStore_UpdateTxForRebroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxForRebroadcast' -type EvmTxStore_UpdateTxForRebroadcast_Call struct { +// EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxFatalErrorAndDeleteAttempts' +type EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call struct { *mock.Call } -// UpdateTxForRebroadcast is a helper method to define mock.On call +// UpdateTxFatalErrorAndDeleteAttempts is a helper method to define mock.On call // - ctx context.Context -// - etx types.Tx[*big.Int,common.Address,common.Hash,common.Hash,evmtypes.Nonce,gas.EvmFee] -// - etxAttempt types.TxAttempt[*big.Int,common.Address,common.Hash,common.Hash,evmtypes.Nonce,gas.EvmFee] -func (_e *EvmTxStore_Expecter) UpdateTxForRebroadcast(ctx interface{}, etx interface{}, etxAttempt interface{}) *EvmTxStore_UpdateTxForRebroadcast_Call { - return &EvmTxStore_UpdateTxForRebroadcast_Call{Call: _e.mock.On("UpdateTxForRebroadcast", ctx, etx, etxAttempt)} +// - etx *types.Tx[*big.Int,common.Address,common.Hash,common.Hash,evmtypes.Nonce,gas.EvmFee] +func (_e *EvmTxStore_Expecter) UpdateTxFatalErrorAndDeleteAttempts(ctx interface{}, etx interface{}) *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call { + return &EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call{Call: _e.mock.On("UpdateTxFatalErrorAndDeleteAttempts", ctx, etx)} } -func (_c *EvmTxStore_UpdateTxForRebroadcast_Call) Run(run func(ctx context.Context, etx types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], etxAttempt types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) *EvmTxStore_UpdateTxForRebroadcast_Call { +func (_c *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call) Run(run func(ctx context.Context, etx *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]), args[2].(types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) + run(args[0].(context.Context), args[1].(*types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee])) }) return _c } -func (_c *EvmTxStore_UpdateTxForRebroadcast_Call) Return(_a0 error) *EvmTxStore_UpdateTxForRebroadcast_Call { +func (_c *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call) Return(_a0 error) *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call { _c.Call.Return(_a0) return _c } -func (_c *EvmTxStore_UpdateTxForRebroadcast_Call) RunAndReturn(run func(context.Context, types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], types.TxAttempt[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error) *EvmTxStore_UpdateTxForRebroadcast_Call { +func (_c *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call) RunAndReturn(run func(context.Context, *types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) error) *EvmTxStore_UpdateTxFatalErrorAndDeleteAttempts_Call { _c.Call.Return(run) return _c } -// UpdateTxStatesToFinalizedUsingReceiptIds provides a mock function with given fields: ctx, etxIDs, chainId -func (_m *EvmTxStore) UpdateTxStatesToFinalizedUsingReceiptIds(ctx context.Context, etxIDs []int64, chainId *big.Int) error { - ret := _m.Called(ctx, etxIDs, chainId) +// UpdateTxStatesToFinalizedUsingTxHashes provides a mock function with given fields: ctx, txHashes, chainID +func (_m *EvmTxStore) UpdateTxStatesToFinalizedUsingTxHashes(ctx context.Context, txHashes []common.Hash, chainID *big.Int) error { + ret := _m.Called(ctx, txHashes, chainID) if len(ret) == 0 { - panic("no return value specified for UpdateTxStatesToFinalizedUsingReceiptIds") + panic("no return value specified for UpdateTxStatesToFinalizedUsingTxHashes") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []int64, *big.Int) error); ok { - r0 = rf(ctx, etxIDs, chainId) + if rf, ok := ret.Get(0).(func(context.Context, []common.Hash, *big.Int) error); ok { + r0 = rf(ctx, txHashes, chainID) } else { r0 = ret.Error(0) } @@ -3218,32 +3282,32 @@ func (_m *EvmTxStore) UpdateTxStatesToFinalizedUsingReceiptIds(ctx context.Conte return r0 } -// EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxStatesToFinalizedUsingReceiptIds' -type EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call struct { +// EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxStatesToFinalizedUsingTxHashes' +type EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call struct { *mock.Call } -// UpdateTxStatesToFinalizedUsingReceiptIds is a helper method to define mock.On call +// UpdateTxStatesToFinalizedUsingTxHashes is a helper method to define mock.On call // - ctx context.Context -// - etxIDs []int64 -// - chainId *big.Int -func (_e *EvmTxStore_Expecter) UpdateTxStatesToFinalizedUsingReceiptIds(ctx interface{}, etxIDs interface{}, chainId interface{}) *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call { - return &EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call{Call: _e.mock.On("UpdateTxStatesToFinalizedUsingReceiptIds", ctx, etxIDs, chainId)} +// - txHashes []common.Hash +// - chainID *big.Int +func (_e *EvmTxStore_Expecter) UpdateTxStatesToFinalizedUsingTxHashes(ctx interface{}, txHashes interface{}, chainID interface{}) *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call { + return &EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call{Call: _e.mock.On("UpdateTxStatesToFinalizedUsingTxHashes", ctx, txHashes, chainID)} } -func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call) Run(run func(ctx context.Context, etxIDs []int64, chainId *big.Int)) *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call { +func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call) Run(run func(ctx context.Context, txHashes []common.Hash, chainID *big.Int)) *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]int64), args[2].(*big.Int)) + run(args[0].(context.Context), args[1].([]common.Hash), args[2].(*big.Int)) }) return _c } -func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call) Return(_a0 error) *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call { +func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call) Return(_a0 error) *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call { _c.Call.Return(_a0) return _c } -func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call) RunAndReturn(run func(context.Context, []int64, *big.Int) error) *EvmTxStore_UpdateTxStatesToFinalizedUsingReceiptIds_Call { +func (_c *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call) RunAndReturn(run func(context.Context, []common.Hash, *big.Int) error) *EvmTxStore_UpdateTxStatesToFinalizedUsingTxHashes_Call { _c.Call.Return(run) return _c } @@ -3296,9 +3360,57 @@ func (_c *EvmTxStore_UpdateTxUnstartedToInProgress_Call) RunAndReturn(run func(c return _c } -// UpdateTxsUnconfirmed provides a mock function with given fields: ctx, ids -func (_m *EvmTxStore) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) error { - ret := _m.Called(ctx, ids) +// UpdateTxsForRebroadcast provides a mock function with given fields: ctx, etxIDs, attemptIDs +func (_m *EvmTxStore) UpdateTxsForRebroadcast(ctx context.Context, etxIDs []int64, attemptIDs []int64) error { + ret := _m.Called(ctx, etxIDs, attemptIDs) + + if len(ret) == 0 { + panic("no return value specified for UpdateTxsForRebroadcast") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, []int64, []int64) error); ok { + r0 = rf(ctx, etxIDs, attemptIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EvmTxStore_UpdateTxsForRebroadcast_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateTxsForRebroadcast' +type EvmTxStore_UpdateTxsForRebroadcast_Call struct { + *mock.Call +} + +// UpdateTxsForRebroadcast is a helper method to define mock.On call +// - ctx context.Context +// - etxIDs []int64 +// - attemptIDs []int64 +func (_e *EvmTxStore_Expecter) UpdateTxsForRebroadcast(ctx interface{}, etxIDs interface{}, attemptIDs interface{}) *EvmTxStore_UpdateTxsForRebroadcast_Call { + return &EvmTxStore_UpdateTxsForRebroadcast_Call{Call: _e.mock.On("UpdateTxsForRebroadcast", ctx, etxIDs, attemptIDs)} +} + +func (_c *EvmTxStore_UpdateTxsForRebroadcast_Call) Run(run func(ctx context.Context, etxIDs []int64, attemptIDs []int64)) *EvmTxStore_UpdateTxsForRebroadcast_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]int64), args[2].([]int64)) + }) + return _c +} + +func (_c *EvmTxStore_UpdateTxsForRebroadcast_Call) Return(_a0 error) *EvmTxStore_UpdateTxsForRebroadcast_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EvmTxStore_UpdateTxsForRebroadcast_Call) RunAndReturn(run func(context.Context, []int64, []int64) error) *EvmTxStore_UpdateTxsForRebroadcast_Call { + _c.Call.Return(run) + return _c +} + +// UpdateTxsUnconfirmed provides a mock function with given fields: ctx, etxIDs +func (_m *EvmTxStore) UpdateTxsUnconfirmed(ctx context.Context, etxIDs []int64) error { + ret := _m.Called(ctx, etxIDs) if len(ret) == 0 { panic("no return value specified for UpdateTxsUnconfirmed") @@ -3306,7 +3418,7 @@ func (_m *EvmTxStore) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) err var r0 error if rf, ok := ret.Get(0).(func(context.Context, []int64) error); ok { - r0 = rf(ctx, ids) + r0 = rf(ctx, etxIDs) } else { r0 = ret.Error(0) } @@ -3321,12 +3433,12 @@ type EvmTxStore_UpdateTxsUnconfirmed_Call struct { // UpdateTxsUnconfirmed is a helper method to define mock.On call // - ctx context.Context -// - ids []int64 -func (_e *EvmTxStore_Expecter) UpdateTxsUnconfirmed(ctx interface{}, ids interface{}) *EvmTxStore_UpdateTxsUnconfirmed_Call { - return &EvmTxStore_UpdateTxsUnconfirmed_Call{Call: _e.mock.On("UpdateTxsUnconfirmed", ctx, ids)} +// - etxIDs []int64 +func (_e *EvmTxStore_Expecter) UpdateTxsUnconfirmed(ctx interface{}, etxIDs interface{}) *EvmTxStore_UpdateTxsUnconfirmed_Call { + return &EvmTxStore_UpdateTxsUnconfirmed_Call{Call: _e.mock.On("UpdateTxsUnconfirmed", ctx, etxIDs)} } -func (_c *EvmTxStore_UpdateTxsUnconfirmed_Call) Run(run func(ctx context.Context, ids []int64)) *EvmTxStore_UpdateTxsUnconfirmed_Call { +func (_c *EvmTxStore_UpdateTxsUnconfirmed_Call) Run(run func(ctx context.Context, etxIDs []int64)) *EvmTxStore_UpdateTxsUnconfirmed_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].([]int64)) }) diff --git a/core/chains/evm/txmgr/nonce_tracker.go b/core/chains/evm/txmgr/nonce_tracker.go index 873f2595dbf..16d14308023 100644 --- a/core/chains/evm/txmgr/nonce_tracker.go +++ b/core/chains/evm/txmgr/nonce_tracker.go @@ -68,6 +68,7 @@ func (s *nonceTracker) getSequenceForAddr(ctx context.Context, address common.Ad seq, err = s.txStore.FindLatestSequence(ctx, address, s.chainID) if err == nil { seq++ + s.lggr.Debugw("found next nonce using stored transactions", "address", address.Hex(), "nonce", seq.String()) return seq, nil } // Look for nonce on-chain if no tx found for address in TxStore or if error occurred. @@ -78,6 +79,7 @@ func (s *nonceTracker) getSequenceForAddr(ctx context.Context, address common.Ad // If that occurs, there could be short term noise in the logs surfacing that a transaction expired without ever getting a receipt. nonce, err := s.client.SequenceAt(ctx, address, nil) if err == nil { + s.lggr.Debugw("found next nonce using RPC", "address", address.Hex(), "nonce", nonce.String()) return nonce, nil } s.lggr.Criticalw("failed to retrieve next sequence from on-chain for address: ", "address", address.String()) diff --git a/core/chains/evm/txmgr/txmgr_test.go b/core/chains/evm/txmgr/txmgr_test.go index 7052a694719..9ee2396846d 100644 --- a/core/chains/evm/txmgr/txmgr_test.go +++ b/core/chains/evm/txmgr/txmgr_test.go @@ -899,19 +899,6 @@ func mustInsertConfirmedEthTxWithReceipt(t *testing.T, txStore txmgr.TestEvmTxSt return etx } -func mustInsertConfirmedEthTxBySaveFetchedReceipts(t *testing.T, txStore txmgr.TestEvmTxStore, fromAddress common.Address, nonce int64, blockNum int64, chainID big.Int) (etx txmgr.Tx) { - etx = cltest.MustInsertConfirmedEthTxWithLegacyAttempt(t, txStore, nonce, blockNum, fromAddress) - receipt := evmtypes.Receipt{ - TxHash: etx.TxAttempts[0].Hash, - BlockHash: utils.NewHash(), - BlockNumber: big.NewInt(nonce), - TransactionIndex: uint(1), - } - err := txStore.SaveFetchedReceipts(tests.Context(t), []*evmtypes.Receipt{&receipt}, txmgrcommon.TxConfirmed, nil, &chainID) - require.NoError(t, err) - return etx -} - func mustInsertFatalErrorEthTx(t *testing.T, txStore txmgr.TestEvmTxStore, fromAddress common.Address) txmgr.Tx { etx := cltest.NewEthTx(fromAddress) etx.Error = null.StringFrom("something exploded") diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 50411e10d42..f6b8db43123 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -676,11 +676,10 @@ func (s *Shell) RebroadcastTransactions(c *cli.Context) (err error) { orm := txmgr.NewTxStore(app.GetDB(), lggr) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), chain.Config().EVM().GasEstimator(), keyStore.Eth(), nil) - cfg := txmgr.NewEvmTxmConfig(chain.Config().EVM()) feeCfg := txmgr.NewEvmTxmFeeConfig(chain.Config().EVM().GasEstimator()) stuckTxDetector := txmgr.NewStuckTxDetector(lggr, ethClient.ConfiguredChainID(), "", assets.NewWei(assets.NewEth(100).ToInt()), chain.Config().EVM().Transactions().AutoPurge(), nil, orm, ethClient) ec := txmgr.NewEvmConfirmer(orm, txmgr.NewEvmTxmClient(ethClient, chain.Config().EVM().NodePool().Errors()), - cfg, feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, chain.HeadTracker()) + feeCfg, chain.Config().EVM().Transactions(), app.GetConfig().Database(), keyStore.Eth(), txBuilder, chain.Logger(), stuckTxDetector, chain.HeadTracker()) totalNonces := endingNonce - beginningNonce + 1 nonces := make([]evmtypes.Nonce, totalNonces) for i := int64(0); i < totalNonces; i++ { diff --git a/core/internal/features/features_test.go b/core/internal/features/features_test.go index bf7b2e4ccba..919b01f3364 100644 --- a/core/internal/features/features_test.go +++ b/core/internal/features/features_test.go @@ -526,10 +526,15 @@ observationSource = """ assert.Equal(t, []*string(nil), run.Errors) testutils.WaitForLogMessage(t, o, "Sending transaction") - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - testutils.WaitForLogMessage(t, o, "Resume run success") + gomega.NewWithT(t).Eventually(func() bool { + b.Commit() // Process new head until tx confirmed, receipt is fetched, and task resumed + for _, l := range o.All() { + if strings.Contains(l.Message, "Resume run success") { + return true + } + } + return false + }, testutils.WaitTimeout(t), 1*time.Second).Should(gomega.BeTrue()) pipelineRuns := cltest.WaitForPipelineComplete(t, 0, j.ID, 1, 1, app.JobORM(), testutils.WaitTimeout(t), time.Second) @@ -572,10 +577,15 @@ observationSource = """ assert.Equal(t, []*string(nil), run.Errors) testutils.WaitForLogMessage(t, o, "Sending transaction") - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - testutils.WaitForLogMessage(t, o, "Resume run success") + gomega.NewWithT(t).Eventually(func() bool { + b.Commit() // Process new head until tx confirmed, receipt is fetched, and task resumed + for _, l := range o.All() { + if strings.Contains(l.Message, "Resume run success") { + return true + } + } + return false + }, testutils.WaitTimeout(t), 1*time.Second).Should(gomega.BeTrue()) pipelineRuns := cltest.WaitForPipelineError(t, 0, j.ID, 1, 1, app.JobORM(), testutils.WaitTimeout(t), time.Second) @@ -610,10 +620,15 @@ observationSource = """ assert.Equal(t, []*string(nil), run.Errors) testutils.WaitForLogMessage(t, o, "Sending transaction") - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - b.Commit() // Needs at least two confirmations - testutils.WaitForLogMessage(t, o, "Resume run success") + gomega.NewWithT(t).Eventually(func() bool { + b.Commit() // Process new head until tx confirmed, receipt is fetched, and task resumed + for _, l := range o.All() { + if strings.Contains(l.Message, "Resume run success") { + return true + } + } + return false + }, testutils.WaitTimeout(t), 1*time.Second).Should(gomega.BeTrue()) pipelineRuns := cltest.WaitForPipelineComplete(t, 0, j.ID, 1, 1, app.JobORM(), testutils.WaitTimeout(t), time.Second) @@ -625,7 +640,7 @@ observationSource = """ require.Len(t, outputs, 1) output := outputs[0] receipt := output.(map[string]interface{}) - assert.Equal(t, "0x11", receipt["blockNumber"]) + assert.Equal(t, "0x13", receipt["blockNumber"]) assert.Equal(t, "0x7a120", receipt["gasUsed"]) assert.Equal(t, "0x0", receipt["status"]) }) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 63167093ed4..97fcfe46ec5 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -410,7 +410,7 @@ require ( github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241017133723-5277829bd53f // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e // indirect github.com/smartcontractkit/chainlink-feeds v0.1.1 // indirect - github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0 // indirect + github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0 // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241115191142-8b8369c1f44e // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241017135645-176a23722fd8 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 719ea492a88..0678468c384 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1417,8 +1417,8 @@ github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241018134907-a00ba3729b5e/go.mod h1:iK3BNHKCLgSgkOyiu3iE7sfZ20Qnuk7xwjV/yO/6gnQ= github.com/smartcontractkit/chainlink-feeds v0.1.1 h1:JzvUOM/OgGQA1sOqTXXl52R6AnNt+Wg64sVG+XSA49c= github.com/smartcontractkit/chainlink-feeds v0.1.1/go.mod h1:55EZ94HlKCfAsUiKUTNI7QlE/3d3IwTlsU3YNa/nBb4= -github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0 h1:1xTm8UGeDUAjvCXRh08+4xBRX33owH5MqC522JdelM0= -github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0/go.mod h1:/dVVLXrsp+V0AbcYGJo3XMzKg3CkELsweA/TTopCsKE= +github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 h1:0ewLMbAz3rZrovdRUCgd028yOXX8KigB4FndAUdI2kM= +github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0/go.mod h1:/dVVLXrsp+V0AbcYGJo3XMzKg3CkELsweA/TTopCsKE= github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0 h1:PBUaFfPLm+Efq7H9kdfGBivH+QhJ6vB5EZTR/sCZsxI= github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0/go.mod h1:m/A3lqD7ms/RsQ9BT5P2uceYY0QX5mIt4KQxT2G6qEo= github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241115191142-8b8369c1f44e h1:XxTWJ9VIXK+XuAjP5131PqqBn0NEt5lBvnRAWRdqy8A= diff --git a/core/services/vrf/v2/integration_helpers_test.go b/core/services/vrf/v2/integration_helpers_test.go index 48e1ffdd69c..1a46fc1e334 100644 --- a/core/services/vrf/v2/integration_helpers_test.go +++ b/core/services/vrf/v2/integration_helpers_test.go @@ -445,13 +445,13 @@ func testMultipleConsumersNeedTrustedBHS( topUpSubscription(t, consumer, consumerContract, uni.backend, big.NewInt(5e18 /* 5 LINK */), nativePayment) // Wait for fulfillment to be queued. - gomega.NewGomegaWithT(t).Eventually(func() bool { + require.Eventually(t, func() bool { uni.backend.Commit() runs, err := app.PipelineORM().GetAllRuns(ctx) require.NoError(t, err) t.Log("runs", len(runs)) - return len(runs) == 1 - }, testutils.WaitTimeout(t), time.Second).Should(gomega.BeTrue()) + return len(runs) >= 1 + }, testutils.WaitTimeout(t), time.Second) mine(t, requestID, subID, uni.backend, db, vrfVersion, testutils.SimulatedChainID) @@ -1020,7 +1020,7 @@ func testSingleConsumerForcedFulfillment( uni.backend.Commit() // Wait for force-fulfillment to be queued. - gomega.NewGomegaWithT(t).Eventually(func() bool { + require.Eventually(t, func() bool { uni.backend.Commit() commitment, err2 := uni.oldRootContract.GetCommitment(nil, requestID) require.NoError(t, err2) @@ -1036,7 +1036,7 @@ func testSingleConsumerForcedFulfillment( } t.Log("num RandomWordsForced logs:", i) return utils.IsEmpty(commitment[:]) - }, testutils.WaitTimeout(t), time.Second).Should(gomega.BeTrue()) + }, testutils.WaitTimeout(t), time.Second) // Mine the fulfillment that was queued. mine(t, requestID, subID, uni.backend, db, vrfVersion, testutils.SimulatedChainID) diff --git a/core/services/vrf/v2/integration_v2_plus_test.go b/core/services/vrf/v2/integration_v2_plus_test.go index 151bbced6dd..75cffe1057c 100644 --- a/core/services/vrf/v2/integration_v2_plus_test.go +++ b/core/services/vrf/v2/integration_v2_plus_test.go @@ -13,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" gethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/onsi/gomega" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1216,13 +1215,13 @@ func TestVRFV2PlusIntegration_Migration(t *testing.T) { requestID, _ := requestRandomnessAndAssertRandomWordsRequestedEvent(t, consumerContract, consumer, keyHash, subID, numWords, 500_000, uni.rootContract, uni.backend, false) // Wait for fulfillment to be queued. - gomega.NewGomegaWithT(t).Eventually(func() bool { + require.Eventually(t, func() bool { uni.backend.Commit() runs, err := app.PipelineORM().GetAllRuns(ctx) require.NoError(t, err) t.Log("runs", len(runs)) return len(runs) == 1 - }, testutils.WaitTimeout(t), time.Second).Should(gomega.BeTrue()) + }, testutils.WaitTimeout(t), time.Second) mine(t, requestID, subID, uni.backend, db, vrfcommon.V2Plus, testutils.SimulatedChainID) assertRandomWordsFulfilled(t, requestID, true, uni.rootContract, false) diff --git a/core/services/vrf/v2/integration_v2_reverted_txns_test.go b/core/services/vrf/v2/integration_v2_reverted_txns_test.go index af4a135ccd3..67716d440e3 100644 --- a/core/services/vrf/v2/integration_v2_reverted_txns_test.go +++ b/core/services/vrf/v2/integration_v2_reverted_txns_test.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/google/uuid" - "github.com/onsi/gomega" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -199,14 +198,14 @@ func waitForForceFulfillment(t *testing.T, requestID := req.requestID // Wait for force-fulfillment to be queued. - gomega.NewGomegaWithT(t).Eventually(func() bool { + require.Eventually(t, func() bool { uni.backend.Commit() commitment, err := coordinator.GetCommitment(nil, requestID) require.NoError(t, err) t.Log("commitment is:", hexutil.Encode(commitment[:]), ", requestID: ", common.BigToHash(requestID).Hex()) checkForForceFulfilledEvent(t, th, req, sub, -1) return utils.IsEmpty(commitment[:]) - }, testutils.WaitTimeout(t), time.Second).Should(gomega.BeTrue()) + }, testutils.WaitTimeout(t), time.Second) // Mine the fulfillment that was queued. mineForceFulfilled(t, requestID, sub.subID, forceFulfilledCount, *uni, th.db) diff --git a/core/services/vrf/v2/integration_v2_test.go b/core/services/vrf/v2/integration_v2_test.go index fcf894911dc..d9086a52a33 100644 --- a/core/services/vrf/v2/integration_v2_test.go +++ b/core/services/vrf/v2/integration_v2_test.go @@ -808,9 +808,12 @@ func mine(t *testing.T, requestID, subID *big.Int, backend evmtypes.Backend, db return assert.Eventually(t, func() bool { backend.Commit() - txes, err := txstore.FindTxesByMetaFieldAndStates(testutils.Context(t), metaField, subID.String(), []txmgrtypes.TxState{txmgrcommon.TxConfirmed}, chainID) + txes, err := txstore.FindTxesByMetaFieldAndStates(testutils.Context(t), metaField, subID.String(), []txmgrtypes.TxState{txmgrcommon.TxConfirmed, txmgrcommon.TxFinalized}, chainID) require.NoError(t, err) for _, tx := range txes { + if !checkForReceipt(t, db, tx.ID) { + return false + } meta, err := tx.GetMeta() require.NoError(t, err) if meta.RequestID.String() == common.BytesToHash(requestID.Bytes()).String() { @@ -837,9 +840,12 @@ func mineBatch(t *testing.T, requestIDs []*big.Int, subID *big.Int, backend evmt } return assert.Eventually(t, func() bool { backend.Commit() - txes, err := txstore.FindTxesByMetaFieldAndStates(testutils.Context(t), metaField, subID.String(), []txmgrtypes.TxState{txmgrcommon.TxConfirmed}, chainID) + txes, err := txstore.FindTxesByMetaFieldAndStates(testutils.Context(t), metaField, subID.String(), []txmgrtypes.TxState{txmgrcommon.TxConfirmed, txmgrcommon.TxFinalized}, chainID) require.NoError(t, err) for _, tx := range txes { + if !checkForReceipt(t, db, tx.ID) { + return false + } meta, err := tx.GetMeta() require.NoError(t, err) for _, requestID := range meta.RequestIDs { @@ -863,16 +869,40 @@ func mineForceFulfilled(t *testing.T, requestID *big.Int, subID uint64, forceFul var txs []txmgr.DbEthTx err := db.Select(&txs, ` SELECT * FROM evm.txes - WHERE evm.txes.state = 'confirmed' + WHERE evm.txes.state IN ('confirmed', 'finalized') AND evm.txes.meta->>'RequestID' = $1 AND CAST(evm.txes.meta->>'SubId' AS NUMERIC) = $2 ORDER BY created_at DESC `, common.BytesToHash(requestID.Bytes()).String(), subID) require.NoError(t, err) t.Log("num txs", len(txs)) - return len(txs) == int(forceFulfilledCount) + for _, tx := range txs { + if !checkForReceipt(t, db, tx.ID) { + return false + } + } + return len(txs) >= int(forceFulfilledCount) }, testutils.WaitTimeout(t), time.Second) } +func checkForReceipt(t *testing.T, db *sqlx.DB, txID int64) bool { + // Confirm receipt is fetched and stored for transaction to consider it mined + var count uint32 + sql := ` + SELECT count(*) FROM evm.receipts + JOIN evm.tx_attempts ON evm.tx_attempts.hash = evm.receipts.tx_hash + JOIN evm.txes ON evm.txes.ID = evm.tx_attempts.eth_tx_id + WHERE evm.txes.ID = $1 AND evm.txes.state IN ('confirmed', 'finalized')` + if txID != -1 { + err := db.GetContext(testutils.Context(t), &count, sql, txID) + require.NoError(t, err) + } else { + sql = strings.Replace(sql, "evm.txes.ID = $1", "evm.txes.meta->>'ForceFulfilled' IS NOT NULL", 1) + err := db.GetContext(testutils.Context(t), &count, sql, txID) + require.NoError(t, err) + } + return count > 0 +} + func TestVRFV2Integration_SingleConsumer_ForceFulfillment(t *testing.T) { t.Parallel() ownerKey := cltest.MustGenerateRandomKey(t) @@ -1825,13 +1855,16 @@ func TestIntegrationVRFV2(t *testing.T) { linkWeiCharged.BigInt(), }) - // We should see the response count present - require.NoError(t, err) - var counts map[string]uint64 - counts, err = listenerV2.GetStartingResponseCountsV2(ctx) - require.NoError(t, err) - t.Log(counts, rf[0].RequestID().String()) - assert.Equal(t, uint64(1), counts[rf[0].RequestID().String()]) + // We should see the response count present after receipt is fetched and stored for transaction + // Check periodically for receipt in case it is fetched and stored after more than 1 block + require.Eventually(t, func() bool { + uni.backend.Commit() + var counts map[string]uint64 + counts, err = listenerV2.GetStartingResponseCountsV2(ctx) + require.NoError(t, err) + t.Log(counts, rf[0].RequestID().String()) + return uint64(1) == counts[rf[0].RequestID().String()] + }, testutils.WaitTimeout(t), 1*time.Second) } func TestMaliciousConsumer(t *testing.T) { diff --git a/core/web/assets/index.html b/core/web/assets/index.html index d5b24848b5d..d7d5959f285 100644 --- a/core/web/assets/index.html +++ b/core/web/assets/index.html @@ -1 +1 @@ -