diff --git a/integration-tests/contracts/ethereum_contracts_automation_seth.go b/integration-tests/contracts/ethereum_contracts_automation_seth.go index d91e092d720..062586dd918 100644 --- a/integration-tests/contracts/ethereum_contracts_automation_seth.go +++ b/integration-tests/contracts/ethereum_contracts_automation_seth.go @@ -7,6 +7,7 @@ import ( "math/big" "strconv" "strings" + "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -2278,7 +2279,21 @@ func (v *EthereumAutomationConsumerBenchmark) GetUpkeepCount(ctx context.Context }, id) } +// DeployKeeperConsumerBenchmark deploys a keeper consumer benchmark contract with a standard contract backend func DeployKeeperConsumerBenchmark(client *seth.Client) (AutomationConsumerBenchmark, error) { + return deployKeeperConsumerBenchmarkWithWrapperFn(client, func(client *seth.Client) *wrappers.WrappedContractBackend { + return wrappers.MustNewWrappedContractBackend(nil, client) + }) +} + +// DeployKeeperConsumerBenchmarkWithRetry deploys a keeper consumer benchmark contract with a read-only operations retrying contract backend +func DeployKeeperConsumerBenchmarkWithRetry(client *seth.Client, logger zerolog.Logger, maxAttempts uint, retryDelay time.Duration) (AutomationConsumerBenchmark, error) { + return deployKeeperConsumerBenchmarkWithWrapperFn(client, func(client *seth.Client) *wrappers.WrappedContractBackend { + return wrappers.MustNewRetryingWrappedContractBackend(client, logger, maxAttempts, retryDelay) + }) +} + +func deployKeeperConsumerBenchmarkWithWrapperFn(client *seth.Client, wrapperConstrFn func(client *seth.Client) *wrappers.WrappedContractBackend) (AutomationConsumerBenchmark, error) { abi, err := automation_consumer_benchmark.AutomationConsumerBenchmarkMetaData.GetAbi() if err != nil { return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("failed to get AutomationConsumerBenchmark ABI: %w", err) @@ -2288,7 +2303,7 @@ func DeployKeeperConsumerBenchmark(client *seth.Client) (AutomationConsumerBench return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("AutomationConsumerBenchmark instance deployment have failed: %w", err) } - instance, err := automation_consumer_benchmark.NewAutomationConsumerBenchmark(data.Address, wrappers.MustNewWrappedContractBackend(nil, client)) + instance, err := automation_consumer_benchmark.NewAutomationConsumerBenchmark(data.Address, wrapperConstrFn(client)) if err != nil { return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("failed to instantiate AutomationConsumerBenchmark instance: %w", err) } @@ -2325,9 +2340,9 @@ type KeeperConsumerBenchmarkUpkeepObserver struct { l zerolog.Logger } -// NewKeeperConsumerBenchmarkkUpkeepObserver provides a new instance of a KeeperConsumerBenchmarkkUpkeepObserver +// NewKeeperConsumerBenchmarkUpkeepObserver provides a new instance of a NewKeeperConsumerBenchmarkUpkeepObserver // Used to track and log benchmark test results for keepers -func NewKeeperConsumerBenchmarkkUpkeepObserver( +func NewKeeperConsumerBenchmarkUpkeepObserver( contract AutomationConsumerBenchmark, registry KeeperRegistry, upkeepID *big.Int, diff --git a/integration-tests/go.mod b/integration-tests/go.mod index e338302734b..8cd8ae9a021 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -6,6 +6,7 @@ go 1.21.7 replace github.com/smartcontractkit/chainlink/v2 => ../ require ( + github.com/avast/retry-go/v4 v4.5.1 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/cli/go-gh/v2 v2.0.0 github.com/ethereum/go-ethereum v1.13.8 @@ -88,7 +89,6 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect - github.com/avast/retry-go/v4 v4.5.1 // indirect github.com/aws/aws-sdk-go v1.45.25 // indirect github.com/aws/constructs-go/constructs/v10 v10.1.255 // indirect github.com/aws/jsii-runtime-go v1.75.0 // indirect diff --git a/integration-tests/testconfig/keeper/config.go b/integration-tests/testconfig/keeper/config.go index da6cd7acc98..0e11266d39f 100644 --- a/integration-tests/testconfig/keeper/config.go +++ b/integration-tests/testconfig/keeper/config.go @@ -2,17 +2,26 @@ package keeper import ( "errors" + + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" ) type Config struct { - Common *Common `toml:"Common"` + Common *Common `toml:"Common"` + Resiliency *ResiliencyConfig `toml:"Resiliency"` } func (c *Config) Validate() error { if c.Common == nil { return nil } - return c.Common.Validate() + if err := c.Common.Validate(); err != nil { + return err + } + if c.Resiliency == nil { + return nil + } + return c.Resiliency.Validate() } type Common struct { @@ -83,3 +92,19 @@ func (c *Common) Validate() error { } return nil } + +type ResiliencyConfig struct { + ContractCallLimit *uint `toml:"contract_call_limit"` + ContractCallInterval *blockchain.StrDuration `toml:"contract_call_interval"` +} + +func (c *ResiliencyConfig) Validate() error { + if c.ContractCallLimit == nil { + return errors.New("contract_call_limit must be set") + } + if c.ContractCallInterval == nil { + return errors.New("contract_call_interval must be set") + } + + return nil +} diff --git a/integration-tests/testconfig/keeper/example.toml b/integration-tests/testconfig/keeper/example.toml index d76fff343e7..0bda9982988 100644 --- a/integration-tests/testconfig/keeper/example.toml +++ b/integration-tests/testconfig/keeper/example.toml @@ -85,4 +85,13 @@ max_perform_gas = 5000000 block_range = 3600 block_interval = 20 forces_single_tx_key = false -delete_jobs_on_end = true \ No newline at end of file +delete_jobs_on_end = true + +# If present will wrap keeper benchmakr consumers in retrying contract backend +# that retries read-only operations on failure related to network issues or node unavailability +# To disable simply remove this section or set any of the values to 0 +[Keeper.Resiliency] +# number of retries before giving up +contract_call_limit = 500 +# static interval between retries +contract_call_interval = "5s" \ No newline at end of file diff --git a/integration-tests/testconfig/keeper/keeper.toml b/integration-tests/testconfig/keeper/keeper.toml index 49db54b88f0..228ea077bd3 100644 --- a/integration-tests/testconfig/keeper/keeper.toml +++ b/integration-tests/testconfig/keeper/keeper.toml @@ -16,6 +16,13 @@ block_interval = 20 forces_single_tx_key = false delete_jobs_on_end = true +# will retry roughly for 1h before giving up (900 * 4s) +[Keeper.Resiliency] +# number of retries before giving up +contract_call_limit = 900 +# static interval between retries +contract_call_interval = "4s" + [Seth] # keeper benchmark running on simulated network requires 100k per node root_key_funds_buffer = 700_000 \ No newline at end of file diff --git a/integration-tests/testsetups/keeper_benchmark.go b/integration-tests/testsetups/keeper_benchmark.go index 9435f804c18..18c12d35c92 100644 --- a/integration-tests/testsetups/keeper_benchmark.go +++ b/integration-tests/testsetups/keeper_benchmark.go @@ -29,6 +29,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/k8s/environment" "github.com/smartcontractkit/chainlink-testing-framework/logging" reportModel "github.com/smartcontractkit/chainlink-testing-framework/testreporters" + "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -36,6 +37,7 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" + keepertestconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig/keeper" "github.com/smartcontractkit/chainlink/integration-tests/testreporters" tt "github.com/smartcontractkit/chainlink/integration-tests/types" ) @@ -128,6 +130,14 @@ func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config tt.Keep k.upkeepIDs = make([][]*big.Int, len(inputs.RegistryVersions)) k.log.Debug().Interface("TestInputs", inputs).Msg("Setting up benchmark test") + // if not present disable it + if k.testConfig.GetKeeperConfig().Resiliency == nil { + k.testConfig.GetKeeperConfig().Resiliency = &keepertestconfig.ResiliencyConfig{ + ContractCallLimit: ptr.Ptr(uint(0)), + ContractCallInterval: ptr.Ptr(blockchain.StrDuration{Duration: 0 * time.Second}), + } + } + var err error // Connect to networks and prepare for contract deployment k.chainlinkNodes, err = client.ConnectChainlinkNodes(k.env) @@ -303,7 +313,7 @@ func (k *KeeperBenchmarkTest) Run() { sub.Unsubscribe() return case err := <-sub.Err(): - // no need to unsubscribe, subscripion errored + // no need to unsubscribe, subscription errored k.log.Error().Err(err).Msg("header subscription failed. Trying to reconnect...") connectionLostAt := time.Now() // we use infinite loop here on purposes, these nodes can be down for extended periods of time ¯\_(ツ)_/¯ @@ -333,7 +343,7 @@ func (k *KeeperBenchmarkTest) Run() { startedObservations.Add(1) k.log.Info().Int("Channel index", chIndex).Str("UpkeepID", upkeepIDCopy.String()).Msg("Starting upkeep observation") - confirmer := contracts.NewKeeperConsumerBenchmarkkUpkeepObserver( + confirmer := contracts.NewKeeperConsumerBenchmarkUpkeepObserver( k.keeperConsumerContracts[registryIndex], k.keeperRegistries[registryIndex], upkeepIDCopy, @@ -802,11 +812,24 @@ func (k *KeeperBenchmarkTest) DeployBenchmarkKeeperContracts(index int) { func (k *KeeperBenchmarkTest) DeployKeeperConsumersBenchmark() contracts.AutomationConsumerBenchmark { // Deploy consumer - keeperConsumerInstance, err := contracts.DeployKeeperConsumerBenchmark(k.chainClient) - if err != nil { - k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance %d shouldn't fail") + var err error + var keeperConsumerInstance contracts.AutomationConsumerBenchmark + if *k.testConfig.GetKeeperConfig().Resiliency.ContractCallLimit != 0 && k.testConfig.GetKeeperConfig().Resiliency.ContractCallInterval.Duration != 0 { + maxRetryAttempts := *k.testConfig.GetKeeperConfig().Resiliency.ContractCallLimit + callRetryDelay := k.testConfig.GetKeeperConfig().Resiliency.ContractCallInterval.Duration + keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmarkWithRetry(k.chainClient, k.log, maxRetryAttempts, callRetryDelay) + if err != nil { + k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance shouldn't fail") + keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmarkWithRetry(k.chainClient, k.log, maxRetryAttempts, callRetryDelay) + require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark") + } + } else { keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmark(k.chainClient) - require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark") + if err != nil { + k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance %d shouldn't fail") + keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmark(k.chainClient) + require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark") + } } k.log.Debug(). Str("Contract Address", keeperConsumerInstance.Address()). diff --git a/integration-tests/wrappers/contract_caller.go b/integration-tests/wrappers/contract_caller.go index 07028eeba54..952fe79c941 100644 --- a/integration-tests/wrappers/contract_caller.go +++ b/integration-tests/wrappers/contract_caller.go @@ -2,19 +2,25 @@ package wrappers import ( "context" + "fmt" "math/big" + "strings" + "time" + "github.com/avast/retry-go/v4" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/smartcontractkit/seth" - evmClient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" - "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + + evmClient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" ) // WrappedContractBackend is a wrapper around the go-ethereum ContractBackend interface. It's a thin wrapper @@ -22,8 +28,12 @@ import ( // methods that send data both in "input" and "data" field for backwards compatibility with older clients. Other methods // are passed through to the underlying client. type WrappedContractBackend struct { - evmClient blockchain.EVMClient - sethClient *seth.Client + evmClient blockchain.EVMClient + sethClient *seth.Client + logger zerolog.Logger + maxAttempts uint + retryDelay time.Duration + withRetries bool } // MustNewWrappedContractBackend creates a new WrappedContractBackend with the given clients @@ -38,6 +48,22 @@ func MustNewWrappedContractBackend(evmClient blockchain.EVMClient, sethClient *s } } +// MustNewRetryingWrappedContractBackend creates a new WrappedContractBackend, which retries read-only operations every 'retryDelay' until +// 'maxAttempts' are reached. It works only with Seth, because EVMClient has some retrying capability already included. +func MustNewRetryingWrappedContractBackend(sethClient *seth.Client, logger zerolog.Logger, maxAttempts uint, retryDelay time.Duration) *WrappedContractBackend { + if sethClient == nil { + panic("Must provide at Seth client reference") + } + + return &WrappedContractBackend{ + sethClient: sethClient, + logger: logger, + maxAttempts: maxAttempts, + retryDelay: retryDelay, + withRetries: true, + } +} + func (w *WrappedContractBackend) getGethClient() *ethclient.Client { if w.sethClient != nil { return w.sethClient.Client @@ -55,8 +81,13 @@ func (w *WrappedContractBackend) CodeAt(ctx context.Context, contract common.Add return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call CodeAt") } - client := w.getGethClient() - return client.CodeAt(ctx, contract, blockNumber) + var fn = func() ([]byte, error) { + client := w.getGethClient() + return client.CodeAt(ctx, contract, blockNumber) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("CodeAt", fn) } func (w *WrappedContractBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { @@ -64,8 +95,13 @@ func (w *WrappedContractBackend) PendingCodeAt(ctx context.Context, contract com return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call PendingCodeAt") } - client := w.getGethClient() - return client.PendingCodeAt(ctx, contract) + var fn = func() ([]byte, error) { + client := w.getGethClient() + return client.PendingCodeAt(ctx, contract) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("PendingCodeAt", fn) } func (w *WrappedContractBackend) CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error) { @@ -73,8 +109,13 @@ func (w *WrappedContractBackend) CodeAtHash(ctx context.Context, contract common return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call CodeAtHash") } - client := w.getGethClient() - return client.CodeAtHash(ctx, contract, blockHash) + var fn = func() ([]byte, error) { + client := w.getGethClient() + return client.CodeAtHash(ctx, contract, blockHash) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("CodeAtHash", fn) } func (w *WrappedContractBackend) CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error) { @@ -82,8 +123,13 @@ func (w *WrappedContractBackend) CallContractAtHash(ctx context.Context, call et return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call CallContractAtHash") } - client := w.getGethClient() - return client.CallContractAtHash(ctx, call, blockHash) + var fn = func() ([]byte, error) { + client := w.getGethClient() + return client.CallContractAtHash(ctx, call, blockHash) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("CallContractAtHash", fn) } func (w *WrappedContractBackend) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { @@ -91,8 +137,13 @@ func (w *WrappedContractBackend) HeaderByNumber(ctx context.Context, number *big return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call HeaderByNumber") } - client := w.getGethClient() - return client.HeaderByNumber(ctx, number) + var fn = func() (*types.Header, error) { + client := w.getGethClient() + return client.HeaderByNumber(ctx, number) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[*types.Header](w) + return ethHeadBanger.retry("HeaderByNumber", fn) } func (w *WrappedContractBackend) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { @@ -100,8 +151,13 @@ func (w *WrappedContractBackend) PendingNonceAt(ctx context.Context, account com return 0, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call PendingNonceAt") } - client := w.getGethClient() - return client.PendingNonceAt(ctx, account) + var fn = func() (uint64, error) { + client := w.getGethClient() + return client.PendingNonceAt(ctx, account) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[uint64](w) + return ethHeadBanger.retry("PendingNonceAt", fn) } func (w *WrappedContractBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { @@ -109,8 +165,13 @@ func (w *WrappedContractBackend) SuggestGasPrice(ctx context.Context) (*big.Int, return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call SuggestGasPrice") } - client := w.getGethClient() - return client.SuggestGasPrice(ctx) + var fn = func() (*big.Int, error) { + client := w.getGethClient() + return client.SuggestGasPrice(ctx) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[*big.Int](w) + return ethHeadBanger.retry("SuggestGasPrice", fn) } func (w *WrappedContractBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { @@ -118,8 +179,13 @@ func (w *WrappedContractBackend) SuggestGasTipCap(ctx context.Context) (*big.Int return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call SuggestGasTipCap") } - client := w.getGethClient() - return client.SuggestGasTipCap(ctx) + var fn = func() (*big.Int, error) { + client := w.getGethClient() + return client.SuggestGasTipCap(ctx) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[*big.Int](w) + return ethHeadBanger.retry("SuggestGasTipCap", fn) } func (w *WrappedContractBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { @@ -127,8 +193,13 @@ func (w *WrappedContractBackend) EstimateGas(ctx context.Context, call ethereum. return 0, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call EstimateGas") } - client := w.getGethClient() - return client.EstimateGas(ctx, call) + var fn = func() (uint64, error) { + client := w.getGethClient() + return client.EstimateGas(ctx, call) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[uint64](w) + return ethHeadBanger.retry("EstimateGas", fn) } func (w *WrappedContractBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error { @@ -145,8 +216,13 @@ func (w *WrappedContractBackend) FilterLogs(ctx context.Context, query ethereum. return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call FilterLogs") } - client := w.getGethClient() - return client.FilterLogs(ctx, query) + var fn = func() ([]types.Log, error) { + client := w.getGethClient() + return client.FilterLogs(ctx, query) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[[]types.Log](w) + return ethHeadBanger.retry("FilterLogs", fn) } func (w *WrappedContractBackend) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { @@ -154,8 +230,13 @@ func (w *WrappedContractBackend) SubscribeFilterLogs(ctx context.Context, query return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call SubscribeFilterLogs") } - client := w.getGethClient() - return client.SubscribeFilterLogs(ctx, query, ch) + var fn = func() (ethereum.Subscription, error) { + client := w.getGethClient() + return client.SubscribeFilterLogs(ctx, query, ch) + } + + ethHeadBanger := newEthHeadBangerFromWrapper[ethereum.Subscription](w) + return ethHeadBanger.retry("SubscribeFilterLogs", fn) } func (w *WrappedContractBackend) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { @@ -163,13 +244,18 @@ func (w *WrappedContractBackend) CallContract(ctx context.Context, msg ethereum. return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call CallContract") } - var hex hexutil.Bytes - client := w.getGethClient() - err := client.Client().CallContext(ctx, &hex, "eth_call", evmClient.ToBackwardCompatibleCallArg(msg), evmClient.ToBackwardCompatibleBlockNumArg(blockNumber)) - if err != nil { - return nil, err + var fn = func() ([]byte, error) { + var hex hexutil.Bytes + client := w.getGethClient() + err := client.Client().CallContext(ctx, &hex, "eth_call", evmClient.ToBackwardCompatibleCallArg(msg), evmClient.ToBackwardCompatibleBlockNumArg(blockNumber)) + if err != nil { + return nil, err + } + return hex, nil } - return hex, nil + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("CallContract", fn) } func (w *WrappedContractBackend) PendingCallContract(ctx context.Context, msg ethereum.CallMsg) ([]byte, error) { @@ -177,13 +263,18 @@ func (w *WrappedContractBackend) PendingCallContract(ctx context.Context, msg et return nil, errors.Wrapf(ctxErr, "the context you passed had an error set. Won't call PendingCallContract") } - var hex hexutil.Bytes - client := w.getGethClient() - err := client.Client().CallContext(ctx, &hex, "eth_call", evmClient.ToBackwardCompatibleCallArg(msg), "pending") - if err != nil { - return nil, err + var fn = func() ([]byte, error) { + var hex hexutil.Bytes + client := w.getGethClient() + err := client.Client().CallContext(ctx, &hex, "eth_call", evmClient.ToBackwardCompatibleCallArg(msg), "pending") + if err != nil { + return nil, err + } + return hex, nil } - return hex, nil + + ethHeadBanger := newEthHeadBangerFromWrapper[[]byte](w) + return ethHeadBanger.retry("PendingCallContract", fn) } func (w *WrappedContractBackend) getErrorFromContext(ctx context.Context) error { @@ -199,3 +290,51 @@ func (w *WrappedContractBackend) getErrorFromContext(ctx context.Context) error return nil } + +// ethHeadBanger is just a fancy name for a struct that retries a function a number of times with a delay between each attempt +type ethHeadBanger[ReturnType any] struct { + logger zerolog.Logger + maxAttempts uint + retryDelay time.Duration +} + +func newEthHeadBangerFromWrapper[ResultType any](wrapper *WrappedContractBackend) ethHeadBanger[ResultType] { + return ethHeadBanger[ResultType]{ + logger: wrapper.logger, + maxAttempts: wrapper.maxAttempts, + retryDelay: wrapper.retryDelay, + } +} + +func (e ethHeadBanger[ReturnType]) retry(functionName string, fnToRetry func() (ReturnType, error)) (ReturnType, error) { + var result ReturnType + err := retry.Do(func() error { + var err error + result, err = fnToRetry() + + return err + }, + retry.RetryIf(func(err error) bool { + if err.Error() == rpc.ErrClientQuit.Error() || + err.Error() == rpc.ErrBadResult.Error() || + strings.Contains(err.Error(), "connection") || + strings.Contains(err.Error(), "EOF") { + return true + } + + e.logger.Error().Err(err).Msgf("Error in %s. Not retrying.", functionName) + + return false + }), + retry.Attempts(e.maxAttempts), + retry.Delay(e.retryDelay), + retry.OnRetry(func(n uint, err error) { + e.logger.Info(). + Str("Attempt", fmt.Sprintf("%d/%d", n+1, 10)). + Str("Error", err.Error()). + Msgf("Retrying %s", functionName) + }), + ) + + return result, err +}