optional retrying of read-only contract calls; used only by keeper be…

…nchmark test

integration-tests/contracts/ethereum_contracts_automation_seth.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -2278,7 +2279,21 @@ func (v *EthereumAutomationConsumerBenchmark) GetUpkeepCount(ctx context.Context
 	}, id)
 }
 
+// DeployKeeperConsumerBenchmark deploys a keeper consumer benchmark contract with a standard contract backend
 func DeployKeeperConsumerBenchmark(client *seth.Client) (AutomationConsumerBenchmark, error) {
+	return deployKeeperConsumerBenchmarkWithWrapperFn(client, func(client *seth.Client) *wrappers.WrappedContractBackend {
+		return wrappers.MustNewWrappedContractBackend(nil, client)
+	})
+}
+
+// DeployKeeperConsumerBenchmarkWithRetry deploys a keeper consumer benchmark contract with a read-only operations retrying contract backend
+func DeployKeeperConsumerBenchmarkWithRetry(client *seth.Client, logger zerolog.Logger, maxAttempts uint, retryDelay time.Duration) (AutomationConsumerBenchmark, error) {
+	return deployKeeperConsumerBenchmarkWithWrapperFn(client, func(client *seth.Client) *wrappers.WrappedContractBackend {
+		return wrappers.MustNewRetryingWrappedContractBackend(client, logger, maxAttempts, retryDelay)
+	})
+}
+
+func deployKeeperConsumerBenchmarkWithWrapperFn(client *seth.Client, wrapperConstrFn func(client *seth.Client) *wrappers.WrappedContractBackend) (AutomationConsumerBenchmark, error) {
 	abi, err := automation_consumer_benchmark.AutomationConsumerBenchmarkMetaData.GetAbi()
 	if err != nil {
 		return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("failed to get AutomationConsumerBenchmark ABI: %w", err)
@@ -2288,7 +2303,7 @@ func DeployKeeperConsumerBenchmark(client *seth.Client) (AutomationConsumerBench
 		return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("AutomationConsumerBenchmark instance deployment have failed: %w", err)
 	}
 
-	instance, err := automation_consumer_benchmark.NewAutomationConsumerBenchmark(data.Address, wrappers.MustNewWrappedContractBackend(nil, client))
+	instance, err := automation_consumer_benchmark.NewAutomationConsumerBenchmark(data.Address, wrapperConstrFn(client))
 	if err != nil {
 		return &EthereumAutomationConsumerBenchmark{}, fmt.Errorf("failed to instantiate AutomationConsumerBenchmark instance: %w", err)
 	}
@@ -2325,9 +2340,9 @@ type KeeperConsumerBenchmarkUpkeepObserver struct {
 	l                       zerolog.Logger
 }
 
-// NewKeeperConsumerBenchmarkkUpkeepObserver provides a new instance of a KeeperConsumerBenchmarkkUpkeepObserver
+// NewKeeperConsumerBenchmarkUpkeepObserver provides a new instance of a NewKeeperConsumerBenchmarkUpkeepObserver
 // Used to track and log benchmark test results for keepers
-func NewKeeperConsumerBenchmarkkUpkeepObserver(
+func NewKeeperConsumerBenchmarkUpkeepObserver(
 	contract AutomationConsumerBenchmark,
 	registry KeeperRegistry,
 	upkeepID *big.Int,

integration-tests/go.mod

@@ -6,6 +6,7 @@ go 1.21.7
 replace github.com/smartcontractkit/chainlink/v2 => ../
 
 require (
+	github.com/avast/retry-go/v4 v4.5.1
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/cli/go-gh/v2 v2.0.0
 	github.com/ethereum/go-ethereum v1.13.8
@@ -88,7 +89,6 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/avast/retry-go v3.0.0+incompatible // indirect
-	github.com/avast/retry-go/v4 v4.5.1 // indirect
 	github.com/aws/aws-sdk-go v1.45.25 // indirect
 	github.com/aws/constructs-go/constructs/v10 v10.1.255 // indirect
 	github.com/aws/jsii-runtime-go v1.75.0 // indirect

integration-tests/testconfig/keeper/config.go

@@ -2,17 +2,26 @@ package keeper
 
 import (
 	"errors"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 )
 
 type Config struct {
-	Common *Common `toml:"Common"`
+	Common     *Common           `toml:"Common"`
+	Resiliency *ResiliencyConfig `toml:"Resiliency"`
 }
 
 func (c *Config) Validate() error {
 	if c.Common == nil {
 		return nil
 	}
-	return c.Common.Validate()
+	if err := c.Common.Validate(); err != nil {
+		return err
+	}
+	if c.Resiliency == nil {
+		return nil
+	}
+	return c.Resiliency.Validate()
 }
 
 type Common struct {
@@ -83,3 +92,19 @@ func (c *Common) Validate() error {
 	}
 	return nil
 }
+
+type ResiliencyConfig struct {
+	ContractCallLimit    *uint                   `toml:"contract_call_limit"`
+	ContractCallInterval *blockchain.StrDuration `toml:"contract_call_interval"`
+}
+
+func (c *ResiliencyConfig) Validate() error {
+	if c.ContractCallLimit == nil {
+		return errors.New("contract_call_limit must be set")
+	}
+	if c.ContractCallInterval == nil {
+		return errors.New("contract_call_interval must be set")
+	}
+
+	return nil
+}

integration-tests/testconfig/keeper/example.toml

@@ -85,4 +85,13 @@ max_perform_gas = 5000000
 block_range = 3600
 block_interval = 20
 forces_single_tx_key = false
-delete_jobs_on_end = true
+delete_jobs_on_end = true
+
+# If present will wrap keeper benchmakr consumers in retrying contract backend
+# that retries read-only operations on failure related to network issues or node unavailability
+# To disable simply remove this section or set any of the values to 0
+[Keeper.Resiliency]
+# number of retries before giving up
+contract_call_limit = 500
+# static interval between retries
+contract_call_interval = "5s"

integration-tests/testconfig/keeper/keeper.toml

@@ -16,6 +16,13 @@ block_interval = 20
 forces_single_tx_key = false
 delete_jobs_on_end = true
 
+# will retry roughly for 1h before giving up (900 * 4s)
+[Keeper.Resiliency]
+# number of retries before giving up
+contract_call_limit = 900
+# static interval between retries
+contract_call_interval = "4s"
+
 [Seth]
 # keeper benchmark running on simulated network requires 100k per node
 root_key_funds_buffer = 700_000

integration-tests/testsetups/keeper_benchmark.go

@@ -29,13 +29,15 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s/environment"
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	reportModel "github.com/smartcontractkit/chainlink-testing-framework/testreporters"
+	"github.com/smartcontractkit/chainlink-testing-framework/utils/ptr"
 	"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/actions"
 	actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
+	keepertestconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig/keeper"
 	"github.com/smartcontractkit/chainlink/integration-tests/testreporters"
 	tt "github.com/smartcontractkit/chainlink/integration-tests/types"
 )
@@ -128,6 +130,14 @@ func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config tt.Keep
 	k.upkeepIDs = make([][]*big.Int, len(inputs.RegistryVersions))
 	k.log.Debug().Interface("TestInputs", inputs).Msg("Setting up benchmark test")
 
+	// if not present disable it
+	if k.testConfig.GetKeeperConfig().Resiliency == nil {
+		k.testConfig.GetKeeperConfig().Resiliency = &keepertestconfig.ResiliencyConfig{
+			ContractCallLimit:    ptr.Ptr(uint(0)),
+			ContractCallInterval: ptr.Ptr(blockchain.StrDuration{Duration: 0 * time.Second}),
+		}
+	}
+
 	var err error
 	// Connect to networks and prepare for contract deployment
 	k.chainlinkNodes, err = client.ConnectChainlinkNodes(k.env)
@@ -303,7 +313,7 @@ func (k *KeeperBenchmarkTest) Run() {
 				sub.Unsubscribe()
 				return
 			case err := <-sub.Err():
-				// no need to unsubscribe, subscripion errored
+				// no need to unsubscribe, subscription errored
 				k.log.Error().Err(err).Msg("header subscription failed. Trying to reconnect...")
 				connectionLostAt := time.Now()
 				// we use infinite loop here on purposes, these nodes can be down for extended periods of time ¯\_(ツ)_/¯
@@ -333,7 +343,7 @@ func (k *KeeperBenchmarkTest) Run() {
 				startedObservations.Add(1)
 				k.log.Info().Int("Channel index", chIndex).Str("UpkeepID", upkeepIDCopy.String()).Msg("Starting upkeep observation")
 
-				confirmer := contracts.NewKeeperConsumerBenchmarkkUpkeepObserver(
+				confirmer := contracts.NewKeeperConsumerBenchmarkUpkeepObserver(
 					k.keeperConsumerContracts[registryIndex],
 					k.keeperRegistries[registryIndex],
 					upkeepIDCopy,
@@ -802,11 +812,24 @@ func (k *KeeperBenchmarkTest) DeployBenchmarkKeeperContracts(index int) {
 
 func (k *KeeperBenchmarkTest) DeployKeeperConsumersBenchmark() contracts.AutomationConsumerBenchmark {
 	// Deploy consumer
-	keeperConsumerInstance, err := contracts.DeployKeeperConsumerBenchmark(k.chainClient)
-	if err != nil {
-		k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance %d shouldn't fail")
+	var err error
+	var keeperConsumerInstance contracts.AutomationConsumerBenchmark
+	if *k.testConfig.GetKeeperConfig().Resiliency.ContractCallLimit != 0 && k.testConfig.GetKeeperConfig().Resiliency.ContractCallInterval.Duration != 0 {
+		maxRetryAttempts := *k.testConfig.GetKeeperConfig().Resiliency.ContractCallLimit
+		callRetryDelay := k.testConfig.GetKeeperConfig().Resiliency.ContractCallInterval.Duration
+		keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmarkWithRetry(k.chainClient, k.log, maxRetryAttempts, callRetryDelay)
+		if err != nil {
+			k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance shouldn't fail")
+			keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmarkWithRetry(k.chainClient, k.log, maxRetryAttempts, callRetryDelay)
+			require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark")
+		}
+	} else {
 		keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmark(k.chainClient)
-		require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark")
+		if err != nil {
+			k.log.Error().Err(err).Msg("Deploying AutomationConsumerBenchmark instance %d shouldn't fail")
+			keeperConsumerInstance, err = contracts.DeployKeeperConsumerBenchmark(k.chainClient)
+			require.NoError(k.t, err, "Error deploying AutomationConsumerBenchmark")
+		}
 	}
 	k.log.Debug().
 		Str("Contract Address", keeperConsumerInstance.Address()).