From 39b938ed5c1a014aec8fc81541ff477b152e6cd0 Mon Sep 17 00:00:00 2001 From: skudasov Date: Wed, 22 May 2024 18:51:14 +0200 Subject: [PATCH] block gas limit halving test --- .../action.yml | 4 + .github/workflows/ccip-load-tests.yml | 32 ++++--- integration-tests/ccip-tests/README.md | 5 ++ .../ccip-tests/chaos/gas_suite.go | 87 +++++++++++++++++-- integration-tests/ccip-tests/load/README.md | 0 .../ccip-tests/load/ccip_test.go | 25 +++++- .../ccip-tests/testconfig/ccip.go | 13 +++ .../tomls/ccip-gas-block-halving-dst.toml | 73 ++++++++++++++++ .../tomls/ccip-gas-block-halving-src.toml | 73 ++++++++++++++++ 9 files changed, 290 insertions(+), 22 deletions(-) create mode 100644 integration-tests/ccip-tests/load/README.md create mode 100644 integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-dst.toml create mode 100644 integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-src.toml diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml index 072473aea2..3f4d86c636 100644 --- a/.github/actions/setup-create-base64-config-ccip/action.yml +++ b/.github/actions/setup-create-base64-config-ccip/action.yml @@ -181,6 +181,10 @@ runs: GasRaisePercentage = 0.7 Spike = true Duration = '3m' + + [CCIP.Groups.load.ChaosGasLimitProfile] + TargetChain = "src" + BlockGasLimitPercentage = 0.1 EOF diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml index d7cc3fbb9d..54040c6605 100644 --- a/.github/workflows/ccip-load-tests.yml +++ b/.github/workflows/ccip-load-tests.yml @@ -1,11 +1,11 @@ name: CCIP Load Test on: push: -# branches: -# - ccip-develop + # branches: + # - ccip-develop workflow_dispatch: inputs: - base64_test_input : # base64 encoded toml for test input + base64_test_input: # base64 encoded toml for test input description: 'Base64 encoded toml test input' required: false @@ -156,6 +156,14 @@ jobs: os: ubuntu-latest run: ^TestLoadCCIPStableRPSGasSpike$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-gas-spike-fast-dst.toml + - name: ccip-gas-limit-block-capacity-halving-src + os: ubuntu-latest + run: ^TestLoadCCIPStableRPSChangeBlockGasLimit$ + config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-src.toml + - name: ccip-gas-limit-block-capacity-halving-dst + os: ubuntu-latest + run: ^TestLoadCCIPStableRPSChangeBlockGasLimit$ + config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-dst.toml runs-on: ${{ matrix.type.os }} name: CCIP ${{ matrix.type.name }} steps: @@ -236,12 +244,12 @@ jobs: cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" should_cleanup: "true" - ## Notify in slack if the job fails - - name: Notify Slack - if: failure() && github.event_name != 'workflow_dispatch' - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - with: - channel-id: "#ccip-testing" - slack-message: ":x: :mild-panic-intensifies: CCIP load tests failed: ${{ job.html_url }}\n${{ format('https://github.com/smartcontractkit/chainlink/actions/runs/{0}', github.run_id) }}" +# ## Notify in slack if the job fails +# - name: Notify Slack +# if: failure() && github.event_name != 'workflow_dispatch' +# uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 +# env: +# SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} +# with: +# channel-id: "#ccip-testing" +# slack-message: ":x: :mild-panic-intensifies: CCIP load tests failed: ${{ job.html_url }}\n${{ format('https://github.com/smartcontractkit/chainlink/actions/runs/{0}', github.run_id) }}" diff --git a/integration-tests/ccip-tests/README.md b/integration-tests/ccip-tests/README.md index 53a89736e3..631775356d 100644 --- a/integration-tests/ccip-tests/README.md +++ b/integration-tests/ccip-tests/README.md @@ -95,6 +95,11 @@ make test_smoke_ccip_default testname=TestSmokeCCIPForBidirectionalLane secret_t ``` Currently other types of tests like load and chaos can only be run using remote kubernetes cluster. +### Load + Chaos tests +We test CCIP with multiple blockchain chaos experiments + + + ### Using remote kubernetes cluster These tests remain bound to a Kubernetes run environment, and require more complex setup and running instructions. We endeavor to make these easier to run and configure, but for the time being please seek a member of the QA/Test Tooling team if you want to run these. \ No newline at end of file diff --git a/integration-tests/ccip-tests/chaos/gas_suite.go b/integration-tests/ccip-tests/chaos/gas_suite.go index f61be1cbaf..1aae91007e 100644 --- a/integration-tests/ccip-tests/chaos/gas_suite.go +++ b/integration-tests/ccip-tests/chaos/gas_suite.go @@ -1,7 +1,10 @@ package chaos import ( + "context" "fmt" + "github.com/ethereum/go-ethereum/ethclient" + "math" "testing" "time" @@ -18,8 +21,10 @@ type GasSuite struct { t *testing.T Cfg *GasSuiteConfig Logger zerolog.Logger - SrcClient *client.RPCClient - DstClient *client.RPCClient + SrcRPCClient *client.RPCClient + DstRPCClient *client.RPCClient + SrcEVMClient *ethclient.Client + DstEVMClient *ethclient.Client GrafanaClient *grafana.Client } @@ -44,18 +49,28 @@ func NewGasSuite(t *testing.T, cfg *GasSuiteConfig) (*GasSuite, error) { if err := cfg.Validate(); err != nil { return nil, err } + srcEVMClient, err := ethclient.Dial(cfg.SrcGethHTTPURL) + if err != nil { + return nil, err + } + dstEVMClient, err := ethclient.Dial(cfg.DstGethHTTPURL) + if err != nil { + return nil, err + } return &GasSuite{ t: t, Cfg: cfg, Logger: l, - SrcClient: client.NewRPCClient(cfg.SrcGethHTTPURL), - DstClient: client.NewRPCClient(cfg.DstGethHTTPURL), + SrcRPCClient: client.NewRPCClient(cfg.SrcGethHTTPURL), + DstRPCClient: client.NewRPCClient(cfg.DstGethHTTPURL), + SrcEVMClient: srcEVMClient, + DstEVMClient: dstEVMClient, GrafanaClient: grafana.NewGrafanaClient(cfg.GrafanaURL, cfg.GrafanaToken), }, nil } -// RaiseGas simulates slow or fast gas spike -func (r *GasSuite) RaiseGas(chain string, from int64, percentage float64, duration time.Duration, spike bool) { +// ChangeBlockGasBaseFee simulates slow or fast gas spike +func (r *GasSuite) ChangeBlockGasBaseFee(chain string, from int64, percentage float64, duration time.Duration, spike bool) { go func() { err := PostGrafanaAnnotation( r.Logger, @@ -67,10 +82,10 @@ func (r *GasSuite) RaiseGas(chain string, from int64, percentage float64, durati assert.NoError(r.t, err) switch chain { case "src": - err := r.SrcClient.ModulateBaseFeeOverDuration(r.Logger, from, percentage, duration, spike) + err := r.SrcRPCClient.ModulateBaseFeeOverDuration(r.Logger, from, percentage, duration, spike) assert.NoError(r.t, err) case "dst": - err := r.DstClient.ModulateBaseFeeOverDuration(r.Logger, from, percentage, duration, spike) + err := r.DstRPCClient.ModulateBaseFeeOverDuration(r.Logger, from, percentage, duration, spike) assert.NoError(r.t, err) default: r.t.Errorf("chain can be 'src' or 'dst'") @@ -85,3 +100,59 @@ func (r *GasSuite) RaiseGas(chain string, from int64, percentage float64, durati assert.NoError(r.t, err) }() } + +// ChangeNextBlockGasLimit changes next block gas limit, +// sets it to percentage of last gasUsed in previous block creating congestion +func (r *GasSuite) ChangeNextBlockGasLimit(startIn time.Duration, wait time.Duration, chain string, percentage float64) { + go func() { + time.Sleep(startIn) + latestBlock, err := r.SrcEVMClient.BlockByNumber(context.Background(), nil) + assert.NoError(r.t, err) + newGasLimit := int64(math.Ceil(float64(latestBlock.GasUsed()) * percentage)) + r.Logger.Info(). + Str("Network", chain). + Int64("GasLimit", newGasLimit). + Uint64("GasUsed", latestBlock.GasUsed()). + Msg("Setting next block gas limit") + err = PostGrafanaAnnotation( + r.Logger, + r.GrafanaClient, + r.Cfg.dashboardUID, + fmt.Sprintf("changed block gas limit, now: %d, was used in last block: %d, network: %s", newGasLimit, latestBlock.GasUsed(), chain), + []string{"gas-limit"}, + ) + assert.NoError(r.t, err) + switch chain { + case "src": + err := r.SrcRPCClient.AnvilSetBlockGasLimit([]interface{}{newGasLimit}) + assert.NoError(r.t, err) + case "dst": + err := r.DstRPCClient.AnvilSetBlockGasLimit([]interface{}{newGasLimit}) + assert.NoError(r.t, err) + default: + r.t.Errorf("chain can be 'src' or 'dst'") + } + time.Sleep(wait) + r.Logger.Info(). + Str("Network", chain). + Uint64("GasLimit", latestBlock.GasLimit()). + Msg("Returning old gas limit") + switch chain { + case "src": + err := r.SrcRPCClient.AnvilSetBlockGasLimit([]interface{}{latestBlock.GasLimit()}) + assert.NoError(r.t, err) + case "dst": + err := r.DstRPCClient.AnvilSetBlockGasLimit([]interface{}{latestBlock.GasLimit()}) + assert.NoError(r.t, err) + default: + r.t.Errorf("chain can be 'src' or 'dst'") + } + err = PostGrafanaAnnotation( + r.Logger, + r.GrafanaClient, + r.Cfg.dashboardUID, + fmt.Sprintf("changed block gas limit, now: %d, network: %s", newGasLimit, chain), + []string{"gas-limit"}, + ) + }() +} diff --git a/integration-tests/ccip-tests/load/README.md b/integration-tests/ccip-tests/load/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/ccip-tests/load/ccip_test.go b/integration-tests/ccip-tests/load/ccip_test.go index a3e3982d70..ab1e22ef77 100644 --- a/integration-tests/ccip-tests/load/ccip_test.go +++ b/integration-tests/ccip-tests/load/ccip_test.go @@ -158,9 +158,30 @@ func TestLoadCCIPStableRPSGasSpike(t *testing.T) { }) chcfg := testArgs.TestCfg.TestGroupInput.ChaosGasProfile - log.Warn().Any("Config", chcfg).Msg("Gas config") gs := setupGasSuite(t, testArgs) - gs.RaiseGas(chcfg.TargetChain, chcfg.StartGasPrice, chcfg.GasRaisePercentage, chcfg.Duration.Duration(), chcfg.Spike) + gs.ChangeBlockGasBaseFee(chcfg.TargetChain, chcfg.StartGasPrice, chcfg.GasRaisePercentage, chcfg.Duration.Duration(), chcfg.Spike) + + testArgs.TriggerLoadByLane() + testArgs.Wait() +} + +func TestLoadCCIPStableRPSChangeBlockGasLimit(t *testing.T) { + t.Parallel() + lggr := logging.GetTestLogger(t) + testArgs := NewLoadArgs(t, lggr) + testArgs.Setup() + // if the test runs on remote runner + if len(testArgs.TestSetupArgs.Lanes) == 0 { + return + } + t.Cleanup(func() { + log.Info().Msg("Tearing down the environment") + require.NoError(t, testArgs.TestSetupArgs.TearDown()) + }) + + chcfg := testArgs.TestCfg.TestGroupInput.ChaosGasLimitProfile + gs := setupGasSuite(t, testArgs) + gs.ChangeNextBlockGasLimit(1*time.Minute, 1*time.Minute, chcfg.TargetChain, chcfg.BlockGasLimitPercentage) testArgs.TriggerLoadByLane() testArgs.Wait() diff --git a/integration-tests/ccip-tests/testconfig/ccip.go b/integration-tests/ccip-tests/testconfig/ccip.go index 2681c5e20a..33973607fb 100644 --- a/integration-tests/ccip-tests/testconfig/ccip.go +++ b/integration-tests/ccip-tests/testconfig/ccip.go @@ -241,6 +241,18 @@ func (l *LoadProfile) SetTestRunName(name string) { } } +type ChaosGasLimitProfile struct { + TargetChain string `toml:",omitempty"` + BlockGasLimitPercentage float64 `toml:",omitempty"` +} + +func (gp *ChaosGasLimitProfile) Validate() error { + if gp.TargetChain != "src" && gp.TargetChain != "dst" { + return fmt.Errorf("target chain for gas chaos should be 'src' or 'dst'") + } + return nil +} + type ChaosGasProfile struct { TargetChain string `toml:",omitempty"` StartGasPrice int64 `toml:",omitempty"` @@ -294,6 +306,7 @@ type CCIPTestConfig struct { StoreLaneConfig *bool `toml:",omitempty"` LoadProfile *LoadProfile `toml:",omitempty"` ChaosGasProfile *ChaosGasProfile `toml:",omitempty"` + ChaosGasLimitProfile *ChaosGasLimitProfile `toml:",omitempty"` ChaosReorgProfile *ChaosReorgProfile `toml:",omitempty"` } diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-dst.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-dst.toml new file mode 100644 index 0000000000..18271e3078 --- /dev/null +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-dst.toml @@ -0,0 +1,73 @@ +[CCIP] +[CCIP.Env] +Mockserver = 'http://mockserver:1080' +[CCIP.Env.Network] +selected_networks= ['source-chain', 'dest-chain'] + +[CCIP.Env.Network.AnvilConfigs.source-chain] +block_time = 1 + +[CCIP.Env.Network.AnvilConfigs.dest-chain] +block_time = 1 + +[CCIP.Env.Network.EVMNetworks.source-chain] +evm_name = 'source-chain' +evm_chain_id = 1337 +evm_urls = ['ws://127.0.0.1:9546'] +evm_http_urls = ['http://127.0.0.1:9544'] +evm_keys = ['59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[CCIP.Env.Network.EVMNetworks.dest-chain] +evm_name = 'dest-chain' +evm_chain_id = 2337 +evm_urls = ['ws://127.0.0.1:7546'] +evm_http_urls = ['http://127.0.0.1:7544'] +evm_keys = ['ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[CCIP.Env.NewCLCluster] +NoOfNodes = 6 +NodeMemory = '4Gi' +NodeCPU = '2' +DBMemory = '4Gi' +DBCPU = '2' +DBCapacity = '10Gi' +IsStateful = true +DBArgs = ['shared_buffers=1536MB', 'effective_cache_size=4096MB', 'work_mem=64MB'] + +[CCIP.Env.NewCLCluster.Common] +CommonChainConfigTOML = """ +[HeadTracker] +HistoryDepth = 50 + +[GasEstimator] +PriceMax = '200 gwei' +LimitDefault = 6000000 +FeeCapDefault = '200 gwei' +""" + + +[CCIP.Groups.load.LoadProfile] +TestDuration = '5m' +FailOnFirstErrorInLoad = true + +[CCIP.Groups.load.ChaosGasLimitProfile] +TargetChain = "dst" +BlockGasLimitPercentage = 0.5 \ No newline at end of file diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-src.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-src.toml new file mode 100644 index 0000000000..0387f90b50 --- /dev/null +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip-gas-block-halving-src.toml @@ -0,0 +1,73 @@ +[CCIP] +[CCIP.Env] +Mockserver = 'http://mockserver:1080' +[CCIP.Env.Network] +selected_networks= ['source-chain', 'dest-chain'] + +[CCIP.Env.Network.AnvilConfigs.source-chain] +block_time = 1 + +[CCIP.Env.Network.AnvilConfigs.dest-chain] +block_time = 1 + +[CCIP.Env.Network.EVMNetworks.source-chain] +evm_name = 'source-chain' +evm_chain_id = 1337 +evm_urls = ['ws://127.0.0.1:9546'] +evm_http_urls = ['http://127.0.0.1:9544'] +evm_keys = ['59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[CCIP.Env.Network.EVMNetworks.dest-chain] +evm_name = 'dest-chain' +evm_chain_id = 2337 +evm_urls = ['ws://127.0.0.1:7546'] +evm_http_urls = ['http://127.0.0.1:7544'] +evm_keys = ['ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 5000 +evm_transaction_timeout = '3m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[CCIP.Env.NewCLCluster] +NoOfNodes = 6 +NodeMemory = '4Gi' +NodeCPU = '2' +DBMemory = '4Gi' +DBCPU = '2' +DBCapacity = '10Gi' +IsStateful = true +DBArgs = ['shared_buffers=1536MB', 'effective_cache_size=4096MB', 'work_mem=64MB'] + +[CCIP.Env.NewCLCluster.Common] +CommonChainConfigTOML = """ +[HeadTracker] +HistoryDepth = 50 + +[GasEstimator] +PriceMax = '200 gwei' +LimitDefault = 6000000 +FeeCapDefault = '200 gwei' +""" + + +[CCIP.Groups.load.LoadProfile] +TestDuration = '5m' +FailOnFirstErrorInLoad = true + +[CCIP.Groups.load.ChaosGasLimitProfile] +TargetChain = "src" +BlockGasLimitPercentage = 0.5 \ No newline at end of file