diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a4d82ab67f..0b2e42c34d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -373,6 +373,13 @@ jobs: file: ccip run: -run ^TestSmokeCCIPForBidirectionalLane$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml + - name: ccip-smoke-reorg + nodes: 3 + dir: ccip-tests/smoke + os: ubuntu-latest + file: ccip + run: -run ^TestSmokeCCIPReorg + config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml - name: runlog id: runlog nodes: 2 diff --git a/integration-tests/ccip-tests/smoke/ccip_test.go b/integration-tests/ccip-tests/smoke/ccip_test.go index 8e36f9a7d1..64aaacfcb1 100644 --- a/integration-tests/ccip-tests/smoke/ccip_test.go +++ b/integration-tests/ccip-tests/smoke/ccip_test.go @@ -2,6 +2,7 @@ package smoke import ( "fmt" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "math/big" "testing" @@ -887,7 +888,7 @@ func TestSmokeCCIPReorgBelowFinality(t *testing.T) { // sending multiple request and expect all should go through though there is below finality reorg err := lane.SendRequests(5, gasLimit) require.NoError(t, err) - rs := testsetups.SetupReorgSuite(t, &log, setUpOutput, TestCfg) + rs := SetupReorgSuite(t, &log, setUpOutput) // run below finality reorg in both source and destination chain blocksBackSrc := int(rs.Cfg.SrcFinalityDepth) - rs.Cfg.FinalityDelta blocksBackDst := int(rs.Cfg.DstFinalityDepth) - rs.Cfg.FinalityDelta @@ -898,10 +899,30 @@ func TestSmokeCCIPReorgBelowFinality(t *testing.T) { }) } -// Test expects to generate above finality reorg in both destination and -// expect CCIP transaction doesn't go through successful and node detects reorg successfully -func TestSmokeCCIPReorgAboveFinality(t *testing.T) { +// Test creates above finality reorg at destination and +// expects ccip transactions in-flight and the one initiated after reorg +// doesn't go through and verifies every node is able to detect reorg. +// Note: LogPollInterval interval is set as 1s to detect the reorg immediately +func TestSmokeCCIPReorgAboveFinalityAtDestination(t *testing.T) { t.Parallel() + t.Run("Above finality reorg in destination chain", func(t *testing.T) { + performReorgAndValidate(t, "Destination") + }) +} + +// Test creates above finality reorg at destination and +// expects ccip transactions in-flight doesn't go through, the transaction initiated after reorg +// shouldn't even get initiated and verifies every node is able to detect reorg. +// Note: LogPollInterval interval is set as 1s to detect the reorg immediately +func TestSmokeCCIPReorgAboveFinalityAtSource(t *testing.T) { + t.Parallel() + t.Run("Above finality reorg in source chain", func(t *testing.T) { + performReorgAndValidate(t, "Source") + }) +} + +// performReorgAndValidate is to perform the above finality reorg test +func performReorgAndValidate(t *testing.T, network string) { log := logging.GetTestLogger(t) TestCfg := testsetups.NewCCIPTestConfig(t, log, testconfig.Smoke) require.NotNil(t, TestCfg.TestGroupInput.MsgDetails.DestGasLimit) @@ -911,46 +932,54 @@ func TestSmokeCCIPReorgAboveFinality(t *testing.T) { log.Info().Msg("No lanes found") return } - t.Cleanup(func() { require.NoError(t, setUpOutput.TearDown()) }) - + rs := SetupReorgSuite(t, &log, setUpOutput) lane := setUpOutput.Lanes[0].ForwardLane log.Info(). Str("Source", lane.SourceNetworkName). Str("Destination", lane.DestNetworkName). - Msg("Starting CCIP reorg test") - t.Run(fmt.Sprintf("CCIP reorg above finality test from network %s to network %s", - lane.SourceNetworkName, lane.DestNetworkName), func(t *testing.T) { - t.Parallel() - lane.Test = t - lane.RecordStateBeforeTransfer() - rs := testsetups.SetupReorgSuite(t, &log, setUpOutput, TestCfg) - err := lane.SendRequests(1, gasLimit) - require.NoError(t, err) - // run above finality reorg in destination chain - blocksBackDst := int(rs.Cfg.DstFinalityDepth) + rs.Cfg.FinalityDelta - //blocksBackSrc := int(rs.Cfg.SrcFinalityDepth) + rs.Cfg.FinalityDelta - //rs.RunReorg(rs.SrcClient, blocksBackSrc, "Source", 2*time.Second) - rs.RunReorg(rs.DstClient, blocksBackDst, "Destination", 2*time.Second) - clNodes := setUpOutput.Env.CLNodes - assert.Eventually(t, func() bool { - for _, node := range clNodes { - resp, _, err := node.Health() - require.NoError(t, err) - for _, d := range resp.Data { - if d.Attributes.Name == "EVM.2337.LogPoller" && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" { - log.Info().Msg("Finality violated is reported by node") - return true - } + Msg("Starting ccip reorg test") + lane.Test = t + lane.RecordStateBeforeTransfer() + err := lane.SendRequests(1, gasLimit) + require.NoError(t, err) + logPollerName := "" + if network == "Destination" { + logPollerName = "EVM.2337.LogPoller" + rs.RunReorg(rs.DstClient, int(rs.Cfg.DstFinalityDepth)+rs.Cfg.FinalityDelta, network, 2*time.Second) + } else { + logPollerName = "EVM.1337.LogPoller" + rs.RunReorg(rs.SrcClient, int(rs.Cfg.SrcFinalityDepth)+rs.Cfg.FinalityDelta, network, 2*time.Second) + } + clNodes := setUpOutput.Env.CLNodes + // assert every node is detecting the reorg (LogPollInterval is set as 1s for faster detection in the test) + assert.Eventually(t, func() bool { + violationDetectedByNodeCount := 0 + for _, node := range clNodes { + resp, _, err := node.Health() + require.NoError(t, err) + for _, d := range resp.Data { + if d.Attributes.Name == logPollerName && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" { + log.Debug().Msg("Finality violated is detected by node") + violationDetectedByNodeCount++ } } - log.Info().Msg("Finality violated is not yet reported by node") - return false - }, 3*time.Minute, 20*time.Second, "Reorg above finality depth is not detected by node") - lane.ValidateRequests(actions.ExpectAnyPhaseToFail(actions.WithTimeout(time.Minute))) - }) + } + return violationDetectedByNodeCount == len(clNodes) + }, 3*time.Minute, 20*time.Second, "Reorg above finality depth is not detected by node") + // send another request and verify it fails + err = lane.SendRequests(1, gasLimit) + if network == "Source" { + // if it is source chain reorg, the transaction will not even be initiated + require.Error(t, err) + } else { + // if it is destination chain reorg, the transaction will be initiated and will fail in the process + require.NoError(t, err) + } + + lane.ValidateRequests(actions.ExpectAnyPhaseToFail(actions.WithTimeout(time.Minute))) } // add liquidity to pools on both networks @@ -1104,3 +1133,36 @@ func testOffRampRateLimits(t *testing.T, rateLimiterConfig contracts.RateLimiter } } + +// SetupReorgSuite defines the setup required to perform re-org step +func SetupReorgSuite(t *testing.T, lggr *zerolog.Logger, setupOutput *testsetups.CCIPTestSetUpOutputs) *ReorgSuite { + var finalitySrc uint64 + var finalityDst uint64 + if setupOutput.Cfg.SelectedNetworks[0].FinalityTag { + finalitySrc = 10 + } else { + finalitySrc = setupOutput.Cfg.SelectedNetworks[0].FinalityDepth + } + if setupOutput.Cfg.SelectedNetworks[1].FinalityTag { + finalityDst = 10 + } else { + finalityDst = setupOutput.Cfg.SelectedNetworks[1].FinalityDepth + } + var srcGethHTTPURL, dstGethHTTPURL string + if setupOutput.Env.LocalCluster != nil { + srcGethHTTPURL = setupOutput.Env.LocalCluster.EVMNetworks[0].HTTPURLs[0] + dstGethHTTPURL = setupOutput.Env.LocalCluster.EVMNetworks[1].HTTPURLs[0] + } else { + srcGethHTTPURL = setupOutput.Env.K8Env.URLs["source-chain_http"][0] + dstGethHTTPURL = setupOutput.Env.K8Env.URLs["dest-chain_http"][0] + } + rs, err := NewReorgSuite(t, lggr, &ReorgConfig{ + SrcGethHTTPURL: srcGethHTTPURL, + DstGethHTTPURL: dstGethHTTPURL, + SrcFinalityDepth: finalitySrc, + DstFinalityDepth: finalityDst, + FinalityDelta: setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.FinalityDelta, + }) + require.NoError(t, err) + return rs +} diff --git a/integration-tests/ccip-tests/chaos/reorg_suite.go b/integration-tests/ccip-tests/smoke/reorg_suite.go similarity index 69% rename from integration-tests/ccip-tests/chaos/reorg_suite.go rename to integration-tests/ccip-tests/smoke/reorg_suite.go index c23447f362..1b089c2af0 100644 --- a/integration-tests/ccip-tests/chaos/reorg_suite.go +++ b/integration-tests/ccip-tests/smoke/reorg_suite.go @@ -1,4 +1,4 @@ -package chaos +package smoke import ( "fmt" @@ -9,17 +9,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/smartcontractkit/chainlink-testing-framework/client" - "github.com/smartcontractkit/chainlink-testing-framework/grafana" ) // ReorgSuite is a test suite that generates reorgs on source/dest chains type ReorgSuite struct { - t *testing.T - Cfg *ReorgConfig - Logger *zerolog.Logger - SrcClient *client.RPCClient - DstClient *client.RPCClient - GrafanaClient *grafana.Client + t *testing.T + Cfg *ReorgConfig + Logger *zerolog.Logger + SrcClient *client.RPCClient + DstClient *client.RPCClient } // ReorgConfig is a configuration for reorg tests @@ -34,10 +32,6 @@ type ReorgConfig struct { DstFinalityDepth uint64 // FinalityDelta blocks to rewind below or above finality FinalityDelta int - // ExperimentDuration experiment duration - ExperimentDuration time.Duration - // GrafanaConfig is common Grafana config - *GrafanaConfig } // Validate validates ReorgConfig params @@ -48,7 +42,7 @@ func (rc *ReorgConfig) Validate() error { rc.FinalityDelta, rc.SrcFinalityDepth, rc.DstFinalityDepth, ) } - return rc.GrafanaConfig.Validate() + return nil } // NewReorgSuite creates new reorg suite with source/dest RPC clients, works only with Geth @@ -57,12 +51,11 @@ func NewReorgSuite(t *testing.T, lggr *zerolog.Logger, cfg *ReorgConfig) (*Reorg return nil, err } return &ReorgSuite{ - t: t, - Cfg: cfg, - Logger: lggr, - SrcClient: client.NewRPCClient(cfg.SrcGethHTTPURL), - DstClient: client.NewRPCClient(cfg.DstGethHTTPURL), - GrafanaClient: grafana.NewGrafanaClient(cfg.GrafanaURL, cfg.GrafanaToken), + t: t, + Cfg: cfg, + Logger: lggr, + SrcClient: client.NewRPCClient(cfg.SrcGethHTTPURL), + DstClient: client.NewRPCClient(cfg.DstGethHTTPURL), }, nil } @@ -89,14 +82,6 @@ func (r *ReorgSuite) RunReorg(client *client.RPCClient, blocksBack int, network Int64("Number", blockNumber). Str("Network", network). Msg("Block number after rewinding") - err = PostGrafanaAnnotation( - r.Logger, - r.GrafanaClient, - r.Cfg.dashboardUID, - fmt.Sprintf("rewinded %s chain for %d blocks back, finality in source is: %d and finality in destination is: %d", - network, blocksBack, r.Cfg.SrcFinalityDepth, r.Cfg.SrcFinalityDepth), - nil, - ) assert.NoError(r.t, err) }() } diff --git a/integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml b/integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml index 6e4c4412eb..f639c71e27 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml @@ -48,6 +48,7 @@ DBArgs = ['shared_buffers=1536MB', 'effective_cache_size=4096MB', 'work_mem=64MB [CCIP.Env.NewCLCluster.Common] CommonChainConfigTOML = """ +LogPollInterval = '1s' [HeadTracker] HistoryDepth = 30 @@ -56,19 +57,10 @@ PriceMax = '200 gwei' LimitDefault = 6000000 FeeCapDefault = '200 gwei' """ -#[CCIP.Groups.load] -#BiDirectionalLane = true -#[CCIP.Groups.load.LoadProfile] -#RequestPerUnitTime = [1] # number of ccip requests to be sent per unit time -#TimeUnit = '10s' # unit of time for RequestPerUnitTime -#TestDuration = '10m' # load test duration, not used for smoke tests -#WaitBetweenChaosDuringLoad = '2m' # Duration to wait between each chaos injection during load test; only valid for chaos tests -#NetworkChaosDelay = '100ms' # Duration for network chaos delay; only valid for chaos tests using network chaos -#FailOnFirstErrorInLoad = true [CCIP.Groups.smoke] BiDirectionalLane = true -PhaseTimeout = '5m' # Duration to wait for the each phase validation(SendRequested, Commit, RMN Blessing, Execution) to time-out. +PhaseTimeout = '3m' # Duration to wait for the each phase validation(SendRequested, Commit, RMN Blessing, Execution) to time-out. LocalCluster = false [CCIP.Groups.smoke.ChaosReorgProfile] diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index 358d1fa4c6..a42ddaa7c0 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -3,7 +3,6 @@ package testsetups import ( "context" "fmt" - ch "github.com/smartcontractkit/ccip/integration-tests/ccip-tests/chaos" "math/big" "math/rand" "os" @@ -1225,40 +1224,6 @@ func CCIPDefaultTestSetUp( return setUpArgs } -// SetupReorgSuite defines the setup required to perform re-org step -func SetupReorgSuite(t *testing.T, lggr *zerolog.Logger, setupOutput *CCIPTestSetUpOutputs, testCfg *CCIPTestConfig) *ch.ReorgSuite { - var finalitySrc uint64 - var finalityDst uint64 - if setupOutput.Cfg.SelectedNetworks[0].FinalityTag { - finalitySrc = 10 - } else { - finalitySrc = setupOutput.Cfg.SelectedNetworks[0].FinalityDepth - } - if setupOutput.Cfg.SelectedNetworks[1].FinalityTag { - finalityDst = 10 - } else { - finalityDst = setupOutput.Cfg.SelectedNetworks[1].FinalityDepth - } - rs, err := ch.NewReorgSuite(t, lggr, &ch.ReorgConfig{ - SrcGethHTTPURL: setupOutput.Env.K8Env.URLs["source-chain_http"][0], - DstGethHTTPURL: setupOutput.Env.K8Env.URLs["dest-chain_http"][0], - // enable the below set of lines for local docker run - //SrcGethHTTPURL: setupOutput.Env.LocalCluster.EVMNetworks[0].HTTPURLs[0], - //DstGethHTTPURL: setupOutput.Env.LocalCluster.EVMNetworks[1].HTTPURLs[0], - SrcFinalityDepth: finalitySrc, - DstFinalityDepth: finalityDst, - FinalityDelta: setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.FinalityDelta, - ExperimentDuration: setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.Duration.Duration(), - GrafanaConfig: &ch.GrafanaConfig{ - GrafanaURL: *testCfg.EnvInput.Logging.Grafana.BaseUrl, - GrafanaToken: *testCfg.EnvInput.Logging.Grafana.BearerToken, - DashboardURL: *testCfg.EnvInput.Logging.Grafana.DashboardUrl, - }, - }) - require.NoError(t, err) - return rs -} - // CreateEnvironment creates the environment for the test and registers the test clean-up function to tear down the set-up environment // It returns the map of chainID to EVMClient func (o *CCIPTestSetUpOutputs) CreateEnvironment(