Assertion updates and cleanup

.github/workflows/integration-tests.yml

@@ -373,6 +373,13 @@ jobs:
             file: ccip
             run: -run ^TestSmokeCCIPForBidirectionalLane$
             config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml
+          - name: ccip-smoke-reorg
+            nodes: 3
+            dir: ccip-tests/smoke
+            os: ubuntu-latest
+            file: ccip
+            run: -run ^TestSmokeCCIPReorg
+            config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
           - name: runlog
             id: runlog
             nodes: 2

integration-tests/ccip-tests/smoke/ccip_test.go

@@ -2,6 +2,7 @@ package smoke
 
 import (
 	"fmt"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"math/big"
 	"testing"
@@ -887,7 +888,7 @@ func TestSmokeCCIPReorgBelowFinality(t *testing.T) {
 		// sending multiple request and expect all should go through though there is below finality reorg
 		err := lane.SendRequests(5, gasLimit)
 		require.NoError(t, err)
-		rs := testsetups.SetupReorgSuite(t, &log, setUpOutput, TestCfg)
+		rs := SetupReorgSuite(t, &log, setUpOutput)
 		// run below finality reorg in both source and destination chain
 		blocksBackSrc := int(rs.Cfg.SrcFinalityDepth) - rs.Cfg.FinalityDelta
 		blocksBackDst := int(rs.Cfg.DstFinalityDepth) - rs.Cfg.FinalityDelta
@@ -898,10 +899,30 @@ func TestSmokeCCIPReorgBelowFinality(t *testing.T) {
 	})
 }
 
-// Test expects to generate above finality reorg in both destination and
-// expect CCIP transaction doesn't go through successful and node detects reorg successfully
-func TestSmokeCCIPReorgAboveFinality(t *testing.T) {
+// Test creates above finality reorg at destination and
+// expects ccip transactions in-flight and the one initiated after reorg
+// doesn't go through and verifies every node is able to detect reorg.
+// Note: LogPollInterval interval is set as 1s to detect the reorg immediately
+func TestSmokeCCIPReorgAboveFinalityAtDestination(t *testing.T) {
 	t.Parallel()
+	t.Run("Above finality reorg in destination chain", func(t *testing.T) {
+		performReorgAndValidate(t, "Destination")
+	})
+}
+
+// Test creates above finality reorg at destination and
+// expects ccip transactions in-flight doesn't go through, the transaction initiated after reorg
+// shouldn't even get initiated and verifies every node is able to detect reorg.
+// Note: LogPollInterval interval is set as 1s to detect the reorg immediately
+func TestSmokeCCIPReorgAboveFinalityAtSource(t *testing.T) {
+	t.Parallel()
+	t.Run("Above finality reorg in source chain", func(t *testing.T) {
+		performReorgAndValidate(t, "Source")
+	})
+}
+
+// performReorgAndValidate is to perform the above finality reorg test
+func performReorgAndValidate(t *testing.T, network string) {
 	log := logging.GetTestLogger(t)
 	TestCfg := testsetups.NewCCIPTestConfig(t, log, testconfig.Smoke)
 	require.NotNil(t, TestCfg.TestGroupInput.MsgDetails.DestGasLimit)
@@ -911,46 +932,54 @@ func TestSmokeCCIPReorgAboveFinality(t *testing.T) {
 		log.Info().Msg("No lanes found")
 		return
 	}
-
 	t.Cleanup(func() {
 		require.NoError(t, setUpOutput.TearDown())
 	})
-
+	rs := SetupReorgSuite(t, &log, setUpOutput)
 	lane := setUpOutput.Lanes[0].ForwardLane
 	log.Info().
 		Str("Source", lane.SourceNetworkName).
 		Str("Destination", lane.DestNetworkName).
-		Msg("Starting CCIP reorg test")
-	t.Run(fmt.Sprintf("CCIP reorg above finality test from network %s to network %s",
-		lane.SourceNetworkName, lane.DestNetworkName), func(t *testing.T) {
-		t.Parallel()
-		lane.Test = t
-		lane.RecordStateBeforeTransfer()
-		rs := testsetups.SetupReorgSuite(t, &log, setUpOutput, TestCfg)
-		err := lane.SendRequests(1, gasLimit)
-		require.NoError(t, err)
-		// run above finality reorg in destination chain
-		blocksBackDst := int(rs.Cfg.DstFinalityDepth) + rs.Cfg.FinalityDelta
-		//blocksBackSrc := int(rs.Cfg.SrcFinalityDepth) + rs.Cfg.FinalityDelta
-		//rs.RunReorg(rs.SrcClient, blocksBackSrc, "Source", 2*time.Second)
-		rs.RunReorg(rs.DstClient, blocksBackDst, "Destination", 2*time.Second)
-		clNodes := setUpOutput.Env.CLNodes
-		assert.Eventually(t, func() bool {
-			for _, node := range clNodes {
-				resp, _, err := node.Health()
-				require.NoError(t, err)
-				for _, d := range resp.Data {
-					if d.Attributes.Name == "EVM.2337.LogPoller" && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" {
-						log.Info().Msg("Finality violated is reported by node")
-						return true
-					}
+		Msg("Starting ccip reorg test")
+	lane.Test = t
+	lane.RecordStateBeforeTransfer()
+	err := lane.SendRequests(1, gasLimit)
+	require.NoError(t, err)
+	logPollerName := ""
+	if network == "Destination" {
+		logPollerName = "EVM.2337.LogPoller"
+		rs.RunReorg(rs.DstClient, int(rs.Cfg.DstFinalityDepth)+rs.Cfg.FinalityDelta, network, 2*time.Second)
+	} else {
+		logPollerName = "EVM.1337.LogPoller"
+		rs.RunReorg(rs.SrcClient, int(rs.Cfg.SrcFinalityDepth)+rs.Cfg.FinalityDelta, network, 2*time.Second)
+	}
+	clNodes := setUpOutput.Env.CLNodes
+	// assert every node is detecting the reorg (LogPollInterval is set as 1s for faster detection in the test)
+	assert.Eventually(t, func() bool {
+		violationDetectedByNodeCount := 0
+		for _, node := range clNodes {
+			resp, _, err := node.Health()
+			require.NoError(t, err)
+			for _, d := range resp.Data {
+				if d.Attributes.Name == logPollerName && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" {
+					log.Debug().Msg("Finality violated is detected by node")
+					violationDetectedByNodeCount++
 				}
 			}
-			log.Info().Msg("Finality violated is not yet reported by node")
-			return false
-		}, 3*time.Minute, 20*time.Second, "Reorg above finality depth is not detected by node")
-		lane.ValidateRequests(actions.ExpectAnyPhaseToFail(actions.WithTimeout(time.Minute)))
-	})
+		}
+		return violationDetectedByNodeCount == len(clNodes)
+	}, 3*time.Minute, 20*time.Second, "Reorg above finality depth is not detected by node")
+	// send another request and verify it fails
+	err = lane.SendRequests(1, gasLimit)
+	if network == "Source" {
+		// if it is source chain reorg, the transaction will not even be initiated
+		require.Error(t, err)
+	} else {
+		// if it is destination chain reorg, the transaction will be initiated and will fail in the process
+		require.NoError(t, err)
+	}
+
+	lane.ValidateRequests(actions.ExpectAnyPhaseToFail(actions.WithTimeout(time.Minute)))
 }
 
 // add liquidity to pools on both networks
@@ -1104,3 +1133,36 @@ func testOffRampRateLimits(t *testing.T, rateLimiterConfig contracts.RateLimiter
 	}
 
 }
+
+// SetupReorgSuite defines the setup required to perform re-org step
+func SetupReorgSuite(t *testing.T, lggr *zerolog.Logger, setupOutput *testsetups.CCIPTestSetUpOutputs) *ReorgSuite {
+	var finalitySrc uint64
+	var finalityDst uint64
+	if setupOutput.Cfg.SelectedNetworks[0].FinalityTag {
+		finalitySrc = 10
+	} else {
+		finalitySrc = setupOutput.Cfg.SelectedNetworks[0].FinalityDepth
+	}
+	if setupOutput.Cfg.SelectedNetworks[1].FinalityTag {
+		finalityDst = 10
+	} else {
+		finalityDst = setupOutput.Cfg.SelectedNetworks[1].FinalityDepth
+	}
+	var srcGethHTTPURL, dstGethHTTPURL string
+	if setupOutput.Env.LocalCluster != nil {
+		srcGethHTTPURL = setupOutput.Env.LocalCluster.EVMNetworks[0].HTTPURLs[0]
+		dstGethHTTPURL = setupOutput.Env.LocalCluster.EVMNetworks[1].HTTPURLs[0]
+	} else {
+		srcGethHTTPURL = setupOutput.Env.K8Env.URLs["source-chain_http"][0]
+		dstGethHTTPURL = setupOutput.Env.K8Env.URLs["dest-chain_http"][0]
+	}
+	rs, err := NewReorgSuite(t, lggr, &ReorgConfig{
+		SrcGethHTTPURL:   srcGethHTTPURL,
+		DstGethHTTPURL:   dstGethHTTPURL,
+		SrcFinalityDepth: finalitySrc,
+		DstFinalityDepth: finalityDst,
+		FinalityDelta:    setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.FinalityDelta,
+	})
+	require.NoError(t, err)
+	return rs
+}

integration-tests/ccip-tests/chaos/reorg_suite.go → integration-tests/ccip-tests/smoke/reorg_suite.go

@@ -1,4 +1,4 @@
-package chaos
+package smoke
 
 import (
 	"fmt"
@@ -9,17 +9,15 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/client"
-	"github.com/smartcontractkit/chainlink-testing-framework/grafana"
 )
 
 // ReorgSuite is a test suite that generates reorgs on source/dest chains
 type ReorgSuite struct {
-	t             *testing.T
-	Cfg           *ReorgConfig
-	Logger        *zerolog.Logger
-	SrcClient     *client.RPCClient
-	DstClient     *client.RPCClient
-	GrafanaClient *grafana.Client
+	t         *testing.T
+	Cfg       *ReorgConfig
+	Logger    *zerolog.Logger
+	SrcClient *client.RPCClient
+	DstClient *client.RPCClient
 }
 
 // ReorgConfig is a configuration for reorg tests
@@ -34,10 +32,6 @@ type ReorgConfig struct {
 	DstFinalityDepth uint64
 	// FinalityDelta blocks to rewind below or above finality
 	FinalityDelta int
-	// ExperimentDuration experiment duration
-	ExperimentDuration time.Duration
-	// GrafanaConfig is common Grafana config
-	*GrafanaConfig
 }
 
 // Validate validates ReorgConfig params
@@ -48,7 +42,7 @@ func (rc *ReorgConfig) Validate() error {
 			rc.FinalityDelta, rc.SrcFinalityDepth, rc.DstFinalityDepth,
 		)
 	}
-	return rc.GrafanaConfig.Validate()
+	return nil
 }
 
 // NewReorgSuite creates new reorg suite with source/dest RPC clients, works only with Geth
@@ -57,12 +51,11 @@ func NewReorgSuite(t *testing.T, lggr *zerolog.Logger, cfg *ReorgConfig) (*Reorg
 		return nil, err
 	}
 	return &ReorgSuite{
-		t:             t,
-		Cfg:           cfg,
-		Logger:        lggr,
-		SrcClient:     client.NewRPCClient(cfg.SrcGethHTTPURL),
-		DstClient:     client.NewRPCClient(cfg.DstGethHTTPURL),
-		GrafanaClient: grafana.NewGrafanaClient(cfg.GrafanaURL, cfg.GrafanaToken),
+		t:         t,
+		Cfg:       cfg,
+		Logger:    lggr,
+		SrcClient: client.NewRPCClient(cfg.SrcGethHTTPURL),
+		DstClient: client.NewRPCClient(cfg.DstGethHTTPURL),
 	}, nil
 }
 
@@ -89,14 +82,6 @@ func (r *ReorgSuite) RunReorg(client *client.RPCClient, blocksBack int, network
 			Int64("Number", blockNumber).
 			Str("Network", network).
 			Msg("Block number after rewinding")
-		err = PostGrafanaAnnotation(
-			r.Logger,
-			r.GrafanaClient,
-			r.Cfg.dashboardUID,
-			fmt.Sprintf("rewinded %s chain for %d blocks back, finality in source is: %d and finality in destination is: %d",
-				network, blocksBack, r.Cfg.SrcFinalityDepth, r.Cfg.SrcFinalityDepth),
-			nil,
-		)
 		assert.NoError(r.t, err)
 	}()
 }

integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml

@@ -48,6 +48,7 @@ DBArgs = ['shared_buffers=1536MB', 'effective_cache_size=4096MB', 'work_mem=64MB
 
 [CCIP.Env.NewCLCluster.Common]
 CommonChainConfigTOML = """
+LogPollInterval = '1s'
 [HeadTracker]
 HistoryDepth = 30
 
@@ -56,19 +57,10 @@ PriceMax = '200 gwei'
 LimitDefault = 6000000
 FeeCapDefault = '200 gwei'
 """
-#[CCIP.Groups.load]
-#BiDirectionalLane = true
-#[CCIP.Groups.load.LoadProfile]
-#RequestPerUnitTime = [1]          # number of ccip requests to be sent per unit time
-#TimeUnit = '10s'                  # unit of time for RequestPerUnitTime
-#TestDuration = '10m'              # load test duration, not used for smoke tests
-#WaitBetweenChaosDuringLoad = '2m' # Duration to wait between each chaos injection during load test; only valid for chaos tests
-#NetworkChaosDelay = '100ms'       # Duration for network chaos delay; only valid for chaos tests using network chaos
-#FailOnFirstErrorInLoad = true
 
 [CCIP.Groups.smoke]
 BiDirectionalLane = true
-PhaseTimeout = '5m'       # Duration to wait for the each phase validation(SendRequested, Commit, RMN Blessing, Execution) to time-out.
+PhaseTimeout = '3m'       # Duration to wait for the each phase validation(SendRequested, Commit, RMN Blessing, Execution) to time-out.
 LocalCluster = false
 
 [CCIP.Groups.smoke.ChaosReorgProfile]

integration-tests/ccip-tests/testsetups/ccip.go

@@ -3,7 +3,6 @@ package testsetups
 import (
 	"context"
 	"fmt"
-	ch "github.com/smartcontractkit/ccip/integration-tests/ccip-tests/chaos"
 	"math/big"
 	"math/rand"
 	"os"
@@ -1225,40 +1224,6 @@ func CCIPDefaultTestSetUp(
 	return setUpArgs
 }
 
-// SetupReorgSuite defines the setup required to perform re-org step
-func SetupReorgSuite(t *testing.T, lggr *zerolog.Logger, setupOutput *CCIPTestSetUpOutputs, testCfg *CCIPTestConfig) *ch.ReorgSuite {
-	var finalitySrc uint64
-	var finalityDst uint64
-	if setupOutput.Cfg.SelectedNetworks[0].FinalityTag {
-		finalitySrc = 10
-	} else {
-		finalitySrc = setupOutput.Cfg.SelectedNetworks[0].FinalityDepth
-	}
-	if setupOutput.Cfg.SelectedNetworks[1].FinalityTag {
-		finalityDst = 10
-	} else {
-		finalityDst = setupOutput.Cfg.SelectedNetworks[1].FinalityDepth
-	}
-	rs, err := ch.NewReorgSuite(t, lggr, &ch.ReorgConfig{
-		SrcGethHTTPURL: setupOutput.Env.K8Env.URLs["source-chain_http"][0],
-		DstGethHTTPURL: setupOutput.Env.K8Env.URLs["dest-chain_http"][0],
-		// enable the below set of lines for local docker run
-		//SrcGethHTTPURL:     setupOutput.Env.LocalCluster.EVMNetworks[0].HTTPURLs[0],
-		//DstGethHTTPURL:     setupOutput.Env.LocalCluster.EVMNetworks[1].HTTPURLs[0],
-		SrcFinalityDepth:   finalitySrc,
-		DstFinalityDepth:   finalityDst,
-		FinalityDelta:      setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.FinalityDelta,
-		ExperimentDuration: setupOutput.Cfg.TestGroupInput.ChaosReorgProfile.Duration.Duration(),
-		GrafanaConfig: &ch.GrafanaConfig{
-			GrafanaURL:   *testCfg.EnvInput.Logging.Grafana.BaseUrl,
-			GrafanaToken: *testCfg.EnvInput.Logging.Grafana.BearerToken,
-			DashboardURL: *testCfg.EnvInput.Logging.Grafana.DashboardUrl,
-		},
-	})
-	require.NoError(t, err)
-	return rs
-}
-
 // CreateEnvironment creates the environment for the test and registers the test clean-up function to tear down the set-up environment
 // It returns the map of chainID to EVMClient
 func (o *CCIPTestSetUpOutputs) CreateEnvironment(