Move all test config to TOML from env vars (#231)

.github/workflows/integration-tests.yml

@@ -253,7 +253,6 @@ jobs:
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}
       TEST_LOG_LEVEL: debug
-      CCIP_DEPLOY_ON_LOCAL: true
     strategy:
       fail-fast: false
       matrix:

.gitignore

@@ -60,6 +60,8 @@ core/web/assets
 # Integration Tests
 integration-tests/**/logs/
 integration-tests/**/tmp_*
+integration-tests/**/testconfig/**/override.toml
+integration-tests/**/testconfig/**/**.env
 tests-*.xml
 *.test
 tmp-manifest-*.yaml

integration-tests/ccip-tests/Makefile

@@ -1,7 +1,15 @@
+## To Override the default config,
+# example usage: make override_config overrideconfig="" override_toml=../config/config.toml env=../.env
+.PHONY: override_config
+override_config:
+	cd ./testconfig/override && \
+	go run . --overridewith=$(overrideconfig) --path=$(override_toml) --output=$(env) && \
+	cd ../..
+
 # example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos
 .PHONY: test_load_ccip_simulated_k8
-test_load_ccip_simulated_k8:
-	source ./load-test.env && \
+test_load_ccip_simulated_k8:override_config
+	source ./testconfig/override/$(env) && \
     CHAINLINK_IMAGE=$(image)  \
     CHAINLINK_VERSION=$(tag)  \
     ENV_JOB_IMAGE=$(testimage)  \
@@ -10,42 +18,33 @@ test_load_ccip_simulated_k8:
 
 # example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest
 .PHONY: test_load_ccip_testnet_k8
-test_load_ccip_testnet_k8:
-	source ./load-test.env && \
-	SELECTED_NETWORKS="SIMULATED,SEPOLIA,OPTIMISM_GOERLI,ARBITRUM_GOERLI,AVALANCHE_FUJI,BSC_TESTNET,MUMBAI,BASE_GOERLI"  \
-	CCIP_NO_OF_NETWORKS=7  \
-	CCIP_CHAINLINK_NODE_FUNDING=10  \
+include test-config.env
+overrideconfig := "CCIP.Env.Networks = ['SEPOLIA', 'OPTIMISM_GOERLI', 'ARBITRUM_GOERLI', 'AVALANCHE_FUJI', 'BSC_TESTNET', 'MUMBAI', 'BASE_GOERLI]"
+override overrideconfig += "CCIP.Groups.load.NodeFunding = 10"
+test_load_ccip_testnet_k8: override_config
+	source ./testconfig/override/$(env) && \
     CHAINLINK_IMAGE=$(image)  \
     CHAINLINK_VERSION=$(tag)  \
     ENV_JOB_IMAGE=$(testimage)  \
 	go test -timeout 24h -count=1 -v -run ^TestLoadCCIPStableRPS$$ ./load
 
-# example usage: make test_smoke_ccip_simulated_local image=chainlink-ccip tag=latest testname=TestSmokeCCIPForBidirectionalLane
+# example usage: make test_smoke_ccip_simulated_local image=chainlink-ccip tag=latest testname=TestSmokeCCIPForBidirectionalLane override_toml=../config/config.toml env=test-config.env
 .PHONY: test_smoke_ccip_simulated_local
-test_smoke_ccip_simulated_local:
-	DATABASE_URL="postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable" \
-    SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2"  \
-    CCIP_DEPLOY_ON_LOCAL=True  \
-    CCIP_NETWORK_PAIRS="" \
-    CCIP_NO_OF_NETWORKS=2  \
-    CCIP_NO_OF_ROUTERS_PER_PAIR=1  \
-    CCIP_MSG_TYPE=WithToken \
+include test-config.env
+test_smoke_ccip_simulated_local: override_config
+	source ./testconfig/override/$(env) && \
     CHAINLINK_IMAGE=$(image)  \
     CHAINLINK_VERSION=$(tag)  \
-    CCIP_TESTS_ON_EXISTING_DEPLOYMENT=False  \
     ENV_JOB_IMAGE=""  \
 	go test -timeout 24h -count=1 -v -run ^$(testname)$$ ./smoke
 
-# example usage: make test_smoke_ccip_simulated_local_besu image=chainlink tag=latest
+# example usage: make test_smoke_ccip_simulated_local_besu image=chainlink-ccip tag=latest override_toml=../config/config.toml env=test-config.env
 .PHONY: test_smoke_ccip_simulated_local_besu
-test_smoke_ccip_simulated_local_besu:
-	DATABASE_URL="postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable" \
-    SELECTED_NETWORKS="SIMULATED,SIMULATED_BESU_NONDEV_1,SIMULATED_BESU_NONDEV_2"  \
-    CCIP_DEPLOY_ON_LOCAL=True  \
-    CCIP_NO_OF_NETWORKS=2  \
-    CCIP_MSG_TYPE=WithToken \
-    CHAINLINK_IMAGE=$(image)  \
-    CHAINLINK_VERSION=$(tag)  \
-    CCIP_TESTS_ON_EXISTING_DEPLOYMENT=False  \
+include test-config.env
+overrideconfig := "CCIP.Env.Networks = ['SIMULATED_BESU_NONDEV_1', 'SIMULATED_BESU_NONDEV_2']"
+test_smoke_ccip_simulated_local_besu: override_config
+	source ./testconfig/override/$(env) && \
+  	CHAINLINK_IMAGE=$(image)  \
+  	CHAINLINK_VERSION=$(tag)  \
     ENV_JOB_IMAGE=""  \
 	go test -timeout 24h -count=1 -v -run ^TestSmokeCCIPForBidirectionalLane$$ ./smoke

integration-tests/ccip-tests/chaos/ccip_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/stretchr/testify/require"
 
+	"github.com/smartcontractkit/ccip/integration-tests/ccip-tests/testconfig"
 	"github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/actions"
 	"github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testsetups"
 )
@@ -100,15 +101,11 @@ func TestChaosCCIP(t *testing.T) {
 		t.Run(in.testName, func(t *testing.T) {
 			t.Parallel()
 			l := logging.GetTestLogger(t)
-			testCfg := testsetups.NewCCIPTestConfig(t, l, testsetups.Chaos)
-			var (
-				numOfCommitNodes = 5
-				numOfRequests    = 3
-			)
+			testCfg := testsetups.NewCCIPTestConfig(t, l, testconfig.Chaos)
+			var numOfRequests = 3
 
 			setUpArgs := testsetups.CCIPDefaultTestSetUp(
-				t, l, "chaos-ccip", 12,
-				nil, numOfCommitNodes, false, false, testCfg)
+				t, l, "chaos-ccip", nil, testCfg)
 
 			if len(setUpArgs.Lanes) == 0 {
 				return
@@ -125,7 +122,7 @@ func TestChaosCCIP(t *testing.T) {
 
 			lane.RecordStateBeforeTransfer()
 			// Send the ccip-request and verify ocr2 is running
-			err := lane.SendRequests(1, testCfg.MsgType)
+			err := lane.SendRequests(1, testCfg.TestGroupInput.MsgType)
 			require.NoError(t, err)
 			lane.ValidateRequests()
 
@@ -140,7 +137,7 @@ func TestChaosCCIP(t *testing.T) {
 			})
 			lane.RecordStateBeforeTransfer()
 			// Now send the ccip-request while the chaos is at play
-			err = lane.SendRequests(numOfRequests, testCfg.MsgType)
+			err = lane.SendRequests(numOfRequests, testCfg.TestGroupInput.MsgType)
 			require.NoError(t, err)
 			if in.waitForChaosRecovery {
 				// wait for chaos to be recovered before further validation

integration-tests/ccip-tests/contracts/laneconfig/parse_contracts.go

@@ -168,9 +168,13 @@ func (l *Lanes) WriteLaneConfig(networkA string, cfg *LaneConfig) error {
 	return nil
 }
 
-func ReadLanesFromExistingDeployment() (*Lanes, error) {
+func ReadLanesFromExistingDeployment(contracts []byte) (*Lanes, error) {
+	// if contracts is empty, use the existing contracts from contracts.json
+	if len(contracts) == 0 {
+		contracts = ExistingContracts
+	}
 	var lanes Lanes
-	if err := json.Unmarshal(ExistingContracts, &lanes); err != nil {
+	if err := json.Unmarshal(contracts, &lanes); err != nil {
 		return nil, err
 	}
 	return &lanes, nil

integration-tests/ccip-tests/load/ccip_test.go

@@ -11,14 +11,16 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/stretchr/testify/require"
 
+	"github.com/smartcontractkit/ccip/integration-tests/utils"
 	"github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/actions"
+	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
 
 func TestLoadCCIPStableRPS(t *testing.T) {
 	t.Parallel()
 	lggr := logging.GetTestLogger(t)
 	testArgs := NewLoadArgs(t, lggr, context.Background())
-	testArgs.Setup(true, 5, 5)
+	testArgs.Setup()
 	// if the test runs on remote runner
 	if len(testArgs.TestSetupArgs.Lanes) == 0 {
 		return
@@ -36,11 +38,11 @@ func TestLoadCCIPSequentialLaneAdd(t *testing.T) {
 	t.Skipf("test needs maintenance")
 	lggr := logging.GetTestLogger(t)
 	testArgs := NewLoadArgs(t, lggr, context.Background())
-	testArgs.TestCfg.SequentialLaneAddition = true
+	testArgs.TestCfg.TestGroupInput.SequentialLaneAddition = utils.Ptr(true)
 	if len(testArgs.TestCfg.NetworkPairs) <= 1 {
 		t.Skip("Skipping the test as there are not enough network pairs to run the test")
 	}
-	testArgs.Setup(true, 5, 5)
+	testArgs.Setup()
 	// if the test runs on remote runner
 	if len(testArgs.TestSetupArgs.Lanes) == 0 {
 		return
@@ -58,7 +60,7 @@ func TestLoadCCIPStableRequestTriggeringWithNetworkChaos(t *testing.T) {
 	t.Parallel()
 	lggr := logging.GetTestLogger(t)
 	testArgs := NewLoadArgs(t, lggr, context.Background())
-	testArgs.Setup(true, 16, 16)
+	testArgs.Setup()
 	// if the test runs on remote runner
 	if len(testArgs.TestSetupArgs.Lanes) == 0 {
 		return
@@ -82,7 +84,7 @@ func TestLoadCCIPStableRequestTriggeringWithNetworkChaos(t *testing.T) {
 			testEnv.K8Env.Cfg.Namespace, &chaos.Props{
 				FromLabels:  &map[string]*string{"geth": a.Str(actions.ChaosGroupCCIPGeth)},
 				ToLabels:    &map[string]*string{"app": a.Str("chainlink-0")},
-				DurationStr: testArgs.TestCfg.TestDuration.String(),
+				DurationStr: testArgs.TestCfg.TestGroupInput.TestDuration.String(),
 				Delay:       "300ms",
 			}))
 	require.NoError(t, err)
@@ -116,23 +118,24 @@ func TestLoadCCIPStableWithMajorityNodeFailure(t *testing.T) {
 		},
 	}
 
+	lggr := logging.GetTestLogger(t)
+	testArgs := NewLoadArgs(t, lggr, context.Background(), inputs...)
+
 	var allChaosDur time.Duration
 	// to override the default duration of chaos with test input
-	for _, ch := range inputs {
-		dur, err := ch.SetInput()
-		require.NoError(t, err)
-		allChaosDur += dur
+	for i := range inputs {
+		inputs[i].ChaosProps.DurationStr = testArgs.TestCfg.TestGroupInput.ChaosDuration.String()
+		allChaosDur += testArgs.TestCfg.TestGroupInput.ChaosDuration.Duration()
+		inputs[i].WaitBetweenChaos = testArgs.TestCfg.TestGroupInput.WaitBetweenChaosDuringLoad.Duration()
+		allChaosDur += inputs[i].WaitBetweenChaos
 	}
 
-	lggr := logging.GetTestLogger(t)
-	testArgs := NewLoadArgs(t, lggr, context.Background(), inputs...)
-
 	// the duration of load test should be greater than the duration of chaos
-	if testArgs.TestCfg.TestDuration < allChaosDur+2*time.Minute {
+	if testArgs.TestCfg.TestGroupInput.TestDuration.Duration() < allChaosDur+2*time.Minute {
 		t.Fatalf("Skipping the test as the test duration is less than the chaos duration")
 	}
 
-	testArgs.Setup(true, 16, 16)
+	testArgs.Setup()
 	// if the test runs on remote runner
 	if len(testArgs.TestSetupArgs.Lanes) == 0 {
 		return
@@ -169,23 +172,24 @@ func TestLoadCCIPStableWithMinorityNodeFailure(t *testing.T) {
 		},
 	}
 
+	lggr := logging.GetTestLogger(t)
+	testArgs := NewLoadArgs(t, lggr, context.Background(), inputs...)
+
 	var allChaosDur time.Duration
 	// to override the default duration of chaos with test input
-	for _, ch := range inputs {
-		dur, err := ch.SetInput()
-		require.NoError(t, err)
-		allChaosDur += dur
+	for i := range inputs {
+		inputs[i].ChaosProps.DurationStr = testArgs.TestCfg.TestGroupInput.ChaosDuration.String()
+		allChaosDur += testArgs.TestCfg.TestGroupInput.ChaosDuration.Duration()
+		inputs[i].WaitBetweenChaos = testArgs.TestCfg.TestGroupInput.WaitBetweenChaosDuringLoad.Duration()
+		allChaosDur += inputs[i].WaitBetweenChaos
 	}
 
-	lggr := logging.GetTestLogger(t)
-	testArgs := NewLoadArgs(t, lggr, context.Background(), inputs...)
-
 	// the duration of load test should be greater than the duration of chaos
-	if testArgs.TestCfg.TestDuration < allChaosDur+2*time.Minute {
+	if testArgs.TestCfg.TestGroupInput.TestDuration.Duration() < allChaosDur+2*time.Minute {
 		t.Fatalf("Skipping the test as the test duration is less than the chaos duration")
 	}
 
-	testArgs.Setup(true, 16, 16)
+	testArgs.Setup()
 	// if the test runs on remote runner
 	if len(testArgs.TestSetupArgs.Lanes) == 0 {
 		return
@@ -249,11 +253,11 @@ func TestLoadCCIPStableWithPodChaosDiffCommitAndExec(t *testing.T) {
 			t.Parallel()
 			lggr := logging.GetTestLogger(t)
 			testArgs := NewLoadArgs(t, lggr, context.Background(), in)
-			testArgs.TestCfg.TestDuration = 5 * time.Minute
-			testArgs.TestCfg.Load.TimeUnit = 1 * time.Second
-			testArgs.TestCfg.Load.RequestPerUnitTime = []int64{2}
+			testArgs.TestCfg.TestGroupInput.TestDuration = models.MustNewDuration(5 * time.Minute)
+			testArgs.TestCfg.TestGroupInput.TimeUnit = models.MustNewDuration(1 * time.Second)
+			testArgs.TestCfg.TestGroupInput.RequestPerUnitTime = []int64{2}
 
-			testArgs.Setup(false, 5, 5)
+			testArgs.Setup()
 			// if the test runs on remote runner
 			if len(testArgs.TestSetupArgs.Lanes) == 0 {
 				return