From cb1f98434964f30cf1f9de2b13294a7a7b2e0ad8 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Mon, 25 Nov 2024 20:25:27 +0900 Subject: [PATCH 1/8] add new ccip load test, compiles --- deployment/environment.go | 1 + integration-tests/load/ccip/ccip_load.go | 170 ++++++++++++++++++ integration-tests/load/ccip/commit.go | 11 ++ .../load/ccip/destination_gun.go | 146 +++++++++++++++ integration-tests/load/ccip/helpers.go | 55 ++++++ integration-tests/testconfig/ccip/config.go | 1 + integration-tests/testconfig/ccip/load.go | 5 + 7 files changed, 389 insertions(+) create mode 100644 integration-tests/load/ccip/ccip_load.go create mode 100644 integration-tests/load/ccip/commit.go create mode 100644 integration-tests/load/ccip/destination_gun.go create mode 100644 integration-tests/load/ccip/helpers.go create mode 100644 integration-tests/testconfig/ccip/load.go diff --git a/deployment/environment.go b/deployment/environment.go index 0823404da2d..7b5d1b6c9e3 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -35,6 +35,7 @@ type OnchainClient interface { bind.DeployBackend BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) + BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) } // OffchainClient interacts with the job-distributor diff --git a/integration-tests/load/ccip/ccip_load.go b/integration-tests/load/ccip/ccip_load.go new file mode 100644 index 00000000000..52e4bbdb039 --- /dev/null +++ b/integration-tests/load/ccip/ccip_load.go @@ -0,0 +1,170 @@ +package ccip + +import ( + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/stretchr/testify/require" + "math/big" + "sync" + "testing" + "time" +) + +var ( + CommonTestLabels = map[string]string{ + "branch": "ccip_load_crib", + "commit": "ccip_load_crib", + } + wg sync.WaitGroup +) + +// step 1: setup +// Parse the test config, initialize CRIB with configurations defined +// step 2: load +// Use wasp to initiate load +// step 3: parse logs +// Parse all events from the simulated chains, send to Loki +// step 4: teardown +// Stop the chains, cleanup the environment +func LoadTestCCIP(t *testing.T) { + ctx := tests.Context(t) + l := logging.GetTestLogger(t) + + config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) + if err != nil { + t.Fatal(err) + } + l.Info().Interface("loadedTestConfig", config).Msg("Loaded Test Config") + + l.Info().Msg("Starting ccip load test") + l.Info(). + Int("Number of Nodes", *(config.CCIP.Load.NoOfNodes)). + Interface("config", config.CCIP.Load). + Msg("Test Config") + + var env = deployment.Environment{} + // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) + // require.NoError(t, err) + // env, err = crib.DeployPrerequisites(output, config.CCIP) + // merge addressbooks + // env, err := crib.DeployCCIPContracts(output, config.CCIP) + + // Need to keep track of the block number for each chain so that event subscription can be done from that block. + startBlocks := make(map[uint64]*uint64) + state, err := ccipdeployment.LoadOnchainState(env) + require.NoError(t, err) + + cfgl := config.Logging.Loki + + // Parse all events from the simulated chains, send to Loki + loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) + if err != nil { + l.Error().Err(err).Msg("Failed to create Loki client") + return + } + defer loki.StopNow() + + // Based on the config, initiate a DestinationGun + p := wasp.NewProfile() + for selector, chain := range env.Chains { + latesthdr, err := chain.Client.HeaderByNumber(ctx, nil) + require.NoError(t, err) + block := latesthdr.Number.Uint64() + startBlocks[selector] = &block + + p.Add(wasp.NewGenerator(&wasp.Config{ + T: t, + GenName: "ccipLoad", + LoadType: wasp.RPS, + CallTimeout: 20 * time.Second, + Schedule: wasp.Plain(1, 20*time.Second), + // will need to be divided by number of chains + // this schedule is per generator + // in this example, it would be 1 request per 10seconds per generator (dest chain) + // so if there are 3 generators, it would be 3 requests per 10 seconds over the network + Gun: NewDestinationGun(l, selector, env, state.Chains[selector].Receiver.Address(), loki), + Labels: CommonTestLabels, + LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), + // use the same loki client using `NewLokiClient` with the same config for sending events + })) + } + + _, err = p.Run(true) + + lokiLabels := map[string]string{} + for chainSelector, startBlock := range startBlocks { + wg.Add(1) + go func(chainSelector uint64, startBlock *uint64) { + defer wg.Done() + filterOpts := &bind.FilterOpts{ + Start: *startBlock, + End: nil, // To the latest block + Context: ctx, + } + + offRamp := state.Chains[chainSelector].OffRamp + // Filter CommitReportAccepted events + commitIterator, err := offRamp.FilterCommitReportAccepted(filterOpts) + require.NoError(t, err) + + for commitIterator.Next() { + event := commitIterator.Event + fmt.Printf("CommitReportAccepted event: %+v\n", event) + + blockNum := commitIterator.Event.Raw.BlockNumber + block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + require.NoError(t, err) + timestamp := time.Unix(int64(block.Time()), 0) + + for _, root := range event.MerkleRoots { + lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) + + for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { + // todo: push loki calls to channel? + SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + EventType: committed, + Timestamp: timestamp, + SequenceNumber: i, + }) + } + } + } + + // Filter ExecutionStateChanged events + execIterator, err := state.Chains[chainSelector].OffRamp.FilterExecutionStateChanged(filterOpts, []uint64{chainSelector}, nil, nil) + require.NoError(t, err) + for execIterator.Next() { + event := execIterator.Event + fmt.Printf("ExecutionStateChanged event: %+v\n", event) + + blockNum := execIterator.Event.Raw.BlockNumber + block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + require.NoError(t, err) + timestamp := time.Unix(int64(block.Time()), 0) + + // todo: push loki calls to channel? + lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + EventType: executed, + Timestamp: timestamp, + GasUsed: execIterator.Event.GasUsed.Uint64(), + SequenceNumber: execIterator.Event.SequenceNumber, + }) + + } + }(chainSelector, startBlock) + } + + wg.Wait() + + // Stop the chains, cleanup the environment + + // crib.StopChains(env) + // crib.StopNodes(env) +} diff --git a/integration-tests/load/ccip/commit.go b/integration-tests/load/ccip/commit.go new file mode 100644 index 00000000000..6c05d5980b2 --- /dev/null +++ b/integration-tests/load/ccip/commit.go @@ -0,0 +1,11 @@ +package ccip + +import ( + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" +) + +func ProcessCommitReportEvents(destCS uint64, offRamp offramp.OffRamp, filterOpts *bind.FilterOpts) error { + + return err +} diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go new file mode 100644 index 00000000000..455f6dcc186 --- /dev/null +++ b/integration-tests/load/ccip/destination_gun.go @@ -0,0 +1,146 @@ +package ccip + +import ( + "context" + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "sync/atomic" + "time" +) + +type ChainSelectorPair struct { + src uint64 + dst uint64 +} + +type DestinationGun struct { + l zerolog.Logger + env deployment.Environment + seqNums map[ChainSelectorPair]*atomic.Uint64 + roundNum *atomic.Int32 + chainSelector uint64 + receiver common.Address + loki *wasp.LokiClient +} + +func NewDestinationGun(l zerolog.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { + seqNums := make(map[ChainSelectorPair]*atomic.Uint64) + for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { + + seqNums[ChainSelectorPair{ + src: cs, + dst: chainSelector, + }] = &atomic.Uint64{} + seqNums[ChainSelectorPair{ + src: chainSelector, + dst: cs, + }].Store(1) + } + return &DestinationGun{ + l: l, + env: env, + seqNums: seqNums, + roundNum: &atomic.Int32{}, + chainSelector: chainSelector, + receiver: receiver, + loki: loki, + } +} + +func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { + m.roundNum.Add(1) + requestedRound := m.roundNum.Load() + + waspGroup := fmt.Sprintf("%d-%s", m.chainSelector, "messageOnly") + + state, err := ccipdeployment.LoadOnchainState(m.env) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + src, err := m.MustSourceChain() + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + csPair := ChainSelectorPair{ + src: src, + dst: m.chainSelector, + } + m.seqNums[csPair].Add(1) + m.l.Info(). + Int32("RoundNum", requestedRound). + Uint64("Destination ChainSelector", m.chainSelector). + Uint64("Source ChainSelector", src). + Uint64("SequenceNumber", m.seqNums[csPair].Load()). + Msg("starting transmit") + + r := state.Chains[src].Router + + msg, err := m.GetMessage() + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + fee, err := r.GetFee( + &bind.CallOpts{Context: context.Background()}, src, msg) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + if msg.FeeToken == common.HexToAddress("0x0") { + m.env.Chains[src].DeployerKey.Value = fee + defer func() { m.env.Chains[src].DeployerKey.Value = nil }() + } + _, err = r.CcipSend( + m.env.Chains[src].DeployerKey, + m.chainSelector, + msg) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + lokiLabels := setLokiLabels(src, m.chainSelector) + SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ + EventType: transmitted, + Timestamp: time.Now(), + SequenceNumber: m.seqNums[csPair].Load(), + }) + + return &wasp.Response{Failed: false, Group: waspGroup} +} + +// GetCycledClient will return a valid client from a random source chain +func (m *DestinationGun) MustSourceChain() (uint64, error) { + + // TODO: check if this chain has sent a message recently, if so, switch to the next chain + otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) + if len(otherCS) == 0 { + return 0, fmt.Errorf("no other chains to send from") + } + index := m.roundNum.Load() % int32(len(otherCS)) + return otherCS[index], nil +} + +// GetMessage will return the message to be sent while considering expected load of different messages +// TODO: implement randomness and different types of messages +func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { + rcv, err := utils.ABIEncode(`[{"type":"address"}]`, m.receiver) + if err != nil { + m.l.Error().Err(err).Msg("Error encoding receiver address") + } + + return router.ClientEVM2AnyMessage{ + Receiver: rcv, + Data: common.Hex2Bytes("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }, nil +} diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go new file mode 100644 index 00000000000..e7afe6883e4 --- /dev/null +++ b/integration-tests/load/ccip/helpers.go @@ -0,0 +1,55 @@ +package ccip + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + "time" +) + +const ( + transmitted = iota + committed + executed + LokiLoadLabel = "ccip_load_test" + ErrLokiClient = "failed to create Loki client for monitoring" + ErrLokiPush = "failed to push metrics to Loki" +) + +// todo: Have a different struct for commit/exec? +type LokiMetric struct { + EventType int `json:"event_type"` + Timestamp time.Time `json:"timestamp"` + GasUsed uint64 `json:"gas_used"` + SequenceNumber uint64 `json:"sequence_number"` +} + +func GetAddressFromTypeAndVersion(ab deployment.AddressBook, cs uint64, tv string) (common.Address, error) { + allAddr, err := ab.AddressesForChain(cs) + if err != nil { + return common.Address{}, err + } + for addr, tv := range allAddr { + if tv.Type == tv.Type && tv.Version == tv.Version { + return common.HexToAddress(addr), nil + } + } + + return common.Address{}, fmt.Errorf("address not found for chain selector %d and typeAndVersion %s", cs, tv) +} + +func SendMetricsToLoki(l zerolog.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { + if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { + l.Error().Err(err).Msg(ErrLokiPush) + } +} + +func setLokiLabels(src, dst uint64) map[string]string { + return map[string]string{ + "sourceSelector": fmt.Sprintf("%d", src), + "destinationSelector": fmt.Sprintf("%d", dst), + "testType": LokiLoadLabel, + } +} diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index 70c850fd591..374322041ec 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -38,6 +38,7 @@ type Config struct { HomeChainSelector *string `toml:",omitempty"` FeedChainSelector *string `toml:",omitempty"` RMNConfig RMNConfig `toml:",omitempty"` + Load *LoadConfig `toml:",omitempty"` } type RMNConfig struct { diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go new file mode 100644 index 00000000000..af2e89db79a --- /dev/null +++ b/integration-tests/testconfig/ccip/load.go @@ -0,0 +1,5 @@ +package ccip + +type LoadConfig struct { + NoOfNodes *int +} From b38d7da0648fb00ae83ea0a001f232d020e0f669 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 00:07:50 +0900 Subject: [PATCH 2/8] fix build --- deployment/environment/memory/sim.go | 4 ++++ integration-tests/load/ccip/commit.go | 11 ----------- 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 integration-tests/load/ccip/commit.go diff --git a/deployment/environment/memory/sim.go b/deployment/environment/memory/sim.go index c0fba87e2b3..93f77a0d29b 100644 --- a/deployment/environment/memory/sim.go +++ b/deployment/environment/memory/sim.go @@ -80,6 +80,10 @@ func (b *Backend) NonceAt(ctx context.Context, account common.Address, blockNumb return b.Sim.Client().NonceAt(ctx, account, blockNumber) } +func (b *Backend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + return b.Sim.Client().BlockByNumber(ctx, number) +} + func NewBackend(sim *simulated.Backend) *Backend { if sim == nil { panic("simulated backend is nil") diff --git a/integration-tests/load/ccip/commit.go b/integration-tests/load/ccip/commit.go deleted file mode 100644 index 6c05d5980b2..00000000000 --- a/integration-tests/load/ccip/commit.go +++ /dev/null @@ -1,11 +0,0 @@ -package ccip - -import ( - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" -) - -func ProcessCommitReportEvents(destCS uint64, offRamp offramp.OffRamp, filterOpts *bind.FilterOpts) error { - - return err -} From af850195bd9105ff43738d69d685694224871b77 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 00:13:49 +0900 Subject: [PATCH 3/8] make test runnable --- integration-tests/load/ccip/{ccip_load.go => ccip_test.go} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename integration-tests/load/ccip/{ccip_load.go => ccip_test.go} (99%) diff --git a/integration-tests/load/ccip/ccip_load.go b/integration-tests/load/ccip/ccip_test.go similarity index 99% rename from integration-tests/load/ccip/ccip_load.go rename to integration-tests/load/ccip/ccip_test.go index 52e4bbdb039..d9782218519 100644 --- a/integration-tests/load/ccip/ccip_load.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -32,7 +32,7 @@ var ( // Parse all events from the simulated chains, send to Loki // step 4: teardown // Stop the chains, cleanup the environment -func LoadTestCCIP(t *testing.T) { +func TestCCIP_Load(t *testing.T) { ctx := tests.Context(t) l := logging.GetTestLogger(t) From 648bf6870ab59054463f97e705fa10abdebf04e0 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 17:40:26 +0900 Subject: [PATCH 4/8] rename --- integration-tests/load/ccip/ccip_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index d9782218519..3fc130b855d 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -32,7 +32,7 @@ var ( // Parse all events from the simulated chains, send to Loki // step 4: teardown // Stop the chains, cleanup the environment -func TestCCIP_Load(t *testing.T) { +func TestCCIPLoad_RPS(t *testing.T) { ctx := tests.Context(t) l := logging.GetTestLogger(t) From 1d87264e9cd48e0c0355fe5a73840c47d0c9fb1e Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 27 Nov 2024 21:38:08 +0900 Subject: [PATCH 5/8] in progress --- integration-tests/load/ccip/ccip_test.go | 12 +++- integration-tests/load/ccip/helpers.go | 70 +++++++++++++++++++ .../ccip-v2-scripts-address-book.json | 1 + .../ccip-v2-scripts-nodes-details.json | 1 + 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json create mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 3fc130b855d..79c87106960 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -21,7 +21,9 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup + wg sync.WaitGroup + abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" + nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" ) // step 1: setup @@ -48,6 +50,8 @@ func TestCCIPLoad_RPS(t *testing.T) { Interface("config", config.CCIP.Load). Msg("Test Config") + var env = generateEnvironment() + var env = deployment.Environment{} // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) // require.NoError(t, err) @@ -168,3 +172,9 @@ func TestCCIPLoad_RPS(t *testing.T) { // crib.StopChains(env) // crib.StopNodes(env) } + +func generateEnvironment() { + ab := readAddressBook(abPath) + nIds := readNodeIds(nodeIdsPath) + +} diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index e7afe6883e4..0ef76126967 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -1,11 +1,15 @@ package ccip import ( + "encoding/json" "fmt" "github.com/ethereum/go-ethereum/common" "github.com/rs/zerolog" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "io" + "os" "time" ) @@ -53,3 +57,69 @@ func setLokiLabels(src, dst uint64) map[string]string { "testType": LokiLoadLabel, } } + +func readFile(inputDir string, fileName string) []byte { + file, err := os.Open(fmt.Sprintf("%s/%s", inputDir, fileName)) + if err != nil { + fmt.Println("Error opening file:", err) + panic(err) + } + defer file.Close() + + // Read the file's content into a byte slice + byteValue, err := io.ReadAll(file) + if err != nil { + fmt.Println("Error reading file:", err) + panic(err) + } + return byteValue +} + +func ReadAddressBook(inputDir string) *deployment.AddressBookMap { + byteValue := readFile(inputDir, AddressBookFileName) + + var result map[uint64]map[string]deployment.TypeAndVersion + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + // Print the deserialized map + fmt.Println(result) + return deployment.NewMemoryAddressBookFromMap(result) +} + +func ReadNodesDetails(inputDir string) NodesDetails { + byteValue := readFile(inputDir, NodesDetailsFileName) + + var result NodesDetails + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + // Print the deserialized map + fmt.Println(result) + return result +} + +func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.AddressBook, nodeIDs []string) (*deployment.Environment, error) { + chains, err := devenv.NewChains(lggr, output.Chains) + if err != nil { + return nil, err + } + return deployment.NewEnvironment( + "Crib Environment", + lggr, + ab, + chains, + nodeIDs, + nil, // todo: populate the offchain client using output.DON + ), nil +} diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json new file mode 100644 index 00000000000..e4b2672cb5f --- /dev/null +++ b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json @@ -0,0 +1 @@ +{"12922642891491394802":{"0x05Aa229Aec102f78CE0E852A812a388F076Aa555":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x0D4ff719551E23185Aeb16FFbF2ABEbB90635942":{"Type":"TestRouter","Version":"1.2.0"},"0x0f5D1ef48f12b6f691401bfe88c2037c690a6afe":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x2dE080e97B0caE9825375D31f5D0eD5751fDf16D":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x2fc631e4B3018258759C52AF169200213e84ABab":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x5C7c905B505f0Cf40Ab6600d05e677F717916F6B":{"Type":"Router","Version":"1.2.0"},"0x63cf2Cd54fE91e3545D1379abf5bfd194545259d":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x712516e61C8B383dF4A63CFe83d7701Bce54B03e":{"Type":"LinkToken","Version":"1.0.0"},"0x71C95911E9a5D330f4D621842EC243EE1343292e":{"Type":"PriceFeed","Version":"1.0.0"},"0x73eccD6288e117cAcA738BDAD4FEC51312166C1A":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x8464135c8F25Da09e49BC8782676a84730C318bC":{"Type":"PriceFeed","Version":"1.0.0"},"0x85C5Dd61585773423e378146D4bEC6f8D149E248":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x948B3c65b89DF0B4894ABE91E6D02FE579834F8F":{"Type":"WETH9","Version":"1.0.0"},"0xAfe1b5bdEbD4ae65AF2024738bf0735fbb65d44b":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xC6bA8C3233eCF65B761049ef63466945c362EdD2":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0xbCF26943C0197d2eE0E5D05c716Be60cc2761508":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0xcA03Dc4665A8C3603cb4Fd5Ce71Af9649dC00d44":{"Type":"RBACTimelock","Version":"1.0.0"},"0xe6b98F104c1BEf218F3893ADab4160Dc73Eb8367":{"Type":"ARMProxy","Version":"1.0.0"},"0xfbAb4aa40C202E4e80390171E82379824f7372dd":{"Type":"NonceManager","Version":"1.6.0-dev"}},"3379446385462418246":{"0x09635F643e140090A9A8Dcd712eD6285858ceBef":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x0B306BF915C4d645ff596e518fAf3F9669b97016":{"Type":"LinkToken","Version":"1.0.0"},"0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6":{"Type":"CCIPHome","Version":"1.6.0-dev"},"0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x3Aa5ebB10DC797CAC828524e59A333d0A371443c":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0x4A679253410272dd5232B3Ff7cF5dbB88f295319":{"Type":"RBACTimelock","Version":"1.0.0"},"0x59b670e9fA9D0A427751Af201D676719a970857b":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x67d269191c92Caf3cD7723F116c85e6E9bf55933":{"Type":"ARMProxy","Version":"1.0.0"},"0x7a2088a1bFc9d81c55368AE168C2C02570cB814F":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x851356ae760d987E095750cCeb3bC6014560891C":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x8A791620dd6260079BF849Dc5567aDC3F2FdC318":{"Type":"RMNHome","Version":"1.6.0-dev"},"0x9A676e781A523b5d0C0e43731313A708CB607508":{"Type":"WETH9","Version":"1.0.0"},"0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0x9E545E3C0baAB3E08CdfD552C960A1050f373042":{"Type":"NonceManager","Version":"1.6.0-dev"},"0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E":{"Type":"Router","Version":"1.2.0"},"0xa513E6E4b8f2a923D98304ec87F64353C4D5C853":{"Type":"CapabilitiesRegistry","Version":"1.0.0"},"0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690":{"Type":"TestRouter","Version":"1.2.0"}}} diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json new file mode 100644 index 00000000000..477ae0527b1 --- /dev/null +++ b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json @@ -0,0 +1 @@ +{"NodeIDs":["node_2URuou3RXmtZu5gLQX8qd","node_m9TTQbUxBx3WjDEjmpVDL","node_4FiKVPtuQjCTvHnS7QpES","node_A4VTgecDwMoG2YYicyjuG","node_jQFpzXDadzaADq147nThS"]} From 4f214045311fb346c0e9c729050a8f65475152e1 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 17 Dec 2024 15:21:26 +0800 Subject: [PATCH 6/8] build --- deployment/environment.go | 1 - deployment/environment/memory/sim.go | 4 - integration-tests/load/ccip/ccip_test.go | 57 +++++-------- .../load/ccip/destination_gun.go | 25 +++--- integration-tests/load/ccip/helpers.go | 84 +++++++------------ 5 files changed, 63 insertions(+), 108 deletions(-) diff --git a/deployment/environment.go b/deployment/environment.go index 7b5d1b6c9e3..0823404da2d 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -35,7 +35,6 @@ type OnchainClient interface { bind.DeployBackend BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) - BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) } // OffchainClient interacts with the job-distributor diff --git a/deployment/environment/memory/sim.go b/deployment/environment/memory/sim.go index 93f77a0d29b..c0fba87e2b3 100644 --- a/deployment/environment/memory/sim.go +++ b/deployment/environment/memory/sim.go @@ -80,10 +80,6 @@ func (b *Backend) NonceAt(ctx context.Context, account common.Address, blockNumb return b.Sim.Client().NonceAt(ctx, account, blockNumber) } -func (b *Backend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - return b.Sim.Client().BlockByNumber(ctx, number) -} - func NewBackend(sim *simulated.Backend) *Backend { if sim == nil { panic("simulated backend is nil") diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 79c87106960..6f364252fb9 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -3,11 +3,10 @@ package ccip import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink-testing-framework/wasp" - "github.com/smartcontractkit/chainlink/deployment" - ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" "github.com/stretchr/testify/require" "math/big" @@ -21,9 +20,7 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup - abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" - nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" + wg sync.WaitGroup ) // step 1: setup @@ -36,32 +33,24 @@ var ( // Stop the chains, cleanup the environment func TestCCIPLoad_RPS(t *testing.T) { ctx := tests.Context(t) - l := logging.GetTestLogger(t) + lggr := logger.Test(t) config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) - if err != nil { - t.Fatal(err) - } - l.Info().Interface("loadedTestConfig", config).Msg("Loaded Test Config") - - l.Info().Msg("Starting ccip load test") - l.Info(). - Int("Number of Nodes", *(config.CCIP.Load.NoOfNodes)). - Interface("config", config.CCIP.Load). - Msg("Test Config") + require.NoError(t, err) + lggr.Infof("loaded test config: %+v", config) - var env = generateEnvironment() + lggr.Info("starting ccip load test") + lggr.Infof("Number of nodes: %d", *(config.CCIP.Load.NoOfNodes)) + lggr.Infof("config: %+v", config.CCIP.Load) + lggr.Info("Test Config") - var env = deployment.Environment{} - // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) - // require.NoError(t, err) - // env, err = crib.DeployPrerequisites(output, config.CCIP) - // merge addressbooks - // env, err := crib.DeployCCIPContracts(output, config.CCIP) + env, err := CreateEnvironmentFromCribOutput(t, lggr) + require.NoError(t, err) + require.NotNil(t, env) // Need to keep track of the block number for each chain so that event subscription can be done from that block. startBlocks := make(map[uint64]*uint64) - state, err := ccipdeployment.LoadOnchainState(env) + state, err := ccipchangeset.LoadOnchainState(*env) require.NoError(t, err) cfgl := config.Logging.Loki @@ -69,7 +58,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // Parse all events from the simulated chains, send to Loki loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) if err != nil { - l.Error().Err(err).Msg("Failed to create Loki client") + env.Logger.Error("failed to create loki client") return } defer loki.StopNow() @@ -92,7 +81,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) // so if there are 3 generators, it would be 3 requests per 10 seconds over the network - Gun: NewDestinationGun(l, selector, env, state.Chains[selector].Receiver.Address(), loki), + Gun: NewDestinationGun(env.Logger, selector, env, state.Chains[selector].Receiver.Address(), loki), Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), // use the same loki client using `NewLokiClient` with the same config for sending events @@ -122,9 +111,9 @@ func TestCCIPLoad_RPS(t *testing.T) { fmt.Printf("CommitReportAccepted event: %+v\n", event) blockNum := commitIterator.Event.Raw.BlockNumber - block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(block.Time()), 0) + timestamp := time.Unix(int64(header.Time()), 0) for _, root := range event.MerkleRoots { lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) @@ -148,9 +137,9 @@ func TestCCIPLoad_RPS(t *testing.T) { fmt.Printf("ExecutionStateChanged event: %+v\n", event) blockNum := execIterator.Event.Raw.BlockNumber - block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(block.Time()), 0) + timestamp := time.Unix(int64(header.Time), 0) // todo: push loki calls to channel? lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) @@ -172,9 +161,3 @@ func TestCCIPLoad_RPS(t *testing.T) { // crib.StopChains(env) // crib.StopNodes(env) } - -func generateEnvironment() { - ab := readAddressBook(abPath) - nIds := readNodeIds(nodeIdsPath) - -} diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 455f6dcc186..3ba9e2416ad 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -5,12 +5,13 @@ import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" - ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "sync/atomic" "time" ) @@ -21,7 +22,7 @@ type ChainSelectorPair struct { } type DestinationGun struct { - l zerolog.Logger + l logger.Logger env deployment.Environment seqNums map[ChainSelectorPair]*atomic.Uint64 roundNum *atomic.Int32 @@ -30,7 +31,7 @@ type DestinationGun struct { loki *wasp.LokiClient } -func NewDestinationGun(l zerolog.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { +func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { seqNums := make(map[ChainSelectorPair]*atomic.Uint64) for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { @@ -60,7 +61,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { waspGroup := fmt.Sprintf("%d-%s", m.chainSelector, "messageOnly") - state, err := ccipdeployment.LoadOnchainState(m.env) + state, err := ccipchangeset.LoadOnchainState(m.env) if err != nil { return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } @@ -75,12 +76,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { dst: m.chainSelector, } m.seqNums[csPair].Add(1) - m.l.Info(). - Int32("RoundNum", requestedRound). - Uint64("Destination ChainSelector", m.chainSelector). - Uint64("Source ChainSelector", src). - Uint64("SequenceNumber", m.seqNums[csPair].Load()). - Msg("starting transmit") + m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Destination ChainSelector", m.chainSelector, "Source ChainSelector", src, "SequenceNumber", m.seqNums[csPair].Load()) r := state.Chains[src].Router @@ -116,10 +112,10 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { return &wasp.Response{Failed: false, Group: waspGroup} } -// GetCycledClient will return a valid client from a random source chain +// MustSourceChain will return a chain selector to send a message from func (m *DestinationGun) MustSourceChain() (uint64, error) { - // TODO: check if this chain has sent a message recently, if so, switch to the next chain + // TODO: make this smarter by checking if this chain has sent a message recently, if so, switch to the next chain otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) if len(otherCS) == 0 { return 0, fmt.Errorf("no other chains to send from") @@ -133,7 +129,8 @@ func (m *DestinationGun) MustSourceChain() (uint64, error) { func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { rcv, err := utils.ABIEncode(`[{"type":"address"}]`, m.receiver) if err != nil { - m.l.Error().Err(err).Msg("Error encoding receiver address") + m.l.Error("Error encoding receiver address") + return router.ClientEVM2AnyMessage{}, err } return router.ClientEVM2AnyMessage{ diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index 0ef76126967..0df27431963 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -1,15 +1,17 @@ package ccip import ( + "context" "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/stretchr/testify/require" "io" "os" + "testing" "time" ) @@ -17,9 +19,12 @@ const ( transmitted = iota committed executed - LokiLoadLabel = "ccip_load_test" - ErrLokiClient = "failed to create Loki client for monitoring" - ErrLokiPush = "failed to push metrics to Loki" + LokiLoadLabel = "ccip_load_test" + ErrLokiClient = "failed to create Loki client for monitoring" + ErrLokiPush = "failed to push metrics to Loki" + abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" + nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" + chainConfigPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-chains-details.json" ) // todo: Have a different struct for commit/exec? @@ -30,36 +35,23 @@ type LokiMetric struct { SequenceNumber uint64 `json:"sequence_number"` } -func GetAddressFromTypeAndVersion(ab deployment.AddressBook, cs uint64, tv string) (common.Address, error) { - allAddr, err := ab.AddressesForChain(cs) - if err != nil { - return common.Address{}, err - } - for addr, tv := range allAddr { - if tv.Type == tv.Type && tv.Version == tv.Version { - return common.HexToAddress(addr), nil - } - } - - return common.Address{}, fmt.Errorf("address not found for chain selector %d and typeAndVersion %s", cs, tv) -} - -func SendMetricsToLoki(l zerolog.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { +func SendMetricsToLoki(l logger.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { - l.Error().Err(err).Msg(ErrLokiPush) + l.Error(ErrLokiPush) } } func setLokiLabels(src, dst uint64) map[string]string { return map[string]string{ - "sourceSelector": fmt.Sprintf("%d", src), + "sourceEvmChainId": fmt.Sprintf("%d", src), + "destEvmChainId": fmt.Sprintf("%d", src), "destinationSelector": fmt.Sprintf("%d", dst), "testType": LokiLoadLabel, } } -func readFile(inputDir string, fileName string) []byte { - file, err := os.Open(fmt.Sprintf("%s/%s", inputDir, fileName)) +func readFile(filePath string) []byte { + file, err := os.Open(filePath) if err != nil { fmt.Println("Error opening file:", err) panic(err) @@ -75,42 +67,28 @@ func readFile(inputDir string, fileName string) []byte { return byteValue } -func ReadAddressBook(inputDir string) *deployment.AddressBookMap { - byteValue := readFile(inputDir, AddressBookFileName) - - var result map[uint64]map[string]deployment.TypeAndVersion +func readFromFile[T []string | *deployment.AddressBookMap | []devenv.ChainConfig](t *testing.T, inputDir string) T { + byteValue := readFile(inputDir) + var result T // Unmarshal the JSON into the map err := json.Unmarshal(byteValue, &result) - if err != nil { - fmt.Println("Error unmarshalling JSON:", err) - panic(err) - } + require.NoError(t, err) // Print the deserialized map fmt.Println(result) - return deployment.NewMemoryAddressBookFromMap(result) + return result } -func ReadNodesDetails(inputDir string) NodesDetails { - byteValue := readFile(inputDir, NodesDetailsFileName) +func CreateEnvironmentFromCribOutput(t *testing.T, lggr logger.Logger) (*deployment.Environment, error) { + ab := readFromFile[*deployment.AddressBookMap](t, abPath) + nodeIds := readFromFile[[]string](t, nodeIdsPath) + chainDetails := readFromFile[[]devenv.ChainConfig](t, chainConfigPath) - var result NodesDetails - - // Unmarshal the JSON into the map - err := json.Unmarshal(byteValue, &result) - if err != nil { - fmt.Println("Error unmarshalling JSON:", err) - panic(err) - } - - // Print the deserialized map - fmt.Println(result) - return result -} + // todo: make sure to call chainDetails.SetDeployerKey() for each chain + // where private keys should be stored in env vars or .toml -func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.AddressBook, nodeIDs []string) (*deployment.Environment, error) { - chains, err := devenv.NewChains(lggr, output.Chains) + chains, err := devenv.NewChains(lggr, chainDetails) if err != nil { return nil, err } @@ -119,7 +97,9 @@ func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.Addres lggr, ab, chains, - nodeIDs, - nil, // todo: populate the offchain client using output.DON + nodeIds, + nil, + func() context.Context { return context.Background() }, + deployment.XXXGenerateTestOCRSecrets(), ), nil } From 8c01cc2818f2f57939655c87915106a0e55abe69 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 17 Dec 2024 19:36:18 +0800 Subject: [PATCH 7/8] load test transmitting --- deployment/ccip/changeset/state.go | 2 ++ integration-tests/load/ccip/ccip_test.go | 34 ++++++++----------- .../load/ccip/destination_gun.go | 9 ++--- integration-tests/testconfig/ccip/ccip.toml | 5 +++ integration-tests/testconfig/ccip/load.go | 3 +- 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/deployment/ccip/changeset/state.go b/deployment/ccip/changeset/state.go index 2403f3f7cc2..7ce9eeb5abb 100644 --- a/deployment/ccip/changeset/state.go +++ b/deployment/ccip/changeset/state.go @@ -380,6 +380,8 @@ func LoadChainState(chain deployment.Chain, addresses map[string]deployment.Type return state, err } state.OffRamp = offRamp + case deployment.NewTypeAndVersion(ARMProxy, deployment.Version1_6_0_dev).String(): + fallthrough case deployment.NewTypeAndVersion(ARMProxy, deployment.Version1_0_0).String(): armProxy, err := rmn_proxy_contract.NewRMNProxyContract(common.HexToAddress(address), chain.Client) if err != nil { diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 6f364252fb9..cdc98dc133a 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -7,6 +7,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-testing-framework/wasp" ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + crib "github.com/smartcontractkit/chainlink/deployment/environment/crib" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" "github.com/stretchr/testify/require" "math/big" @@ -23,6 +24,8 @@ var ( wg sync.WaitGroup ) +const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" + // step 1: setup // Parse the test config, initialize CRIB with configurations defined // step 2: load @@ -37,14 +40,12 @@ func TestCCIPLoad_RPS(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) require.NoError(t, err) - lggr.Infof("loaded test config: %+v", config) + lggr.Infof("loaded ccip test config: %+v", config.CCIP.Load) - lggr.Info("starting ccip load test") - lggr.Infof("Number of nodes: %d", *(config.CCIP.Load.NoOfNodes)) - lggr.Infof("config: %+v", config.CCIP.Load) - lggr.Info("Test Config") + cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) - env, err := CreateEnvironmentFromCribOutput(t, lggr) + cribDeployOutput := cribEnv.GetConfig() + env, err := crib.NewDeployEnvironmentFromCribOutput(lggr, cribDeployOutput) require.NoError(t, err) require.NotNil(t, env) @@ -53,17 +54,12 @@ func TestCCIPLoad_RPS(t *testing.T) { state, err := ccipchangeset.LoadOnchainState(*env) require.NoError(t, err) - cfgl := config.Logging.Loki - // Parse all events from the simulated chains, send to Loki - loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) - if err != nil { - env.Logger.Error("failed to create loki client") - return - } + loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil)) + require.NoError(t, err) defer loki.StopNow() - // Based on the config, initiate a DestinationGun + // Based on the config, initialize DestinationGun p := wasp.NewProfile() for selector, chain := range env.Chains { latesthdr, err := chain.Client.HeaderByNumber(ctx, nil) @@ -81,9 +77,9 @@ func TestCCIPLoad_RPS(t *testing.T) { // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) // so if there are 3 generators, it would be 3 requests per 10 seconds over the network - Gun: NewDestinationGun(env.Logger, selector, env, state.Chains[selector].Receiver.Address(), loki), + Gun: NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), loki), Labels: CommonTestLabels, - LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), + LokiConfig: wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil), // use the same loki client using `NewLokiClient` with the same config for sending events })) } @@ -113,14 +109,14 @@ func TestCCIPLoad_RPS(t *testing.T) { blockNum := commitIterator.Event.Raw.BlockNumber header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(header.Time()), 0) + timestamp := time.Unix(int64(header.Time), 0) for _, root := range event.MerkleRoots { lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? - SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: committed, Timestamp: timestamp, SequenceNumber: i, @@ -143,7 +139,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // todo: push loki calls to channel? lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) - SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: executed, Timestamp: timestamp, GasUsed: execIterator.Event.GasUsed.Uint64(), diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 3ba9e2416ad..acc33b93bdc 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -11,8 +11,7 @@ import ( ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" - - "sync/atomic" + "go.uber.org/atomic" "time" ) @@ -38,11 +37,7 @@ func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Env seqNums[ChainSelectorPair{ src: cs, dst: chainSelector, - }] = &atomic.Uint64{} - seqNums[ChainSelectorPair{ - src: chainSelector, - dst: cs, - }].Store(1) + }] = atomic.NewUint64(1) } return &DestinationGun{ l: l, diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 3f4ba43c48c..b3e3b944b77 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -240,3 +240,8 @@ addresses_to_fund = [ [Seth] # Seth specific configuration, no need for generating ephemeral addresses for ccip-tests. ephemeral_addresses_number = 0 + + +[Load.CCIP.Load] +LokiEndpoint = "http://localhost:3030/loki/api/v1/push" +NoOfNodes = 4 diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go index af2e89db79a..5b226d6e7eb 100644 --- a/integration-tests/testconfig/ccip/load.go +++ b/integration-tests/testconfig/ccip/load.go @@ -1,5 +1,6 @@ package ccip type LoadConfig struct { - NoOfNodes *int + NoOfNodes *int + LokiEndpoint *string } From 44152cb31baa45a91994b46f5f828cd3f14dc4a6 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 18 Dec 2024 18:48:32 +0800 Subject: [PATCH 8/8] load test, have transaction, need to use event watcher --- deployment/environment/crib/env.go | 18 +++++++++++++-- deployment/environment/devenv/chain.go | 1 - integration-tests/load/ccip/ccip_test.go | 23 ++++++++++++++----- .../load/ccip/destination_gun.go | 12 +++++++--- integration-tests/load/ccip/helpers.go | 17 ++++++++++---- 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/deployment/environment/crib/env.go b/deployment/environment/crib/env.go index 3af1acaf754..281565480df 100644 --- a/deployment/environment/crib/env.go +++ b/deployment/environment/crib/env.go @@ -1,5 +1,7 @@ package crib +import "fmt" + const ( AddressBookFileName = "ccip-v2-scripts-address-book.json" NodesDetailsFileName = "ccip-v2-scripts-nodes-details.json" @@ -16,15 +18,27 @@ func NewDevspaceEnvFromStateDir(envStateDir string) CRIBEnv { } } -func (c CRIBEnv) GetConfig() DeployOutput { +func (c CRIBEnv) GetConfig(deployerKeys map[uint64]string) (DeployOutput, error) { reader := NewOutputReader(c.envStateDir) nodesDetails := reader.ReadNodesDetails() chainConfigs := reader.ReadChainConfigs() + for i, chain := range chainConfigs { + key, ok := deployerKeys[chain.ChainID] + if !ok { + return DeployOutput{}, fmt.Errorf("could not find deployer key for %s", key) + } + err := chain.SetDeployerKey(&key) + if err != nil { + return DeployOutput{}, err + } + chainConfigs[i] = chain + } + return DeployOutput{ AddressBook: reader.ReadAddressBook(), NodeIDs: nodesDetails.NodeIDs, Chains: chainConfigs, - } + }, nil } type RPC struct { diff --git a/deployment/environment/devenv/chain.go b/deployment/environment/devenv/chain.go index 5c6c4336ed7..a5c706ecd3c 100644 --- a/deployment/environment/devenv/chain.go +++ b/deployment/environment/devenv/chain.go @@ -67,7 +67,6 @@ func (c *ChainConfig) SetDeployerKey(pvtKeyStr *string) error { if err != nil { return fmt.Errorf("failed to create transactor: %w", err) } - fmt.Printf("Deployer Address: %s for chain id %d\n", deployer.From.Hex(), c.ChainID) c.DeployerKey = deployer return nil } diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index cdc98dc133a..8f5233fcfda 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -21,7 +21,11 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup + wg sync.WaitGroup + SIM_CHAIN_PRIVATE_KEYS = map[uint64]string{ + 1337: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + 2337: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + } ) const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" @@ -44,7 +48,8 @@ func TestCCIPLoad_RPS(t *testing.T) { cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) - cribDeployOutput := cribEnv.GetConfig() + cribDeployOutput, err := cribEnv.GetConfig(SIM_CHAIN_PRIVATE_KEYS) + require.NoError(t, err) env, err := crib.NewDeployEnvironmentFromCribOutput(lggr, cribDeployOutput) require.NoError(t, err) require.NotNil(t, env) @@ -71,8 +76,8 @@ func TestCCIPLoad_RPS(t *testing.T) { T: t, GenName: "ccipLoad", LoadType: wasp.RPS, - CallTimeout: 20 * time.Second, - Schedule: wasp.Plain(1, 20*time.Second), + CallTimeout: 5 * time.Second, + Schedule: wasp.Plain(1, 5*time.Second), // will need to be divided by number of chains // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) @@ -91,6 +96,8 @@ func TestCCIPLoad_RPS(t *testing.T) { wg.Add(1) go func(chainSelector uint64, startBlock *uint64) { defer wg.Done() + lggr.Infow("Starting to query for events on ", "chainSelector", chainSelector, "startblock", startBlock) + filterOpts := &bind.FilterOpts{ Start: *startBlock, End: nil, // To the latest block @@ -102,6 +109,8 @@ func TestCCIPLoad_RPS(t *testing.T) { commitIterator, err := offRamp.FilterCommitReportAccepted(filterOpts) require.NoError(t, err) + fmt.Printf("Events on commitIterator %+v", commitIterator) + for commitIterator.Next() { event := commitIterator.Event fmt.Printf("CommitReportAccepted event: %+v\n", event) @@ -112,7 +121,8 @@ func TestCCIPLoad_RPS(t *testing.T) { timestamp := time.Unix(int64(header.Time), 0) for _, root := range event.MerkleRoots { - lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) + lokiLabels, err = setLokiLabels(root.SourceChainSelector, chainSelector) + require.NoError(t, err) for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? @@ -138,7 +148,8 @@ func TestCCIPLoad_RPS(t *testing.T) { timestamp := time.Unix(int64(header.Time), 0) // todo: push loki calls to channel? - lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + lokiLabels, err = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + require.NoError(t, err) SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: executed, Timestamp: timestamp, diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index acc33b93bdc..c517cdd0ca2 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -81,23 +81,29 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { } fee, err := r.GetFee( - &bind.CallOpts{Context: context.Background()}, src, msg) + &bind.CallOpts{Context: context.Background()}, m.chainSelector, msg) if err != nil { + m.l.Errorw("could not get fee ", "dstChainSelector", m.chainSelector, "msg", msg, "fee", fee) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } + m.l.Debugw("setting fee for ", "srcChain", src, "dstChain", m.chainSelector, "fee", fee, "msg", msg) if msg.FeeToken == common.HexToAddress("0x0") { m.env.Chains[src].DeployerKey.Value = fee defer func() { m.env.Chains[src].DeployerKey.Value = nil }() } - _, err = r.CcipSend( + tx, err := r.CcipSend( m.env.Chains[src].DeployerKey, m.chainSelector, msg) if err != nil { + m.l.Errorw("execution reverted from ", "sourceChain", src, "destchain", m.chainSelector, "err", err, "tx", tx) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } - lokiLabels := setLokiLabels(src, m.chainSelector) + lokiLabels, err := setLokiLabels(src, m.chainSelector) + if err != nil { + m.l.Errorw("Failed setting loki labels", "error", err) + } SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ EventType: transmitted, Timestamp: time.Now(), diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index 0df27431963..faeede61e7a 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" @@ -41,13 +42,21 @@ func SendMetricsToLoki(l logger.Logger, lc *wasp.LokiClient, updatedLabels map[s } } -func setLokiLabels(src, dst uint64) map[string]string { +func setLokiLabels(src, dst uint64) (map[string]string, error) { + srcChainId, err := chainselectors.GetChainIDFromSelector(src) + if err != nil { + return nil, err + } + dstChainId, err := chainselectors.GetChainIDFromSelector(dst) + if err != nil { + return nil, err + } return map[string]string{ - "sourceEvmChainId": fmt.Sprintf("%d", src), - "destEvmChainId": fmt.Sprintf("%d", src), + "sourceEvmChainId": fmt.Sprintf("%s", srcChainId), + "destEvmChainId": fmt.Sprintf("%s", dstChainId), "destinationSelector": fmt.Sprintf("%d", dst), "testType": LokiLoadLabel, - } + }, nil } func readFile(filePath string) []byte {