Skip to content

Commit

Permalink
Cleanup the ccip e2e test (#1231)
Browse files Browse the repository at this point in the history 
While reading the test some things were not obvious.
This PR has some small improvements.
  • Loading branch information
@dimkouv
dimkouv authored Jul 30, 2024
1 parent 451984a commit cdb2d54
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 77 deletions.
5 changes: 3 additions & 2 deletions core/services/ccipcapability/oraclecreator/inprocess.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ import (
"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types"
ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types"

cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"

commitocr3 "github.com/smartcontractkit/chainlink-ccip/commit"
execocr3 "github.com/smartcontractkit/chainlink-ccip/execute"
ccipreaderpkg "github.com/smartcontractkit/chainlink-ccip/pkg/reader"
cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"

"github.com/smartcontractkit/chainlink-common/pkg/types"
"github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm"
Expand Down Expand Up @@ -249,7 +250,7 @@ func (i *inprocessOracleCreator) CreatePluginOracle(pluginType cctypes.PluginTyp
return nil, fmt.Errorf("no transmitter found for dest relay ID %s, can't create contract transmitter", destRelayID)
}

//TODO: Extract the correct transmitter address from the destsFromAccount
// TODO: Extract the correct transmitter address from the destsFromAccount
var factory ocr3types.ReportingPluginFactory[[]byte]
var transmitter ocr3types.ContractTransmitter[[]byte]
if config.Config.PluginType == uint8(cctypes.PluginTypeCCIPCommit) {
Expand Down
34 changes: 18 additions & 16 deletions core/services/ocr3/plugins/ccip_integration_tests/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,13 @@ type chainBase struct {
// 2. Deploys the CCIP contracts to all chains.
// 3. Sets up the initial configurations for the contracts on all chains.
// 4. Wires the chains together.
//
// Conceptually one universe is ONE chain with all the contracts deployed on it and all the dependencies initialized.
func createUniverses(
t *testing.T,
numUniverses int,
numChains int,
) (homeChainUni homeChain, universes map[uint64]onchainUniverse) {
chains := createChains(t, numUniverses)
chains := createChains(t, numChains)

homeChainBase, ok := chains[homeChainID]
require.True(t, ok, "home chain backend not available")
Expand All @@ -119,9 +121,9 @@ func createUniverses(
tokenAdminRegistry := deployTokenAdminRegistry(t, owner, backend, chainID)
nonceManager := deployNonceManager(t, owner, backend, chainID)

//======================================================================
// ======================================================================
// OnRamp
//======================================================================
// ======================================================================
onRampAddr, _, _, err := evm_2_evm_multi_onramp.DeployEVM2EVMMultiOnRamp(
owner,
backend,
Expand All @@ -134,7 +136,7 @@ func createUniverses(
evm_2_evm_multi_onramp.EVM2EVMMultiOnRampDynamicConfig{
Router: rout.Address(),
PriceRegistry: priceRegistry.Address(),
//`withdrawFeeTokens` onRamp function is not part of the message flow
// `withdrawFeeTokens` onRamp function is not part of the message flow
// so we can set this to any address
FeeAggregator: testutils.NewAddress(),
},
Expand All @@ -144,9 +146,9 @@ func createUniverses(
onramp, err := evm_2_evm_multi_onramp.NewEVM2EVMMultiOnRamp(onRampAddr, backend)
require.NoError(t, err)

//======================================================================
// ======================================================================
// OffRamp
//======================================================================
// ======================================================================
offrampAddr, _, _, err := evm_2_evm_multi_offramp.DeployEVM2EVMMultiOffRamp(
owner,
backend,
Expand Down Expand Up @@ -506,25 +508,25 @@ func connectUniverses(
// 2. Set the price registry with local token prices
// 3. Authorize the onRamp and offRamp on the nonce manager
func setupUniverseBasics(t *testing.T, uni onchainUniverse) {
//=============================================================================
// =============================================================================
// Universe specific updates/configs
// These updates are specific to each universe and are set up here
// These updates don't depend on other chains
//=============================================================================
// =============================================================================
owner := uni.owner
//=============================================================================
// =============================================================================
// Mint 1000 LINK to owner
//=============================================================================
// =============================================================================
_, err := uni.linkToken.GrantMintRole(owner, owner.From)
require.NoError(t, err)
_, err = uni.linkToken.Mint(owner, owner.From, e18Mult(1000))
require.NoError(t, err)
uni.backend.Commit()

//=============================================================================
// =============================================================================
// Price updates for tokens
// These are the prices of the fee tokens of local chain in USD
//=============================================================================
// =============================================================================
tokenPriceUpdates := []price_registry.InternalTokenPriceUpdate{
{
SourceToken: uni.linkToken.Address(),
Expand All @@ -549,10 +551,10 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) {
require.NoError(t, err, "failed to authorize offramp on price registry")
uni.backend.Commit()

//=============================================================================
// =============================================================================
// Authorize OnRamp & OffRamp on NonceManager
// Otherwise the onramp will not be able to call the nonceManager to get next Nonce
//=============================================================================
// =============================================================================
authorizedCallersAuthorizedCallerArgs := nonce_manager.AuthorizedCallersAuthorizedCallerArgs{
AddedCallers: []common.Address{
uni.onramp.Address(),
Expand Down Expand Up @@ -767,7 +769,7 @@ func deployPriceRegistry(
[]price_registry.PriceRegistryTokenTransferFeeConfigArgs{},
[]price_registry.PriceRegistryPremiumMultiplierWeiPerEthArgs{
{
PremiumMultiplierWeiPerEth: 9e17, //0.9 ETH
PremiumMultiplierWeiPerEth: 9e17, // 0.9 ETH
Token: linkAddr,
},
{
Expand Down
111 changes: 60 additions & 51 deletions core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/hashicorp/consul/sdk/freeport"
"go.uber.org/zap/zapcore"

"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ping_pong_demo"

Expand All @@ -18,35 +19,45 @@ import (
"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey"

"github.com/smartcontractkit/libocr/commontypes"
confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types"

"github.com/stretchr/testify/require"
)

func TestIntegration_OCR3Nodes(t *testing.T) {
numChains := 3
const (
numChains = 3 // number of chains that this test will run on
numNodes = 4 // number of OCR3 nodes, test assumes that every node supports every chain

simulatedBackendBlockTime = 900 * time.Millisecond // Simulated backend blocks committing interval
oraclesBootWaitTime = 30 * time.Second // Time to wait for oracles to come up (HACK)
fChain = 1 // fChain value for all the chains
oracleLogLevel = zapcore.InfoLevel // Log level for the oracle / plugins.
)

t.Logf("creating %d universes", numChains)
homeChainUni, universes := createUniverses(t, numChains)
numNodes := 4
t.Log("creating ocr3 nodes")

var (
oracles = make(map[uint64][]confighelper2.OracleIdentityExtra)
apps []chainlink.Application
nodes []*ocr3Node
p2pIDs [][32]byte

// The bootstrap node will be the first node (index 0)
// The bootstrap node will be: nodes[0]
bootstrapPort int
bootstrapP2PID p2pkey.PeerID
bootstrappers []commontypes.BootstrapperLocator
)

ports := freeport.GetN(t, numNodes)
ctx := testutils.Context(t)
callCtx := &bind.CallOpts{Context: ctx}

for i := 0; i < numNodes; i++ {
node := setupNodeOCR3(t, ports[i], bootstrappers, universes, homeChainUni)
t.Logf("Setting up ocr3 node:%d at port:%d", i, ports[i])
node := setupNodeOCR3(t, ports[i], universes, homeChainUni, oracleLogLevel)

apps = append(apps, node.app)
for chainID, transmitter := range node.transmitters {
identity := confighelper2.OracleIdentityExtra{
OracleIdentity: confighelper2.OracleIdentity{
Expand All @@ -59,101 +70,95 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
}
oracles[chainID] = append(oracles[chainID], identity)
}

apps = append(apps, node.app)
nodes = append(nodes, node)

peerID, err := p2pkey.MakePeerID(node.peerID)
require.NoError(t, err)
p2pIDs = append(p2pIDs, peerID)

// First Node is the bootstrap node
if i == 0 {
bootstrapPort = ports[i]
bootstrapP2PID = peerID
bootstrappers = []commontypes.BootstrapperLocator{
{PeerID: node.peerID, Addrs: []string{
fmt.Sprintf("127.0.0.1:%d", bootstrapPort),
}},
}
}
}

bootstrapPort = ports[0]
bootstrapP2PID = p2pIDs[0]
bootstrapAddr := fmt.Sprintf("127.0.0.1:%d", bootstrapPort)
t.Logf("[bootstrap node] peerID:%s p2pID:%d address:%s", nodes[0].peerID, bootstrapP2PID, bootstrapAddr)

// Start committing periodically in the background for all the chains
tick := time.NewTicker(900 * time.Millisecond)
tick := time.NewTicker(simulatedBackendBlockTime)
defer tick.Stop()
commitBlocksBackground(t, universes, tick)

ctx := testutils.Context(t)

ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(&bind.CallOpts{
Context: ctx,
}, CapabilityLabelledName, CapabilityVersion)
ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(
callCtx, CapabilityLabelledName, CapabilityVersion)
require.NoError(t, err, "failed to get hashed capability id for ccip")
require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty")

// Need to Add nodes and assign capabilities to them before creating DONS
homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID})

// Add homechain configs
for _, uni := range universes {
AddChainConfig(t, homeChainUni, getSelector(uni.chainID), p2pIDs, 1)
t.Logf("Adding chainconfig for chain %d", uni.chainID)
AddChainConfig(t, homeChainUni, getSelector(uni.chainID), p2pIDs, fChain)
}

cfgs, err3 := homeChainUni.ccipConfig.GetAllChainConfigs(&bind.CallOpts{})
require.NoError(t, err3)
t.Logf("homechain_configs %+v", cfgs)
cfgs, err := homeChainUni.ccipConfig.GetAllChainConfigs(callCtx)
require.NoError(t, err)
t.Logf("Got all homechain configs %#v", cfgs)
require.Len(t, cfgs, numChains)

// Create a DON for each chain
for _, uni := range universes {
// Add nodes and give them the capability
t.Log("AddingDON for universe: ", uni.chainID)
t.Log("Adding DON for universe: ", uni.chainID)
chainSelector := getSelector(uni.chainID)
homeChainUni.AddDON(
t,
ccipCapabilityID,
chainSelector,
uni,
1, // f
fChain,
bootstrapP2PID,
p2pIDs,
oracles[uni.chainID],
)
}

t.Log("creating ocr3 jobs")
t.Log("Creating ocr3 jobs, starting oracles")
for i := 0; i < len(nodes); i++ {
err1 := nodes[i].app.Start(ctx)
require.NoError(t, err1)
tApp := apps[i]
t.Cleanup(func() {
require.NoError(t, tApp.Stop())
})
t.Cleanup(func() { require.NoError(t, tApp.Stop()) })

jb := mustGetJobSpec(t, bootstrapP2PID, bootstrapPort, nodes[i].peerID, nodes[i].keybundle.ID())
require.NoErrorf(t, tApp.AddJobV2(ctx, &jb), "Wasn't able to create ccip job for node %d", i)
}

// sourceChain map[uint64], destChain [32]byte
var messageIDs = make(map[uint64]map[uint64][32]byte)
// map[uint64] chainID, blocks
var replayBlocks = make(map[uint64]uint64)
t.Logf("Initializing PingPong contracts")
pingPongs := initializePingPongContracts(t, universes)

// NOTE: messageIDs are populated in the sendPingPong function
var messageIDs = make(map[uint64]map[uint64][32]byte) // sourceChain->destChain->messageID
var replayBlocks = make(map[uint64]uint64) // chainID -> blocksToReplay

t.Logf("Sending ping pong from each chain to each other")
sendPingPong(t, universes, pingPongs, messageIDs, replayBlocks, 1)
// HACK: wait for the oracles to come up.
// Need some data driven way to do this.
time.Sleep(30 * time.Second)

// replay the log poller on all the chains so that the logs are in the db.
// Wait for the oracles to come up.
// TODO: We need some data driven way to do this e.g. wait until LP filters to be registered.
time.Sleep(oraclesBootWaitTime)

// Replay the log poller on all the chains so that the logs are in the db.
// otherwise the plugins won't pick them up.
// TODO: this is happening too early, we need to wait for the chain readers to get their config
// and register the LP filters before this has any effect.
for _, node := range nodes {
for chainID, replayBlock := range replayBlocks {
t.Logf("Replaying logs for chain %d from block %d", chainID, replayBlock)
require.NoError(t, node.app.ReplayFromBlock(big.NewInt(int64(chainID)), replayBlock, false), "failed to replay logs")
}
}

// Wait for the commit reports to be generated and reported on all chains.
numUnis := len(universes)
var wg sync.WaitGroup
for _, uni := range universes {
Expand All @@ -164,15 +169,17 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
}(uni)
}

t.Log("waiting for commit reports")
tStart := time.Now()
t.Log("Waiting for commit reports")
wg.Wait()
t.Logf("Commit reports received after %s", time.Since(tStart))

var preRequestBlocks = make(map[uint64]uint64)
for _, uni := range universes {
preRequestBlocks[uni.chainID] = uni.backend.Blockchain().CurrentBlock().Number.Uint64()
}

t.Log("PingPong AGAIN")
t.Log("Sending ping pong from each chain to each other again for a second time")
sendPingPong(t, universes, pingPongs, messageIDs, replayBlocks, 2)

for _, uni := range universes {
Expand All @@ -184,8 +191,10 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
}(uni, &startBlock)
}

t.Log("waiting for second batch of commit reports")
tStart = time.Now()
t.Log("Waiting for second batch of commit reports")
wg.Wait()
t.Logf("Second batch of commit reports received after %s", time.Since(tStart))
}

func sendPingPong(t *testing.T, universes map[uint64]onchainUniverse, pingPongs map[uint64]map[uint64]*ping_pong_demo.PingPongDemo, messageIDs map[uint64]map[uint64][32]byte, replayBlocks map[uint64]uint64, expectedSeqNum uint64) {
Expand Down Expand Up @@ -250,7 +259,7 @@ func sendPingPong(t *testing.T, universes map[uint64]onchainUniverse, pingPongs

func waitForCommit(t *testing.T, uni onchainUniverse, numUnis int, startBlock *uint64) {
sink := make(chan *evm_2_evm_multi_offramp.EVM2EVMMultiOffRampCommitReportAccepted)
subscipriton, err := uni.offramp.WatchCommitReportAccepted(&bind.WatchOpts{
subscription, err := uni.offramp.WatchCommitReportAccepted(&bind.WatchOpts{
Start: startBlock,
Context: testutils.Context(t),
}, sink)
Expand All @@ -260,7 +269,7 @@ func waitForCommit(t *testing.T, uni onchainUniverse, numUnis int, startBlock *u
select {
case <-time.After(5 * time.Second):
t.Logf("Waiting for commit report on chain id %d (selector %d)", uni.chainID, getSelector(uni.chainID))
case subErr := <-subscipriton.Err():
case subErr := <-subscription.Err():
t.Fatalf("Subscription error: %+v", subErr)
case report := <-sink:
if len(report.Report.MerkleRoots) > 0 {
Expand Down
Loading

0 comments on commit cdb2d54

Please sign in to comment.