From 1a9d185ddd0c58158872c7c59d561c041aa17362 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:58:31 +0100 Subject: [PATCH 1/8] Implement InboundTracker; minor refactoring --- zetaclient/chains/ton/observer/inbound.go | 100 ++++++++++++--- .../chains/ton/observer/inbound_test.go | 120 +++++++++++++----- zetaclient/chains/ton/observer/observer.go | 19 ++- .../chains/ton/observer/observer_test.go | 21 +++ 4 files changed, 200 insertions(+), 60 deletions(-) diff --git a/zetaclient/chains/ton/observer/inbound.go b/zetaclient/chains/ton/observer/inbound.go index 881853241a..145438ea2b 100644 --- a/zetaclient/chains/ton/observer/inbound.go +++ b/zetaclient/chains/ton/observer/inbound.go @@ -19,32 +19,37 @@ import ( ) const ( - // MaxTransactionsPerTick is the maximum number of transactions to process on a ticker - MaxTransactionsPerTick = 100 + // maximum number of transactions to process on a ticker + maxTransactionsPerTick = 100 + logSampleRate = 10 ) // watchInbound watches for new txs to Gateway's account. func (ob *Observer) watchInbound(ctx context.Context) error { + return ob.inboundTicker(ctx, "WatchInbound", ob.observeGateway) +} + +func (ob *Observer) watchInboundTracker(ctx context.Context) error { + return ob.inboundTicker(ctx, "WatchInboundTracker", ob.processInboundTrackers) +} + +func (ob *Observer) inboundTicker(ctx context.Context, taskName string, taskFunc func(context.Context) error) error { app, err := zctx.FromContext(ctx) if err != nil { return err } - var ( - initialInterval = ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) - sampledLogger = ob.Logger().Inbound.Sample(&zerolog.BasicSampler{N: 10}) - ) - - ob.Logger().Inbound.Info().Msgf("WatchInbound started") + initialInterval := ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) + sampledLogger := ob.Logger().Inbound.Sample(&zerolog.BasicSampler{N: logSampleRate}) task := func(ctx context.Context, t *ticker.Ticker) error { if !app.IsInboundObservationEnabled() { - sampledLogger.Info().Msg("WatchInbound: inbound observation is disabled") + sampledLogger.Info().Msgf("%s: inbound observation is disabled", taskName) return nil } - if err := ob.observeGateway(ctx); err != nil { - ob.Logger().Inbound.Err(err).Msg("WatchInbound: observeGateway error") + if err := taskFunc(ctx); err != nil { + ob.Logger().Inbound.Err(err).Msgf("%s failed", taskName) } newInterval := ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) @@ -58,7 +63,7 @@ func (ob *Observer) watchInbound(ctx context.Context) error { initialInterval, task, ticker.WithStopChan(ob.StopChannel()), - ticker.WithLogger(ob.Logger().Inbound, "WatchInbound"), + ticker.WithLogger(ob.Logger().Inbound, taskName), ) } @@ -86,11 +91,11 @@ func (ob *Observer) observeGateway(ctx context.Context) error { case len(txs) == 0: // noop return nil - case len(txs) > MaxTransactionsPerTick: + case len(txs) > maxTransactionsPerTick: ob.Logger().Inbound.Info(). - Msgf("observeGateway: got %d transactions. Taking first %d", len(txs), MaxTransactionsPerTick) + Msgf("observeGateway: got %d transactions. Taking first %d", len(txs), maxTransactionsPerTick) - txs = txs[:MaxTransactionsPerTick] + txs = txs[:maxTransactionsPerTick] default: ob.Logger().Inbound.Info().Msgf("observeGateway: got %d transactions", len(txs)) } @@ -156,6 +161,63 @@ func (ob *Observer) observeGateway(ctx context.Context) error { return nil } +// processInboundTrackers handles adhoc trackers that were somehow missed by +func (ob *Observer) processInboundTrackers(ctx context.Context) error { + trackers, err := ob.ZetacoreClient().GetInboundTrackersForChain(ctx, ob.Chain().ChainId) + if err != nil { + return errors.Wrap(err, "unable to get inbound trackers") + } + + // noop + if len(trackers) == 0 { + return nil + } + + gatewayAccountID := ob.gateway.AccountID() + + // a single error should not block other trackers + for _, tracker := range trackers { + txHash := tracker.TxHash + + lt, hash, err := liteapi.TransactionHashFromString(txHash) + if err != nil { + ob.logSkippedTracker(txHash, "unable_to_parse_hash", err) + continue + } + + raw, err := ob.client.GetTransaction(ctx, gatewayAccountID, lt, hash) + if err != nil { + ob.logSkippedTracker(txHash, "unable_to_get_tx", err) + continue + } + + tx, err := ob.gateway.ParseTransaction(raw) + + switch { + case errors.Is(err, toncontracts.ErrParse) || errors.Is(err, toncontracts.ErrUnknownOp): + ob.logSkippedTracker(txHash, "unrelated_tx", err) + continue + case err != nil: + // should not happen + ob.logSkippedTracker(txHash, "unexpected_error", err) + continue + case tx.ExitCode != 0: + ob.logSkippedTracker(txHash, "failed_tx", nil) + continue + case tx.IsOutbound(): + ob.logSkippedTracker(txHash, "outbound_tx", nil) + continue + } + + if _, err := ob.voteInbound(ctx, tx); err != nil { + ob.logSkippedTracker(txHash, "vote_failed", err) + continue + } + } + + return nil +} + // Sends PostVoteInbound to zetacore func (ob *Observer) voteInbound(ctx context.Context, tx *toncontracts.Transaction) (string, error) { // noop @@ -283,6 +345,14 @@ func (ob *Observer) setLastScannedTX(tx *toncontracts.Transaction) { Msg("setLastScannedTX: WriteLastTxScannedToDB") } +func (ob *Observer) logSkippedTracker(hash string, reason string, err error) { + ob.Logger().Inbound.Warn(). + Str("transaction.hash", hash). + Str("skip_reason", reason). + Err(err). + Msg("Skipping tracker") +} + func txLogFields(tx *toncontracts.Transaction) map[string]any { return map[string]any{ "transaction.hash": liteapi.TransactionToHashString(tx.Transaction), diff --git a/zetaclient/chains/ton/observer/inbound_test.go b/zetaclient/chains/ton/observer/inbound_test.go index 953db3c744..e0b7478cfa 100644 --- a/zetaclient/chains/ton/observer/inbound_test.go +++ b/zetaclient/chains/ton/observer/inbound_test.go @@ -2,6 +2,7 @@ package observer import ( "encoding/hex" + "fmt" "testing" "github.com/pkg/errors" @@ -11,14 +12,11 @@ import ( "github.com/tonkeeper/tongo/ton" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" "github.com/zeta-chain/node/testutil/sample" + cc "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/ton/liteapi" ) func TestInbound(t *testing.T) { - gw := toncontracts.NewGateway( - ton.MustParseAccountID("0:997d889c815aeac21c47f86ae0e38383efc3c3463067582f6263ad48c5a1485b"), - ) - t.Run("No gateway provided", func(t *testing.T) { ts := newTestSuite(t) @@ -32,11 +30,11 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) // Given mocked lite client call - ts.OnGetFirstTransaction(gw.AccountID(), nil, 0, errors.New("oops")).Once() + ts.OnGetFirstTransaction(ts.gateway.AccountID(), nil, 0, errors.New("oops")).Once() // ACT // Observe inbounds once @@ -52,16 +50,16 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given mocked lite client calls - firstTX := sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + firstTX := sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "1"), }) - ts.OnGetFirstTransaction(gw.AccountID(), &firstTX, 0, nil).Once() - ts.OnGetTransactionsSince(gw.AccountID(), firstTX.Lt, txHash(firstTX), nil, nil).Once() + ts.OnGetFirstTransaction(ts.gateway.AccountID(), &firstTX, 0, nil).Once() + ts.OnGetTransactionsSince(ts.gateway.AccountID(), firstTX.Lt, txHash(firstTX), nil, nil).Once() // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) // ACT @@ -88,13 +86,13 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls - donation := sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + donation := sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "12"), }) @@ -102,7 +100,7 @@ func TestInbound(t *testing.T) { txs := []ton.Transaction{donation} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() // ACT @@ -124,10 +122,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls deposit := toncontracts.Deposit{ @@ -136,11 +134,11 @@ func TestInbound(t *testing.T) { Recipient: sample.EthAddress(), } - depositTX := sample.TONDeposit(t, gw.AccountID(), deposit) + depositTX := sample.TONDeposit(t, ts.gateway.AccountID(), deposit) txs := []ton.Transaction{depositTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() ts.MockGetBlockHeader(depositTX.BlockID) @@ -182,10 +180,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls const callData = "hey there" @@ -198,11 +196,11 @@ func TestInbound(t *testing.T) { CallData: []byte(callData), } - depositAndCallTX := sample.TONDepositAndCall(t, gw.AccountID(), depositAndCall) + depositAndCallTX := sample.TONDepositAndCall(t, ts.gateway.AccountID(), depositAndCall) txs := []ton.Transaction{depositAndCallTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() ts.MockGetBlockHeader(depositAndCallTX.BlockID) @@ -251,10 +249,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls withdrawal := toncontracts.Withdrawal{ @@ -269,11 +267,11 @@ func TestInbound(t *testing.T) { require.NoError(t, err) require.Equal(t, ob.TSS().EVMAddress().Hex(), withdrawalSigner.Hex()) - withdrawalTX := sample.TONWithdrawal(t, gw.AccountID(), withdrawal) + withdrawalTX := sample.TONWithdrawal(t, ts.gateway.AccountID(), withdrawal) txs := []ton.Transaction{withdrawalTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() // ACT @@ -299,10 +297,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given several transactions withdrawal := toncontracts.Withdrawal{ @@ -314,39 +312,39 @@ func TestInbound(t *testing.T) { txs := []ton.Transaction{ // should be skipped - sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "1"), }), // should be voted - sample.TONDeposit(t, gw.AccountID(), toncontracts.Deposit{ + sample.TONDeposit(t, ts.gateway.AccountID(), toncontracts.Deposit{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "3"), Recipient: sample.EthAddress(), }), // should be skipped (invalid inbound message) sample.TONTransaction(t, sample.TONTransactionProps{ - Account: gw.AccountID(), + Account: ts.gateway.AccountID(), Input: &tlb.Message{}, }), // should be voted - sample.TONDeposit(t, gw.AccountID(), toncontracts.Deposit{ + sample.TONDeposit(t, ts.gateway.AccountID(), toncontracts.Deposit{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "3"), Recipient: sample.EthAddress(), }), // a tracker should be added - sample.TONWithdrawal(t, gw.AccountID(), withdrawal), + sample.TONWithdrawal(t, ts.gateway.AccountID(), withdrawal), // should be skipped (invalid inbound/outbound messages) sample.TONTransaction(t, sample.TONTransactionProps{ - Account: gw.AccountID(), + Account: ts.gateway.AccountID(), Input: &tlb.Message{}, Output: &tlb.Message{}, }), } ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() for _, tx := range txs { @@ -391,6 +389,58 @@ func TestInbound(t *testing.T) { }) } +func TestInboundTracker(t *testing.T) { + // ARRANGE + ts := newTestSuite(t) + + // Given observer + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) + require.NoError(t, err) + + // Given TON gateway transactions + // should be voted + deposit := toncontracts.Deposit{ + Sender: sample.GenerateTONAccountID(), + Amount: toncontracts.Coins(5), + Recipient: sample.EthAddress(), + } + + txDeposit := sample.TONDeposit(t, ts.gateway.AccountID(), deposit) + ts.MockGetTransaction(ts.gateway.AccountID(), txDeposit) + ts.MockGetBlockHeader(txDeposit.BlockID) + + // Should be skipped (I doubt anyone would vote for this gov proposal, but let’s still put up rail guards) + txWithdrawal := sample.TONWithdrawal(t, ts.gateway.AccountID(), toncontracts.Withdrawal{ + Recipient: sample.GenerateTONAccountID(), + Amount: toncontracts.Coins(5), + Seqno: 1, + }) + ts.MockGetTransaction(ts.gateway.AccountID(), txWithdrawal) + ts.MockGetBlockHeader(txWithdrawal.BlockID) + + // Given inbound trackers from zetacore + trackers := []cc.InboundTracker{ + ts.TxToInboundTracker(txDeposit), + ts.TxToInboundTracker(txWithdrawal), + } + + ts.OnGetInboundTrackersForChain(trackers).Once() + + // ACT + err = ob.processInboundTrackers(ts.ctx) + + // ARRANGE + require.NoError(t, err) + require.Len(t, ts.votesBag, 1) + + vote := ts.votesBag[0] + assert.Equal(t, deposit.Amount, vote.Amount) + assert.Equal(t, deposit.Sender.ToRaw(), vote.Sender) + + // zevm recipient bytes == memo bytes + assert.Equal(t, fmt.Sprintf("%x", deposit.Recipient), vote.Message) +} + func txHash(tx ton.Transaction) ton.Bits256 { return ton.Bits256(tx.Hash()) } diff --git a/zetaclient/chains/ton/observer/observer.go b/zetaclient/chains/ton/observer/observer.go index 41523d5759..565a544073 100644 --- a/zetaclient/chains/ton/observer/observer.go +++ b/zetaclient/chains/ton/observer/observer.go @@ -86,17 +86,16 @@ func (ob *Observer) Start(ctx context.Context) { ob.Logger().Chain.Info().Msg("observer is starting") - start := func(job func(ctx context.Context) error, name string, log zerolog.Logger) { - bg.Work(ctx, job, bg.WithName(name), bg.WithLogger(log)) - } - - // TODO: watchInboundTracker - // https://github.com/zeta-chain/node/issues/2935 + start(ctx, ob.watchInbound, "WatchInbound", ob.Logger().Inbound) + start(ctx, ob.watchInboundTracker, "WatchInboundTracker", ob.Logger().Inbound) + start(ctx, ob.watchOutbound, "WatchOutbound", ob.Logger().Outbound) + start(ctx, ob.watchGasPrice, "WatchGasPrice", ob.Logger().GasPrice) + start(ctx, ob.watchRPCStatus, "WatchRPCStatus", ob.Logger().Chain) +} - start(ob.watchInbound, "WatchInbound", ob.Logger().Inbound) - start(ob.watchOutbound, "WatchOutbound", ob.Logger().Outbound) - start(ob.watchGasPrice, "WatchGasPrice", ob.Logger().GasPrice) - start(ob.watchRPCStatus, "WatchRPCStatus", ob.Logger().Chain) +// fire goroutine task +func start(ctx context.Context, task func(ctx context.Context) error, name string, log zerolog.Logger) { + bg.Work(ctx, task, bg.WithName(name), bg.WithLogger(log)) } // watchGasPrice observes TON gas price and votes it to Zetacore. diff --git a/zetaclient/chains/ton/observer/observer_test.go b/zetaclient/chains/ton/observer/observer_test.go index 502a269267..290e34081a 100644 --- a/zetaclient/chains/ton/observer/observer_test.go +++ b/zetaclient/chains/ton/observer/observer_test.go @@ -13,6 +13,7 @@ import ( "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/ton" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" "github.com/zeta-chain/node/testutil/sample" cc "github.com/zeta-chain/node/x/crosschain/types" @@ -31,6 +32,7 @@ type testSuite struct { chain chains.Chain chainParams *observertypes.ChainParams + gateway *toncontracts.Gateway liteClient *mocks.LiteClient zetacore *mocks.ZetacoreClient @@ -55,6 +57,10 @@ func newTestSuite(t *testing.T) *testSuite { chain = chains.TONTestnet chainParams = sample.ChainParams(chain.ChainId) + gateway = toncontracts.NewGateway(ton.MustParseAccountID( + "0:997d889c815aeac21c47f86ae0e38383efc3c3463067582f6263ad48c5a1485b", + )) + liteClient = mocks.NewLiteClient(t) tss = mocks.NewGeneratedTSS(t, chain) @@ -90,6 +96,7 @@ func newTestSuite(t *testing.T) *testSuite { chainParams: chainParams, liteClient: liteClient, + gateway: gateway, zetacore: zetacore, tss: tss, @@ -168,6 +175,20 @@ func (ts *testSuite) MockGetBlockHeader(id ton.BlockIDExt) *mock.Call { Return(blockInfo, nil) } +func (ts *testSuite) OnGetInboundTrackersForChain(trackers []cc.InboundTracker) *mock.Call { + return ts.zetacore. + On("GetInboundTrackersForChain", mock.Anything, ts.chain.ChainId). + Return(trackers, nil) +} + +func (ts *testSuite) TxToInboundTracker(tx ton.Transaction) cc.InboundTracker { + return cc.InboundTracker{ + ChainId: ts.chain.ChainId, + TxHash: liteapi.TransactionToHashString(tx), + CoinType: coin.CoinType_Gas, + } +} + type signable interface { Hash() ([32]byte, error) SetSignature([65]byte) From b72b8d91bbd736c2311a13c91083ee5551904f73 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Thu, 31 Oct 2024 21:01:07 +0100 Subject: [PATCH 2/8] E2E: Concurrent withdrawals [WIP] --- cmd/zetae2e/local/local.go | 8 +- e2e/e2etests/e2etests.go | 13 ++- e2e/e2etests/test_ton_withdrawal.go | 7 +- .../test_ton_withdrawal_concurrent.go | 80 +++++++++++++++++++ e2e/runner/setup_ton.go | 2 +- e2e/runner/ton.go | 16 +++- e2e/runner/ton/deployer.go | 11 ++- 7 files changed, 119 insertions(+), 18 deletions(-) create mode 100644 e2e/e2etests/test_ton_withdrawal_concurrent.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 7cde1a92ad..506bd45541 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -416,9 +416,11 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } tonTests := []string{ - e2etests.TestTONDepositName, - e2etests.TestTONDepositAndCallName, - e2etests.TestTONWithdrawName, + // todo + //e2etests.TestTONDepositName, + //e2etests.TestTONDepositAndCallName, + //e2etests.TestTONWithdrawName, + e2etests.TestTONWithdrawConcurrentName, } eg.Go(tonTestRoutine(conf, deployerRunner, verbose, tonTests...)) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 30c25b33a3..53ac316e65 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -65,9 +65,10 @@ const ( /** * TON tests */ - TestTONDepositName = "ton_deposit" - TestTONDepositAndCallName = "ton_deposit_and_call" - TestTONWithdrawName = "ton_withdraw" + TestTONDepositName = "ton_deposit" + TestTONDepositAndCallName = "ton_deposit_and_call" + TestTONWithdrawName = "ton_withdraw" + TestTONWithdrawConcurrentName = "ton_withdraw_concurrent" /* Bitcoin tests @@ -480,6 +481,12 @@ var AllE2ETests = []runner.E2ETest{ }, TestTONWithdraw, ), + runner.NewE2ETest( + TestTONWithdrawConcurrentName, + "withdraw TON from ZEVM for several recipients simultaneously", + []runner.ArgDefinition{}, + TestTONWithdrawConcurrent, + ), /* Bitcoin tests */ diff --git a/e2e/e2etests/test_ton_withdrawal.go b/e2e/e2etests/test_ton_withdrawal.go index d78f624716..db75fa4d28 100644 --- a/e2e/e2etests/test_ton_withdrawal.go +++ b/e2e/e2etests/test_ton_withdrawal.go @@ -12,9 +12,6 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/ton/liteapi" ) -// TODO: Add "withdraw_many_concurrent" test -// https://github.com/zeta-chain/node/issues/3044 - func TestTONWithdraw(r *runner.E2ERunner, args []string) { // ARRANGE require.Len(r, args, 1) @@ -34,7 +31,7 @@ func TestTONWithdraw(r *runner.E2ERunner, args []string) { tonRecipient, err := deployer.CreateWallet(r.Ctx, toncontracts.Coins(1)) require.NoError(r, err) - tonRecipientBalanceBefore, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress()) + tonRecipientBalanceBefore, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress(), true) require.NoError(r, err) r.Logger.Info("Recipient's TON balance before withdrawal: %s", toncontracts.FormatCoins(tonRecipientBalanceBefore)) @@ -61,7 +58,7 @@ func TestTONWithdraw(r *runner.E2ERunner, args []string) { ) // Make sure that recipient's TON balance has increased - tonRecipientBalanceAfter, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress()) + tonRecipientBalanceAfter, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress(), true) require.NoError(r, err) r.Logger.Info("Recipient's balance after withdrawal: %s", toncontracts.FormatCoins(tonRecipientBalanceAfter)) diff --git a/e2e/e2etests/test_ton_withdrawal_concurrent.go b/e2e/e2etests/test_ton_withdrawal_concurrent.go new file mode 100644 index 0000000000..a01cd43e44 --- /dev/null +++ b/e2e/e2etests/test_ton_withdrawal_concurrent.go @@ -0,0 +1,80 @@ +package e2etests + +import ( + "math/rand" + "sync" + + "cosmossdk.io/math" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/require" + "github.com/tonkeeper/tongo/ton" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" + "github.com/zeta-chain/node/testutil/sample" + cc "github.com/zeta-chain/node/x/crosschain/types" +) + +func TestTONWithdrawConcurrent(r *runner.E2ERunner, _ []string) { + // ARRANGE + // Given a deployer + _, deployer := r.Ctx, r.TONDeployer + + const recipientsCount = 10 + type withdrawal struct { + recipient ton.AccountID + amount math.Uint + } + + var ( + testCases []withdrawal + wg sync.WaitGroup + ) + + // Given multiple recipients WITHOUT deployed wallet-contracts + // and sample withdrawal amounts between 1 and 5 TON + for i := 0; i < recipientsCount; i++ { + // #nosec G404: it's a test + amount := 1 + rand.Intn(5) + testCases = append(testCases, withdrawal{ + // #nosec G115 test - always in range + amount: toncontracts.Coins(uint64(amount)), + recipient: sample.GenerateTONAccountID(), + }) + } + + // ACT + // Fire withdrawals. Note that zevm sender is r.ZEVMAuth + for i, tc := range testCases { + r.Logger.Info( + "Withdrawal #%d: sending %s to %s", + i+1, + toncontracts.FormatCoins(tc.amount), + tc.recipient.ToRaw(), + ) + + approvedAmount := tc.amount.Add(toncontracts.Coins(1)) + tx := r.SendWithdrawTONZRC20(tc.recipient, tc.amount.BigInt(), approvedAmount.BigInt()) + + wg.Add(1) + + go func(number int, tx *ethtypes.Transaction) { + defer wg.Done() + + // wait for the cctx to be mined + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.CctxTimeout) + + // ASSERT + utils.RequireCCTXStatus(r, cctx, cc.CctxStatus_OutboundMined) + r.Logger.Info("Withdrawal #%d complete! cctx index: %s", number, cctx.Index) + + // Check recipient's balance ON TON + balance, err := deployer.GetBalanceOf(r.Ctx, tc.recipient, false) + require.NoError(r, err, "failed to get balance of %s", tc.recipient.ToRaw()) + require.Equal(r, tc.amount.Uint64(), balance.Uint64()) + }(i+1, tx) + } + + wg.Wait() +} diff --git a/e2e/runner/setup_ton.go b/e2e/runner/setup_ton.go index 49feb5d95a..0d87e5f825 100644 --- a/e2e/runner/setup_ton.go +++ b/e2e/runner/setup_ton.go @@ -54,7 +54,7 @@ func (r *E2ERunner) SetupTON() error { ) // 3. Check that the gateway indeed was deployed and has desired TON balance. - gwBalance, err := deployer.GetBalanceOf(ctx, gwAccount.ID) + gwBalance, err := deployer.GetBalanceOf(ctx, gwAccount.ID, true) if err != nil { return errors.Wrap(err, "unable to get balance of TON gateway") } diff --git a/e2e/runner/ton.go b/e2e/runner/ton.go index eb8e5346ac..feb579f381 100644 --- a/e2e/runner/ton.go +++ b/e2e/runner/ton.go @@ -6,6 +6,7 @@ import ( "cosmossdk.io/math" eth "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/pkg/errors" "github.com/stretchr/testify/require" "github.com/tonkeeper/tongo/ton" @@ -101,8 +102,12 @@ func (r *E2ERunner) TONDepositAndCall( return cctx, nil } -// WithdrawTONZRC20 withdraws an amount of ZRC20 TON tokens -func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveAmount *big.Int) *cctypes.CrossChainTx { +// SendWithdrawTONZRC20 sends withdraw tx of TON ZRC20 tokens +func (r *E2ERunner) SendWithdrawTONZRC20( + to ton.AccountID, + amount *big.Int, + approveAmount *big.Int, +) *ethtypes.Transaction { // approve tx, err := r.TONZRC20.Approve(r.ZEVMAuth, r.TONZRC20Addr, approveAmount) require.NoError(r, err) @@ -119,6 +124,13 @@ func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveA utils.RequireTxSuccessful(r, receipt, "withdraw") r.Logger.Info("Receipt txhash %s status %d", receipt.TxHash, receipt.Status) + return tx +} + +// WithdrawTONZRC20 withdraws an amount of ZRC20 TON tokens and waits for the cctx to be mined +func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveAmount *big.Int) *cctypes.CrossChainTx { + tx := r.SendWithdrawTONZRC20(to, amount, approveAmount) + // wait for the cctx to be mined cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.CctxTimeout) utils.RequireCCTXStatus(r, cctx, cctypes.CctxStatus_OutboundMined) diff --git a/e2e/runner/ton/deployer.go b/e2e/runner/ton/deployer.go index 4735d3be10..3f12f784da 100644 --- a/e2e/runner/ton/deployer.go +++ b/e2e/runner/ton/deployer.go @@ -57,10 +57,13 @@ func (d *Deployer) Seqno(ctx context.Context) (uint32, error) { return d.blockchain.GetSeqno(ctx, d.GetAddress()) } -// GetBalanceOf returns the balance of the given account. -func (d *Deployer) GetBalanceOf(ctx context.Context, id ton.AccountID) (math.Uint, error) { - if err := d.waitForAccountActivation(ctx, id); err != nil { - return math.Uint{}, errors.Wrap(err, "failed to wait for account activation") +// GetBalanceOf returns the balance of a given account. +// wait=true waits for account activation. +func (d *Deployer) GetBalanceOf(ctx context.Context, id ton.AccountID, wait bool) (math.Uint, error) { + if wait { + if err := d.waitForAccountActivation(ctx, id); err != nil { + return math.Uint{}, errors.Wrap(err, "failed to wait for account activation") + } } state, err := d.blockchain.GetAccountState(ctx, id) From e3688ebc9f95c78573280ad291301e370ad85525 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:05:44 +0100 Subject: [PATCH 3/8] E2E: Concurrent withdrawals --- e2e/runner/setup_ton.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/e2e/runner/setup_ton.go b/e2e/runner/setup_ton.go index 0d87e5f825..10954b5f43 100644 --- a/e2e/runner/setup_ton.go +++ b/e2e/runner/setup_ton.go @@ -11,6 +11,7 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/pkg/constant" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" + cctxtypes "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" ) @@ -55,28 +56,34 @@ func (r *E2ERunner) SetupTON() error { // 3. Check that the gateway indeed was deployed and has desired TON balance. gwBalance, err := deployer.GetBalanceOf(ctx, gwAccount.ID, true) - if err != nil { + switch { + case err != nil: return errors.Wrap(err, "unable to get balance of TON gateway") + case gwBalance.IsZero(): + return fmt.Errorf("TON gateway balance is zero") } - if gwBalance.IsZero() { - return fmt.Errorf("TON gateway balance is zero") + // 4. Set chain params & chain nonce + if err := r.ensureTONChainParams(gwAccount); err != nil { + return errors.Wrap(err, "unable to ensure TON chain params") } - // 4. Deposit 100 TON deployer to Zevm Auth - gw := toncontracts.NewGateway(gwAccount.ID) - veryFirstDeposit := toncontracts.Coins(1000) + r.TONDeployer = deployer + r.TONGateway = toncontracts.NewGateway(gwAccount.ID) + + // 5. Deposit 10000 TON deployer to Zevm Auth + veryFirstDeposit := toncontracts.Coins(10000) zevmRecipient := r.ZEVMAuth.From - err = gw.SendDeposit(ctx, &deployer.Wallet, veryFirstDeposit, zevmRecipient, 0) - if err != nil { - return errors.Wrap(err, "unable to send deposit to TON gateway") + gwDeposit, err := r.TONDeposit(&deployer.Wallet, veryFirstDeposit, zevmRecipient) + switch { + case err != nil: + return errors.Wrap(err, "unable to deposit TON to Zevm Auth") + case gwDeposit.CctxStatus.Status != cctxtypes.CctxStatus_OutboundMined: + return errors.New("gateway deposit CCTX is not mined") } - r.TONDeployer = deployer - r.TONGateway = gw - - return r.ensureTONChainParams(gwAccount) + return nil } func (r *E2ERunner) ensureTONChainParams(gw *ton.AccountInit) error { From ca1beda22e847d6394be97cdb46dcf13cdb40ba9 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:07:49 +0100 Subject: [PATCH 4/8] Improve signer broadcasting & logging --- pkg/contracts/ton/gateway_msg.go | 9 +++ zetaclient/chains/ton/signer/signer.go | 84 +++++++++++++++++---- zetaclient/chains/ton/signer/signer_test.go | 25 ++++++ zetaclient/orchestrator/orchestrator.go | 7 +- 4 files changed, 111 insertions(+), 14 deletions(-) diff --git a/pkg/contracts/ton/gateway_msg.go b/pkg/contracts/ton/gateway_msg.go index 543f7d5a22..3a4716e0b4 100644 --- a/pkg/contracts/ton/gateway_msg.go +++ b/pkg/contracts/ton/gateway_msg.go @@ -24,6 +24,15 @@ const ( const OpWithdraw Op = 200 +// ExitCode represents an error code. Might be TVM or custom. +// TVM: https://docs.ton.org/v3/documentation/tvm/tvm-exit-codes +// Zeta: https://github.com/zeta-chain/protocol-contracts-ton/blob/main/contracts/common/errors.fc +type ExitCode uint32 + +const ( + ExitCodeInvalidSeqno ExitCode = 109 +) + // Donation represents a donation operation type Donation struct { Sender ton.AccountID diff --git a/zetaclient/chains/ton/signer/signer.go b/zetaclient/chains/ton/signer/signer.go index 7542fb37f6..cc43969c62 100644 --- a/zetaclient/chains/ton/signer/signer.go +++ b/zetaclient/chains/ton/signer/signer.go @@ -2,10 +2,14 @@ package signer import ( "context" + "regexp" + "strconv" + "strings" ethcommon "github.com/ethereum/go-ethereum/common" lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" + "github.com/tonkeeper/tongo/liteclient" "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/ton" @@ -84,13 +88,19 @@ func (s *Signer) TryProcessOutbound( }() outcome, err := s.ProcessOutbound(ctx, cctx, zetacore, zetaBlockHeight) - if err != nil { - s.Logger().Std.Error(). - Err(err). + switch { + case err != nil: + s.Logger().Std.Error().Err(err). Str("outbound.id", outboundID). Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). Str("outbound.outcome", string(outcome)). Msg("Unable to ProcessOutbound") + case outcome != Success: + s.Logger().Std.Warn(). + Str("outbound.id", outboundID). + Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). + Str("outbound.outcome", string(outcome)). + Msg("Unsuccessful outcome for ProcessOutbound") } } @@ -147,16 +157,8 @@ func (s *Signer) ProcessOutbound( // Example: If a cctx has amount of 5 TON, the recipient will receive 5 TON, // and gateway's balance will be decreased by 5 TON + txFees. exitCode, err := s.gateway.SendExternalMessage(ctx, s.client, withdrawal) - switch { - case err != nil: - return Fail, errors.Wrap(err, "unable to send external message") - case exitCode != 0: - // Might happen if msg's nonce is too high; retry later. - lf["outbound.exit_code"] = exitCode - lf["outbound.outcome"] = string(Invalid) - s.Logger().Std.Info().Fields(lf).Msg("Unable to send external message") - - return Invalid, nil + if err != nil || exitCode != 0 { + return s.handleSendError(exitCode, err, lf) } // it's okay to run this in the same goroutine @@ -211,6 +213,62 @@ func (s *Signer) setSignature(hash [32]byte, sig [65]byte) { s.signaturesCache.Add(hash, sig) } +// Sample (from local ton): +// error code: 0 message: cannot apply external message to current state: +// External message was not accepted Cannot run message on account: +// inbound external message rejected by transaction ...: exitcode=109, steps=108, gas_used=0\ +// VM Log (truncated): ... +var exitCodeErrorRegex = regexp.MustCompile(`exitcode=(\d+)`) + +// handleSendError tries to figure out the reason of the send error. +func (s *Signer) handleSendError(exitCode uint32, err error, logFields map[string]any) (Outcome, error) { + if err != nil { + // Might be possible if 2 concurrent zeta clients + // are trying to broadcast the same message. + if strings.Contains(err.Error(), "duplicate message") { + s.Logger().Std.Warn().Fields(logFields).Msg("Message already sent") + return Invalid, nil + } + + var errLiteClient liteclient.LiteServerErrorC + if errors.As(err, &errLiteClient) { + logFields["outbound.error.message"] = errLiteClient.Message + exitCode = errLiteClient.Code + } + + if code, ok := extractExitCode(err.Error()); ok { + exitCode = code + } + } + + switch { + case exitCode == uint32(toncontracts.ExitCodeInvalidSeqno): + // Might be possible if zeta clients send several seq. numbers concurrently. + // In the current implementation, Gateway supports only 1 nonce per block. + logFields["outbound.error.exit_code"] = exitCode + s.Logger().Std.Warn().Fields(logFields).Msg("Invalid nonce, retry later") + return Invalid, nil + case err != nil: + return Fail, errors.Wrap(err, "unable to send external message") + default: + return Fail, errors.Errorf("unable to send external message: exit code %d", exitCode) + } +} + +func extractExitCode(text string) (uint32, bool) { + match := exitCodeErrorRegex.FindStringSubmatch(text) + if len(match) < 2 { + return 0, false + } + + exitCode, err := strconv.ParseUint(match[1], 10, 32) + if err != nil { + return 0, false + } + + return uint32(exitCode), true +} + // GetGatewayAddress returns gateway address as raw TON address "0:ABC..." func (s *Signer) GetGatewayAddress() string { return s.gateway.AccountID().ToRaw() diff --git a/zetaclient/chains/ton/signer/signer_test.go b/zetaclient/chains/ton/signer/signer_test.go index 9ac85f281a..dcd78c9817 100644 --- a/zetaclient/chains/ton/signer/signer_test.go +++ b/zetaclient/chains/ton/signer/signer_test.go @@ -85,6 +85,31 @@ func TestSigner(t *testing.T) { require.Equal(t, liteapi.TransactionToHashString(withdrawalTX), tracker.hash) } +func TestExitCodeRegex(t *testing.T) { + for _, tt := range []string{ + `unable to send external message: error code: 0 message: + cannot apply external message to current state : + External message was not accepted\nCannot run message on account: inbound external message rejected by + transaction CC8803E21EDA7E6487D191380725A82CD75316E1C131496E1A5636751CE60347: + \nexitcode=109, steps=108, gas_used=0\nVM Log (truncated):\n...INT 0\nexecute THROWIFNOT + 105\nexecute MYADDR\nexecute XCHG s1,s4\nexecute SDEQ\nexecute THROWIF 112\nexecute OVER\nexecute + EQINT 0\nexecute THROWIF 106\nexecute GETGLOB + 3\nexecute NEQ\nexecute THROWIF 109\ndefault exception handler, terminating vm with exit code 109\n`, + + `unable to send external message: error code: 0 message: cannot apply external message to current state : + External message was not accepted\nCannot run message on account: + inbound external message rejected by transaction + 6CCBB83C7D9BFBFDB40541F35AD069714856F18B4850C1273A117DF6BFADE1C6:\nexitcode=109, steps=108, + gas_used=0\nVM Log (truncated):\n...INT 0....`, + } { + require.True(t, exitCodeErrorRegex.MatchString(tt)) + + exitCode, ok := extractExitCode(tt) + require.True(t, ok) + require.Equal(t, uint32(109), exitCode) + } +} + type testSuite struct { ctx context.Context t *testing.T diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index b632b9234a..fe1c26b76a 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -756,7 +756,12 @@ func (oc *Orchestrator) ScheduleCCTXTON( } // fire async task - taskLogger := oc.logger.Logger.With().Str("outbound.id", outboundID).Logger() + taskLogger := oc.logger.Logger.With(). + Int64(logs.FieldChain, chainID). + Str("outbound.id", outboundID). + Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). + Logger() + bg.Work(ctx, task, bg.WithName("TryProcessOutbound"), bg.WithLogger(taskLogger)) } } From 7148e22b24359d4365038514447a52746f0605b5 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:37:26 +0100 Subject: [PATCH 5/8] Add e2e for deposit & refund --- cmd/zetae2e/local/local.go | 8 ++-- e2e/e2etests/e2etests.go | 9 +++++ e2e/e2etests/test_ton_deposit_refund.go | 50 +++++++++++++++++++++++++ e2e/runner/ton.go | 31 +++++++++++++-- e2e/runner/zeta.go | 9 ++++- 5 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 e2e/e2etests/test_ton_deposit_refund.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 506bd45541..fb85225501 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -416,10 +416,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } tonTests := []string{ - // todo - //e2etests.TestTONDepositName, - //e2etests.TestTONDepositAndCallName, - //e2etests.TestTONWithdrawName, + e2etests.TestTONDepositName, + e2etests.TestTONDepositAndCallName, + e2etests.TestTONDepositRefundName, + e2etests.TestTONWithdrawName, e2etests.TestTONWithdrawConcurrentName, } diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 53ac316e65..d3dcd9e70a 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -67,6 +67,7 @@ const ( */ TestTONDepositName = "ton_deposit" TestTONDepositAndCallName = "ton_deposit_and_call" + TestTONDepositRefundName = "ton_deposit_refund" TestTONWithdrawName = "ton_withdraw" TestTONWithdrawConcurrentName = "ton_withdraw_concurrent" @@ -473,6 +474,14 @@ var AllE2ETests = []runner.E2ETest{ }, TestTONDepositAndCall, ), + runner.NewE2ETest( + TestTONDepositRefundName, + "deposit TON into ZEVM; expect refund", + []runner.ArgDefinition{ + {Description: "amount in nano tons", DefaultValue: "1000000000"}, // 1.0 TON + }, + TestTONDepositAndCallRefund, + ), runner.NewE2ETest( TestTONWithdrawName, "withdraw TON from ZEVM", diff --git a/e2e/e2etests/test_ton_deposit_refund.go b/e2e/e2etests/test_ton_deposit_refund.go new file mode 100644 index 0000000000..3259e1d1d6 --- /dev/null +++ b/e2e/e2etests/test_ton_deposit_refund.go @@ -0,0 +1,50 @@ +package e2etests + +import ( + "github.com/stretchr/testify/require" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + testcontract "github.com/zeta-chain/node/testutil/contracts" + cctypes "github.com/zeta-chain/node/x/crosschain/types" +) + +func TestTONDepositAndCallRefund(r *runner.E2ERunner, args []string) { + require.Len(r, args, 1) + + // Given amount and arbitrary call data + var ( + amount = parseUint(r, args[0]) + data = []byte("hello reverter") + ) + + // Given deployer mock revert contract + // deploy a reverter contract in ZEVM + reverterAddr, _, _, err := testcontract.DeployReverter(r.ZEVMAuth, r.ZEVMClient) + require.NoError(r, err) + r.Logger.Info("Reverter contract deployed at: %s", reverterAddr.String()) + + // ACT + // Send a deposit and call transaction from the deployer (faucet) + // to the reverter contract + cctx, err := r.TONDepositAndCall( + &r.TONDeployer.Wallet, + amount, + reverterAddr, + data, + runner.TONExpectStatus(cctypes.CctxStatus_Reverted), + ) + + // ASSERT + require.NoError(r, err) + r.Logger.CCTX(*cctx, "ton_deposit_and_refund") + + // Check the error carries the revert executed. + // tolerate the error in both the ErrorMessage field and the StatusMessage field + if cctx.CctxStatus.ErrorMessage != "" { + require.Contains(r, cctx.CctxStatus.ErrorMessage, "revert executed") + return + } + + require.Contains(r, cctx.CctxStatus.StatusMessage, utils.ErrHashRevertFoo) +} diff --git a/e2e/runner/ton.go b/e2e/runner/ton.go index feb579f381..369bfa0df3 100644 --- a/e2e/runner/ton.go +++ b/e2e/runner/ton.go @@ -1,6 +1,7 @@ package runner import ( + "encoding/hex" "math/big" "time" @@ -24,6 +25,20 @@ import ( // https://docs.ton.org/develop/smart-contracts/guidelines/message-modes-cookbook const tonDepositSendCode = toncontracts.SendFlagSeparateFees + toncontracts.SendFlagIgnoreErrors +// currently implemented only for DepositAndCall, +// can be adopted for all TON ops +type tonOpts struct { + expectedStatus cctypes.CctxStatus +} + +type TONOpt func(t *tonOpts) + +func TONExpectStatus(status cctypes.CctxStatus) TONOpt { + return func(t *tonOpts) { + t.expectedStatus = status + } +} + // TONDeposit deposit TON to Gateway contract func (r *E2ERunner) TONDeposit( sender *wallet.Wallet, @@ -57,7 +72,7 @@ func (r *E2ERunner) TONDeposit( } // Wait for cctx - cctx := r.WaitForSpecificCCTX(filter, time.Minute) + cctx := r.WaitForSpecificCCTX(filter, cctypes.CctxStatus_OutboundMined, time.Minute) return cctx, nil } @@ -68,7 +83,13 @@ func (r *E2ERunner) TONDepositAndCall( amount math.Uint, zevmRecipient eth.Address, callData []byte, + opts ...TONOpt, ) (*cctypes.CrossChainTx, error) { + cfg := &tonOpts{expectedStatus: cctypes.CctxStatus_OutboundMined} + for _, opt := range opts { + opt(cfg) + } + chain := chains.TONLocalnet require.NotNil(r, r.TONGateway, "TON Gateway is not initialized") @@ -92,12 +113,16 @@ func (r *E2ERunner) TONDepositAndCall( } filter := func(cctx *cctypes.CrossChainTx) bool { + memo := zevmRecipient.Bytes() + memo = append(memo, callData...) + return cctx.InboundParams.SenderChainId == chain.ChainId && - cctx.InboundParams.Sender == sender.GetAddress().ToRaw() + cctx.InboundParams.Sender == sender.GetAddress().ToRaw() && + cctx.RelayedMessage == hex.EncodeToString(memo) } // Wait for cctx - cctx := r.WaitForSpecificCCTX(filter, time.Minute) + cctx := r.WaitForSpecificCCTX(filter, cfg.expectedStatus, time.Minute) return cctx, nil } diff --git a/e2e/runner/zeta.go b/e2e/runner/zeta.go index 1df7e676e3..d59ef85645 100644 --- a/e2e/runner/zeta.go +++ b/e2e/runner/zeta.go @@ -99,11 +99,15 @@ func (r *E2ERunner) WaitForMinedCCTX(txHash ethcommon.Hash) { // WaitForMinedCCTXFromIndex waits for a cctx to be mined from its index func (r *E2ERunner) WaitForMinedCCTXFromIndex(index string) *types.CrossChainTx { + return r.waitForMinedCCTXFromIndex(index, types.CctxStatus_OutboundMined) +} + +func (r *E2ERunner) waitForMinedCCTXFromIndex(index string, status types.CctxStatus) *types.CrossChainTx { r.Lock() defer r.Unlock() cctx := utils.WaitCCTXMinedByIndex(r.Ctx, index, r.CctxClient, r.Logger, r.CctxTimeout) - utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + utils.RequireCCTXStatus(r, cctx, status) return cctx } @@ -111,6 +115,7 @@ func (r *E2ERunner) WaitForMinedCCTXFromIndex(index string) *types.CrossChainTx // WaitForSpecificCCTX scans for cctx by filters and ensures it's mined func (r *E2ERunner) WaitForSpecificCCTX( filter func(*types.CrossChainTx) bool, + status types.CctxStatus, timeout time.Duration, ) *types.CrossChainTx { var ( @@ -128,7 +133,7 @@ func (r *E2ERunner) WaitForSpecificCCTX( for i := range res.CrossChainTx { tx := res.CrossChainTx[i] if filter(tx) { - return r.WaitForMinedCCTXFromIndex(tx.Index) + return r.waitForMinedCCTXFromIndex(tx.Index, status) } } From 1e5604d03b1ae1b7373d6b0fe7845fb7d75088fa Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:38:51 +0100 Subject: [PATCH 6/8] Update changelog --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 4e58815e99..19d3e1addf 100644 --- a/changelog.md +++ b/changelog.md @@ -51,6 +51,7 @@ * [2894](https://github.com/zeta-chain/node/pull/2894) - increase gas limit for TSS vote tx * [2932](https://github.com/zeta-chain/node/pull/2932) - add gateway upgrade as part of the upgrade test * [2947](https://github.com/zeta-chain/node/pull/2947) - initialize simulation tests +* [3075](https://github.com/zeta-chain/node/pull/3075) - ton: withdraw concurrent, deposit & revert. ### Fixes From f13daa2414136d1c386a97801cbbdf71d9ef3eb0 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:55:57 +0100 Subject: [PATCH 7/8] Address PR comments --- changelog.md | 4 +++- cmd/zetae2e/local/local.go | 2 +- e2e/e2etests/e2etests.go | 14 +++++++------- e2e/e2etests/test_ton_withdrawal_concurrent.go | 3 +++ zetaclient/chains/ton/observer/inbound.go | 5 ++++- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/changelog.md b/changelog.md index 19d3e1addf..b89d99e17d 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,9 @@ ## Unreleased +### Tests +* [3075](https://github.com/zeta-chain/node/pull/3075) - ton: withdraw concurrent, deposit & revert. + ## v21.0.0 ### Features @@ -51,7 +54,6 @@ * [2894](https://github.com/zeta-chain/node/pull/2894) - increase gas limit for TSS vote tx * [2932](https://github.com/zeta-chain/node/pull/2932) - add gateway upgrade as part of the upgrade test * [2947](https://github.com/zeta-chain/node/pull/2947) - initialize simulation tests -* [3075](https://github.com/zeta-chain/node/pull/3075) - ton: withdraw concurrent, deposit & revert. ### Fixes diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index fb85225501..01d34e53af 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -418,7 +418,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { tonTests := []string{ e2etests.TestTONDepositName, e2etests.TestTONDepositAndCallName, - e2etests.TestTONDepositRefundName, + e2etests.TestTONDepositAndCallRefundName, e2etests.TestTONWithdrawName, e2etests.TestTONWithdrawConcurrentName, } diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index d3dcd9e70a..e1d1467013 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -65,11 +65,11 @@ const ( /** * TON tests */ - TestTONDepositName = "ton_deposit" - TestTONDepositAndCallName = "ton_deposit_and_call" - TestTONDepositRefundName = "ton_deposit_refund" - TestTONWithdrawName = "ton_withdraw" - TestTONWithdrawConcurrentName = "ton_withdraw_concurrent" + TestTONDepositName = "ton_deposit" + TestTONDepositAndCallName = "ton_deposit_and_call" + TestTONDepositAndCallRefundName = "ton_deposit_refund" + TestTONWithdrawName = "ton_withdraw" + TestTONWithdrawConcurrentName = "ton_withdraw_concurrent" /* Bitcoin tests @@ -475,8 +475,8 @@ var AllE2ETests = []runner.E2ETest{ TestTONDepositAndCall, ), runner.NewE2ETest( - TestTONDepositRefundName, - "deposit TON into ZEVM; expect refund", + TestTONDepositAndCallRefundName, + "deposit TON into ZEVM and call a smart contract that reverts; expect refund", []runner.ArgDefinition{ {Description: "amount in nano tons", DefaultValue: "1000000000"}, // 1.0 TON }, diff --git a/e2e/e2etests/test_ton_withdrawal_concurrent.go b/e2e/e2etests/test_ton_withdrawal_concurrent.go index a01cd43e44..691841c20c 100644 --- a/e2e/e2etests/test_ton_withdrawal_concurrent.go +++ b/e2e/e2etests/test_ton_withdrawal_concurrent.go @@ -16,6 +16,9 @@ import ( cc "github.com/zeta-chain/node/x/crosschain/types" ) +// TestTONWithdrawConcurrent makes sure that multiple concurrent +// withdrawals will be eventually processed by sequentially increasing Gateway nonce +// and that zetaclient tolerates "invalid nonce" error from RPC. func TestTONWithdrawConcurrent(r *runner.E2ERunner, _ []string) { // ARRANGE // Given a deployer diff --git a/zetaclient/chains/ton/observer/inbound.go b/zetaclient/chains/ton/observer/inbound.go index 145438ea2b..092bfa0e9d 100644 --- a/zetaclient/chains/ton/observer/inbound.go +++ b/zetaclient/chains/ton/observer/inbound.go @@ -20,8 +20,11 @@ import ( const ( // maximum number of transactions to process on a ticker + // TODO: move to config + // https://github.com/zeta-chain/node/issues/3086 maxTransactionsPerTick = 100 - logSampleRate = 10 + // zero log sample rate for sampled logger (to avoid spamming logs) + logSampleRate = 10 ) // watchInbound watches for new txs to Gateway's account. From 280010fc1729edf7ae1ccfd2474c161cd6b43bdd Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:19:57 +0100 Subject: [PATCH 8/8] Address PR comments [2] --- .../test_ton_withdrawal_concurrent.go | 51 ++++++++----------- zetaclient/chains/ton/signer/signer.go | 21 ++++---- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/e2e/e2etests/test_ton_withdrawal_concurrent.go b/e2e/e2etests/test_ton_withdrawal_concurrent.go index 691841c20c..fb4217013d 100644 --- a/e2e/e2etests/test_ton_withdrawal_concurrent.go +++ b/e2e/e2etests/test_ton_withdrawal_concurrent.go @@ -25,44 +25,35 @@ func TestTONWithdrawConcurrent(r *runner.E2ERunner, _ []string) { _, deployer := r.Ctx, r.TONDeployer const recipientsCount = 10 - type withdrawal struct { - recipient ton.AccountID - amount math.Uint - } - - var ( - testCases []withdrawal - wg sync.WaitGroup - ) - // Given multiple recipients WITHOUT deployed wallet-contracts - // and sample withdrawal amounts between 1 and 5 TON + // Fire withdrawals. Note that zevm sender is r.ZEVMAuth + var wg sync.WaitGroup for i := 0; i < recipientsCount; i++ { - // #nosec G404: it's a test - amount := 1 + rand.Intn(5) - testCases = append(testCases, withdrawal{ + // ARRANGE + // Given multiple recipients WITHOUT deployed wallet-contracts + // and withdrawal amounts between 1 and 5 TON + var ( + // #nosec G404: it's a test + amountCoins = 1 + rand.Intn(5) // #nosec G115 test - always in range - amount: toncontracts.Coins(uint64(amount)), - recipient: sample.GenerateTONAccountID(), - }) - } + amount = toncontracts.Coins(uint64(amountCoins)) + recipient = sample.GenerateTONAccountID() + ) - // ACT - // Fire withdrawals. Note that zevm sender is r.ZEVMAuth - for i, tc := range testCases { + // ACT r.Logger.Info( "Withdrawal #%d: sending %s to %s", i+1, - toncontracts.FormatCoins(tc.amount), - tc.recipient.ToRaw(), + toncontracts.FormatCoins(amount), + recipient.ToRaw(), ) - approvedAmount := tc.amount.Add(toncontracts.Coins(1)) - tx := r.SendWithdrawTONZRC20(tc.recipient, tc.amount.BigInt(), approvedAmount.BigInt()) + approvedAmount := amount.Add(toncontracts.Coins(1)) + tx := r.SendWithdrawTONZRC20(recipient, amount.BigInt(), approvedAmount.BigInt()) wg.Add(1) - go func(number int, tx *ethtypes.Transaction) { + go func(number int, recipient ton.AccountID, amount math.Uint, tx *ethtypes.Transaction) { defer wg.Done() // wait for the cctx to be mined @@ -73,10 +64,10 @@ func TestTONWithdrawConcurrent(r *runner.E2ERunner, _ []string) { r.Logger.Info("Withdrawal #%d complete! cctx index: %s", number, cctx.Index) // Check recipient's balance ON TON - balance, err := deployer.GetBalanceOf(r.Ctx, tc.recipient, false) - require.NoError(r, err, "failed to get balance of %s", tc.recipient.ToRaw()) - require.Equal(r, tc.amount.Uint64(), balance.Uint64()) - }(i+1, tx) + balance, err := deployer.GetBalanceOf(r.Ctx, recipient, false) + require.NoError(r, err, "failed to get balance of %s", recipient.ToRaw()) + require.Equal(r, amount.Uint64(), balance.Uint64()) + }(i+1, recipient, amount, tx) } wg.Wait() diff --git a/zetaclient/chains/ton/signer/signer.go b/zetaclient/chains/ton/signer/signer.go index cc43969c62..bdd25c0c18 100644 --- a/zetaclient/chains/ton/signer/signer.go +++ b/zetaclient/chains/ton/signer/signer.go @@ -88,19 +88,20 @@ func (s *Signer) TryProcessOutbound( }() outcome, err := s.ProcessOutbound(ctx, cctx, zetacore, zetaBlockHeight) + + lf := map[string]any{ + "outbound.id": outboundID, + "outbound.nonce": cctx.GetCurrentOutboundParam().TssNonce, + "outbound.outcome": string(outcome), + } + switch { case err != nil: - s.Logger().Std.Error().Err(err). - Str("outbound.id", outboundID). - Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). - Str("outbound.outcome", string(outcome)). - Msg("Unable to ProcessOutbound") + s.Logger().Std.Error().Err(err).Fields(lf).Msg("Unable to ProcessOutbound") case outcome != Success: - s.Logger().Std.Warn(). - Str("outbound.id", outboundID). - Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). - Str("outbound.outcome", string(outcome)). - Msg("Unsuccessful outcome for ProcessOutbound") + s.Logger().Std.Warn().Fields(lf).Msg("Unsuccessful outcome for ProcessOutbound") + default: + s.Logger().Std.Info().Fields(lf).Msg("Processed outbound") } }