add some RPC diagnosis log

zeta-chain · Jan 23, 2024 · b0832e9 · b0832e9
1 parent 5271d93
commit b0832e9
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 3 deletions.
diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go
@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	cosmosmath "cosmossdk.io/math"
 	"github.com/btcsuite/btcd/btcjson"
@@ -199,6 +200,54 @@ func (ob *BitcoinChainClient) Start() {
 	go ob.WatchUTXOS()
 	go ob.WatchGasPrice()
 	go ob.ExternalChainWatcherForNewInboundTrackerSuggestions()
+	go ob.RPCStatus()
+}
+
+func (ob *BitcoinChainClient) RPCStatus() {
+	ob.logger.ChainLogger.Info().Msgf("RPCStatus is starting")
+	ticker := time.NewTicker(60 * time.Second)
+
+	for {
+		select {
+		case <-ticker.C:
+			//ob.logger.ChainLogger.Info().Msgf("RPCStatus is running")
+			bn, err := ob.rpcClient.GetBlockCount()
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ")
+				continue
+			}
+			hash, err := ob.rpcClient.GetBlockHash(bn)
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ")
+				continue
+			}
+			header, err := ob.rpcClient.GetBlockHeader(hash)
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ")
+				continue
+			}
+			blockTime := header.Timestamp
+			elapsedSeconds := time.Since(blockTime).Seconds()
+			if elapsedSeconds > 1200 {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ")
+				continue
+			}
+			tssAddr := ob.Tss.BTCAddressWitnessPubkeyHash()
+			res, err := ob.rpcClient.ListUnspentMinMaxAddresses(0, 1000000, []btcutil.Address{tssAddr})
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: can't list utxos of TSS address; wallet or loaded? TSS address is not imported? ")
+				continue
+			}
+			if len(res) == 0 {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: TSS address has no utxos; TSS address is not importede? ")
+				continue
+			}
+			ob.logger.ChainLogger.Info().Msgf("[OK] RPC status check: latest block number %d, timestamp %s (%.fs ago), tss addr %s, #utxos: %d", bn, blockTime, elapsedSeconds, tssAddr, len(res))
+
+		case <-ob.stop:
+			return
+		}
+	}
 }
 
 func (ob *BitcoinChainClient) Stop() {

diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go
@@ -282,6 +282,41 @@ func (ob *EVMChainClient) Start() {
 	go ob.ExternalChainWatcher() // Observes external Chains for incoming trasnactions
 	go ob.WatchGasPrice()        // Observes external Chains for Gas prices and posts to core
 	go ob.observeOutTx()         // Populates receipts and confirmed outbound transactions
+	go ob.ExternalChainRPCStatus()
+}
+
+func (ob *EVMChainClient) ExternalChainRPCStatus() {
+	ob.logger.ChainLogger.Info().Msgf("Starting RPC status check for chain %s", ob.chain.String())
+	ticker := time.NewTicker(60 * time.Second)
+	for {
+		select {
+		case <-ticker.C:
+			bn, err := ob.evmClient.BlockNumber(context.Background())
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?")
+				continue
+			}
+			gasPrice, err := ob.evmClient.SuggestGasPrice(context.Background())
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?")
+				continue
+			}
+			header, err := ob.evmClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(bn))
+			if err != nil {
+				ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?")
+				continue
+			}
+			blockTime := time.Unix(int64(header.Time), 0).UTC()
+			elapsedSeconds := time.Since(blockTime).Seconds()
+			if elapsedSeconds > 100 {
+				ob.logger.ChainLogger.Warn().Msgf("RPC Status Check warning: RPC stale or chain stuck (check explorer)? Latest block %d timestamp is %.0fs ago", bn, elapsedSeconds)
+				continue
+			}
+			ob.logger.ChainLogger.Info().Msgf("[OK] RPC status: latest block num %d, timestamp %s ( %.0fs ago), suggested gas price %d", header.Number, blockTime.String(), elapsedSeconds, gasPrice.Uint64())
+		case <-ob.stop:
+			return
+		}
+	}
 }
 
 func (ob *EVMChainClient) Stop() {
@@ -1078,7 +1113,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint
 // returns the last block successfully scanned
 func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags observertypes.CrosschainFlags) uint64 {
 	if !ob.GetChainParams().IsSupported {
-		ob.logger.ExternalChainWatcher.Warn().Msgf("observeTssRecvd: chain %d is not supported", ob.chain.ChainId)
+		//ob.logger.ExternalChainWatcher.Warn().Msgf("observeTssRecvd: chain %d is not supported", ob.chain.ChainId)
 		return startBlock - 1 // lastScanned
 	}
 	// check TSS address (after keygen, ob.Tss.pubkey will be updated)
@@ -1153,7 +1188,7 @@ func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags obse
 }
 
 func (ob *EVMChainClient) WatchGasPrice() {
-
+	ob.logger.WatchGasPrice.Info().Msg("WatchGasPrice starting...")
 	err := ob.PostGasPrice()
 	if err != nil {
 		height, err := ob.zetaClient.GetBlockHeight()
@@ -1169,6 +1204,7 @@ func (ob *EVMChainClient) WatchGasPrice() {
 		ob.logger.WatchGasPrice.Error().Err(err).Msg("NewDynamicTicker error")
 		return
 	}
+	ob.logger.WatchGasPrice.Info().Msgf("WatchGasPrice started with interval %d", ob.GetChainParams().GasPriceTicker)
 
 	defer ticker.Stop()
 	for {
@@ -1192,6 +1228,7 @@ func (ob *EVMChainClient) WatchGasPrice() {
 }
 
 func (ob *EVMChainClient) PostGasPrice() error {
+
 	// GAS PRICE
 	gasPrice, err := ob.evmClient.SuggestGasPrice(context.TODO())
 	if err != nil {
@@ -1213,7 +1250,6 @@ func (ob *EVMChainClient) PostGasPrice() error {
 		return err
 	}
 	_ = zetaHash
-	//ob.logger.WatchGasPrice.Debug().Msgf("PostGasPrice zeta tx: %s", zetaHash)
 
 	return nil
 }

diff --git a/zetaclient/telemetry.go b/zetaclient/telemetry.go
@@ -133,6 +133,7 @@ func (t *TelemetryServer) Handlers() http.Handler {
 	router.Handle("/status", http.HandlerFunc(t.statusHandler)).Methods(http.MethodGet)
 	router.Handle("/ip", http.HandlerFunc(t.ipHandler)).Methods(http.MethodGet)
 	router.Handle("/hotkeyburnrate", http.HandlerFunc(t.hotKeyFeeBurnRate)).Methods(http.MethodGet)
+
 	// router.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
 	// router.Handle("/debug/pprof/heap", pprof.Handler("heap"))
 	// router.HandleFunc("/debug/pprof/", pprof.Index)