feat: add coordinated zetaclient restart (#3252)

* feat: add coordinated zetaclient restart

feedback updates

* add test coverage

* add genesis and import/export logic

* add basic zetaclient implementation

* complete e2e test logic

* zetaclient test coverage

* make generate

* updates

cmd/zetaclientd/start.go

@@ -115,6 +115,11 @@ func Start(_ *cobra.Command, _ []string) error {
 		graceful.ShutdownNow()
 	})
 
+	maintenance.NewShutdownListener(zetacoreClient, logger.Std).Listen(ctx, func() {
+		logger.Std.Info().Msg("Shutdown listener received an action to shutdown zetaclientd.")
+		graceful.ShutdownNow()
+	})
+
 	// CreateSignerMap: This creates a map of all signers for each chain.
 	// Each signer is responsible for signing transactions for a particular chain
 	signerMap, err := orchestrator.CreateSignerMap(ctx, tss, logger)

cmd/zetacored/parse_genesis.go

@@ -205,6 +205,8 @@ func ModifyObserverState(
 	currentGenState := observertypes.GetGenesisStateFromAppState(cdc, appState)
 	currentGenState.Ballots = importedObserverGenState.Ballots
 	currentGenState.NonceToCctx = importedObserverGenState.NonceToCctx
+	// zero operational flags as they are network specific
+	currentGenState.OperationalFlags = observertypes.OperationalFlags{}
 
 	currentGenStateBz, err := cdc.MarshalJSON(&currentGenState)
 	if err != nil {

cmd/zetae2e/config/clients.go

@@ -59,14 +59,15 @@ func getClientsFromConfig(ctx context.Context, conf config.Config, account confi
 	}
 
 	return runner.Clients{
-		Zetacore: zetaCoreClients,
-		BtcRPC:   btcRPCClient,
-		Solana:   solanaClient,
-		TON:      tonClient,
-		Evm:      evmClient,
-		EvmAuth:  evmAuth,
-		Zevm:     zevmClient,
-		ZevmAuth: zevmAuth,
+		Zetacore:          zetaCoreClients,
+		BtcRPC:            btcRPCClient,
+		Solana:            solanaClient,
+		TON:               tonClient,
+		Evm:               evmClient,
+		EvmAuth:           evmAuth,
+		Zevm:              zevmClient,
+		ZevmAuth:          zevmAuth,
+		ZetaclientMetrics: &runner.MetricsClient{URL: conf.RPCs.ZetaclientMetrics},
 	}, nil
 }
 

cmd/zetae2e/config/local.yml

@@ -109,6 +109,7 @@ rpcs:
   zetacore_rpc: "http://localhost:26657"
   solana: "http://localhost:8899"
   ton_sidecar_url: "http://localhost:8111"
+  zetaclient_metrics: "http://localhost:8886"
 contracts:
   zevm:
     system_contract: "0x91d18e54DAf4F677cB28167158d6dd21F6aB3921"

cmd/zetae2e/config/localnet.yml

@@ -107,6 +107,7 @@ rpcs:
   ton_sidecar_url: "http://ton:8000"
   zetacore_grpc: "zetacore0:9090"
   zetacore_rpc: "http://zetacore0:26657"
+  zetaclient_metrics: "http://zetaclient0:8886"
 contracts:
 # configure localnet solana gateway program id
   solana:

cmd/zetae2e/local/local.go

@@ -362,6 +362,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 
 	if testAdmin {
 		eg.Go(adminTestRoutine(conf, deployerRunner, verbose,
+			e2etests.TestOperationalFlagsName,
 			e2etests.TestWhitelistERC20Name,
 			e2etests.TestPauseZRC20Name,
 			e2etests.TestUpdateBytecodeZRC20Name,

contrib/localnet/docker-compose.yml

@@ -114,6 +114,7 @@ services:
     restart: always
     ports:
       - "6061:6061" # pprof
+      - "8886:8886" # metrics
     volumes:
       - ssh:/root/.ssh
       - preparams:/root/preparams

docs/cli/zetacored/cli.md

@@ -5394,6 +5394,7 @@ zetacored query observer [flags]
 * [zetacored query observer show-keygen](#zetacored-query-observer-show-keygen)	 - shows keygen
 * [zetacored query observer show-node-account](#zetacored-query-observer-show-node-account)	 - shows a NodeAccount
 * [zetacored query observer show-observer-count](#zetacored-query-observer-show-observer-count)	 - Query show-observer-count
+* [zetacored query observer show-operational-flags](#zetacored-query-observer-show-operational-flags)	 - shows the operational flags
 * [zetacored query observer show-tss](#zetacored-query-observer-show-tss)	 - shows a TSS
 * [zetacored query observer show-tss-funds-migrator](#zetacored-query-observer-show-tss-funds-migrator)	 - show the tss funds migrator for a chain
 
@@ -6101,6 +6102,40 @@ zetacored query observer show-observer-count [flags]
 
 * [zetacored query observer](#zetacored-query-observer)	 - Querying commands for the observer module
 
+## zetacored query observer show-operational-flags
+
+shows the operational flags
+
+```
+zetacored query observer show-operational-flags [flags]
+```
+
+### Options
+
+```
+      --grpc-addr string   the gRPC endpoint to use for this chain
+      --grpc-insecure      allow gRPC over insecure channels, if not TLS the server must use TLS
+      --height int         Use a specific height to query state at (this can error if the node is pruning state)
+  -h, --help               help for show-operational-flags
+      --node string        [host]:[port] to Tendermint RPC interface for this chain 
+  -o, --output string      Output format (text|json) 
+```
+
+### Options inherited from parent commands
+
+```
+      --chain-id string     The network chain ID
+      --home string         directory for config and data 
+      --log_format string   The logging format (json|plain) 
+      --log_level string    The logging level (trace|debug|info|warn|error|fatal|panic) 
+      --log_no_color        Disable colored logs
+      --trace               print out full stack trace on errors
+```
+
+### SEE ALSO
+
+* [zetacored query observer](#zetacored-query-observer)	 - Querying commands for the observer module
+
 ## zetacored query observer show-tss
 
 shows a TSS
@@ -12767,6 +12802,7 @@ zetacored tx observer [flags]
 * [zetacored tx observer update-gas-price-increase-flags](#zetacored-tx-observer-update-gas-price-increase-flags)	 - Update the gas price increase flags
 * [zetacored tx observer update-keygen](#zetacored-tx-observer-update-keygen)	 - command to update the keygen block via a group proposal
 * [zetacored tx observer update-observer](#zetacored-tx-observer-update-observer)	 - Broadcast message add-observer
+* [zetacored tx observer update-operational-flags](#zetacored-tx-observer-update-operational-flags)	 - Broadcast message UpdateOperationalFlags
 * [zetacored tx observer vote-blame](#zetacored-tx-observer-vote-blame)	 - Broadcast message vote-blame
 * [zetacored tx observer vote-tss](#zetacored-tx-observer-vote-tss)	 - Vote for a new TSS creation
 
@@ -13300,6 +13336,61 @@ zetacored tx observer update-observer [old-observer-address] [new-observer-addre
 
 * [zetacored tx observer](#zetacored-tx-observer)	 - observer transactions subcommands
 
+## zetacored tx observer update-operational-flags
+
+Broadcast message UpdateOperationalFlags
+
+```
+zetacored tx observer update-operational-flags [flags]
+```
+
+### Options
+
+```
+  -a, --account-number uint      The account number of the signing account (offline mode only)
+      --aux                      Generate aux signer data instead of sending a tx
+  -b, --broadcast-mode string    Transaction broadcasting mode (sync|async) 
+      --chain-id string          The network chain ID
+      --dry-run                  ignore the --gas flag and perform a simulation of a transaction, but don't broadcast it (when enabled, the local Keybase is not accessible)
+      --fee-granter string       Fee granter grants fees for the transaction
+      --fee-payer string         Fee payer pays fees for the transaction instead of deducting from the signer
+      --fees string              Fees to pay along with transaction; eg: 10uatom
+      --file string              Path to a JSON file containing OperationalFlags
+      --from string              Name or address of private key with which to sign
+      --gas string               gas limit to set per-transaction; set to "auto" to calculate sufficient gas automatically. Note: "auto" option doesn't always report accurate results. Set a valid coin value to adjust the result. Can be used instead of "fees". (default 200000)
+      --gas-adjustment float     adjustment factor to be multiplied against the estimate returned by the tx simulation; if the gas limit is set manually this flag is ignored  (default 1)
+      --gas-prices string        Gas prices in decimal format to determine the transaction fee (e.g. 0.1uatom)
+      --generate-only            Build an unsigned transaction and write it to STDOUT (when enabled, the local Keybase only accessed when providing a key name)
+  -h, --help                     help for update-operational-flags
+      --keyring-backend string   Select keyring's backend (os|file|kwallet|pass|test|memory) 
+      --keyring-dir string       The client Keyring directory; if omitted, the default 'home' directory will be used
+      --ledger                   Use a connected Ledger device
+      --node string              [host]:[port] to tendermint rpc interface for this chain 
+      --note string              Note to add a description to the transaction (previously --memo)
+      --offline                  Offline mode (does not allow any online functionality)
+  -o, --output string            Output format (text|json) 
+      --restart-height int       Height for a coordinated zetaclient restart
+  -s, --sequence uint            The sequence number of the signing account (offline mode only)
+      --sign-mode string         Choose sign mode (direct|amino-json|direct-aux), this is an advanced feature
+      --timeout-height uint      Set a block timeout height to prevent the tx from being committed past a certain height
+      --tip string               Tip is the amount that is going to be transferred to the fee payer on the target chain. This flag is only valid when used with --aux, and is ignored if the target chain didn't enable the TipDecorator
+  -y, --yes                      Skip tx broadcasting prompt confirmation
+```
+
+### Options inherited from parent commands
+
+```
+      --home string         directory for config and data 
+      --log_format string   The logging format (json|plain) 
+      --log_level string    The logging level (trace|debug|info|warn|error|fatal|panic) 
+      --log_no_color        Disable colored logs
+      --trace               print out full stack trace on errors
+```
+
+### SEE ALSO
+
+* [zetacored tx observer](#zetacored-tx-observer)	 - observer transactions subcommands
+
 ## zetacored tx observer vote-blame
 
 Broadcast message vote-blame

docs/openapi/openapi.swagger.yaml

@@ -30431,6 +30431,21 @@ paths:
             $ref: '#/definitions/googlerpcStatus'
       tags:
         - Query
+  /zeta-chain/observer/operationalFlags:
+    get:
+      summary: Queries operational flags
+      operationId: Query_OperationalFlags
+      responses:
+        "200":
+          description: A successful response.
+          schema:
+            $ref: '#/definitions/observerQueryOperationalFlagsResponse'
+        default:
+          description: An unexpected error response.
+          schema:
+            $ref: '#/definitions/googlerpcStatus'
+      tags:
+        - Query
   /zeta-chain/observer/pendingNonces:
     get:
       operationId: Query_PendingNoncesAll
@@ -58018,6 +58033,8 @@ definitions:
     type: object
   observerMsgUpdateObserverResponse:
     type: object
+  observerMsgUpdateOperationalFlagsResponse:
+    type: object
   observerMsgVoteBlameResponse:
     type: object
   observerMsgVoteBlockHeaderResponse:
@@ -58084,6 +58101,16 @@ definitions:
       - Tombstoned
       - AdminUpdate
     default: Undefined
+  observerOperationalFlags:
+    type: object
+    properties:
+      restart_height:
+        type: string
+        format: int64
+        description: |-
+          Height for a coordinated zetaclient restart.
+          Will be ignored if missed.
+    description: Flags for the top-level operation of zetaclient.
   observerPendingNonces:
     type: object
     properties:
@@ -58227,6 +58254,11 @@ definitions:
         type: array
         items:
           type: string
+  observerQueryOperationalFlagsResponse:
+    type: object
+    properties:
+      operational_flags:
+        $ref: '#/definitions/observerOperationalFlags'
   observerQueryPendingNoncesByChainResponse:
     type: object
     properties:

docs/spec/observer/messages.md

@@ -164,3 +164,12 @@ message MsgUpdateGasPriceIncreaseFlags {
 }
 ```
 
+## MsgUpdateOperationalFlags
+
+```proto
+message MsgUpdateOperationalFlags {
+	string creator = 1;
+	OperationalFlags operationalFlags = 2;
+}
+```
+

e2e/config/config.go

@@ -94,13 +94,14 @@ type ObserverRelayerAccounts struct {
 
 // RPCs contains the configuration for the RPC endpoints
 type RPCs struct {
-	Zevm          string     `yaml:"zevm"`
-	EVM           string     `yaml:"evm"`
-	Bitcoin       BitcoinRPC `yaml:"bitcoin"`
-	Solana        string     `yaml:"solana"`
-	TONSidecarURL string     `yaml:"ton_sidecar_url"`
-	ZetaCoreGRPC  string     `yaml:"zetacore_grpc"`
-	ZetaCoreRPC   string     `yaml:"zetacore_rpc"`
+	Zevm              string     `yaml:"zevm"`
+	EVM               string     `yaml:"evm"`
+	Bitcoin           BitcoinRPC `yaml:"bitcoin"`
+	Solana            string     `yaml:"solana"`
+	TONSidecarURL     string     `yaml:"ton_sidecar_url"`
+	ZetaCoreGRPC      string     `yaml:"zetacore_grpc"`
+	ZetaCoreRPC       string     `yaml:"zetacore_rpc"`
+	ZetaclientMetrics string     `yaml:"zetaclient_metrics"`
 }
 
 // BitcoinRPC contains the configuration for the Bitcoin RPC endpoint

e2e/e2etests/e2etests.go

@@ -130,8 +130,9 @@ const (
 	TestCriticalAdminTransactionsName = "critical_admin_transactions"
 	TestPauseERC20CustodyName         = "pause_erc20_custody"
 	TestMigrateERC20CustodyFundsName  = "migrate_erc20_custody_funds"
-	TestMigrateTSSName                = "migrate_TSS"
+	TestMigrateTSSName                = "migrate_tss"
 	TestSolanaWhitelistSPLName        = "solana_whitelist_spl"
+	TestOperationalFlagsName          = "operational_flags"
 
 	/*
 	 Operational tests
@@ -878,7 +879,12 @@ var AllE2ETests = []runner.E2ETest{
 		[]runner.ArgDefinition{},
 		TestMigrateERC20CustodyFunds,
 	),
-
+	runner.NewE2ETest(
+		TestOperationalFlagsName,
+		"operational flags functionality",
+		[]runner.ArgDefinition{},
+		TestOperationalFlags,
+	),
 	/*
 	 Special tests
 	*/

e2e/e2etests/test_operational_flags.go

@@ -0,0 +1,62 @@
+package e2etests
+
+import (
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zeta-chain/node/e2e/runner"
+	"github.com/zeta-chain/node/e2e/utils"
+	observertypes "github.com/zeta-chain/node/x/observer/types"
+)
+
+const (
+	startTimestampMetricName = "zetaclient_last_start_timestamp_seconds"
+)
+
+// TestOperationalFlags tests the functionality of operations flags.
+func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
+	_, err := r.Clients.Zetacore.Observer.OperationalFlags(
+		r.Ctx,
+		&observertypes.QueryOperationalFlagsRequest{},
+	)
+	require.NoError(r, err)
+
+	currentHeight, err := r.Clients.Zetacore.GetBlockHeight(r.Ctx)
+	require.NoError(r, err)
+
+	// schedule a restart for 5 blocks in the future
+	restartHeight := currentHeight + 5
+	updateMsg := observertypes.NewMsgUpdateOperationalFlags(
+		r.ZetaTxServer.MustGetAccountAddressFromName(utils.OperationalPolicyName),
+		observertypes.OperationalFlags{
+			RestartHeight: restartHeight,
+		},
+	)
+
+	_, err = r.ZetaTxServer.BroadcastTx(utils.OperationalPolicyName, updateMsg)
+	require.NoError(r, err)
+
+	operationalFlagsRes, err := r.Clients.Zetacore.Observer.OperationalFlags(
+		r.Ctx,
+		&observertypes.QueryOperationalFlagsRequest{},
+	)
+	require.NoError(r, err)
+	require.Equal(r, restartHeight, operationalFlagsRes.OperationalFlags.RestartHeight)
+
+	originalStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName)
+	require.NoError(r, err, "fetching zetaclient metric name")
+
+	// wait for height above restart height
+	// wait for a few extra block to account for shutdown and startup time
+	require.Eventually(r, func() bool {
+		height, err := r.Clients.Zetacore.GetBlockHeight(r.Ctx)
+		require.NoError(r, err)
+		return height > restartHeight+3
+	}, time.Minute, time.Second)
+
+	currentStartTime, err := r.Clients.ZetaclientMetrics.FetchGauge(startTimestampMetricName)
+	require.NoError(r, err)
+
+	require.Greater(r, currentStartTime, originalStartTime+1)
+}