feat: add configurable signer latency correction

cmd/zetae2e/local/local.go

@@ -362,7 +362,8 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 
 	if testAdmin {
 		eg.Go(adminTestRoutine(conf, deployerRunner, verbose,
-			e2etests.TestOperationalFlagsName,
+			e2etests.TestZetaclientSignerOffsetName,
+			e2etests.TestZetaclientRestartHeightName,
 			e2etests.TestWhitelistERC20Name,
 			e2etests.TestPauseZRC20Name,
 			e2etests.TestUpdateBytecodeZRC20Name,

e2e/e2etests/e2etests.go

@@ -132,7 +132,8 @@ const (
 	TestMigrateERC20CustodyFundsName  = "migrate_erc20_custody_funds"
 	TestMigrateTSSName                = "migrate_tss"
 	TestSolanaWhitelistSPLName        = "solana_whitelist_spl"
-	TestOperationalFlagsName          = "operational_flags"
+	TestZetaclientRestartHeightName   = "zetaclient_restart_height"
+	TestZetaclientSignerOffsetName    = "zetaclient_signer_offset"
 
 	/*
 	 Operational tests
@@ -880,10 +881,16 @@ var AllE2ETests = []runner.E2ETest{
 		TestMigrateERC20CustodyFunds,
 	),
 	runner.NewE2ETest(
-		TestOperationalFlagsName,
-		"operational flags functionality",
+		TestZetaclientRestartHeightName,
+		"zetaclient scheduled restart height",
 		[]runner.ArgDefinition{},
-		TestOperationalFlags,
+		TestZetaclientRestartHeight,
+	),
+	runner.NewE2ETest(
+		TestZetaclientSignerOffsetName,
+		"zetaclient signer offset",
+		[]runner.ArgDefinition{},
+		TestZetaclientSignerOffset,
 	),
 	/*
 	 Special tests

e2e/e2etests/test_operational_flags.go

@@ -11,11 +11,13 @@ import (
 )
 
 const (
-	startTimestampMetricName = "zetaclient_last_start_timestamp_seconds"
+	startTimestampMetricName        = "zetaclient_last_start_timestamp_seconds"
+	blockTimeLatencyMetricName      = "zetaclient_core_block_latency"
+	blockTimeLatencySleepMetricName = "zetaclient_core_block_latency_sleep"
 )
 
-// TestOperationalFlags tests the functionality of operations flags.
-func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
+// TestZetaclientRestartHeight tests scheduling a zetaclient restart via operational flags
+func TestZetaclientRestartHeight(r *runner.E2ERunner, _ []string) {
 	_, err := r.Clients.Zetacore.Observer.OperationalFlags(
 		r.Ctx,
 		&observertypes.QueryOperationalFlagsRequest{},
@@ -60,3 +62,40 @@ func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
 
 	require.Greater(r, currentStartTime, originalStartTime+1)
 }
+
+// TestZetaclientSignerOffset tests scheduling a zetaclient restart via operational flags
+func TestZetaclientSignerOffset(r *runner.E2ERunner, _ []string) {
+	startBlockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName)
+	require.NoError(r, err)
+	require.InDelta(r, 0, startBlockTimeLatencySleep, .01, "start block time latency should be 0")
+
+	startBlockTimeLatency, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencyMetricName)
+	require.NoError(r, err)
+
+	desiredSignerBlockTimeOffset := time.Duration(startBlockTimeLatency*float64(time.Second)) + time.Millisecond*200
+
+	updateMsg := observertypes.NewMsgUpdateOperationalFlags(
+		r.ZetaTxServer.MustGetAccountAddressFromName(utils.OperationalPolicyName),
+		observertypes.OperationalFlags{
+			SignerBlockTimeOffset: &desiredSignerBlockTimeOffset,
+		},
+	)
+
+	_, err = r.ZetaTxServer.BroadcastTx(utils.OperationalPolicyName, updateMsg)
+	require.NoError(r, err)
+
+	operationalFlagsRes, err := r.Clients.Zetacore.Observer.OperationalFlags(
+		r.Ctx,
+		&observertypes.QueryOperationalFlagsRequest{},
+	)
+	require.NoError(r, err)
+	require.InDelta(r, desiredSignerBlockTimeOffset, *(operationalFlagsRes.OperationalFlags.SignerBlockTimeOffset), .01)
+
+	require.Eventually(r, func() bool {
+		blockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName)
+		if err != nil {
+			return false
+		}
+		return blockTimeLatencySleep > .1
+	}, time.Second*15, time.Second*1)
+}

proto/zetachain/zetacore/observer/operational.proto

@@ -1,11 +1,19 @@
 syntax = "proto3";
 package zetachain.zetacore.observer;
 
+import "gogoproto/gogo.proto";
+import "google/protobuf/duration.proto";
+
 option go_package = "github.com/zeta-chain/node/x/observer/types";
 
 // Flags for the top-level operation of zetaclient.
 message OperationalFlags {
   // Height for a coordinated zetaclient restart.
   // Will be ignored if missed.
   int64 restart_height = 1;
+
+  // Offset from the zetacore block time to initiate signing.
+  // Should be calculated and set based on max(zetaclient_core_block_latency).
+  google.protobuf.Duration signer_block_time_offset = 2
+      [ (gogoproto.stdduration) = true ];
 }

proto/zetachain/zetacore/observer/tx.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 package zetachain.zetacore.observer;
 
 import "gogoproto/gogo.proto";
+import "google/protobuf/field_mask.proto";
 import "zetachain/zetacore/observer/blame.proto";
 import "zetachain/zetacore/observer/crosschain_flags.proto";
 import "zetachain/zetacore/observer/observer.proto";
@@ -143,7 +144,7 @@ message MsgUpdateGasPriceIncreaseFlagsResponse {}
 
 message MsgUpdateOperationalFlags {
   string creator = 1;
-  OperationalFlags operationalFlags = 2 [ (gogoproto.nullable) = false ];
+  OperationalFlags operational_flags = 2 [ (gogoproto.nullable) = false ];
 }
 
 message MsgUpdateOperationalFlagsResponse {}

x/observer/client/cli/tx_update_operational_flags.go

@@ -13,8 +13,9 @@ import (
 )
 
 const (
-	fileFlag          = "file"
-	restartHeightFlag = "restart-height"
+	fileFlag                  = "file"
+	restartHeightFlag         = "restart-height"
+	signerBlockTimeOffsetFlag = "signer-block-time-offset"
 )
 
 func CmdUpdateOperationalFlags() *cobra.Command {
@@ -33,6 +34,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 			flagSet := cmd.Flags()
 			file, _ := flagSet.GetString(fileFlag)
 			restartHeight, _ := flagSet.GetInt64(restartHeightFlag)
+			signerBlockTimeOffset, _ := flagSet.GetDuration(signerBlockTimeOffsetFlag)
 
 			if file != "" {
 				input, err := os.ReadFile(file) // #nosec G304
@@ -45,6 +47,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 				}
 			} else {
 				operationalFlags.RestartHeight = restartHeight
+				operationalFlags.SignerBlockTimeOffset = &signerBlockTimeOffset
 			}
 
 			msg := types.NewMsgUpdateOperationalFlags(
@@ -58,6 +61,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 
 	cmd.Flags().String(fileFlag, "", "Path to a JSON file containing OperationalFlags")
 	cmd.Flags().Int64(restartHeightFlag, 0, "Height for a coordinated zetaclient restart")
+	cmd.Flags().Duration(signerBlockTimeOffsetFlag, 0, "Offset from the zetacore block time to initiate signing")
 	flags.AddTxFlagsToCmd(cmd)
 
 	return cmd