From b864930f6cb649e5a8ba7cc86579b251208700ea Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Wed, 18 Dec 2024 12:47:15 -0800 Subject: [PATCH] feat: add configurable signer latency correction --- cmd/zetae2e/local/local.go | 3 +- e2e/e2etests/e2etests.go | 15 +- e2e/e2etests/test_operational_flags.go | 45 ++++- .../zetacore/observer/operational.proto | 8 + proto/zetachain/zetacore/observer/tx.proto | 3 +- .../client/cli/tx_update_operational_flags.go | 8 +- x/observer/types/operational.pb.go | 97 ++++++++-- x/observer/types/tx.pb.go | 174 +++++++++--------- zetaclient/metrics/metrics.go | 7 + zetaclient/orchestrator/orchestrator.go | 61 ++++++ 10 files changed, 311 insertions(+), 110 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 4385101a24..65b5fdcf44 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -362,7 +362,8 @@ func localE2ETest(cmd *cobra.Command, _ []string) { if testAdmin { eg.Go(adminTestRoutine(conf, deployerRunner, verbose, - e2etests.TestOperationalFlagsName, + e2etests.TestZetaclientSignerOffsetName, + e2etests.TestZetaclientRestartHeightName, e2etests.TestWhitelistERC20Name, e2etests.TestPauseZRC20Name, e2etests.TestUpdateBytecodeZRC20Name, diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index cabc1e394f..1b9f3ef419 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -132,7 +132,8 @@ const ( TestMigrateERC20CustodyFundsName = "migrate_erc20_custody_funds" TestMigrateTSSName = "migrate_tss" TestSolanaWhitelistSPLName = "solana_whitelist_spl" - TestOperationalFlagsName = "operational_flags" + TestZetaclientRestartHeightName = "zetaclient_restart_height" + TestZetaclientSignerOffsetName = "zetaclient_signer_offset" /* Operational tests @@ -880,10 +881,16 @@ var AllE2ETests = []runner.E2ETest{ TestMigrateERC20CustodyFunds, ), runner.NewE2ETest( - TestOperationalFlagsName, - "operational flags functionality", + TestZetaclientRestartHeightName, + "zetaclient scheduled restart height", []runner.ArgDefinition{}, - TestOperationalFlags, + TestZetaclientRestartHeight, + ), + runner.NewE2ETest( + TestZetaclientSignerOffsetName, + "zetaclient signer offset", + []runner.ArgDefinition{}, + TestZetaclientSignerOffset, ), /* Special tests diff --git a/e2e/e2etests/test_operational_flags.go b/e2e/e2etests/test_operational_flags.go index 5084c38b58..fb289b2dc3 100644 --- a/e2e/e2etests/test_operational_flags.go +++ b/e2e/e2etests/test_operational_flags.go @@ -11,11 +11,13 @@ import ( ) const ( - startTimestampMetricName = "zetaclient_last_start_timestamp_seconds" + startTimestampMetricName = "zetaclient_last_start_timestamp_seconds" + blockTimeLatencyMetricName = "zetaclient_core_block_latency" + blockTimeLatencySleepMetricName = "zetaclient_core_block_latency_sleep" ) -// TestOperationalFlags tests the functionality of operations flags. -func TestOperationalFlags(r *runner.E2ERunner, _ []string) { +// TestZetaclientRestartHeight tests scheduling a zetaclient restart via operational flags +func TestZetaclientRestartHeight(r *runner.E2ERunner, _ []string) { _, err := r.Clients.Zetacore.Observer.OperationalFlags( r.Ctx, &observertypes.QueryOperationalFlagsRequest{}, @@ -60,3 +62,40 @@ func TestOperationalFlags(r *runner.E2ERunner, _ []string) { require.Greater(r, currentStartTime, originalStartTime+1) } + +// TestZetaclientSignerOffset tests scheduling a zetaclient restart via operational flags +func TestZetaclientSignerOffset(r *runner.E2ERunner, _ []string) { + startBlockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName) + require.NoError(r, err) + require.InDelta(r, 0, startBlockTimeLatencySleep, .01, "start block time latency should be 0") + + startBlockTimeLatency, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencyMetricName) + require.NoError(r, err) + + desiredSignerBlockTimeOffset := time.Duration(startBlockTimeLatency*float64(time.Second)) + time.Millisecond*200 + + updateMsg := observertypes.NewMsgUpdateOperationalFlags( + r.ZetaTxServer.MustGetAccountAddressFromName(utils.OperationalPolicyName), + observertypes.OperationalFlags{ + SignerBlockTimeOffset: &desiredSignerBlockTimeOffset, + }, + ) + + _, err = r.ZetaTxServer.BroadcastTx(utils.OperationalPolicyName, updateMsg) + require.NoError(r, err) + + operationalFlagsRes, err := r.Clients.Zetacore.Observer.OperationalFlags( + r.Ctx, + &observertypes.QueryOperationalFlagsRequest{}, + ) + require.NoError(r, err) + require.InDelta(r, desiredSignerBlockTimeOffset, *(operationalFlagsRes.OperationalFlags.SignerBlockTimeOffset), .01) + + require.Eventually(r, func() bool { + blockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName) + if err != nil { + return false + } + return blockTimeLatencySleep > .1 + }, time.Second*15, time.Second*1) +} diff --git a/proto/zetachain/zetacore/observer/operational.proto b/proto/zetachain/zetacore/observer/operational.proto index f94c7e7d11..62ff737074 100644 --- a/proto/zetachain/zetacore/observer/operational.proto +++ b/proto/zetachain/zetacore/observer/operational.proto @@ -1,6 +1,9 @@ syntax = "proto3"; package zetachain.zetacore.observer; +import "gogoproto/gogo.proto"; +import "google/protobuf/duration.proto"; + option go_package = "github.com/zeta-chain/node/x/observer/types"; // Flags for the top-level operation of zetaclient. @@ -8,4 +11,9 @@ message OperationalFlags { // Height for a coordinated zetaclient restart. // Will be ignored if missed. int64 restart_height = 1; + + // Offset from the zetacore block time to initiate signing. + // Should be calculated and set based on max(zetaclient_core_block_latency). + google.protobuf.Duration signer_block_time_offset = 2 + [ (gogoproto.stdduration) = true ]; } \ No newline at end of file diff --git a/proto/zetachain/zetacore/observer/tx.proto b/proto/zetachain/zetacore/observer/tx.proto index 0d4ecf275e..aaeb62ba7c 100644 --- a/proto/zetachain/zetacore/observer/tx.proto +++ b/proto/zetachain/zetacore/observer/tx.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package zetachain.zetacore.observer; import "gogoproto/gogo.proto"; +import "google/protobuf/field_mask.proto"; import "zetachain/zetacore/observer/blame.proto"; import "zetachain/zetacore/observer/crosschain_flags.proto"; import "zetachain/zetacore/observer/observer.proto"; @@ -143,7 +144,7 @@ message MsgUpdateGasPriceIncreaseFlagsResponse {} message MsgUpdateOperationalFlags { string creator = 1; - OperationalFlags operationalFlags = 2 [ (gogoproto.nullable) = false ]; + OperationalFlags operational_flags = 2 [ (gogoproto.nullable) = false ]; } message MsgUpdateOperationalFlagsResponse {} diff --git a/x/observer/client/cli/tx_update_operational_flags.go b/x/observer/client/cli/tx_update_operational_flags.go index e730331cf9..6d380706e8 100644 --- a/x/observer/client/cli/tx_update_operational_flags.go +++ b/x/observer/client/cli/tx_update_operational_flags.go @@ -13,8 +13,9 @@ import ( ) const ( - fileFlag = "file" - restartHeightFlag = "restart-height" + fileFlag = "file" + restartHeightFlag = "restart-height" + signerBlockTimeOffsetFlag = "signer-block-time-offset" ) func CmdUpdateOperationalFlags() *cobra.Command { @@ -33,6 +34,7 @@ func CmdUpdateOperationalFlags() *cobra.Command { flagSet := cmd.Flags() file, _ := flagSet.GetString(fileFlag) restartHeight, _ := flagSet.GetInt64(restartHeightFlag) + signerBlockTimeOffset, _ := flagSet.GetDuration(signerBlockTimeOffsetFlag) if file != "" { input, err := os.ReadFile(file) // #nosec G304 @@ -45,6 +47,7 @@ func CmdUpdateOperationalFlags() *cobra.Command { } } else { operationalFlags.RestartHeight = restartHeight + operationalFlags.SignerBlockTimeOffset = &signerBlockTimeOffset } msg := types.NewMsgUpdateOperationalFlags( @@ -58,6 +61,7 @@ func CmdUpdateOperationalFlags() *cobra.Command { cmd.Flags().String(fileFlag, "", "Path to a JSON file containing OperationalFlags") cmd.Flags().Int64(restartHeightFlag, 0, "Height for a coordinated zetaclient restart") + cmd.Flags().Duration(signerBlockTimeOffsetFlag, 0, "Offset from the zetacore block time to initiate signing") flags.AddTxFlagsToCmd(cmd) return cmd diff --git a/x/observer/types/operational.pb.go b/x/observer/types/operational.pb.go index b7af5711cf..e5fb9c3281 100644 --- a/x/observer/types/operational.pb.go +++ b/x/observer/types/operational.pb.go @@ -5,16 +5,21 @@ package types import ( fmt "fmt" + _ "github.com/cosmos/gogoproto/gogoproto" proto "github.com/cosmos/gogoproto/proto" + github_com_cosmos_gogoproto_types "github.com/cosmos/gogoproto/types" + _ "google.golang.org/protobuf/types/known/durationpb" io "io" math "math" math_bits "math/bits" + time "time" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf +var _ = time.Kitchen // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. @@ -27,6 +32,9 @@ type OperationalFlags struct { // Height for a coordinated zetaclient restart. // Will be ignored if missed. RestartHeight int64 `protobuf:"varint,1,opt,name=restart_height,json=restartHeight,proto3" json:"restart_height,omitempty"` + // Offset from the zetacore block time to initiate signing. + // Should be calculated and set based on max(zetaclient_core_block_latency). + SignerBlockTimeOffset *time.Duration `protobuf:"bytes,2,opt,name=signer_block_time_offset,json=signerBlockTimeOffset,proto3,stdduration" json:"signer_block_time_offset,omitempty"` } func (m *OperationalFlags) Reset() { *m = OperationalFlags{} } @@ -69,6 +77,13 @@ func (m *OperationalFlags) GetRestartHeight() int64 { return 0 } +func (m *OperationalFlags) GetSignerBlockTimeOffset() *time.Duration { + if m != nil { + return m.SignerBlockTimeOffset + } + return nil +} + func init() { proto.RegisterType((*OperationalFlags)(nil), "zetachain.zetacore.observer.OperationalFlags") } @@ -78,19 +93,25 @@ func init() { } var fileDescriptor_ea3eed2ec55093b5 = []byte{ - // 184 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0xad, 0x4a, 0x2d, 0x49, - 0x4c, 0xce, 0x48, 0xcc, 0xcc, 0xd3, 0x07, 0xb3, 0xf2, 0x8b, 0x52, 0xf5, 0xf3, 0x93, 0x8a, 0x53, - 0x8b, 0xca, 0x52, 0x8b, 0xf4, 0xf3, 0x0b, 0x52, 0x8b, 0x12, 0x4b, 0x32, 0xf3, 0xf3, 0x12, 0x73, - 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xa4, 0xe1, 0xca, 0xf5, 0x60, 0xca, 0xf5, 0x60, 0xca, - 0x95, 0x2c, 0xb9, 0x04, 0xfc, 0x11, 0x3a, 0xdc, 0x72, 0x12, 0xd3, 0x8b, 0x85, 0x54, 0xb9, 0xf8, - 0x8a, 0x52, 0x8b, 0x4b, 0x12, 0x8b, 0x4a, 0xe2, 0x33, 0x52, 0x33, 0xd3, 0x33, 0x4a, 0x24, 0x18, - 0x15, 0x18, 0x35, 0x98, 0x83, 0x78, 0xa1, 0xa2, 0x1e, 0x60, 0x41, 0x27, 0xd7, 0x13, 0x8f, 0xe4, - 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, - 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0xd2, 0x4e, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, - 0xce, 0xcf, 0x05, 0xbb, 0x50, 0x17, 0xe2, 0xd8, 0xbc, 0xfc, 0x94, 0x54, 0xfd, 0x0a, 0x84, 0x53, - 0x4b, 0x2a, 0x0b, 0x52, 0x8b, 0x93, 0xd8, 0xc0, 0xae, 0x34, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, - 0xcf, 0xd8, 0x7d, 0x54, 0xd6, 0x00, 0x00, 0x00, + // 282 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0xc1, 0x4a, 0xec, 0x30, + 0x14, 0x86, 0x27, 0xf7, 0x8a, 0x8b, 0x8a, 0x22, 0x83, 0x42, 0x1d, 0x21, 0x0e, 0x82, 0x30, 0x20, + 0x93, 0x80, 0xbe, 0x41, 0x51, 0x71, 0x37, 0x30, 0xb8, 0x10, 0x37, 0x25, 0xed, 0x9c, 0xa6, 0xc1, + 0xb6, 0xa7, 0x24, 0xa9, 0xa8, 0x4f, 0x21, 0xb8, 0xf1, 0x91, 0x5c, 0xce, 0xd2, 0x9d, 0xd2, 0xbe, + 0x88, 0x4c, 0x32, 0x75, 0x76, 0x27, 0x27, 0xdf, 0x7f, 0x3e, 0xf8, 0x83, 0xe9, 0x2b, 0x58, 0x91, + 0xe6, 0x42, 0x55, 0xdc, 0x4d, 0xa8, 0x81, 0x63, 0x62, 0x40, 0x3f, 0x81, 0xe6, 0x58, 0x83, 0x16, + 0x56, 0x61, 0x25, 0x0a, 0x56, 0x6b, 0xb4, 0x38, 0x3c, 0xfe, 0xc3, 0x59, 0x8f, 0xb3, 0x1e, 0x1f, + 0x1d, 0x48, 0x94, 0xe8, 0x38, 0xbe, 0x9a, 0x7c, 0x64, 0x44, 0x25, 0xa2, 0x2c, 0x80, 0xbb, 0x57, + 0xd2, 0x64, 0x7c, 0xd1, 0xf8, 0xa3, 0xfe, 0xff, 0xf4, 0x9d, 0x04, 0xfb, 0xb3, 0x8d, 0xe8, 0xa6, + 0x10, 0xd2, 0x0c, 0xcf, 0x82, 0x3d, 0x0d, 0xc6, 0x0a, 0x6d, 0xe3, 0x1c, 0x94, 0xcc, 0x6d, 0x48, + 0xc6, 0x64, 0xf2, 0x7f, 0xbe, 0xbb, 0xde, 0xde, 0xba, 0xe5, 0xf0, 0x3e, 0x08, 0x8d, 0x92, 0x15, + 0xe8, 0x38, 0x29, 0x30, 0x7d, 0x8c, 0xad, 0x2a, 0x21, 0xc6, 0x2c, 0x33, 0x60, 0xc3, 0x7f, 0x63, + 0x32, 0xd9, 0xb9, 0x38, 0x62, 0x5e, 0xcf, 0x7a, 0x3d, 0xbb, 0x5a, 0xeb, 0xa3, 0xad, 0x8f, 0xef, + 0x13, 0x32, 0x3f, 0xf4, 0x07, 0xa2, 0x55, 0xfe, 0x4e, 0x95, 0x30, 0x73, 0xe9, 0xe8, 0xfa, 0xb3, + 0xa5, 0x64, 0xd9, 0x52, 0xf2, 0xd3, 0x52, 0xf2, 0xd6, 0xd1, 0xc1, 0xb2, 0xa3, 0x83, 0xaf, 0x8e, + 0x0e, 0x1e, 0xce, 0xa5, 0xb2, 0x79, 0x93, 0xb0, 0x14, 0x4b, 0x57, 0xd9, 0xd4, 0xb7, 0x57, 0xe1, + 0x02, 0xf8, 0xf3, 0xa6, 0x3b, 0xfb, 0x52, 0x83, 0x49, 0xb6, 0x9d, 0xf6, 0xf2, 0x37, 0x00, 0x00, + 0xff, 0xff, 0xf0, 0xe0, 0x3c, 0x60, 0x67, 0x01, 0x00, 0x00, } func (m *OperationalFlags) Marshal() (dAtA []byte, err error) { @@ -113,6 +134,16 @@ func (m *OperationalFlags) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SignerBlockTimeOffset != nil { + n1, err1 := github_com_cosmos_gogoproto_types.StdDurationMarshalTo(*m.SignerBlockTimeOffset, dAtA[i-github_com_cosmos_gogoproto_types.SizeOfStdDuration(*m.SignerBlockTimeOffset):]) + if err1 != nil { + return 0, err1 + } + i -= n1 + i = encodeVarintOperational(dAtA, i, uint64(n1)) + i-- + dAtA[i] = 0x12 + } if m.RestartHeight != 0 { i = encodeVarintOperational(dAtA, i, uint64(m.RestartHeight)) i-- @@ -141,6 +172,10 @@ func (m *OperationalFlags) Size() (n int) { if m.RestartHeight != 0 { n += 1 + sovOperational(uint64(m.RestartHeight)) } + if m.SignerBlockTimeOffset != nil { + l = github_com_cosmos_gogoproto_types.SizeOfStdDuration(*m.SignerBlockTimeOffset) + n += 1 + l + sovOperational(uint64(l)) + } return n } @@ -198,6 +233,42 @@ func (m *OperationalFlags) Unmarshal(dAtA []byte) error { break } } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SignerBlockTimeOffset", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperational + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOperational + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOperational + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SignerBlockTimeOffset == nil { + m.SignerBlockTimeOffset = new(time.Duration) + } + if err := github_com_cosmos_gogoproto_types.StdDurationUnmarshal(m.SignerBlockTimeOffset, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipOperational(dAtA[iNdEx:]) diff --git a/x/observer/types/tx.pb.go b/x/observer/types/tx.pb.go index 7ea1ae4086..eb779eebc5 100644 --- a/x/observer/types/tx.pb.go +++ b/x/observer/types/tx.pb.go @@ -14,6 +14,7 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + _ "google.golang.org/protobuf/types/known/fieldmaskpb" io "io" math "math" math_bits "math/bits" @@ -1242,7 +1243,7 @@ var xxx_messageInfo_MsgUpdateGasPriceIncreaseFlagsResponse proto.InternalMessage type MsgUpdateOperationalFlags struct { Creator string `protobuf:"bytes,1,opt,name=creator,proto3" json:"creator,omitempty"` - OperationalFlags OperationalFlags `protobuf:"bytes,2,opt,name=operationalFlags,proto3" json:"operationalFlags"` + OperationalFlags OperationalFlags `protobuf:"bytes,2,opt,name=operational_flags,json=operationalFlags,proto3" json:"operational_flags"` } func (m *MsgUpdateOperationalFlags) Reset() { *m = MsgUpdateOperationalFlags{} } @@ -1362,91 +1363,92 @@ func init() { } var fileDescriptor_eda6e3b1d16a4021 = []byte{ - // 1332 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcb, 0x6f, 0xdb, 0xc6, - 0x13, 0x36, 0x7f, 0xce, 0xc3, 0x1e, 0x5b, 0xb2, 0xc3, 0x9f, 0xe3, 0xd8, 0x74, 0xa3, 0xa6, 0x6c, - 0xe2, 0x28, 0x2f, 0x29, 0x56, 0x8a, 0x3e, 0x81, 0x02, 0x8e, 0x93, 0xd8, 0x6e, 0xea, 0x24, 0xa0, - 0xdc, 0xa0, 0xc8, 0x85, 0x58, 0x91, 0x6b, 0x8a, 0x35, 0xbd, 0x2b, 0x70, 0x29, 0x3b, 0x4e, 0x83, - 0x02, 0x3d, 0x16, 0xe8, 0x21, 0xa7, 0x9e, 0x0a, 0xf4, 0xde, 0xff, 0xa0, 0xf7, 0x1e, 0xd2, 0x43, - 0x81, 0x1c, 0x7b, 0x2a, 0x8a, 0xe4, 0xd4, 0xff, 0xa2, 0xe0, 0xee, 0x72, 0x25, 0x51, 0x32, 0x45, - 0xb9, 0xe8, 0x49, 0xe4, 0xec, 0xf7, 0xcd, 0x7c, 0xb3, 0x8f, 0x99, 0x15, 0xe1, 0xe2, 0x33, 0x1c, - 0x21, 0xa7, 0x89, 0x7c, 0x52, 0xe5, 0x4f, 0x34, 0xc4, 0x55, 0xda, 0x60, 0x38, 0xdc, 0xc7, 0x61, - 0x35, 0x7a, 0x5a, 0x69, 0x85, 0x34, 0xa2, 0xfa, 0x92, 0x42, 0x55, 0x12, 0x54, 0x25, 0x41, 0x19, - 0x73, 0x1e, 0xf5, 0x28, 0xc7, 0x55, 0xe3, 0x27, 0x41, 0x31, 0x2e, 0x67, 0x39, 0x6e, 0x04, 0x68, - 0x0f, 0x4b, 0x60, 0x2d, 0x0b, 0xe8, 0x84, 0x94, 0x31, 0x3e, 0x68, 0xef, 0x04, 0xc8, 0x63, 0x92, - 0x73, 0x35, 0x8b, 0x93, 0x3c, 0x48, 0x6c, 0x39, 0x0b, 0xdb, 0x42, 0x21, 0xda, 0x4b, 0xbc, 0xde, - 0xcc, 0x44, 0x62, 0xe2, 0xfa, 0xc4, 0xb3, 0x09, 0x25, 0x0e, 0x4e, 0x18, 0x97, 0x32, 0x67, 0x8f, - 0x25, 0xb0, 0x1b, 0x99, 0x72, 0x5b, 0x38, 0x44, 0x91, 0x4f, 0x09, 0x0a, 0x32, 0xb2, 0x6b, 0xed, - 0x7a, 0x55, 0x6e, 0x62, 0xf2, 0x67, 0x08, 0xb6, 0x15, 0x52, 0xba, 0xc3, 0xe4, 0x8f, 0xc0, 0x9a, - 0x7f, 0x6b, 0x70, 0x66, 0x8b, 0x79, 0x5f, 0xb4, 0x5c, 0x14, 0xe1, 0x87, 0x32, 0xbe, 0xbe, 0x00, - 0xa7, 0x9d, 0x10, 0xa3, 0x88, 0x86, 0x0b, 0xda, 0x05, 0xad, 0x3c, 0x69, 0x25, 0xaf, 0xfa, 0x4d, - 0x98, 0xa3, 0x81, 0x6b, 0x27, 0x4a, 0x6d, 0xe4, 0xba, 0x21, 0x66, 0x6c, 0xe1, 0x7f, 0x1c, 0xa6, - 0xd3, 0xc0, 0x4d, 0x9c, 0xac, 0x8a, 0x91, 0x98, 0x41, 0xf0, 0x41, 0x3f, 0x63, 0x5c, 0x30, 0x08, - 0x3e, 0x48, 0x33, 0x1e, 0x43, 0xa1, 0xcd, 0xf5, 0xd8, 0x21, 0x46, 0x8c, 0x92, 0x85, 0x13, 0x17, - 0xb4, 0x72, 0xb1, 0xb6, 0x52, 0xc9, 0xd8, 0x71, 0x95, 0xc4, 0x89, 0xc8, 0xc4, 0xe2, 0x44, 0x6b, - 0xba, 0xdd, 0xf5, 0x66, 0x2e, 0xc1, 0x62, 0x5f, 0xaa, 0x16, 0x66, 0x2d, 0x4a, 0x18, 0x36, 0x7f, - 0xd3, 0x40, 0xdf, 0x62, 0xde, 0x63, 0x1a, 0xe1, 0xdb, 0x01, 0x75, 0x76, 0x37, 0x30, 0x72, 0x33, - 0x67, 0x62, 0x11, 0x26, 0xc4, 0x26, 0xf4, 0x5d, 0x9e, 0xfd, 0xb8, 0x75, 0x9a, 0xbf, 0x6f, 0xba, - 0xfa, 0x79, 0x80, 0x46, 0xec, 0xc3, 0x6e, 0x22, 0xd6, 0xe4, 0x89, 0x4e, 0x5b, 0x93, 0xdc, 0xb2, - 0x81, 0x58, 0x53, 0x9f, 0x87, 0x53, 0x4d, 0xec, 0x7b, 0xcd, 0x88, 0x27, 0x36, 0x6e, 0xc9, 0x37, - 0x7d, 0x3d, 0xb6, 0xc7, 0x51, 0x17, 0x4e, 0x5e, 0xd0, 0xca, 0x53, 0xb5, 0x2b, 0x83, 0x12, 0x6e, - 0xed, 0x7a, 0x15, 0xb9, 0x82, 0x42, 0xe2, 0x1d, 0x14, 0xa1, 0xdb, 0x27, 0x5e, 0xfe, 0xf9, 0xf6, - 0x98, 0x25, 0xe9, 0xe6, 0x57, 0x60, 0xf4, 0xa7, 0x92, 0x64, 0xaa, 0x5f, 0x82, 0x62, 0x03, 0x05, - 0x01, 0x8d, 0x6c, 0x9e, 0x0a, 0x76, 0x79, 0x66, 0x13, 0x56, 0x41, 0x58, 0xd7, 0x84, 0x31, 0x86, - 0xed, 0xd3, 0x08, 0xdb, 0x3b, 0x3e, 0x41, 0x81, 0xff, 0x0c, 0x8b, 0x2c, 0x27, 0xac, 0x42, 0x6c, - 0xbd, 0x97, 0x18, 0xcd, 0xe7, 0x30, 0xa7, 0x26, 0x75, 0x2d, 0x96, 0xfa, 0x88, 0x1f, 0x9f, 0x8c, - 0x89, 0xfb, 0x0c, 0xa6, 0x9c, 0x0e, 0x90, 0x7b, 0x9d, 0xaa, 0x95, 0x33, 0x17, 0xb7, 0xcb, 0xb1, - 0xd5, 0x4d, 0x36, 0x4b, 0xf0, 0xd6, 0xa0, 0xe8, 0x6a, 0x55, 0xef, 0x73, 0x75, 0x16, 0xde, 0xa3, - 0xfb, 0x39, 0xd5, 0x1d, 0xbd, 0xac, 0x32, 0x58, 0x9f, 0x33, 0x15, 0xec, 0x57, 0x0d, 0x8a, 0x5b, - 0xcc, 0x5b, 0x75, 0xdd, 0x1c, 0x07, 0xe9, 0x0a, 0xcc, 0x1e, 0x71, 0x88, 0x66, 0x68, 0xea, 0x3c, - 0x7c, 0x0c, 0x8b, 0x7c, 0x4a, 0x02, 0x1f, 0x93, 0xc8, 0xf6, 0x42, 0x44, 0x22, 0x8c, 0xed, 0x56, - 0xbb, 0xb1, 0x8b, 0x0f, 0xe5, 0x31, 0x3a, 0xd7, 0x01, 0xac, 0x8b, 0xf1, 0x47, 0x7c, 0x58, 0x5f, - 0x81, 0xb3, 0xc8, 0x75, 0x6d, 0x42, 0x5d, 0x6c, 0x23, 0xc7, 0xa1, 0x6d, 0x12, 0xd9, 0x94, 0x04, - 0x87, 0x7c, 0xeb, 0x4d, 0x58, 0x3a, 0x72, 0xdd, 0x07, 0xd4, 0xc5, 0xab, 0x62, 0xe8, 0x21, 0x09, - 0x0e, 0xcd, 0x05, 0x98, 0xef, 0xcd, 0x42, 0x25, 0xf8, 0xbd, 0x06, 0xd3, 0x6a, 0x63, 0xa1, 0x3d, - 0x7c, 0xbc, 0xd3, 0xb1, 0x1e, 0x9f, 0x0e, 0xb4, 0x87, 0x6d, 0x9f, 0xec, 0x50, 0xae, 0x7f, 0xaa, - 0x66, 0x66, 0x2e, 0x3f, 0x0f, 0x26, 0xf7, 0xf8, 0x24, 0xe7, 0x6e, 0x92, 0x1d, 0x6a, 0xce, 0xf3, - 0xc5, 0x55, 0x6a, 0x94, 0xcc, 0x55, 0x98, 0x51, 0x9b, 0xe2, 0x3e, 0x3e, 0xf4, 0x30, 0xc9, 0x10, - 0x3a, 0x07, 0x27, 0xf9, 0xc9, 0x94, 0x2a, 0xc5, 0x8b, 0xb9, 0x08, 0xe7, 0x52, 0x2e, 0x94, 0xf7, - 0x1f, 0x35, 0xf8, 0x3f, 0xdf, 0x06, 0x0c, 0x47, 0x7c, 0x17, 0x3c, 0xe0, 0xd5, 0xff, 0x78, 0x73, - 0xb1, 0x0c, 0x33, 0x62, 0x88, 0xb7, 0x10, 0x3b, 0xa0, 0x07, 0x7c, 0x42, 0xc6, 0xad, 0x82, 0xa3, - 0x5c, 0x7f, 0x4e, 0x0f, 0xf4, 0x32, 0xcc, 0x76, 0xe3, 0x9a, 0xbe, 0xd7, 0x94, 0xc5, 0xa3, 0xd8, - 0x01, 0x6e, 0xf8, 0x5e, 0xd3, 0x3c, 0x0f, 0x4b, 0x03, 0xd4, 0x29, 0xf5, 0xbf, 0x68, 0x00, 0x72, - 0xd2, 0xb6, 0xeb, 0xf5, 0x0c, 0xd1, 0xe7, 0x01, 0x22, 0xc6, 0x92, 0x5d, 0x26, 0x76, 0xe6, 0x64, - 0xc4, 0x98, 0xdc, 0x57, 0xd7, 0x41, 0xdf, 0xe5, 0xf3, 0x62, 0xc7, 0xcb, 0x65, 0xcb, 0x7a, 0x26, - 0xb4, 0xcf, 0x8a, 0x91, 0x27, 0x38, 0x42, 0x1b, 0xa2, 0xb2, 0xdd, 0x81, 0x53, 0x2c, 0x42, 0x51, - 0x9b, 0xc9, 0x52, 0x7e, 0xfd, 0xa8, 0xca, 0x26, 0xfb, 0x98, 0x85, 0x1d, 0xec, 0xef, 0xe3, 0x3a, - 0xe7, 0x58, 0x92, 0x6b, 0x7e, 0xd7, 0x29, 0xd1, 0xdb, 0xf5, 0xfa, 0x7f, 0x53, 0xcf, 0x62, 0x98, - 0x4c, 0x8c, 0xb5, 0x1d, 0x27, 0x69, 0x54, 0x13, 0x56, 0x41, 0x58, 0xeb, 0xc2, 0x68, 0x1e, 0x40, - 0x61, 0x8b, 0x79, 0x77, 0x09, 0x6a, 0x04, 0x78, 0x6d, 0x6d, 0xfb, 0xcb, 0x8c, 0x99, 0xbc, 0x08, - 0x05, 0xcc, 0x71, 0x9b, 0xa4, 0x41, 0xdb, 0x44, 0xc5, 0xed, 0x31, 0xea, 0xcb, 0x50, 0x14, 0x86, - 0x87, 0xed, 0x48, 0xc0, 0x44, 0xdc, 0x94, 0xd5, 0x3c, 0x07, 0x67, 0x7b, 0x02, 0xab, 0x95, 0x7d, - 0xce, 0x8b, 0xcf, 0x1d, 0x9f, 0xe5, 0x90, 0xb4, 0x0c, 0x45, 0x57, 0x00, 0x7b, 0x35, 0xa5, 0xac, - 0x7a, 0x19, 0x66, 0xa4, 0x25, 0xa5, 0x2a, 0x6d, 0x96, 0x45, 0xa3, 0x2b, 0xba, 0xd2, 0xf5, 0xb3, - 0x06, 0x25, 0x75, 0x96, 0xd6, 0x11, 0x7b, 0x14, 0xfa, 0x0e, 0xde, 0x24, 0xb1, 0x14, 0x86, 0xef, - 0xc5, 0x17, 0xb8, 0x0c, 0xa1, 0x04, 0xce, 0x7a, 0x83, 0x28, 0xb2, 0x6b, 0xd4, 0x32, 0xcb, 0xc6, - 0xc0, 0x60, 0xb2, 0x8c, 0x0c, 0x76, 0x6b, 0x96, 0x61, 0x39, 0x5b, 0xab, 0x4a, 0xeb, 0x07, 0xad, - 0xfb, 0x36, 0xd1, 0xb9, 0xaf, 0x0d, 0xcb, 0xc8, 0x86, 0x59, 0x9a, 0x42, 0xcb, 0x64, 0x6e, 0x64, - 0xdf, 0x6f, 0x52, 0x24, 0x99, 0x47, 0x9f, 0x33, 0xf3, 0x5d, 0x78, 0xe7, 0x48, 0x5d, 0x89, 0xfa, - 0xda, 0xef, 0xd3, 0x30, 0xbe, 0xc5, 0x3c, 0x9d, 0xc2, 0x54, 0x77, 0xbb, 0xba, 0x96, 0x29, 0xa1, - 0xb7, 0x2b, 0x18, 0xb7, 0x46, 0x00, 0xab, 0xc3, 0xfa, 0x14, 0x8a, 0xa9, 0xbb, 0x66, 0x65, 0x98, - 0x9b, 0x5e, 0xbc, 0xf1, 0xfe, 0x68, 0x78, 0x15, 0xf9, 0x5b, 0x0d, 0xce, 0xf4, 0x5f, 0x53, 0x56, - 0xf2, 0x79, 0xeb, 0xa2, 0x18, 0x1f, 0x8d, 0x4c, 0xe9, 0xd1, 0xd0, 0x7f, 0x19, 0x19, 0xaa, 0xa1, - 0x8f, 0x32, 0x5c, 0xc3, 0x91, 0xb7, 0x14, 0xdd, 0x87, 0xc9, 0x4e, 0x03, 0xbf, 0x32, 0xcc, 0x8f, - 0x82, 0x1a, 0x2b, 0xb9, 0xa1, 0x2a, 0x54, 0x08, 0xd3, 0x3d, 0x5d, 0xf8, 0x7a, 0xbe, 0x99, 0x13, - 0x68, 0xe3, 0xbd, 0x51, 0xd0, 0x2a, 0xe6, 0xd7, 0x30, 0x93, 0xbe, 0xc3, 0x57, 0xf3, 0x29, 0x57, - 0x04, 0xe3, 0x83, 0x11, 0x09, 0x2a, 0xf8, 0x37, 0x30, 0xdb, 0x77, 0x2f, 0xb8, 0x39, 0x7c, 0xa9, - 0x7a, 0x19, 0xc6, 0x87, 0xa3, 0x32, 0x54, 0x7c, 0x07, 0x4e, 0x27, 0x9d, 0xfd, 0x72, 0x9e, 0x1c, - 0xb6, 0xeb, 0x75, 0xa3, 0x9a, 0x13, 0xa8, 0x82, 0x04, 0x00, 0x5d, 0x7d, 0xef, 0xea, 0x30, 0x7a, - 0x07, 0x6b, 0xd4, 0xf2, 0x63, 0x55, 0x34, 0x0a, 0x53, 0xdd, 0x3d, 0x6d, 0x68, 0x85, 0xea, 0x02, - 0x0f, 0xaf, 0x50, 0x03, 0xfa, 0x95, 0xfe, 0x93, 0x06, 0x4b, 0x59, 0xcd, 0xea, 0x93, 0x7c, 0xdb, - 0x72, 0x20, 0xd9, 0x58, 0xfb, 0x17, 0x64, 0xa5, 0xf0, 0x85, 0x06, 0xf3, 0x47, 0xf4, 0x9d, 0xbc, - 0xc5, 0x31, 0xc5, 0x33, 0x3e, 0x3d, 0x1e, 0x2f, 0x91, 0x74, 0xfb, 0xee, 0xcb, 0xd7, 0x25, 0xed, - 0xd5, 0xeb, 0x92, 0xf6, 0xd7, 0xeb, 0x92, 0xf6, 0xe2, 0x4d, 0x69, 0xec, 0xd5, 0x9b, 0xd2, 0xd8, - 0x1f, 0x6f, 0x4a, 0x63, 0x4f, 0xae, 0x79, 0x7e, 0xd4, 0x6c, 0x37, 0x2a, 0x0e, 0xdd, 0xe3, 0x5f, - 0x23, 0x6e, 0x88, 0x0f, 0x13, 0xf1, 0x1f, 0x92, 0xea, 0xd3, 0xae, 0x0f, 0x23, 0x87, 0x2d, 0xcc, - 0x1a, 0xa7, 0xf8, 0x47, 0x89, 0x5b, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa3, 0x3d, 0x21, - 0x82, 0x12, 0x00, 0x00, + // 1356 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6f, 0x1b, 0x45, + 0x14, 0xcf, 0x92, 0x7e, 0x24, 0x2f, 0xb1, 0x93, 0x2e, 0x69, 0x9a, 0x6c, 0xa8, 0x09, 0x4b, 0x9b, + 0xba, 0x5f, 0x76, 0xe3, 0x22, 0x3e, 0x25, 0xa4, 0x34, 0x6d, 0x93, 0x50, 0xd2, 0x56, 0xeb, 0x50, + 0xa1, 0x5e, 0x96, 0xf1, 0xee, 0x78, 0xbd, 0x64, 0x3d, 0x63, 0xed, 0xac, 0x93, 0xa6, 0x54, 0x48, + 0x1c, 0x91, 0x38, 0xf4, 0xc6, 0x05, 0x89, 0x3b, 0xff, 0x01, 0x77, 0x0e, 0xe5, 0x80, 0xd4, 0x23, + 0x27, 0x84, 0xda, 0x13, 0xff, 0x05, 0xda, 0x99, 0xd9, 0xf1, 0xfa, 0x23, 0x6b, 0x27, 0x88, 0x53, + 0x3c, 0x6f, 0x7e, 0xbf, 0xf7, 0x7e, 0x6f, 0x3e, 0xde, 0x9b, 0x2c, 0x5c, 0x78, 0x8a, 0x23, 0xe4, + 0x34, 0x90, 0x4f, 0xca, 0xfc, 0x17, 0x0d, 0x71, 0x99, 0xd6, 0x18, 0x0e, 0xf7, 0x70, 0x58, 0x8e, + 0x9e, 0x94, 0x5a, 0x21, 0x8d, 0xa8, 0xbe, 0xa4, 0x50, 0xa5, 0x04, 0x55, 0x4a, 0x50, 0xc6, 0x9c, + 0x47, 0x3d, 0xca, 0x71, 0xe5, 0xf8, 0x97, 0xa0, 0x18, 0xcb, 0x1e, 0xa5, 0x5e, 0x80, 0xcb, 0x7c, + 0x54, 0x6b, 0xd7, 0xcb, 0x75, 0x1f, 0x07, 0xae, 0xdd, 0x44, 0x6c, 0x57, 0x22, 0x2e, 0x65, 0x85, + 0xae, 0x05, 0xa8, 0x89, 0x25, 0xb0, 0x92, 0x05, 0x74, 0x42, 0xca, 0x18, 0x9f, 0xb4, 0xeb, 0x01, + 0xf2, 0x98, 0xe4, 0x5c, 0xc9, 0xe2, 0x24, 0x3f, 0x24, 0xb6, 0x98, 0x85, 0x6d, 0xa1, 0x10, 0x35, + 0x13, 0xaf, 0x37, 0x32, 0x91, 0x98, 0xb8, 0x3e, 0xf1, 0x6c, 0x42, 0x89, 0x83, 0x13, 0xc6, 0xc5, + 0xcc, 0xf5, 0x65, 0x09, 0xec, 0x7a, 0xa6, 0xdc, 0x16, 0x0e, 0x51, 0xe4, 0x53, 0x82, 0x82, 0x8c, + 0xec, 0x5a, 0xbb, 0x5e, 0x99, 0x9b, 0x98, 0xfc, 0x33, 0x04, 0xdb, 0x0a, 0x29, 0xad, 0x33, 0xf9, + 0x47, 0x60, 0xcd, 0x7f, 0x34, 0x38, 0xb3, 0xcd, 0xbc, 0x2f, 0x5a, 0x2e, 0x8a, 0xf0, 0x03, 0x19, + 0x5f, 0x5f, 0x80, 0xd3, 0x4e, 0x88, 0x51, 0x44, 0xc3, 0x05, 0x6d, 0x59, 0x2b, 0x4e, 0x5a, 0xc9, + 0x50, 0xbf, 0x01, 0x73, 0x34, 0x70, 0xed, 0x44, 0xa9, 0x8d, 0x5c, 0x37, 0xc4, 0x8c, 0x2d, 0xbc, + 0xc1, 0x61, 0x3a, 0x0d, 0xdc, 0xc4, 0xc9, 0x9a, 0x98, 0x89, 0x19, 0x04, 0xef, 0xf7, 0x33, 0xc6, + 0x05, 0x83, 0xe0, 0xfd, 0x5e, 0xc6, 0x23, 0xc8, 0xb5, 0xb9, 0x1e, 0x3b, 0xc4, 0x88, 0x51, 0xb2, + 0x70, 0x62, 0x59, 0x2b, 0xe6, 0x2b, 0xab, 0xa5, 0x8c, 0x33, 0x59, 0x4a, 0x9c, 0x88, 0x4c, 0x2c, + 0x4e, 0xb4, 0xa6, 0xdb, 0xa9, 0x91, 0xb9, 0x04, 0x8b, 0x7d, 0xa9, 0x5a, 0x98, 0xb5, 0x28, 0x61, + 0xd8, 0xfc, 0x5d, 0x03, 0x7d, 0x9b, 0x79, 0x8f, 0x68, 0x84, 0x6f, 0x05, 0xd4, 0xd9, 0xdd, 0xc4, + 0xc8, 0xcd, 0x5c, 0x89, 0x45, 0x98, 0x10, 0x87, 0xd0, 0x77, 0x79, 0xf6, 0xe3, 0xd6, 0x69, 0x3e, + 0xde, 0x72, 0xf5, 0xf3, 0x00, 0xb5, 0xd8, 0x87, 0xdd, 0x40, 0xac, 0xc1, 0x13, 0x9d, 0xb6, 0x26, + 0xb9, 0x65, 0x13, 0xb1, 0x86, 0x3e, 0x0f, 0xa7, 0x1a, 0xd8, 0xf7, 0x1a, 0x11, 0x4f, 0x6c, 0xdc, + 0x92, 0x23, 0x7d, 0x23, 0xb6, 0xc7, 0x51, 0x17, 0x4e, 0x2e, 0x6b, 0xc5, 0xa9, 0xca, 0xe5, 0x41, + 0x09, 0xb7, 0x76, 0xbd, 0x92, 0xdc, 0x41, 0x21, 0xf1, 0x36, 0x8a, 0xd0, 0xad, 0x13, 0x2f, 0xfe, + 0x7a, 0x7b, 0xcc, 0x92, 0x74, 0xf3, 0x6b, 0x30, 0xfa, 0x53, 0x49, 0x32, 0xd5, 0x2f, 0x42, 0xbe, + 0x86, 0x82, 0x80, 0x46, 0x36, 0x4f, 0x05, 0xbb, 0x3c, 0xb3, 0x09, 0x2b, 0x27, 0xac, 0xeb, 0xc2, + 0x18, 0xc3, 0xf6, 0x68, 0x84, 0xed, 0xba, 0x4f, 0x50, 0xe0, 0x3f, 0xc5, 0x22, 0xcb, 0x09, 0x2b, + 0x17, 0x5b, 0xef, 0x26, 0x46, 0xf3, 0x19, 0xcc, 0xa9, 0x45, 0x5d, 0x8f, 0xa5, 0x3e, 0xe4, 0xd7, + 0x27, 0x63, 0xe1, 0x3e, 0x83, 0x29, 0xa7, 0x03, 0xe4, 0x5e, 0xa7, 0x2a, 0xc5, 0xcc, 0xcd, 0x4d, + 0x39, 0xb6, 0xd2, 0x64, 0xb3, 0x00, 0x6f, 0x0d, 0x8a, 0xae, 0x76, 0xf5, 0x1e, 0x57, 0x67, 0xe1, + 0x26, 0xdd, 0x1b, 0x51, 0xdd, 0xe1, 0xdb, 0x2a, 0x83, 0xf5, 0x39, 0x53, 0xc1, 0x7e, 0xd3, 0x20, + 0xbf, 0xcd, 0xbc, 0x35, 0xd7, 0x1d, 0xe1, 0x22, 0x5d, 0x86, 0xd9, 0x43, 0x2e, 0xd1, 0x0c, 0xed, + 0xb9, 0x0f, 0x1f, 0xc3, 0x22, 0x5f, 0x92, 0xc0, 0xc7, 0x24, 0xb2, 0xbd, 0x10, 0x91, 0x08, 0x63, + 0xbb, 0xd5, 0xae, 0xed, 0xe2, 0x03, 0x79, 0x8d, 0xce, 0x75, 0x00, 0x1b, 0x62, 0xfe, 0x21, 0x9f, + 0xd6, 0x57, 0xe1, 0x2c, 0x72, 0x5d, 0x9b, 0x50, 0x17, 0xdb, 0xc8, 0x71, 0x68, 0x9b, 0x44, 0x36, + 0x25, 0xc1, 0x01, 0x3f, 0x7a, 0x13, 0x96, 0x8e, 0x5c, 0xf7, 0x3e, 0x75, 0xf1, 0x9a, 0x98, 0x7a, + 0x40, 0x82, 0x03, 0x73, 0x01, 0xe6, 0xbb, 0xb3, 0x50, 0x09, 0xfe, 0xa0, 0xc1, 0xb4, 0x3a, 0x58, + 0xa8, 0x89, 0x8f, 0x77, 0x3b, 0x36, 0xe2, 0xdb, 0x81, 0x9a, 0xd8, 0xf6, 0x49, 0x9d, 0x72, 0xfd, + 0x53, 0x15, 0x33, 0x73, 0xfb, 0x79, 0x30, 0x79, 0xc6, 0x27, 0x39, 0x77, 0x8b, 0xd4, 0xa9, 0x39, + 0xcf, 0x37, 0x57, 0xa9, 0x51, 0x32, 0xd7, 0x60, 0x46, 0x1d, 0x8a, 0x7b, 0xf8, 0xc0, 0xc3, 0x24, + 0x43, 0xe8, 0x1c, 0x9c, 0xe4, 0x37, 0x53, 0xaa, 0x14, 0x03, 0x73, 0x11, 0xce, 0xf5, 0xb8, 0x50, + 0xde, 0x7f, 0xd2, 0xe0, 0x4d, 0x7e, 0x0c, 0x18, 0x8e, 0xf8, 0x29, 0xb8, 0xcf, 0xab, 0xff, 0xf1, + 0xd6, 0x62, 0x05, 0x66, 0xc4, 0x14, 0x6f, 0x21, 0x76, 0x40, 0xf7, 0xf9, 0x82, 0x8c, 0x5b, 0x39, + 0x47, 0xb9, 0xfe, 0x9c, 0xee, 0xeb, 0x45, 0x98, 0x4d, 0xe3, 0x1a, 0xbe, 0xd7, 0x90, 0xc5, 0x23, + 0xdf, 0x01, 0x6e, 0xfa, 0x5e, 0xc3, 0x3c, 0x0f, 0x4b, 0x03, 0xd4, 0x29, 0xf5, 0xbf, 0x6a, 0x00, + 0x72, 0xd1, 0x76, 0xaa, 0xd5, 0x0c, 0xd1, 0xe7, 0x01, 0x22, 0xc6, 0x92, 0x53, 0x26, 0x4e, 0xe6, + 0x64, 0xc4, 0x98, 0x3c, 0x57, 0xd7, 0x40, 0xdf, 0xe5, 0xeb, 0x62, 0xc7, 0xdb, 0x65, 0xcb, 0x7a, + 0x26, 0xb4, 0xcf, 0x8a, 0x99, 0xc7, 0x38, 0x42, 0x9b, 0xa2, 0xb2, 0xdd, 0x86, 0x53, 0x2c, 0x42, + 0x51, 0x9b, 0xc9, 0x52, 0x7e, 0xed, 0xb0, 0xca, 0x26, 0xfb, 0x98, 0x85, 0x1d, 0xec, 0xef, 0xe1, + 0x2a, 0xe7, 0x58, 0x92, 0x6b, 0x7e, 0xdf, 0x29, 0xd1, 0x3b, 0xd5, 0xea, 0xff, 0x53, 0xcf, 0x62, + 0x98, 0x4c, 0x8c, 0xb5, 0x1d, 0x27, 0x69, 0x54, 0x13, 0x56, 0x4e, 0x58, 0xab, 0xc2, 0x68, 0xee, + 0x43, 0x6e, 0x9b, 0x79, 0x77, 0x08, 0xaa, 0x05, 0x78, 0x7d, 0x7d, 0xe7, 0xcb, 0x8c, 0x95, 0xbc, + 0x00, 0x39, 0xcc, 0x71, 0x5b, 0xa4, 0x46, 0xdb, 0x44, 0xc5, 0xed, 0x32, 0xea, 0x2b, 0x90, 0x17, + 0x86, 0x07, 0xed, 0x48, 0xc0, 0x44, 0xdc, 0x1e, 0xab, 0x79, 0x0e, 0xce, 0x76, 0x05, 0x56, 0x3b, + 0xfb, 0x8c, 0x17, 0x9f, 0xdb, 0x3e, 0x1b, 0x41, 0xd2, 0x0a, 0xe4, 0x5d, 0x01, 0xec, 0xd6, 0xd4, + 0x63, 0xd5, 0x8b, 0x30, 0x23, 0x2d, 0x3d, 0xaa, 0x7a, 0xcd, 0xb2, 0x68, 0xa4, 0xa2, 0x2b, 0x5d, + 0xbf, 0x68, 0x50, 0x50, 0x77, 0x69, 0x03, 0xb1, 0x87, 0xa1, 0xef, 0xe0, 0x2d, 0x12, 0x4b, 0x61, + 0xf8, 0x6e, 0xfc, 0x80, 0xcb, 0x10, 0x4a, 0xe0, 0xac, 0x37, 0x88, 0x22, 0xbb, 0x46, 0x25, 0xb3, + 0x6c, 0x0c, 0x0c, 0x26, 0xcb, 0xc8, 0x60, 0xb7, 0x66, 0x11, 0x56, 0xb2, 0xb5, 0xaa, 0xb4, 0x7e, + 0xd4, 0xd2, 0xaf, 0x89, 0xce, 0x7b, 0x6d, 0x58, 0x46, 0x5f, 0xc1, 0x99, 0xd4, 0xeb, 0x4e, 0xbc, + 0x60, 0x65, 0x36, 0xd7, 0xb3, 0x1f, 0x38, 0x3d, 0x31, 0x64, 0x22, 0xb3, 0xb4, 0xc7, 0x6e, 0xbe, + 0x0b, 0xef, 0x1c, 0x2a, 0x2c, 0x91, 0x5f, 0xf9, 0x63, 0x1a, 0xc6, 0xb7, 0x99, 0xa7, 0x53, 0x98, + 0x4a, 0xf7, 0xab, 0xab, 0x99, 0x12, 0xba, 0xdb, 0x82, 0x71, 0xf3, 0x08, 0x60, 0x75, 0x5b, 0x9f, + 0x40, 0xbe, 0xe7, 0xb1, 0x59, 0x1a, 0xe6, 0xa6, 0x1b, 0x6f, 0xbc, 0x7f, 0x34, 0xbc, 0x8a, 0xfc, + 0x9d, 0x06, 0x67, 0xfa, 0xdf, 0x29, 0xab, 0xa3, 0x79, 0x4b, 0x51, 0x8c, 0x8f, 0x8e, 0x4c, 0xe9, + 0xd2, 0xd0, 0xff, 0x1a, 0x19, 0xaa, 0xa1, 0x8f, 0x32, 0x5c, 0xc3, 0xa1, 0xcf, 0x14, 0xdd, 0x87, + 0xc9, 0x4e, 0x07, 0xbf, 0x3c, 0xcc, 0x8f, 0x82, 0x1a, 0xab, 0x23, 0x43, 0x55, 0xa8, 0x10, 0xa6, + 0xbb, 0xda, 0xf0, 0xb5, 0xd1, 0x56, 0x4e, 0xa0, 0x8d, 0xf7, 0x8e, 0x82, 0x56, 0x31, 0xbf, 0x81, + 0x99, 0xde, 0x47, 0x7c, 0x79, 0x34, 0xe5, 0x8a, 0x60, 0x7c, 0x70, 0x44, 0x82, 0x0a, 0xfe, 0x2d, + 0xcc, 0xf6, 0x3d, 0x0c, 0x6e, 0x0c, 0xdf, 0xaa, 0x6e, 0x86, 0xf1, 0xe1, 0x51, 0x19, 0x2a, 0xbe, + 0x03, 0xa7, 0x93, 0xd6, 0x7e, 0x69, 0x94, 0x1c, 0x76, 0xaa, 0x55, 0xa3, 0x3c, 0x22, 0x50, 0x05, + 0x09, 0x00, 0x52, 0x8d, 0xef, 0xca, 0x30, 0x7a, 0x07, 0x6b, 0x54, 0x46, 0xc7, 0xaa, 0x68, 0x14, + 0xa6, 0xd2, 0x4d, 0x6d, 0x68, 0x85, 0x4a, 0x81, 0x87, 0x57, 0xa8, 0x01, 0x0d, 0x4b, 0xff, 0x59, + 0x83, 0xa5, 0xac, 0x6e, 0xf5, 0xc9, 0x68, 0xc7, 0x72, 0x20, 0xd9, 0x58, 0xff, 0x0f, 0x64, 0xa5, + 0xf0, 0xb9, 0x06, 0xf3, 0x87, 0x34, 0x9e, 0x51, 0x8b, 0x63, 0x0f, 0xcf, 0xf8, 0xf4, 0x78, 0xbc, + 0x44, 0xd2, 0xad, 0x3b, 0x2f, 0x5e, 0x15, 0xb4, 0x97, 0xaf, 0x0a, 0xda, 0xdf, 0xaf, 0x0a, 0xda, + 0xf3, 0xd7, 0x85, 0xb1, 0x97, 0xaf, 0x0b, 0x63, 0x7f, 0xbe, 0x2e, 0x8c, 0x3d, 0xbe, 0xea, 0xf9, + 0x51, 0xa3, 0x5d, 0x2b, 0x39, 0xb4, 0xc9, 0x3f, 0x47, 0x5c, 0x17, 0x5f, 0x26, 0xe2, 0xff, 0x48, + 0xca, 0x4f, 0x52, 0x5f, 0x46, 0x0e, 0x5a, 0x98, 0xd5, 0x4e, 0xf1, 0xaf, 0x12, 0x37, 0xff, 0x0d, + 0x00, 0x00, 0xff, 0xff, 0xbb, 0xb6, 0x4e, 0x47, 0xa5, 0x12, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/zetaclient/metrics/metrics.go b/zetaclient/metrics/metrics.go index 1c2e03a21f..5475930707 100644 --- a/zetaclient/metrics/metrics.go +++ b/zetaclient/metrics/metrics.go @@ -94,6 +94,13 @@ var ( Help: "Difference between system time and block time from zetacore", }) + // CoreBlockLatencySleep is a gauge of the duration we sleep before signing + CoreBlockLatencySleep = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: ZetaClientNamespace, + Name: "core_block_latency_sleep", + Help: "The duration we sleep before signing", + }) + // Info is a gauge that contains information about the zetaclient environment Info = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: ZetaClientNamespace, diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index 6e818329b1..3e475fe442 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -68,6 +68,9 @@ type Orchestrator struct { dbDirectory string baseLogger base.Logger + // signerBlockTimeOffset + signerBlockTimeOffset time.Duration + // misc logger multiLogger ts *metrics.TelemetryServer @@ -135,6 +138,12 @@ func (oc *Orchestrator) Start(ctx context.Context) error { bg.Work(ctx, oc.runScheduler, bg.WithName("runScheduler"), bg.WithLogger(oc.logger.Logger)) bg.Work(ctx, oc.runObserverSignerSync, bg.WithName("runObserverSignerSync"), bg.WithLogger(oc.logger.Logger)) bg.Work(ctx, oc.runAppContextUpdater, bg.WithName("runAppContextUpdater"), bg.WithLogger(oc.logger.Logger)) + bg.Work( + ctx, + oc.runSyncObserverOperationalFlags, + bg.WithName("runSyncObserverOperationalFlags"), + bg.WithLogger(oc.logger.Logger), + ) return nil } @@ -318,6 +327,13 @@ func (oc *Orchestrator) runScheduler(ctx context.Context) error { continue } + sleepDuration := time.Until(newBlock.Block.Time.Add(oc.signerBlockTimeOffset)) + if sleepDuration < 0 { + sleepDuration = 0 + } + metrics.CoreBlockLatencySleep.Set(sleepDuration.Seconds()) + time.Sleep(sleepDuration) + balance, err := oc.zetacoreClient.GetZetaHotKeyBalance(ctx) if err != nil { oc.logger.Error().Err(err).Msgf("couldn't get operator balance") @@ -795,3 +811,48 @@ func (oc *Orchestrator) syncObserverSigner(ctx context.Context) error { return nil } + +func (oc *Orchestrator) runSyncObserverOperationalFlags(ctx context.Context) error { + // every other block + const cadence = 2 * constant.ZetaBlockTime + + task := func(ctx context.Context, _ *ticker.Ticker) error { + if err := oc.syncObserverOperationalFlags(ctx); err != nil { + oc.logger.Error().Err(err).Msg("syncObserverOperationalFlags failed") + } + + return nil + } + + return ticker.Run( + ctx, + cadence, + task, + ticker.WithLogger(oc.logger.Logger, "SyncObserverOperationalFlags"), + ticker.WithStopChan(oc.stop), + ) +} + +func (oc *Orchestrator) syncObserverOperationalFlags(ctx context.Context) error { + client := oc.zetacoreClient + flags, err := client.GetOperationalFlags(ctx) + if err != nil { + return fmt.Errorf("get operational flags: %w", err) + } + + oc.mu.Lock() + defer oc.mu.Unlock() + newSignerBlockTimeOffsetPtr := flags.SignerBlockTimeOffset + if newSignerBlockTimeOffsetPtr == nil { + return nil + } + newSignerBlockTimeOffset := *newSignerBlockTimeOffsetPtr + if oc.signerBlockTimeOffset != newSignerBlockTimeOffset { + oc.logger.Info(). + Dur("offset", newSignerBlockTimeOffset). + Msg("block time offset updated") + oc.signerBlockTimeOffset = newSignerBlockTimeOffset + } + + return nil +}