Merge branch 'main' into signature-aggregation-api-acp-118

go.mod

@@ -6,12 +6,12 @@ require (
 	github.com/ava-labs/avalanchego v1.11.11-0.20240813203340-ab83fb41528d
 	github.com/ava-labs/subnet-evm v0.6.9-0.20240816202746-18633729a0cd
 	github.com/ava-labs/teleporter v1.0.5
-	github.com/aws/aws-sdk-go-v2 v1.30.3
+	github.com/aws/aws-sdk-go-v2 v1.30.4
 	github.com/aws/aws-sdk-go-v2/config v1.27.9
-	github.com/aws/aws-sdk-go-v2/service/kms v1.35.3
+	github.com/aws/aws-sdk-go-v2/service/kms v1.35.4
 	github.com/ethereum/go-ethereum v1.13.8
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/onsi/ginkgo/v2 v2.20.0
+	github.com/onsi/ginkgo/v2 v2.20.1
 	github.com/onsi/gomega v1.34.1
 	github.com/pingcap/errors v0.11.4
 	github.com/pkg/errors v0.9.1
@@ -33,15 +33,15 @@ require (
 	github.com/ava-labs/coreth v0.13.8-fixed-genesis-upgrade.0.20240813194342-7635a96aa180 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
-	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect

go.sum

@@ -66,34 +66,34 @@ github.com/ava-labs/subnet-evm v0.6.9-0.20240816202746-18633729a0cd h1:5kJTOhmIh
 github.com/ava-labs/subnet-evm v0.6.9-0.20240816202746-18633729a0cd/go.mod h1:QfIzh7YxKj97jbendOHQbaAxM7SMj5MWdV13o1VLn70=
 github.com/ava-labs/teleporter v1.0.5 h1:51fRzvo7eo9VotYq6cldx+yoAkEk5o9yCXOJ6q3tSEo=
 github.com/ava-labs/teleporter v1.0.5/go.mod h1:C5/ETmPQh/MxZOCARrhPV+J40y4R3TOeT0lvQrjz/RE=
-github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
-github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
+github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
 github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg=
 github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
-github.com/aws/aws-sdk-go-v2/service/kms v1.35.3 h1:UPTdlTOwWUX49fVi7cymEN6hDqCwe3LNv1vi7TXUutk=
-github.com/aws/aws-sdk-go-v2/service/kms v1.35.3/go.mod h1:gjDP16zn+WWalyaUqwCCioQ8gU8lzttCCc9jYsiQI/8=
+github.com/aws/aws-sdk-go-v2/service/kms v1.35.4 h1:mG1MH6yPwT5gNEeBrhig3FHc4mK0QaZOXsmQUbphP6Y=
+github.com/aws/aws-sdk-go-v2/service/kms v1.35.4/go.mod h1:A5CS0VRmxxj2YKYLCY08l/Zzbd01m6JZn0WzxgT1OCA=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
-github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
-github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -494,8 +494,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw=
-github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
+github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo=
+github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=

peers/app_request_netrwork_metrics.go

@@ -0,0 +1,47 @@
+package peers
+
+import (
+	"errors"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	ErrFailedToCreateAppRequestNetworkMetrics = errors.New("failed to create app request network metrics")
+)
+
+type AppRequestNetworkMetrics struct {
+	infoAPICallLatencyMS   prometheus.Histogram
+	pChainAPICallLatencyMS prometheus.Histogram
+}
+
+func newAppRequestNetworkMetrics(registerer prometheus.Registerer) (*AppRequestNetworkMetrics, error) {
+	infoAPICallLatencyMS := prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "info_api_call_latency_ms",
+			Help:    "Latency of calling info api in milliseconds",
+			Buckets: prometheus.ExponentialBucketsRange(100, 10000, 10),
+		},
+	)
+	if infoAPICallLatencyMS == nil {
+		return nil, ErrFailedToCreateAppRequestNetworkMetrics
+	}
+	registerer.MustRegister(infoAPICallLatencyMS)
+
+	pChainAPICallLatencyMS := prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "p_chain_api_call_latency_ms",
+			Help:    "Latency of calling p-chain rpc in milliseconds",
+			Buckets: prometheus.ExponentialBucketsRange(100, 10000, 10),
+		},
+	)
+	if pChainAPICallLatencyMS == nil {
+		return nil, ErrFailedToCreateAppRequestNetworkMetrics
+	}
+	registerer.MustRegister(pChainAPICallLatencyMS)
+
+	return &AppRequestNetworkMetrics{
+		infoAPICallLatencyMS:   infoAPICallLatencyMS,
+		pChainAPICallLatencyMS: pChainAPICallLatencyMS,
+	}, nil
+}

peers/app_request_network.go

@@ -35,6 +35,7 @@ type AppRequestNetwork struct {
 	logger          logging.Logger
 	lock            *sync.Mutex
 	validatorClient *validators.CanonicalValidatorClient
+	metrics         *AppRequestNetworkMetrics
 }
 
 // NewNetwork creates a p2p network client for interacting with validators
@@ -53,8 +54,14 @@ func NewNetwork(
 		),
 	)
 
+	metrics, err := newAppRequestNetworkMetrics(registerer)
+	if err != nil {
+		logger.Fatal("Failed to create app request network metrics", zap.Error(err))
+		panic(err)
+	}
+
 	// Create the handler for handling inbound app responses
-	handler, err := NewRelayerExternalHandler(logger, registerer)
+	handler, err := NewRelayerExternalHandler(logger, prometheus.DefaultRegisterer)
 	if err != nil {
 		logger.Error(
 			"Failed to create p2p network handler",
@@ -98,6 +105,7 @@ func NewNetwork(
 		logger:          logger,
 		lock:            new(sync.Mutex),
 		validatorClient: validatorClient,
+		metrics:         metrics,
 	}
 	go logger.RecoverAndPanic(func() {
 		testNetwork.Dispatch()
@@ -123,8 +131,10 @@ func (n *AppRequestNetwork) ConnectPeers(nodeIDs set.Set[ids.NodeID]) set.Set[id
 	// through connectedPeers for already tracked peers, just iterate through the full list,
 	// re-adding connections to already tracked peers.
 
+	startInfoAPICall := time.Now()
 	// Get the list of peers
 	peers, err := n.infoAPI.Peers(context.Background())
+	n.setInfoAPICallLatencyMS(float64(time.Since(startInfoAPICall).Milliseconds()))
 	if err != nil {
 		n.logger.Error(
 			"Failed to get peers",
@@ -147,13 +157,19 @@ func (n *AppRequestNetwork) ConnectPeers(nodeIDs set.Set[ids.NodeID]) set.Set[id
 
 	// If the Info API node is in nodeIDs, it will not be reflected in the call to info.Peers.
 	// In this case, we need to manually track the API node.
-	if apiNodeID, _, err := n.infoAPI.GetNodeID(context.Background()); err != nil {
+	startInfoAPICall = time.Now()
+	apiNodeID, _, err := n.infoAPI.GetNodeID(context.Background())
+	n.setInfoAPICallLatencyMS(float64(time.Since(startInfoAPICall).Milliseconds()))
+	if err != nil {
 		n.logger.Error(
 			"Failed to get API Node ID",
 			zap.Error(err),
 		)
 	} else if nodeIDs.Contains(apiNodeID) {
-		if apiNodeIPPort, err := n.infoAPI.GetNodeIP(context.Background()); err != nil {
+		startInfoAPICall = time.Now()
+		apiNodeIPPort, err := n.infoAPI.GetNodeIP(context.Background())
+		n.setInfoAPICallLatencyMS(float64(time.Since(startInfoAPICall).Milliseconds()))
+		if err != nil {
 			n.logger.Error(
 				"Failed to get API Node IP",
 				zap.Error(err),
@@ -186,11 +202,12 @@ func (c *ConnectedCanonicalValidators) GetValidator(nodeID ids.NodeID) (*warp.Va
 // validator information
 func (n *AppRequestNetwork) ConnectToCanonicalValidators(subnetID ids.ID) (*ConnectedCanonicalValidators, error) {
 	// Get the subnet's current canonical validator set
+	startPChainAPICall := time.Now()
 	validatorSet, totalValidatorWeight, err := n.validatorClient.GetCurrentCanonicalValidatorSet(subnetID)
+	n.setPChainAPICallLatencyMS(float64(time.Since(startPChainAPICall).Milliseconds()))
 	if err != nil {
 		return nil, err
 	}
-
 	// We make queries to node IDs, not unique validators as represented by a BLS pubkey, so we need this map to track
 	// responses from nodes and populate the signatureMap with the corresponding validator signature
 	// This maps node IDs to the index in the canonical validator set
@@ -241,3 +258,15 @@ func (n *AppRequestNetwork) RegisterRequestID(requestID uint32, numExpectedRespo
 func (n *AppRequestNetwork) GetSubnetID(blockchainID ids.ID) (ids.ID, error) {
 	return n.validatorClient.GetSubnetID(context.Background(), blockchainID)
 }
+
+//
+// Metrics
+//
+
+func (n *AppRequestNetwork) setInfoAPICallLatencyMS(latency float64) {
+	n.metrics.infoAPICallLatencyMS.Observe(latency)
+}
+
+func (n *AppRequestNetwork) setPChainAPICallLatencyMS(latency float64) {
+	n.metrics.pChainAPICallLatencyMS.Observe(latency)
+}

relayer/main/main.go

@@ -148,7 +148,7 @@ func main() {
 
 	network, err := peers.NewNetwork(
 		networkLogLevel,
-		prometheus.DefaultRegisterer,
+		registerer,
 		trackedSubnets,
 		&cfg,
 	)
@@ -282,6 +282,7 @@ func main() {
 			)
 		})
 	}
+	logger.Info("Initialization complete")
 	err = errGroup.Wait()
 	logger.Error("Relayer exiting.", zap.Error(err))
 }

scripts/build.sh

@@ -1,3 +1,7 @@
+#!/usr/bin/env bash
+# Copyright (C) 2024, Ava Labs, Inc. All rights reserved.
+# See the file LICENSE for licensing terms.
+
 # Root directory
 root=$(
     cd "$(dirname "${BASH_SOURCE[0]}")"

signature-aggregator/aggregator/aggregator.go

@@ -119,6 +119,12 @@ func (s *SignatureAggregator) CreateSignedMessage(
 		s.metrics.FailuresToGetValidatorSet.Inc()
 		return nil, fmt.Errorf("%s: %w", msg, err)
 	}
+	s.metrics.ConnectedStakeWeightPercentage.WithLabelValues(
+		signingSubnet.String(),
+	).Set(
+		float64(connectedValidators.ConnectedWeight) /
+			float64(connectedValidators.TotalValidatorWeight) * 100,
+	)
 
 	if !utils.CheckStakeWeightPercentageExceedsThreshold(
 		big.NewInt(0).SetUint64(connectedValidators.ConnectedWeight),

signature-aggregator/main/main.go

@@ -130,6 +130,7 @@ func main() {
 	)
 	healthcheck.HandleHealthCheckRequest()
 
+	logger.Info("Initialization complete")
 	err = http.ListenAndServe(fmt.Sprintf(":%d", cfg.APIPort), nil)
 	if errors.Is(err, http.ErrServerClosed) {
 		logger.Info("server closed")

signature-aggregator/metrics/metrics.go

@@ -31,6 +31,7 @@ var Opts = struct {
 	InvalidSignatureResponses          prometheus.CounterOpts
 	SignatureCacheHits                 prometheus.CounterOpts
 	SignatureCacheMisses               prometheus.CounterOpts
+	ConnectedStakeWeightPercentage     prometheus.GaugeOpts
 }{
 	AggregateSignaturesLatencyMS: prometheus.GaugeOpts{
 		Name: "agg_sigs_latency_ms",
@@ -72,6 +73,10 @@ var Opts = struct {
 		Name: "signature_cache_misses",
 		Help: "Number of signatures that were not found in the cache",
 	},
+	ConnectedStakeWeightPercentage: prometheus.GaugeOpts{
+		Name: "connected_stake_weight_percentage",
+		Help: "The percentage of connected stake weight for a specific subnet",
+	},
 }
 
 type SignatureAggregatorMetrics struct {
@@ -85,6 +90,7 @@ type SignatureAggregatorMetrics struct {
 	InvalidSignatureResponses          prometheus.Counter
 	SignatureCacheHits                 prometheus.Counter
 	SignatureCacheMisses               prometheus.Counter
+	ConnectedStakeWeightPercentage     *prometheus.GaugeVec
 
 	// TODO: consider other failures to monitor. Issue #384 requires
 	// "network failures", but we probably don't handle those directly.
@@ -129,6 +135,10 @@ func NewSignatureAggregatorMetrics(
 		SignatureCacheMisses: prometheus.NewCounter(
 			Opts.SignatureCacheMisses,
 		),
+		ConnectedStakeWeightPercentage: prometheus.NewGaugeVec(
+			Opts.ConnectedStakeWeightPercentage,
+			[]string{"subnetID"},
+		),
 	}
 
 	registerer.MustRegister(m.AggregateSignaturesLatencyMS)
@@ -141,6 +151,7 @@ func NewSignatureAggregatorMetrics(
 	registerer.MustRegister(m.InvalidSignatureResponses)
 	registerer.MustRegister(m.SignatureCacheHits)
 	registerer.MustRegister(m.SignatureCacheMisses)
+	registerer.MustRegister(m.ConnectedStakeWeightPercentage)
 
 	return &m
 }