Add gRPC metrics (#3104)

go.mod

@@ -73,6 +73,7 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/jellydator/ttlcache/v3 v3.2.0 // indirect

go.sum

@@ -127,6 +127,8 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

src/metrics/prometheus.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -23,13 +24,12 @@ var registerer = prometheus.WrapRegistererWith(prometheus.Labels{
 // Push performs a single push of all registered metrics to the pushgateway (if configured).
 func Push(config *core.Configuration) {
 	if family, err := prometheus.DefaultGatherer.Gather(); err == nil {
+		var buf strings.Builder
 		for _, fam := range family {
-			for _, metric := range fam.Metric {
-				if metric.Counter != nil {
-					log.Debug("Metric recorded: %s: %0.0f", *fam.Name, *metric.Counter.Value)
-				} else if metric.Gauge != nil {
-					log.Debug("Metric recorded: %s: %0.0f", *fam.Name, *metric.Gauge.Value)
-				}
+			buf.Reset()
+			if _, err := expfmt.MetricFamilyToText(&buf, fam); err == nil {
+				s := strings.TrimSpace(buf.String())
+				log.Debug("Metric recorded: %s", s[strings.LastIndex(s, "\n")+1:])
 			}
 		}
 	}

src/remote/BUILD

@@ -31,6 +31,7 @@ go_library(
         "///third_party/go/google.golang.org_grpc//status",
         "///third_party/go/google.golang.org_protobuf//proto",
         "///third_party/go/google.golang.org_protobuf//types/known/durationpb",
+        "///third_party/go/github.com_grpc-ecosystem_go-grpc-prometheus//:go-grpc-prometheus",
         "//src/build",
         "//src/cli/logging",
         "//src/core",

src/remote/utils.go

@@ -19,6 +19,7 @@ import (
 	"github.com/bazelbuild/remote-apis-sdks/go/pkg/uploadinfo"
 	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	"github.com/bazelbuild/remote-apis/build/bazel/semver"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	rpcstatus "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -608,6 +609,8 @@ func (c *Client) dialOpts() ([]grpc.DialOption, error) {
 		grpc.WithStatsHandler(c.stats),
 		// Set an arbitrarily large (400MB) max message size so it isn't a limitation.
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(419430400)),
+		grpc.WithChainUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+		grpc.WithChainStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
 	}
 	if c.state.Config.Remote.TokenFile == "" {
 		return opts, nil

third_party/go/BUILD

@@ -680,3 +680,9 @@ go_repo(
     version = "v1.2.4",
     licences = ["MIT"],
 )
+
+go_repo(
+    module = "github.com/grpc-ecosystem/go-grpc-prometheus",
+    version = "v1.2.0",
+    licences = ["Apache-2.0"],
+)