From 5e7e8fcfaa5ca551e73ed211524403b68f939933 Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 14 Feb 2024 16:40:34 +0100 Subject: [PATCH] Add prometheus metrics --- cmd-rpc.go | 4 +++ go.mod | 13 ++++----- go.sum | 25 ++++++++--------- metrics.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++ multiepoch.go | 25 +++++++++++++++-- 5 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 metrics.go diff --git a/cmd-rpc.go b/cmd-rpc.go index 4e269513..b07cd6f9 100644 --- a/cmd-rpc.go +++ b/cmd-rpc.go @@ -190,6 +190,7 @@ func newCmd_rpc() *cli.Command { if err := multi.AddEpoch(epoch.Epoch(), epoch); err != nil { return cli.Exit(fmt.Sprintf("failed to add epoch %d: %s", epoch.Epoch(), err.Error()), 1) } + metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1) } tookInitializingEpochs := time.Since(startedInitiatingEpochsAt) klog.Infof("Initialized %d epochs in %s", len(epochs), tookInitializingEpochs) @@ -249,6 +250,7 @@ func newCmd_rpc() *cli.Command { return } klog.V(2).Infof("Epoch %d added/replaced in %s", epoch.Epoch(), time.Since(startedAt)) + metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1) } case fsnotify.Create: { @@ -271,6 +273,7 @@ func newCmd_rpc() *cli.Command { return } klog.V(2).Infof("Epoch %d added in %s", epoch.Epoch(), time.Since(startedAt)) + metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1) } case fsnotify.Remove: { @@ -282,6 +285,7 @@ func newCmd_rpc() *cli.Command { klog.Errorf("error removing epoch for config file %q: %s", event.Name, err.Error()) } klog.V(2).Infof("Epoch %d removed in %s", epNumber, time.Since(startedAt)) + metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epNumber)).Set(0) } case fsnotify.Rename: klog.V(3).Infof("File %q was renamed; do nothing", event.Name) diff --git a/go.mod b/go.mod index 9deac589..e35bd9b4 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/filecoin-project/go-state-types v0.10.0 // indirect github.com/gagliardetto/binary v0.7.8 github.com/gagliardetto/solana-go v1.8.4 - github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.6.0 github.com/hannahhoward/go-pubsub v1.0.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect @@ -72,6 +71,7 @@ require ( github.com/libp2p/go-reuseport v0.4.0 github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 github.com/mr-tron/base58 v1.2.0 + github.com/prometheus/client_golang v1.18.0 github.com/ronanh/intcomp v1.1.0 github.com/ryanuber/go-glob v1.0.0 github.com/tejzpr/ordered-concurrently/v3 v3.0.1 @@ -167,7 +167,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/miekg/dns v1.1.56 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -191,10 +191,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.4.1 // indirect github.com/quic-go/quic-go v0.40.0 // indirect @@ -220,7 +219,7 @@ require ( go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect diff --git a/go.sum b/go.sum index 5e5443c5..94f6a08f 100644 --- a/go.sum +++ b/go.sum @@ -586,8 +586,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -711,23 +711,23 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= @@ -1127,8 +1127,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1272,7 +1272,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/metrics.go b/metrics.go new file mode 100644 index 00000000..4b63e15e --- /dev/null +++ b/metrics.go @@ -0,0 +1,76 @@ +package main + +import "github.com/prometheus/client_golang/prometheus" + +// - RPC requests by method (counter) +// - Epochs available epoch_available{epoch="200"} = 1 +// - status_code +// - miner ids +// - source type (ipfs/bitwarden/s3/etc) +// - response time histogram + +func init() { + prometheus.MustRegister(metrics_RpcRequestByMethod) + prometheus.MustRegister(metrics_epochsAvailable) + prometheus.MustRegister(metrics_statusCode) + prometheus.MustRegister(metrics_methodToCode) + prometheus.MustRegister(metrics_methodToSuccessOrFailure) + prometheus.MustRegister(metrics_methodToNumProxied) + prometheus.MustRegister(metrics_responseTimeHistogram) +} + +var metrics_RpcRequestByMethod = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_requests_by_method", + Help: "RPC requests by method", + }, + []string{"method"}, +) + +var metrics_epochsAvailable = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "epoch_available", + Help: "Epochs available", + }, + []string{"epoch"}, +) + +var metrics_statusCode = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "status_code", + Help: "Status code", + }, + []string{"code"}, +) + +var metrics_methodToCode = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "method_to_code", + Help: "Method to code", + }, + []string{"method", "code"}, +) + +var metrics_methodToSuccessOrFailure = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "method_to_success_or_failure", + Help: "Method to success or failure", + }, + []string{"method", "status"}, +) + +var metrics_methodToNumProxied = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "method_to_num_proxied", + Help: "Method to num proxied", + }, + []string{"method"}, +) + +var metrics_responseTimeHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "response_time_histogram", + Help: "Response time histogram", + }, + []string{"method"}, +) diff --git a/multiepoch.go b/multiepoch.go index f62b408b..59a36376 100644 --- a/multiepoch.go +++ b/multiepoch.go @@ -12,9 +12,11 @@ import ( "github.com/google/uuid" "github.com/goware/urlx" "github.com/libp2p/go-reuseport" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rpcpool/yellowstone-faithful/ipld/ipldbindcode" "github.com/sourcegraph/jsonrpc2" "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpadaptor" "k8s.io/klog/v2" ) @@ -274,9 +276,20 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx return func(reqCtx *fasthttp.RequestCtx) { startedAt := time.Now() reqID := randomRequestID() + var method string = "" defer func() { - klog.V(2).Infof("[%s] request took %s", reqID, time.Since(startedAt)) + klog.V(2).Infof("[%s] request %q took %s", reqID, sanitizeMethod(method), time.Since(startedAt)) + metrics_statusCode.WithLabelValues(fmt.Sprint(reqCtx.Response.StatusCode())).Inc() + metrics_responseTimeHistogram.WithLabelValues(sanitizeMethod(method)).Observe(time.Since(startedAt).Seconds()) }() + { + // handle the /metrics endpoint + if string(reqCtx.Path()) == "/metrics" { + handler := fasthttpadaptor.NewFastHTTPHandler(promhttp.Handler()) + handler(reqCtx) + return + } + } { // make sure the method is POST if !reqCtx.IsPost() { @@ -317,7 +330,11 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx }) return } - method := rpcRequest.Method + method = rpcRequest.Method + metrics_RpcRequestByMethod.WithLabelValues(sanitizeMethod(method)).Inc() + defer func() { + metrics_methodToCode.WithLabelValues(sanitizeMethod(method), fmt.Sprint(reqCtx.Response.StatusCode())).Inc() + }() klog.V(2).Infof("[%s] method=%q", reqID, sanitizeMethod(method)) klog.V(3).Infof("[%s] received request with body: %q", reqID, strings.TrimSpace(string(body))) @@ -334,6 +351,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx body, reqID, ) + metrics_methodToNumProxied.WithLabelValues(sanitizeMethod(method)).Inc() return } @@ -366,6 +384,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx klog.Errorf("[%s] failed to handle %s: %v", reqID, sanitizeMethod(method), err) } if errorResp != nil { + metrics_methodToSuccessOrFailure.WithLabelValues(sanitizeMethod(method), "failure").Inc() if proxy != nil && lsConf.ProxyConfig.ProxyFailedRequests { klog.Warningf("[%s] Failed local method %q, proxying to %q", reqID, rpcRequest.Method, proxy.Addr) // proxy the request to the target @@ -378,6 +397,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx body, reqID, ) + metrics_methodToNumProxied.WithLabelValues(sanitizeMethod(method)).Inc() return } else { rqCtx.ReplyWithError( @@ -388,6 +408,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx } return } + metrics_methodToSuccessOrFailure.WithLabelValues(sanitizeMethod(method), "success").Inc() } }