Skip to content

Commit

Permalink
Add prometheus metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
gagliardetto committed Feb 14, 2024
1 parent a78c1c3 commit 5e7e8fc
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 22 deletions.
4 changes: 4 additions & 0 deletions cmd-rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ func newCmd_rpc() *cli.Command {
if err := multi.AddEpoch(epoch.Epoch(), epoch); err != nil {
return cli.Exit(fmt.Sprintf("failed to add epoch %d: %s", epoch.Epoch(), err.Error()), 1)
}
metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1)
}
tookInitializingEpochs := time.Since(startedInitiatingEpochsAt)
klog.Infof("Initialized %d epochs in %s", len(epochs), tookInitializingEpochs)
Expand Down Expand Up @@ -249,6 +250,7 @@ func newCmd_rpc() *cli.Command {
return
}
klog.V(2).Infof("Epoch %d added/replaced in %s", epoch.Epoch(), time.Since(startedAt))
metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1)
}
case fsnotify.Create:
{
Expand All @@ -271,6 +273,7 @@ func newCmd_rpc() *cli.Command {
return
}
klog.V(2).Infof("Epoch %d added in %s", epoch.Epoch(), time.Since(startedAt))
metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epoch.Epoch())).Set(1)
}
case fsnotify.Remove:
{
Expand All @@ -282,6 +285,7 @@ func newCmd_rpc() *cli.Command {
klog.Errorf("error removing epoch for config file %q: %s", event.Name, err.Error())
}
klog.V(2).Infof("Epoch %d removed in %s", epNumber, time.Since(startedAt))
metrics_epochsAvailable.WithLabelValues(fmt.Sprintf("%d", epNumber)).Set(0)
}
case fsnotify.Rename:
klog.V(3).Infof("File %q was renamed; do nothing", event.Name)
Expand Down
13 changes: 6 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ require (
github.com/filecoin-project/go-state-types v0.10.0 // indirect
github.com/gagliardetto/binary v0.7.8
github.com/gagliardetto/solana-go v1.8.4
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.6.0
github.com/hannahhoward/go-pubsub v1.0.0 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
Expand Down Expand Up @@ -72,6 +71,7 @@ require (
github.com/libp2p/go-reuseport v0.4.0
github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1
github.com/mr-tron/base58 v1.2.0
github.com/prometheus/client_golang v1.18.0
github.com/ronanh/intcomp v1.1.0
github.com/ryanuber/go-glob v1.0.0
github.com/tejzpr/ordered-concurrently/v3 v3.0.1
Expand Down Expand Up @@ -167,7 +167,7 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
Expand All @@ -191,10 +191,9 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
github.com/quic-go/quic-go v0.40.0 // indirect
Expand All @@ -220,7 +219,7 @@ require (
go.uber.org/zap v1.26.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
Expand Down
25 changes: 12 additions & 13 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -586,8 +586,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
Expand Down Expand Up @@ -711,23 +711,23 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
Expand Down Expand Up @@ -1127,8 +1127,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down Expand Up @@ -1272,7 +1272,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
Expand Down
76 changes: 76 additions & 0 deletions metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package main

import "github.com/prometheus/client_golang/prometheus"

// - RPC requests by method (counter)
// - Epochs available epoch_available{epoch="200"} = 1
// - status_code
// - miner ids
// - source type (ipfs/bitwarden/s3/etc)
// - response time histogram

func init() {
prometheus.MustRegister(metrics_RpcRequestByMethod)
prometheus.MustRegister(metrics_epochsAvailable)
prometheus.MustRegister(metrics_statusCode)
prometheus.MustRegister(metrics_methodToCode)
prometheus.MustRegister(metrics_methodToSuccessOrFailure)
prometheus.MustRegister(metrics_methodToNumProxied)
prometheus.MustRegister(metrics_responseTimeHistogram)
}

var metrics_RpcRequestByMethod = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "rpc_requests_by_method",
Help: "RPC requests by method",
},
[]string{"method"},
)

var metrics_epochsAvailable = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "epoch_available",
Help: "Epochs available",
},
[]string{"epoch"},
)

var metrics_statusCode = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "status_code",
Help: "Status code",
},
[]string{"code"},
)

var metrics_methodToCode = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "method_to_code",
Help: "Method to code",
},
[]string{"method", "code"},
)

var metrics_methodToSuccessOrFailure = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "method_to_success_or_failure",
Help: "Method to success or failure",
},
[]string{"method", "status"},
)

var metrics_methodToNumProxied = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "method_to_num_proxied",
Help: "Method to num proxied",
},
[]string{"method"},
)

var metrics_responseTimeHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "response_time_histogram",
Help: "Response time histogram",
},
[]string{"method"},
)
25 changes: 23 additions & 2 deletions multiepoch.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (
"github.com/google/uuid"
"github.com/goware/urlx"
"github.com/libp2p/go-reuseport"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rpcpool/yellowstone-faithful/ipld/ipldbindcode"
"github.com/sourcegraph/jsonrpc2"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpadaptor"
"k8s.io/klog/v2"
)

Expand Down Expand Up @@ -274,9 +276,20 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
return func(reqCtx *fasthttp.RequestCtx) {
startedAt := time.Now()
reqID := randomRequestID()
var method string = "<unknown>"
defer func() {
klog.V(2).Infof("[%s] request took %s", reqID, time.Since(startedAt))
klog.V(2).Infof("[%s] request %q took %s", reqID, sanitizeMethod(method), time.Since(startedAt))
metrics_statusCode.WithLabelValues(fmt.Sprint(reqCtx.Response.StatusCode())).Inc()
metrics_responseTimeHistogram.WithLabelValues(sanitizeMethod(method)).Observe(time.Since(startedAt).Seconds())
}()
{
// handle the /metrics endpoint
if string(reqCtx.Path()) == "/metrics" {
handler := fasthttpadaptor.NewFastHTTPHandler(promhttp.Handler())
handler(reqCtx)
return
}
}
{
// make sure the method is POST
if !reqCtx.IsPost() {
Expand Down Expand Up @@ -317,7 +330,11 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
})
return
}
method := rpcRequest.Method
method = rpcRequest.Method
metrics_RpcRequestByMethod.WithLabelValues(sanitizeMethod(method)).Inc()
defer func() {
metrics_methodToCode.WithLabelValues(sanitizeMethod(method), fmt.Sprint(reqCtx.Response.StatusCode())).Inc()
}()

klog.V(2).Infof("[%s] method=%q", reqID, sanitizeMethod(method))
klog.V(3).Infof("[%s] received request with body: %q", reqID, strings.TrimSpace(string(body)))
Expand All @@ -334,6 +351,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
body,
reqID,
)
metrics_methodToNumProxied.WithLabelValues(sanitizeMethod(method)).Inc()
return
}

Expand Down Expand Up @@ -366,6 +384,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
klog.Errorf("[%s] failed to handle %s: %v", reqID, sanitizeMethod(method), err)
}
if errorResp != nil {
metrics_methodToSuccessOrFailure.WithLabelValues(sanitizeMethod(method), "failure").Inc()
if proxy != nil && lsConf.ProxyConfig.ProxyFailedRequests {
klog.Warningf("[%s] Failed local method %q, proxying to %q", reqID, rpcRequest.Method, proxy.Addr)
// proxy the request to the target
Expand All @@ -378,6 +397,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
body,
reqID,
)
metrics_methodToNumProxied.WithLabelValues(sanitizeMethod(method)).Inc()
return
} else {
rqCtx.ReplyWithError(
Expand All @@ -388,6 +408,7 @@ func newMultiEpochHandler(handler *MultiEpoch, lsConf *ListenerConfig) func(ctx
}
return
}
metrics_methodToSuccessOrFailure.WithLabelValues(sanitizeMethod(method), "success").Inc()
}
}

Expand Down

0 comments on commit 5e7e8fc

Please sign in to comment.