From 2e3001cc2af2483f5afa5b3b1e24651f44ff2e67 Mon Sep 17 00:00:00 2001 From: "M. Hamzah Khan" Date: Mon, 2 Sep 2024 14:25:35 +0100 Subject: [PATCH] feat: Add Prometheus metrics and health endpoints --- api/api.go | 39 ++++++++++++--- api/metrics.go | 129 +++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 14 ++++-- go.sum | 22 +++++++++ 4 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 api/metrics.go diff --git a/api/api.go b/api/api.go index 5ee28e3..df81e10 100644 --- a/api/api.go +++ b/api/api.go @@ -5,6 +5,8 @@ import ( "time" "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/yukimochi/Activity-Relay/models" "github.com/yukimochi/machinery-v1/v1" @@ -24,6 +26,8 @@ var ( ActorCache *cache.Cache MachineryServer *machinery.Server RelayState models.RelayState + + metrics *defaultMetrics ) func Entrypoint(g *models.RelayConfig, v string) error { @@ -32,6 +36,9 @@ func Entrypoint(g *models.RelayConfig, v string) error { version = v GlobalConfig = g + // Initialize the metrics + metrics = newDefaultMetrics(prometheus.DefaultRegisterer, nil, nil) + err = initialize(GlobalConfig) if err != nil { return err @@ -70,11 +77,31 @@ func initialize(globalConfig *models.RelayConfig) error { } func handlersRegister() { - http.HandleFunc("/.well-known/nodeinfo", handleNodeinfoLink) - http.HandleFunc("/.well-known/webfinger", handleWebfinger) - http.HandleFunc("/nodeinfo/2.1", handleNodeinfo) - http.HandleFunc("/actor", handleRelayActor) - http.HandleFunc("/inbox", func(w http.ResponseWriter, r *http.Request) { + // Register the Prometheus metrics endpoint + http.Handle("/metrics", promhttp.Handler()) + + // Register the new health and readiness endpoints + http.Handle("/-/healthy", metrics.MetricsMiddleware(http.HandlerFunc(handleHealthy))) + http.Handle("/-/ready", metrics.MetricsMiddleware(http.HandlerFunc(handleReady))) + + // Wrap handlers with the metrics middleware, preserving the URL path + http.Handle("/.well-known/nodeinfo", metrics.MetricsMiddleware(http.HandlerFunc(handleNodeinfoLink))) + http.Handle("/.well-known/webfinger", metrics.MetricsMiddleware(http.HandlerFunc(handleWebfinger))) + http.Handle("/nodeinfo/2.1", metrics.MetricsMiddleware(http.HandlerFunc(handleNodeinfo))) + http.Handle("/actor", metrics.MetricsMiddleware(http.HandlerFunc(handleRelayActor))) + http.Handle("/inbox", metrics.MetricsMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { handleInbox(w, r, decodeActivity) - }) + }))) +} + +// handleHealthy returns a health status message +func handleHealthy(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ActivityRelay is Healthy.")) +} + +// handleReady returns a readiness status message +func handleReady(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ActivityRelay is Ready.")) } diff --git a/api/metrics.go b/api/metrics.go new file mode 100644 index 0000000..675963b --- /dev/null +++ b/api/metrics.go @@ -0,0 +1,129 @@ +package api + +import ( + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type defaultMetrics struct { + requestDuration *prometheus.HistogramVec + requestSize *prometheus.HistogramVec + requestsTotal *prometheus.CounterVec + responseSize *prometheus.HistogramVec + inflightHTTPRequests *prometheus.GaugeVec +} + +func newDefaultMetrics(reg prometheus.Registerer, durationBuckets []float64, extraLabels []string) *defaultMetrics { + if durationBuckets == nil { + durationBuckets = []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720} + } + + bytesBuckets := prometheus.ExponentialBuckets(64, 2, 10) + bucketFactor := 1.1 + maxBuckets := uint32(100) + + return &defaultMetrics{ + requestDuration: promauto.With(reg).NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "Tracks the latencies for HTTP requests.", + Buckets: durationBuckets, + NativeHistogramBucketFactor: bucketFactor, + NativeHistogramMaxBucketNumber: maxBuckets, + }, + append([]string{"code", "handler", "method"}, extraLabels...), + ), + + requestSize: promauto.With(reg).NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_size_bytes", + Help: "Tracks the size of HTTP requests.", + Buckets: bytesBuckets, + NativeHistogramBucketFactor: bucketFactor, + NativeHistogramMaxBucketNumber: maxBuckets, + }, + append([]string{"code", "handler", "method"}, extraLabels...), + ), + + requestsTotal: promauto.With(reg).NewCounterVec( + prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Tracks the number of HTTP requests.", + }, + append([]string{"code", "handler", "method"}, extraLabels...), + ), + + responseSize: promauto.With(reg).NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_response_size_bytes", + Help: "Tracks the size of HTTP responses.", + Buckets: bytesBuckets, + NativeHistogramBucketFactor: bucketFactor, + NativeHistogramMaxBucketNumber: maxBuckets, + }, + append([]string{"code", "handler", "method"}, extraLabels...), + ), + + inflightHTTPRequests: promauto.With(reg).NewGaugeVec( + prometheus.GaugeOpts{ + Name: "http_inflight_requests", + Help: "Current number of HTTP requests the handler is responding to.", + }, + append([]string{"handler", "method"}, extraLabels...), + ), + } +} + +func (m *defaultMetrics) MetricsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + // Extract the request path and method + requestPath := r.URL.Path + + // Increment inflight requests gauge + m.inflightHTTPRequests.WithLabelValues(requestPath, r.Method).Inc() + defer m.inflightHTTPRequests.WithLabelValues(requestPath, r.Method).Dec() + + // Capture the response size + rw := &responseWriter{w: w} + next.ServeHTTP(rw, r) + + duration := time.Since(start).Seconds() + statusCode := strconv.Itoa(rw.statusCode) + + // Observe metrics + m.requestDuration.WithLabelValues(statusCode, requestPath, r.Method).Observe(duration) + m.requestSize.WithLabelValues(statusCode, requestPath, r.Method).Observe(float64(r.ContentLength)) + m.requestsTotal.WithLabelValues(statusCode, requestPath, r.Method).Inc() + m.responseSize.WithLabelValues(statusCode, requestPath, r.Method).Observe(float64(rw.size)) + }) +} + +type responseWriter struct { + w http.ResponseWriter + statusCode int + size int +} + +func (rw *responseWriter) Header() http.Header { + return rw.w.Header() +} + +func (rw *responseWriter) Write(b []byte) (int, error) { + if rw.statusCode == 0 { + rw.statusCode = http.StatusOK + } + size, err := rw.w.Write(b) + rw.size += size + return size, err +} + +func (rw *responseWriter) WriteHeader(statusCode int) { + rw.w.WriteHeader(statusCode) + rw.statusCode = statusCode +} diff --git a/go.mod b/go.mod index ed89e47..373f936 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,8 @@ require ( require ( github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect @@ -27,11 +28,17 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rabbitmq/amqp091-go v1.10.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -45,8 +52,9 @@ require ( go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index dfaa4fe..b3557df 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,16 @@ github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9C github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae/go.mod h1:rJJ84PyA/Wlmw1hO+xTzV2wsSUon6J5ktg0g8BF2PuU= github.com/Songmu/go-httpdate v1.0.0 h1:39S00oyg9q+kMso2ahhK4pvD4EXk4zQWzt/AMqGlH3o= github.com/Songmu/go-httpdate v1.0.0/go.mod h1:QPvdlIAR7M8UtklJx5CMOOCIq7hbx2QdxyEPvTF5QVs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,6 +50,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -54,6 +60,8 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -71,6 +79,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= @@ -140,11 +156,17 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=