Merge pull request #95 from tonytheleg/RHCLOUD-34499-add-metrics

RHCLOUD-34499 adds basic metrics to grpc and http servers

cmd/serve/serve.go

@@ -144,7 +144,10 @@ func NewCommand(
 			}
 
 			// construct servers
-			server := server.New(serverConfig, middleware.Authentication(authenticator), logger)
+			server, err := server.New(serverConfig, middleware.Authentication(authenticator), logger)
+			if err != nil {
+				return err
+			}
 
 			// wire together notificationsintegrations handling
 			notifs_repo := notifsrepo.New(db, authorizer, eventingManager)

go.mod

@@ -15,10 +15,16 @@ require (
 	github.com/google/wire v0.6.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/project-kessel/relations-api v0.0.0-20240801131134-0f51350f3c3d
+	github.com/prometheus/client_golang v1.19.1
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/otel v1.28.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.50.0
+	go.opentelemetry.io/otel/metric v1.28.0
+	go.opentelemetry.io/otel/sdk v1.28.0
+	go.opentelemetry.io/otel/sdk/metric v1.28.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117
 	google.golang.org/grpc v1.66.0
 	google.golang.org/protobuf v1.34.2
@@ -30,9 +36,12 @@ require (
 
 require (
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/docker/docker v27.2.0+incompatible // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
@@ -56,10 +65,15 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -68,8 +82,6 @@ require (
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	go.opentelemetry.io/otel v1.28.0 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect

go.sum

@@ -116,8 +116,8 @@ github.com/docker/compose/v2 v2.27.0 h1:FKyClQdErCxUZULC2zo6Jn5ve+epFPe/Y0HaxjmU
 github.com/docker/compose/v2 v2.27.0/go.mod h1:uaqwmY6haO8wXWHk+LAsqqDapX6boH4izRKqj/E7+Bo=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v27.0.0+incompatible h1:JRugTYuelmWlW0M3jakcIadDx2HUoUO6+Tf2C5jVfwA=
-github.com/docker/docker v27.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4=
+github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8=
 github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40=
 github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
@@ -310,6 +310,8 @@ github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZ
 github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
 github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
 github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

internal/server/grpc/grpc.go

@@ -7,25 +7,40 @@ import (
 	"github.com/go-kratos/kratos/v2/middleware/selector"
 	kgrpc "github.com/go-kratos/kratos/v2/transport/grpc"
 	m "github.com/project-kessel/inventory-api/internal/middleware"
+
+	"github.com/go-kratos/kratos/v2/middleware/metrics"
+	"go.opentelemetry.io/otel/metric"
 )
 
 // New creates a new a gRPC server.
-func New(c CompletedConfig, authn middleware.Middleware) *kgrpc.Server {
+func New(c CompletedConfig, authn middleware.Middleware, meter metric.Meter) (*kgrpc.Server, error) {
+	requests, err := metrics.DefaultRequestsCounter(meter, metrics.DefaultServerRequestsCounterName)
+	if err != nil {
+		return nil, err
+	}
+	seconds, err := metrics.DefaultSecondsHistogram(meter, metrics.DefaultServerSecondsHistogramName)
+	if err != nil {
+		return nil, err
+	}
 	validator, err := protovalidate.New()
 	if err != nil {
-		return nil
+		return nil, err
 	}
 	// TODO: pass in health, authn middleware
 	var opts = []kgrpc.ServerOption{
 		kgrpc.Middleware(
 			recovery.Recovery(),
 			m.Validation(validator),
+			metrics.Server(
+				metrics.WithRequests(requests),
+				metrics.WithSeconds(seconds),
+			),
 			selector.Server(
 				authn,
 			).Match(NewWhiteListMatcher).Build(),
 		),
 	}
 	opts = append(opts, c.ServerOptions...)
 	srv := kgrpc.NewServer(opts...)
-	return srv
+	return srv, nil
 }

internal/server/http/http.go

@@ -3,29 +3,51 @@ package http
 import (
 	"github.com/bufbuild/protovalidate-go"
 	"github.com/go-kratos/kratos/v2/middleware"
+	"github.com/go-kratos/kratos/v2/middleware/metrics"
 	"github.com/go-kratos/kratos/v2/middleware/recovery"
 	"github.com/go-kratos/kratos/v2/middleware/selector"
 	"github.com/go-kratos/kratos/v2/transport/http"
 	m "github.com/project-kessel/inventory-api/internal/middleware"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/otel/metric"
 )
 
 // New create a new http server.
-func New(c CompletedConfig, authn middleware.Middleware) *http.Server {
+func New(c CompletedConfig, authn middleware.Middleware, meter metric.Meter) (*http.Server, error) {
+	requests, err := metrics.DefaultRequestsCounter(meter, metrics.DefaultServerRequestsCounterName)
+	if err != nil {
+		return nil, err
+	}
+	seconds, err := metrics.DefaultSecondsHistogram(meter, metrics.DefaultServerSecondsHistogramName)
+	if err != nil {
+		return nil, err
+	}
 	validator, err := protovalidate.New()
 	if err != nil {
-		return nil
+		return nil, err
 	}
 	// TODO: pass in health, authn middleware
 	var opts = []http.ServerOption{
 		http.Middleware(
 			recovery.Recovery(),
 			m.Validation(validator),
+			metrics.Server(
+				metrics.WithSeconds(seconds),
+				metrics.WithRequests(requests),
+			),
 			selector.Server(
 				authn,
 			).Match(NewWhiteListMatcher).Build(),
 		),
 	}
 	opts = append(opts, c.ServerOptions...)
 	srv := http.NewServer(opts...)
-	return srv
+	srv.HandlePrefix("/metrics", promhttp.HandlerFor(
+		prometheus.DefaultGatherer,
+		promhttp.HandlerOpts{
+			EnableOpenMetrics: true,
+		},
+	))
+	return srv, nil
 }

internal/server/otel.go

@@ -0,0 +1,39 @@
+package server
+
+// Taken from Kratos examples: https://github.com/go-kratos/examples/blob/main/otel/internal/dep/otel.go
+
+import (
+	"github.com/go-kratos/kratos/v2/middleware/metrics"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/metric"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
+)
+
+func NewMeter(provider metric.MeterProvider) (metric.Meter, error) {
+	return provider.Meter("inventory-api"), nil
+}
+
+func NewMeterProvider(s *Server) (metric.MeterProvider, error) {
+	exporter, err := prometheus.New()
+	if err != nil {
+		return nil, err
+	}
+
+	provider := sdkmetric.NewMeterProvider(
+		sdkmetric.WithResource(
+			resource.NewWithAttributes(
+				semconv.SchemaURL,
+				semconv.ServiceNameKey.String("inventory-api"),
+			),
+		),
+		sdkmetric.WithReader(exporter),
+		sdkmetric.WithView(
+			metrics.DefaultSecondsHistogramView(metrics.DefaultServerSecondsHistogramName),
+		),
+	)
+	otel.SetMeterProvider(provider)
+	return provider, nil
+}

internal/server/server.go

@@ -24,16 +24,37 @@ type Server struct {
 	Logger log.Logger
 }
 
-func New(c CompletedConfig, authn middleware.Middleware, logger log.Logger) *Server {
+func New(c CompletedConfig, authn middleware.Middleware, logger log.Logger) (*Server, error) {
 	s := &Server{
-		Id:         c.Options.Id,
-		Name:       c.Options.Name,
-		HttpServer: http.New(c.HttpConfig, authn),
-		GrpcServer: grpc.New(c.GrpcConfig, authn),
-		Logger:     log.With(logger, "service.id", c.Options.Id),
+		Id:     c.Options.Id,
+		Name:   c.Options.Name,
+		Logger: log.With(logger, "service.id", c.Options.Id),
 	}
 
-	return s
+	meterProvider, err := NewMeterProvider(s)
+	if err != nil {
+		return nil, err
+	}
+
+	meter, err := NewMeter(meterProvider)
+	if err != nil {
+		return nil, err
+	}
+
+	httpServer, err := http.New(c.HttpConfig, authn, meter)
+	if err != nil {
+		return nil, err
+	}
+
+	grpcServer, err := grpc.New(c.GrpcConfig, authn, meter)
+	if err != nil {
+		return nil, err
+	}
+
+	s.HttpServer = httpServer
+	s.GrpcServer = grpcServer
+
+	return s, nil
 }
 
 func (s *Server) Run(ctx context.Context) error {