Add metrics to operator

wso2 · Mar 15, 2024 · 19580e0 · 19580e0
1 parent 68dead0
commit 19580e0
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 56 deletions.
diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go
@@ -27,14 +27,15 @@ import (
 	xdsv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3"
 	enforcerCallbacks "github.com/wso2/apk/adapter/internal/discovery/xds/enforcercallbacks"
 	routercb "github.com/wso2/apk/adapter/internal/discovery/xds/routercallbacks"
+	"github.com/wso2/apk/adapter/internal/loggers"
 	"github.com/wso2/apk/adapter/internal/operator"
 	apiservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/api"
 	configservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/config"
 	subscriptionservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/subscription"
 	wso2_server "github.com/wso2/apk/adapter/pkg/discovery/protocol/server/v3"
 	"github.com/wso2/apk/adapter/pkg/health"
 	healthservice "github.com/wso2/apk/adapter/pkg/health/api/wso2/health/service"
-	metrics "github.com/wso2/apk/adapter/pkg/metrics"
+	"github.com/wso2/apk/adapter/pkg/metrics"
 	"github.com/wso2/apk/adapter/pkg/utils/tlsutils"
 
 	"context"
@@ -157,12 +158,6 @@ func Run(conf *config.Config) {
 
 	logger.LoggerAPK.Info("Starting adapter ....")
 
-	// Start the metrics server
-	if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) {
-		logger.LoggerAPK.Info("Starting Prometheus Metrics Server ....")
-		go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port)
-	}
-
 	cache := xds.GetXdsCache()
 	enforcerCache := xds.GetEnforcerCache()
 	enforcerAPICache := xds.GetEnforcerAPICache()
@@ -188,7 +183,13 @@ func Run(conf *config.Config) {
 	// Set enforcer startup configs
 	xds.UpdateEnforcerConfig(conf)
 
-	go operator.InitOperator()
+	go operator.InitOperator(conf.Adapter.Metrics.Port)
+
+	// Start the metrics server
+	if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) {
+		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
+		go metrics.StartPrometheusMetricsServer()
+	}
 
 OUTER:
 	for {

diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go
@@ -19,6 +19,7 @@ package operator
 
 import (
 	"flag"
+	"strconv"
 
 	"github.com/wso2/apk/adapter/config"
 	"github.com/wso2/apk/adapter/internal/loggers"
@@ -67,11 +68,12 @@ func init() {
 }
 
 // InitOperator starts the Kubernetes gateway operator
-func InitOperator() {
+func InitOperator(prometheusPort int32) {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	port := strconv.FormatInt(int64(prometheusPort), 10)
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -105,6 +107,7 @@ func InitOperator() {
 		// after the manager stops then its usage might be unsafe.
 		// LeaderElectionReleaseOnCancel: true,
 	})
+
 	if err != nil {
 		loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2600, logging.BLOCKER, "Unable to start manager: %v", err))
 	}

diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go
@@ -20,16 +20,10 @@
 package metrics
 
 import (
-	"fmt"
-	"net/http"
-	"strconv"
-
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	xds "github.com/wso2/apk/adapter/internal/discovery/xds"
-	logger "github.com/wso2/apk/adapter/internal/loggers"
-	"github.com/wso2/apk/adapter/pkg/logging"
 	commonmetrics "github.com/wso2/apk/common-go-libs/pkg/metrics"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var (
@@ -94,17 +88,9 @@ func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus.
-func StartPrometheusMetricsServer(port int32) {
+func StartPrometheusMetricsServer() {
 
 	collector := adapterMetricsCollector()
+	k8smetrics.Registry.MustRegister(collector)
 	prometheus.MustRegister(collector)
-	http.Handle("/metrics", promhttp.Handler())
-	err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil)
-	if err != nil {
-		logger.LoggerAPK.ErrorC(logging.ErrorDetails{
-			Message:   fmt.Sprintln("Prometheus metrics server error:", err),
-			Severity:  logging.MAJOR,
-			ErrorCode: 1110,
-		})
-	}
 }
diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go
@@ -206,12 +206,6 @@ func InitCommonControllerServer(conf *config.Config) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// Start the metrics server
-	if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) {
-		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
-		go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port)
-	}
-
 	loggers.LoggerAPKOperator.Info("Starting common controller ....")
 
 	rateLimiterCache := xds.GetRateLimiterCache()
@@ -225,7 +219,13 @@ func InitCommonControllerServer(conf *config.Config) {
 	// Start Enforcer xDS gRPC server
 	runCommonEnforcerServer(port)
 
-	go operator.InitOperator()
+	go operator.InitOperator(conf.CommonController.Metrics.Port)
+
+	// Start the metrics server
+	if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) {
+		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
+		go metrics.StartPrometheusMetricsServer()
+	}
 
 OUTER:
 	for {

diff --git a/common-controller/go.mod b/common-controller/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/jackc/pgx/v5 v5.5.2
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.10
-	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/sirupsen/logrus v1.9.0
 	k8s.io/apimachinery v0.28.3

diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go
@@ -20,6 +20,7 @@ package operator
 import (
 	"flag"
 	"os"
+	"strconv"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -64,12 +65,13 @@ func init() {
 }
 
 // InitOperator initializes the operator
-func InitOperator() {
+func InitOperator(prometheusPort int32) {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
 	controlPlaneID := uuid.New().String()
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	port := strconv.FormatInt(int64(prometheusPort), 10)
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+

diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go
@@ -20,29 +20,13 @@
 package metrics
 
 import (
-	"fmt"
-	"net/http"
-	"strconv"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/wso2/apk/adapter/pkg/logging"
-	logger "github.com/wso2/apk/common-controller/internal/loggers"
 	metrics "github.com/wso2/apk/common-go-libs/pkg/metrics"
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus.
-func StartPrometheusMetricsServer(port int32) {
+func StartPrometheusMetricsServer() {
 
 	collector := metrics.CustomMetricsCollector()
-	prometheus.MustRegister(collector)
-	http.Handle("/metrics", promhttp.Handler())
-	err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil)
-	if err != nil {
-		logger.LoggerAPK.ErrorC(logging.ErrorDetails{
-			Message:   fmt.Sprintln("Prometheus metrics server error:", err),
-			Severity:  logging.MAJOR,
-			ErrorCode: 1110,
-		})
-	}
+	k8smetrics.Registry.MustRegister(collector)
 }