Refactor metrics code

wso2 · Mar 15, 2024 · ea6115e · ea6115e
1 parent 19580e0
commit ea6115e
Show file tree

Hide file tree

Showing 10 changed files with 54 additions and 47 deletions.
diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go
@@ -32,7 +32,7 @@ var defaultConfig = &Config{
 			Namespaces: nil,
 		},
 		Environment: "Default",
-		Metrics: metrics{
+		Metrics: Metrics{
 			Enabled: false,
 			Type:    "prometheus",
 			Port:    18006,
@@ -192,7 +192,7 @@ var defaultConfig = &Config{
 			MaximumSize: 10000,
 			ExpiryTime:  15,
 		},
-		Metrics: metrics{
+		Metrics: Metrics{
 			Enabled: false,
 			Type:    "azure",
 		},

diff --git a/adapter/config/types.go b/adapter/config/types.go
@@ -94,7 +94,7 @@ type adapter struct {
 	// Environment of the Adapter
 	Environment string
 	// Metric represents configurations to expose/export go metrics
-	Metrics metrics
+	Metrics Metrics
 }
 
 // Envoy Listener Component related configurations.
@@ -159,7 +159,7 @@ type enforcer struct {
 	Management                    management
 	RestServer                    restServer
 	Filters                       []filter
-	Metrics                       metrics
+	Metrics                       Metrics
 	MandateSubscriptionValidation bool
 	Client                        httpClient
 }
@@ -300,7 +300,8 @@ type tracing struct {
 	ConfigProperties map[string]string
 }
 
-type metrics struct {
+// Metrics defines the configuration for metrics collection.
+type Metrics struct {
 	Enabled            bool
 	Type               string
 	Port               int32

diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go
@@ -20,22 +20,19 @@ package adapter
 
 import (
 	"crypto/tls"
-	"strings"
 	"time"
 
 	discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	xdsv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3"
 	enforcerCallbacks "github.com/wso2/apk/adapter/internal/discovery/xds/enforcercallbacks"
 	routercb "github.com/wso2/apk/adapter/internal/discovery/xds/routercallbacks"
-	"github.com/wso2/apk/adapter/internal/loggers"
 	"github.com/wso2/apk/adapter/internal/operator"
 	apiservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/api"
 	configservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/config"
 	subscriptionservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/subscription"
 	wso2_server "github.com/wso2/apk/adapter/pkg/discovery/protocol/server/v3"
 	"github.com/wso2/apk/adapter/pkg/health"
 	healthservice "github.com/wso2/apk/adapter/pkg/health/api/wso2/health/service"
-	"github.com/wso2/apk/adapter/pkg/metrics"
 	"github.com/wso2/apk/adapter/pkg/utils/tlsutils"
 
 	"context"
@@ -183,13 +180,7 @@ func Run(conf *config.Config) {
 	// Set enforcer startup configs
 	xds.UpdateEnforcerConfig(conf)
 
-	go operator.InitOperator(conf.Adapter.Metrics.Port)
-
-	// Start the metrics server
-	if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) {
-		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
-		go metrics.StartPrometheusMetricsServer()
-	}
+	go operator.InitOperator(conf.Adapter.Metrics)
 
 OUTER:
 	for {

diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go
@@ -19,12 +19,14 @@ package operator
 
 import (
 	"flag"
-	"strconv"
+	"fmt"
+	"strings"
 
 	"github.com/wso2/apk/adapter/config"
 	"github.com/wso2/apk/adapter/internal/loggers"
 
 	"github.com/wso2/apk/adapter/pkg/logging"
+	"github.com/wso2/apk/adapter/pkg/metrics"
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -68,12 +70,9 @@ func init() {
 }
 
 // InitOperator starts the Kubernetes gateway operator
-func InitOperator(prometheusPort int32) {
-	var metricsAddr string
+func InitOperator(metricsConfig config.Metrics) {
 	var enableLeaderElection bool
 	var probeAddr string
-	port := strconv.FormatInt(int64(prometheusPort), 10)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -87,9 +86,8 @@ func InitOperator(prometheusPort int32) {
 
 	operatorDataStore := synchronizer.GetOperatorDataStore()
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	options := ctrl.Options{
 		Scheme:                  scheme,
-		MetricsBindAddress:      metricsAddr,
 		Port:                    9443,
 		HealthProbeBindAddress:  probeAddr,
 		LeaderElection:          true,
@@ -106,7 +104,13 @@ func InitOperator(prometheusPort int32) {
 		// if you are doing or is intended to do any operation such as perform cleanups
 		// after the manager stops then its usage might be unsafe.
 		// LeaderElectionReleaseOnCancel: true,
-	})
+	}
+
+	if metricsConfig.Enabled {
+		options.MetricsBindAddress = fmt.Sprintf(":%d", metricsConfig.Port)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
 
 	if err != nil {
 		loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2600, logging.BLOCKER, "Unable to start manager: %v", err))
@@ -140,6 +144,12 @@ func InitOperator(prometheusPort int32) {
 		loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2603, logging.BLOCKER, "Unable to set up ready check: %v", err))
 	}
 
+	// Register the metrics collector
+	if metricsConfig.Enabled && strings.EqualFold(metricsConfig.Type, metrics.PrometheusMetricType) {
+		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
+		go metrics.RegisterPrometheusCollector()
+	}
+
 	go synchronizer.HandleAPILifeCycleEvents(&ch, &successChannel)
 	go synchronizer.HandleGatewayLifeCycleEvents(&gatewaych)
 	if config.ReadConfigs().PartitionServer.Enabled {

diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go
@@ -87,8 +87,8 @@ func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(collector.internalClusterCount, prometheus.GaugeValue, internalClusterCount)
 }
 
-// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus.
-func StartPrometheusMetricsServer() {
+// RegisterPrometheusCollector registers the collector for metrics.
+func RegisterPrometheusCollector() {
 
 	collector := adapterMetricsCollector()
 	k8smetrics.Registry.MustRegister(collector)

diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go
@@ -25,7 +25,6 @@ import (
 	"net"
 	"os"
 	"os/signal"
-	"strings"
 	"time"
 
 	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -41,7 +40,6 @@ import (
 	"github.com/wso2/apk/common-controller/internal/server"
 	utils "github.com/wso2/apk/common-controller/internal/utils"
 	xds "github.com/wso2/apk/common-controller/internal/xds"
-	"github.com/wso2/apk/common-controller/pkg/metrics"
 	apkmgt "github.com/wso2/apk/common-go-libs/pkg/discovery/api/wso2/discovery/service/apkmgt"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -219,13 +217,7 @@ func InitCommonControllerServer(conf *config.Config) {
 	// Start Enforcer xDS gRPC server
 	runCommonEnforcerServer(port)
 
-	go operator.InitOperator(conf.CommonController.Metrics.Port)
-
-	// Start the metrics server
-	if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) {
-		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
-		go metrics.StartPrometheusMetricsServer()
-	}
+	go operator.InitOperator(conf.CommonController.Metrics)
 
 OUTER:
 	for {

diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go
@@ -43,7 +43,7 @@ var defaultConfig = &Config{
 			RetryInterval: 5,
 			Persistence:   persistence{Type: "K8s"},
 		},
-		Metrics: metrics{
+		Metrics: Metrics{
 			Enabled: false,
 			Type:    "prometheus",
 			Port:    18006,

diff --git a/common-controller/internal/config/types.go b/common-controller/internal/config/types.go
@@ -48,7 +48,7 @@ type commoncontroller struct {
 	WebServer         webServer
 	InternalAPIServer internalAPIServer
 	ControlPlane      controlplane
-	Metrics           metrics
+	Metrics           Metrics
 	Database          database
 }
 type controlplane struct {
@@ -105,7 +105,8 @@ type webServer struct {
 	Port int64
 }
 
-type metrics struct {
+// Metrics defines the configuration for metrics collection.
+type Metrics struct {
 	Enabled            bool
 	Type               string
 	Port               int32

diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go
@@ -19,8 +19,9 @@ package operator
 
 import (
 	"flag"
+	"fmt"
 	"os"
-	"strconv"
+	"strings"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -34,6 +35,7 @@ import (
 	"github.com/wso2/apk/common-controller/internal/loggers"
 	cpcontrollers "github.com/wso2/apk/common-controller/internal/operator/controllers/cp"
 	dpcontrollers "github.com/wso2/apk/common-controller/internal/operator/controllers/dp"
+	"github.com/wso2/apk/common-controller/pkg/metrics"
 	cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2"
 	dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1"
 	dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2"
@@ -65,13 +67,11 @@ func init() {
 }
 
 // InitOperator initializes the operator
-func InitOperator(prometheusPort int32) {
-	var metricsAddr string
+// func InitOperator(prometheusPort int32, metricsEnabled bool) {
+func InitOperator(metricsConfig config.Metrics) {
 	var enableLeaderElection bool
 	var probeAddr string
 	controlPlaneID := uuid.New().String()
-	port := strconv.FormatInt(int64(prometheusPort), 10)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -85,9 +85,9 @@ func InitOperator(prometheusPort int32) {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 	ratelimitStore := cache.CreateNewOperatorDataStore()
 	subscriptionStore := cache.CreateNewSubscriptionDataStore()
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+
+	options := ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
 		Port:                   9443,
 		HealthProbeBindAddress: probeAddr,
 		// LeaderElection:         true,
@@ -103,7 +103,13 @@ func InitOperator(prometheusPort int32) {
 		// if you are doing or is intended to do any operation such as perform cleanups
 		// after the manager stops then its usage might be unsafe.
 		// LeaderElectionReleaseOnCancel: true,
-	})
+	}
+
+	if metricsConfig.Enabled {
+		options.MetricsBindAddress = fmt.Sprintf(":%d", metricsConfig.Port)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
 	if err != nil {
 		loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2600, logging.BLOCKER, "Unable to start manager: %v", err))
 		os.Exit(1)
@@ -201,6 +207,12 @@ func InitOperator(prometheusPort int32) {
 		}()
 	}
 
+	// Register the metrics collector
+	if metricsConfig.Enabled && strings.EqualFold(metricsConfig.Type, metrics.PrometheusMetricType) {
+		loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....")
+		go metrics.RegisterPrometheusCollector()
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2604, logging.BLOCKER, "Problem running manager: %v", err))

diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go
@@ -24,8 +24,8 @@ import (
 	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
-// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus.
-func StartPrometheusMetricsServer() {
+// RegisterPrometheusCollector registers the collector for metrics.
+func RegisterPrometheusCollector() {
 
 	collector := metrics.CustomMetricsCollector()
 	k8smetrics.Registry.MustRegister(collector)