From ea6115ea166ccd1d76111b26e34320fcdc08f482 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Fri, 15 Mar 2024 13:22:07 +0530 Subject: [PATCH] Refactor metrics code --- adapter/config/default_config.go | 4 +-- adapter/config/types.go | 7 +++-- adapter/internal/adapter/adapter.go | 11 +------- adapter/internal/operator/operator.go | 26 +++++++++++------ adapter/pkg/metrics/metrics.go | 4 +-- .../commoncontroller/common_controller.go | 10 +------ .../internal/config/default_config.go | 2 +- common-controller/internal/config/types.go | 5 ++-- .../internal/operator/operator.go | 28 +++++++++++++------ common-controller/pkg/metrics/metrics.go | 4 +-- 10 files changed, 54 insertions(+), 47 deletions(-) diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 72dd7cd012..05e3618dc5 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -32,7 +32,7 @@ var defaultConfig = &Config{ Namespaces: nil, }, Environment: "Default", - Metrics: metrics{ + Metrics: Metrics{ Enabled: false, Type: "prometheus", Port: 18006, @@ -192,7 +192,7 @@ var defaultConfig = &Config{ MaximumSize: 10000, ExpiryTime: 15, }, - Metrics: metrics{ + Metrics: Metrics{ Enabled: false, Type: "azure", }, diff --git a/adapter/config/types.go b/adapter/config/types.go index 8e8954f389..6cda6a1cb3 100644 --- a/adapter/config/types.go +++ b/adapter/config/types.go @@ -94,7 +94,7 @@ type adapter struct { // Environment of the Adapter Environment string // Metric represents configurations to expose/export go metrics - Metrics metrics + Metrics Metrics } // Envoy Listener Component related configurations. @@ -159,7 +159,7 @@ type enforcer struct { Management management RestServer restServer Filters []filter - Metrics metrics + Metrics Metrics MandateSubscriptionValidation bool Client httpClient } @@ -300,7 +300,8 @@ type tracing struct { ConfigProperties map[string]string } -type metrics struct { +// Metrics defines the configuration for metrics collection. +type Metrics struct { Enabled bool Type string Port int32 diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index e0e9389ea7..39f9c9be20 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -20,14 +20,12 @@ package adapter import ( "crypto/tls" - "strings" "time" discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" xdsv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" enforcerCallbacks "github.com/wso2/apk/adapter/internal/discovery/xds/enforcercallbacks" routercb "github.com/wso2/apk/adapter/internal/discovery/xds/routercallbacks" - "github.com/wso2/apk/adapter/internal/loggers" "github.com/wso2/apk/adapter/internal/operator" apiservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/api" configservice "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/service/config" @@ -35,7 +33,6 @@ import ( wso2_server "github.com/wso2/apk/adapter/pkg/discovery/protocol/server/v3" "github.com/wso2/apk/adapter/pkg/health" healthservice "github.com/wso2/apk/adapter/pkg/health/api/wso2/health/service" - "github.com/wso2/apk/adapter/pkg/metrics" "github.com/wso2/apk/adapter/pkg/utils/tlsutils" "context" @@ -183,13 +180,7 @@ func Run(conf *config.Config) { // Set enforcer startup configs xds.UpdateEnforcerConfig(conf) - go operator.InitOperator(conf.Adapter.Metrics.Port) - - // Start the metrics server - if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { - loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer() - } + go operator.InitOperator(conf.Adapter.Metrics) OUTER: for { diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index 725b778c83..0abc7106aa 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -19,12 +19,14 @@ package operator import ( "flag" - "strconv" + "fmt" + "strings" "github.com/wso2/apk/adapter/config" "github.com/wso2/apk/adapter/internal/loggers" "github.com/wso2/apk/adapter/pkg/logging" + "github.com/wso2/apk/adapter/pkg/metrics" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -68,12 +70,9 @@ func init() { } // InitOperator starts the Kubernetes gateway operator -func InitOperator(prometheusPort int32) { - var metricsAddr string +func InitOperator(metricsConfig config.Metrics) { var enableLeaderElection bool var probeAddr string - port := strconv.FormatInt(int64(prometheusPort), 10) - flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -87,9 +86,8 @@ func InitOperator(prometheusPort int32) { operatorDataStore := synchronizer.GetOperatorDataStore() - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + options := ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: true, @@ -106,7 +104,13 @@ func InitOperator(prometheusPort int32) { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, - }) + } + + if metricsConfig.Enabled { + options.MetricsBindAddress = fmt.Sprintf(":%d", metricsConfig.Port) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2600, logging.BLOCKER, "Unable to start manager: %v", err)) @@ -140,6 +144,12 @@ func InitOperator(prometheusPort int32) { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2603, logging.BLOCKER, "Unable to set up ready check: %v", err)) } + // Register the metrics collector + if metricsConfig.Enabled && strings.EqualFold(metricsConfig.Type, metrics.PrometheusMetricType) { + loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") + go metrics.RegisterPrometheusCollector() + } + go synchronizer.HandleAPILifeCycleEvents(&ch, &successChannel) go synchronizer.HandleGatewayLifeCycleEvents(&gatewaych) if config.ReadConfigs().PartitionServer.Enabled { diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 0de50025b1..19d28eceff 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -87,8 +87,8 @@ func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(collector.internalClusterCount, prometheus.GaugeValue, internalClusterCount) } -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer() { +// RegisterPrometheusCollector registers the collector for metrics. +func RegisterPrometheusCollector() { collector := adapterMetricsCollector() k8smetrics.Registry.MustRegister(collector) diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index cecdbdb7af..49000b877f 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -25,7 +25,6 @@ import ( "net" "os" "os/signal" - "strings" "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -41,7 +40,6 @@ import ( "github.com/wso2/apk/common-controller/internal/server" utils "github.com/wso2/apk/common-controller/internal/utils" xds "github.com/wso2/apk/common-controller/internal/xds" - "github.com/wso2/apk/common-controller/pkg/metrics" apkmgt "github.com/wso2/apk/common-go-libs/pkg/discovery/api/wso2/discovery/service/apkmgt" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -219,13 +217,7 @@ func InitCommonControllerServer(conf *config.Config) { // Start Enforcer xDS gRPC server runCommonEnforcerServer(port) - go operator.InitOperator(conf.CommonController.Metrics.Port) - - // Start the metrics server - if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { - loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer() - } + go operator.InitOperator(conf.CommonController.Metrics) OUTER: for { diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index bd0c3346ff..8c4019ca34 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -43,7 +43,7 @@ var defaultConfig = &Config{ RetryInterval: 5, Persistence: persistence{Type: "K8s"}, }, - Metrics: metrics{ + Metrics: Metrics{ Enabled: false, Type: "prometheus", Port: 18006, diff --git a/common-controller/internal/config/types.go b/common-controller/internal/config/types.go index bb40d45574..e724ec2445 100644 --- a/common-controller/internal/config/types.go +++ b/common-controller/internal/config/types.go @@ -48,7 +48,7 @@ type commoncontroller struct { WebServer webServer InternalAPIServer internalAPIServer ControlPlane controlplane - Metrics metrics + Metrics Metrics Database database } type controlplane struct { @@ -105,7 +105,8 @@ type webServer struct { Port int64 } -type metrics struct { +// Metrics defines the configuration for metrics collection. +type Metrics struct { Enabled bool Type string Port int32 diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go index 07473972e9..14342bdc60 100644 --- a/common-controller/internal/operator/operator.go +++ b/common-controller/internal/operator/operator.go @@ -19,8 +19,9 @@ package operator import ( "flag" + "fmt" "os" - "strconv" + "strings" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -34,6 +35,7 @@ import ( "github.com/wso2/apk/common-controller/internal/loggers" cpcontrollers "github.com/wso2/apk/common-controller/internal/operator/controllers/cp" dpcontrollers "github.com/wso2/apk/common-controller/internal/operator/controllers/dp" + "github.com/wso2/apk/common-controller/pkg/metrics" cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" @@ -65,13 +67,11 @@ func init() { } // InitOperator initializes the operator -func InitOperator(prometheusPort int32) { - var metricsAddr string +// func InitOperator(prometheusPort int32, metricsEnabled bool) { +func InitOperator(metricsConfig config.Metrics) { var enableLeaderElection bool var probeAddr string controlPlaneID := uuid.New().String() - port := strconv.FormatInt(int64(prometheusPort), 10) - flag.StringVar(&metricsAddr, "metrics-bind-address", ":"+port, "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ @@ -85,9 +85,9 @@ func InitOperator(prometheusPort int32) { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) ratelimitStore := cache.CreateNewOperatorDataStore() subscriptionStore := cache.CreateNewSubscriptionDataStore() - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + + options := ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, Port: 9443, HealthProbeBindAddress: probeAddr, // LeaderElection: true, @@ -103,7 +103,13 @@ func InitOperator(prometheusPort int32) { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, - }) + } + + if metricsConfig.Enabled { + options.MetricsBindAddress = fmt.Sprintf(":%d", metricsConfig.Port) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2600, logging.BLOCKER, "Unable to start manager: %v", err)) os.Exit(1) @@ -201,6 +207,12 @@ func InitOperator(prometheusPort int32) { }() } + // Register the metrics collector + if metricsConfig.Enabled && strings.EqualFold(metricsConfig.Type, metrics.PrometheusMetricType) { + loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") + go metrics.RegisterPrometheusCollector() + } + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2604, logging.BLOCKER, "Problem running manager: %v", err)) diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go index 45ca6b18b7..1fb4fc1ae6 100644 --- a/common-controller/pkg/metrics/metrics.go +++ b/common-controller/pkg/metrics/metrics.go @@ -24,8 +24,8 @@ import ( k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer() { +// RegisterPrometheusCollector registers the collector for metrics. +func RegisterPrometheusCollector() { collector := metrics.CustomMetricsCollector() k8smetrics.Registry.MustRegister(collector)