diff --git a/otelcollector/shared/prometheus-ccp-config-merger.go b/otelcollector/shared/prometheus-ccp-config-merger.go index 7e5455306..b17ae2528 100644 --- a/otelcollector/shared/prometheus-ccp-config-merger.go +++ b/otelcollector/shared/prometheus-ccp-config-merger.go @@ -44,6 +44,10 @@ func loadRegexHash() { } func appendMetricRelabelConfig(yamlConfigFile, keepListRegex string) { + appendMetricRelabelConfigWithExclusions(yamlConfigFile, keepListRegex, "") +} + +func appendMetricRelabelConfigWithExclusions(yamlConfigFile, keepListRegex, dropListRegex string) { fmt.Printf("Adding keep list regex or minimal ingestion regex for %s\n", yamlConfigFile) content, err := os.ReadFile(yamlConfigFile) @@ -68,7 +72,16 @@ func appendMetricRelabelConfig(yamlConfigFile, keepListRegex string) { for _, scfg := range scrapeConfigs { if scfgMap, ok := scfg.(map[interface{}]interface{}); ok { if metricRelabelCfgs, ok := scfgMap["metric_relabel_configs"].([]interface{}); ok { - scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig) + if (dropListRegex == "") { + scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig) + } else { + dropListMetricRelabelConfig := map[string]interface{}{ + "source_labels": []interface{}{"__name__"}, + "action": "drop", + "regex": dropListRegex, + } + scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig, dropListMetricRelabelConfig) + } } else { scfgMap["metric_relabel_configs"] = []interface{}{keepListMetricRelabelConfig} } @@ -123,8 +136,9 @@ func populateDefaultPrometheusConfig() { if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_CONTROLPLANE_APISERVER_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType { controlplaneApiserverKeepListRegex, exists := regexHash["CONTROLPLANE_APISERVER_KEEP_LIST_REGEX"] + controlplaneApiserverDropListRegex, exists := regexHash["CONTROLPLANE_APISERVER_DROP_LIST_REGEX"] if exists && controlplaneApiserverKeepListRegex != "" { - appendMetricRelabelConfig(controlplaneApiserverDefaultFile, controlplaneApiserverKeepListRegex) + appendMetricRelabelConfigWithExclusions(controlplaneApiserverDefaultFile, controlplaneApiserverKeepListRegex, controlplaneApiserverDropListRegex) } contents, err := os.ReadFile(controlplaneApiserverDefaultFile) if err == nil { diff --git a/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go b/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go index 829634561..980d49f88 100644 --- a/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go +++ b/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go @@ -21,6 +21,7 @@ var ( controlplaneKubeSchedulerMinMac = "scheduler_pending_pods|scheduler_unschedulable_pods|scheduler_pod_scheduling_attempts|scheduler_queue_incoming_pods_total|scheduler_preemption_attempts_total|scheduler_preemption_victims|scheduler_scheduling_attempt_duration_seconds|scheduler_schedule_attempts_total|scheduler_pod_scheduling_duration_seconds" //controlplaneApiserverMinMac = "apiserver_request_total|apiserver_cache_list_fetched_objects_total|apiserver_cache_list_returned_objects_total|apiserver_flowcontrol_demand_seats_average|apiserver_flowcontrol_current_limit_seats|apiserver_request_sli_duration_seconds_count|apiserver_request_sli_duration_seconds_sum|process_start_time_seconds|apiserver_request_duration_seconds_count|apiserver_request_duration_seconds_sum|apiserver_storage_list_fetched_objects_total|apiserver_storage_list_returned_objects_total|apiserver_current_inflight_requests" controlplaneApiserverMinMac = "apiserver_request_sli_duration_seconds_count|apiserver_request_sli_duration_seconds_sum" + controlplaneApiserverMinExclusions = "apiserver_request_sli_duration_seconds_bucket|apiserver_request_sli_duration_seconds_bucket" controlplaneClusterAutoscalerMinMac = "rest_client_requests_total|cluster_autoscaler_((last_activity|cluster_safe_to_autoscale|scale_down_in_cooldown|scaled_up_nodes_total|unneeded_nodes_count|unschedulable_pods_count|nodes_count))|cloudprovider_azure_api_request_(errors|duration_seconds_(bucket|count))" controlplaneEtcdMinMac = "etcd_server_has_leader|rest_client_requests_total|etcd_mvcc_db_total_size_in_bytes|etcd_mvcc_db_total_size_in_use_in_bytes|etcd_server_slow_read_indexes_total|etcd_server_slow_apply_total|etcd_network_client_grpc_sent_bytes_total|etcd_server_heartbeat_send_failures_total" ) @@ -163,6 +164,7 @@ func tomlparserCCPTargetsMetricsKeepList() { "CONTROLPLANE_KUBE_CONTROLLER_MANAGER_KEEP_LIST_REGEX": controlplaneKubeControllerManagerRegex, "CONTROLPLANE_KUBE_SCHEDULER_KEEP_LIST_REGEX": controlplaneKubeSchedulerRegex, "CONTROLPLANE_APISERVER_KEEP_LIST_REGEX": controlplaneApiserverRegex, + "CONTROLPLANE_APISERVER_DROP_LIST_REGEX": controlplaneApiserverMinExclusions, "CONTROLPLANE_CLUSTER_AUTOSCALER_KEEP_LIST_REGEX": controlplaneClusterAutoscalerRegex, "CONTROLPLANE_ETCD_KEEP_LIST_REGEX": controlplaneEtcdRegex, }