Drop buckets

Azure · May 12, 2024 · c3ded28 · c3ded28
1 parent 9ce4768
commit c3ded28
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/otelcollector/shared/prometheus-ccp-config-merger.go b/otelcollector/shared/prometheus-ccp-config-merger.go
@@ -44,6 +44,10 @@ func loadRegexHash() {
 }
 
 func appendMetricRelabelConfig(yamlConfigFile, keepListRegex string) {
+	appendMetricRelabelConfigWithExclusions(yamlConfigFile, keepListRegex, "")
+}
+
+func appendMetricRelabelConfigWithExclusions(yamlConfigFile, keepListRegex, dropListRegex string) {
 	fmt.Printf("Adding keep list regex or minimal ingestion regex for %s\n", yamlConfigFile)
 
 	content, err := os.ReadFile(yamlConfigFile)
@@ -68,7 +72,16 @@ func appendMetricRelabelConfig(yamlConfigFile, keepListRegex string) {
 		for _, scfg := range scrapeConfigs {
 			if scfgMap, ok := scfg.(map[interface{}]interface{}); ok {
 				if metricRelabelCfgs, ok := scfgMap["metric_relabel_configs"].([]interface{}); ok {
-					scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig)
+					if (dropListRegex == "") {
+						scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig)
+					} else {
+						dropListMetricRelabelConfig := map[string]interface{}{
+							"source_labels": []interface{}{"__name__"},
+							"action":        "drop",
+							"regex":         dropListRegex,
+						}
+						scfgMap["metric_relabel_configs"] = append(metricRelabelCfgs, keepListMetricRelabelConfig, dropListMetricRelabelConfig)
+					}
 				} else {
 					scfgMap["metric_relabel_configs"] = []interface{}{keepListMetricRelabelConfig}
 				}
@@ -123,8 +136,9 @@ func populateDefaultPrometheusConfig() {
 
 	if enabled, exists := os.LookupEnv("AZMON_PROMETHEUS_CONTROLPLANE_APISERVER_ENABLED"); exists && strings.ToLower(enabled) == "true" && currentControllerType == replicasetControllerType {
 		controlplaneApiserverKeepListRegex, exists := regexHash["CONTROLPLANE_APISERVER_KEEP_LIST_REGEX"]
+		controlplaneApiserverDropListRegex, exists := regexHash["CONTROLPLANE_APISERVER_DROP_LIST_REGEX"]
 		if exists && controlplaneApiserverKeepListRegex != "" {
-			appendMetricRelabelConfig(controlplaneApiserverDefaultFile, controlplaneApiserverKeepListRegex)
+			appendMetricRelabelConfigWithExclusions(controlplaneApiserverDefaultFile, controlplaneApiserverKeepListRegex, controlplaneApiserverDropListRegex)
 		}
 		contents, err := os.ReadFile(controlplaneApiserverDefaultFile)
 		if err == nil {

diff --git a/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go b/otelcollector/shared/tomlparser-ccp-default-targets-metrics-keep-list.go
@@ -21,6 +21,7 @@ var (
 	controlplaneKubeSchedulerMinMac                                        = "scheduler_pending_pods|scheduler_unschedulable_pods|scheduler_pod_scheduling_attempts|scheduler_queue_incoming_pods_total|scheduler_preemption_attempts_total|scheduler_preemption_victims|scheduler_scheduling_attempt_duration_seconds|scheduler_schedule_attempts_total|scheduler_pod_scheduling_duration_seconds"
 	//controlplaneApiserverMinMac                                            = "apiserver_request_total|apiserver_cache_list_fetched_objects_total|apiserver_cache_list_returned_objects_total|apiserver_flowcontrol_demand_seats_average|apiserver_flowcontrol_current_limit_seats|apiserver_request_sli_duration_seconds_count|apiserver_request_sli_duration_seconds_sum|process_start_time_seconds|apiserver_request_duration_seconds_count|apiserver_request_duration_seconds_sum|apiserver_storage_list_fetched_objects_total|apiserver_storage_list_returned_objects_total|apiserver_current_inflight_requests"
 	controlplaneApiserverMinMac                                            = "apiserver_request_sli_duration_seconds_count|apiserver_request_sli_duration_seconds_sum"
+	controlplaneApiserverMinExclusions                                     = "apiserver_request_sli_duration_seconds_bucket|apiserver_request_sli_duration_seconds_bucket"
 	controlplaneClusterAutoscalerMinMac                                    = "rest_client_requests_total|cluster_autoscaler_((last_activity|cluster_safe_to_autoscale|scale_down_in_cooldown|scaled_up_nodes_total|unneeded_nodes_count|unschedulable_pods_count|nodes_count))|cloudprovider_azure_api_request_(errors|duration_seconds_(bucket|count))"
 	controlplaneEtcdMinMac                                                 = "etcd_server_has_leader|rest_client_requests_total|etcd_mvcc_db_total_size_in_bytes|etcd_mvcc_db_total_size_in_use_in_bytes|etcd_server_slow_read_indexes_total|etcd_server_slow_apply_total|etcd_network_client_grpc_sent_bytes_total|etcd_server_heartbeat_send_failures_total"
 )
@@ -163,6 +164,7 @@ func tomlparserCCPTargetsMetricsKeepList() {
 		"CONTROLPLANE_KUBE_CONTROLLER_MANAGER_KEEP_LIST_REGEX": controlplaneKubeControllerManagerRegex,
 		"CONTROLPLANE_KUBE_SCHEDULER_KEEP_LIST_REGEX":          controlplaneKubeSchedulerRegex,
 		"CONTROLPLANE_APISERVER_KEEP_LIST_REGEX":               controlplaneApiserverRegex,
+		"CONTROLPLANE_APISERVER_DROP_LIST_REGEX":               controlplaneApiserverMinExclusions,
 		"CONTROLPLANE_CLUSTER_AUTOSCALER_KEEP_LIST_REGEX":      controlplaneClusterAutoscalerRegex,
 		"CONTROLPLANE_ETCD_KEEP_LIST_REGEX":                    controlplaneEtcdRegex,
 	}