Default metrics filters for new metrics agent (#834)

* Start stubbing out methods for agent config. * Add reconciliation logic, tests. * CRD upgrades, codegen upgrades. * Make type names more concise. * The Endpoint type within the CassandraAgentSpec needs to be a pointer as it must be optional from the perspective of the CR so that it can have defaulting behaviour in the controller. * Replace Cassandra agent `filters` field with `relabels`. * Update metric-collector.yaml to metrics-collector.yaml * Make changes to default metrics so that new metrics name format is correctly parsed by regex.
k8ssandra · Mar 8, 2023 · a7cbbbc · a7cbbbc
1 parent 632ede9
commit a7cbbbc
Show file tree

Hide file tree

Showing 10 changed files with 236 additions and 26 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/apis/telemetry/v1alpha1/telemetry_types.go b/apis/telemetry/v1alpha1/telemetry_types.go
@@ -130,8 +130,8 @@ type McacTelemetrySpec struct {
 }
 
 type CassandraAgentSpec struct {
-	Endpoint Endpoint                `json:"endpoint,omitempty"`
-	Filters  []promapi.RelabelConfig `json:"filters,omitempty"`
+	Endpoint *Endpoint               `json:"endpoint,omitempty"`
+	Relabels []promapi.RelabelConfig `json:"relabels,omitempty"`
 }
 
 type Endpoint struct {

diff --git a/apis/telemetry/v1alpha1/zz_generated.deepcopy.go b/apis/telemetry/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
@@ -13740,7 +13740,7 @@ spec:
                                               port:
                                                 type: string
                                             type: object
-                                          filters:
+                                          relabels:
                                             items:
                                               description: 'RelabelConfig allows dynamic
                                                 rewriting of the label set, being
@@ -14259,7 +14259,7 @@ spec:
                                         port:
                                           type: string
                                       type: object
-                                    filters:
+                                    relabels:
                                       items:
                                         description: 'RelabelConfig allows dynamic
                                           rewriting of the label set, being applied
@@ -16841,7 +16841,7 @@ spec:
                                     port:
                                       type: string
                                   type: object
-                                filters:
+                                relabels:
                                   items:
                                     description: 'RelabelConfig allows dynamic rewriting
                                       of the label set, being applied to samples before
@@ -25518,7 +25518,7 @@ spec:
                               port:
                                 type: string
                             type: object
-                          filters:
+                          relabels:
                             items:
                               description: 'RelabelConfig allows dynamic rewriting
                                 of the label set, being applied to samples before
@@ -28195,7 +28195,7 @@ spec:
                               port:
                                 type: string
                             type: object
-                          filters:
+                          relabels:
                             items:
                               description: 'RelabelConfig allows dynamic rewriting
                                 of the label set, being applied to samples before
@@ -29864,7 +29864,7 @@ spec:
                               port:
                                 type: string
                             type: object
-                          filters:
+                          relabels:
                             items:
                               description: 'RelabelConfig allows dynamic rewriting
                                 of the label set, being applied to samples before

diff --git a/config/crd/bases/reaper.k8ssandra.io_reapers.yaml b/config/crd/bases/reaper.k8ssandra.io_reapers.yaml
@@ -2087,7 +2087,7 @@ spec:
                           port:
                             type: string
                         type: object
-                      filters:
+                      relabels:
                         items:
                           description: 'RelabelConfig allows dynamic rewriting of
                             the label set, being applied to samples before ingestion.

diff --git a/config/crd/bases/stargate.k8ssandra.io_stargates.yaml b/config/crd/bases/stargate.k8ssandra.io_stargates.yaml
@@ -2775,7 +2775,7 @@ spec:
                                 port:
                                   type: string
                               type: object
-                            filters:
+                            relabels:
                               items:
                                 description: 'RelabelConfig allows dynamic rewriting
                                   of the label set, being applied to samples before
@@ -3242,7 +3242,7 @@ spec:
                           port:
                             type: string
                         type: object
-                      filters:
+                      relabels:
                         items:
                           description: 'RelabelConfig allows dynamic rewriting of
                             the label set, being applied to samples before ingestion.

diff --git a/docs/content/en/components/metrics-collector/_index.md b/docs/content/en/components/metrics-collector/_index.md
@@ -82,7 +82,7 @@ Ingress or port forwarding can be used to expose access to the Prometheus and Gr
 
   2. How can I filter out metrics I don't care about?
 
-     Please read the [metric-collector.yaml](https://github.com/datastax/metric-collector-for-apache-cassandra/blob/master/config/metric-collector.yaml) section in the MCAC GitHub repo on how to add filtering rules.
+     Please read the [metrics-collector.yaml](https://github.com/datastax/metric-collector-for-apache-cassandra/blob/master/config/metrics-collector.yaml) section in the MCAC GitHub repo on how to add filtering rules.
 
   3. What is the datalog? And what is it for?
 

diff --git a/docs/content/en/tasks/monitor/_index.md b/docs/content/en/tasks/monitor/_index.md
@@ -3,4 +3,3 @@ title: "Monitor K8ssandra"
 linkTitle: "Monitor"
 weight: 6
 description: "Access tools to monitor your Apache Cassandra® cluster running in Kubernetes."
----
diff --git a/pkg/telemetry/cassandra_agent/cassandra_agent_config.go b/pkg/telemetry/cassandra_agent/cassandra_agent_config.go
@@ -5,12 +5,15 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/adutra/goalesce"
+
 	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
 	k8ssandraapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	telemetryapi "github.com/k8ssandra/k8ssandra-operator/apis/telemetry/v1alpha1"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/labels"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/reconciliation"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/result"
+	promapi "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -21,10 +24,89 @@ import (
 var (
 	agentConfigLocation = "/opt/management-api/configs/metrics-collector.yaml"
 	defaultAgentConfig  = telemetryapi.CassandraAgentSpec{
-		Endpoint: telemetryapi.Endpoint{
+		Endpoint: &telemetryapi.Endpoint{
 			Port:    "9000",
 			Address: "127.0.0.1",
 		},
+		Relabels: []promapi.RelabelConfig{
+			{
+				SourceLabels: []string{"table"},
+				Regex:        ".+",
+				TargetLabel:  "should_drop",
+				Replacement:  "true",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_live_ss_table_count",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_live_disk_space_used",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_memtable.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_all_memtables.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_compaction_bytes_written",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_pending_compactions",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_read_.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_write_.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_range.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_coordinator_.*",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"__name__"},
+				Regex:        "org_apache_cassandra_metrics_table_dropped_mutations",
+				TargetLabel:  "should_drop",
+				Replacement:  "false",
+			},
+			{
+				SourceLabels: []string{"should_drop"},
+				Regex:        "true",
+				Action:       "drop",
+			},
+		},
 	}
 )
 
@@ -42,7 +124,8 @@ func (c Configurator) GetTelemetryAgentConfigMap() (*corev1.ConfigMap, error) {
 	var yamlData []byte
 	var err error
 	if c.TelemetrySpec.Cassandra != nil {
-		yamlData, err = yaml.Marshal(&c.TelemetrySpec.Cassandra)
+		mergedSpec := goalesce.MustDeepMerge(&defaultAgentConfig, c.TelemetrySpec.Cassandra)
+		yamlData, err = yaml.Marshal(&mergedSpec)
 		if err != nil {
 			return &corev1.ConfigMap{}, err
 		}
@@ -79,7 +162,7 @@ func (c Configurator) ReconcileTelemetryAgentConfig(dc *cassdcapi.CassandraDatac
 	recRes := reconciliation.ReconcileObject(c.Ctx, c.RemoteClient, c.RequeueDelay, *desiredCm)
 	switch {
 	case recRes.IsError():
-		fallthrough
+		return recRes
 	case recRes.IsRequeue():
 		return recRes
 	}