[kube-prometheus-stack] Feat: Improve cAdvisor metrics scrape

Improve the collection of cAdvisor metrics by adjusting the scrape interval to match the kubelet hardcoded minimum housekeeping interval. * Set the cAdvisor, and resource, interval to 10s by default. * Use the user configured a kubelet metrics interval if not default. * Enforce honorTimestamps for cAdvisor and resource if timetamp staleness is enabled. Signed-off-by: SuperQ <[email protected]>
prometheus-community · Dec 16, 2024 · 23c1934 · 23c1934
1 parent 8db7c05
commit 23c1934
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 5 deletions.
diff --git a/charts/kube-prometheus-stack/Chart.yaml b/charts/kube-prometheus-stack/Chart.yaml
@@ -23,7 +23,7 @@ name: kube-prometheus-stack
 sources:
   - https://github.com/prometheus-community/helm-charts
   - https://github.com/prometheus-operator/kube-prometheus
-version: 67.1.0
+version: 67.2.0
 appVersion: v0.79.0
 kubeVersion: ">=1.19.0-0"
 home: https://github.com/prometheus-operator/kube-prometheus

diff --git a/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml b/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml
@@ -61,6 +61,8 @@ spec:
     path: /metrics/cadvisor
     {{- if .Values.kubelet.serviceMonitor.interval }}
     interval: {{ .Values.kubelet.serviceMonitor.interval }}
+    {{- else }}
+    interval: {{ .Values.kubelet.serviceMonitor.cAdvisorInterval }}
     {{- end }}
     {{- if .Values.kubelet.serviceMonitor.proxyUrl }}
     proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }}
@@ -69,7 +71,11 @@ spec:
     scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }}
     {{- end }}
     honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }}
+    {{ if .Values.kubelet.serviceMonitor.trackTimestampsStaleness }}
+    honorTimestamps: true
+    {{- else }}
     honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }}
+    {{- end }}
     trackTimestampsStaleness: {{ .Values.kubelet.serviceMonitor.trackTimestampsStaleness }}
     {{- include "kube-prometheus-stack.kubelet.authConfig" . | indent 4 }}
 {{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }}
@@ -95,7 +101,11 @@ spec:
     scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }}
     {{- end }}
     honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }}
+    {{ if .Values.kubelet.serviceMonitor.trackTimestampsStaleness }}
+    honorTimestamps: true
+    {{- else }}
     honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }}
+    {{- end }}
     {{- include "kube-prometheus-stack.kubelet.authConfig" . | indent 4 }}
 {{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }}
     metricRelabelings:
@@ -112,6 +122,8 @@ spec:
     path: {{ .Values.kubelet.serviceMonitor.resourcePath }}
     {{- if .Values.kubelet.serviceMonitor.interval }}
     interval: {{ .Values.kubelet.serviceMonitor.interval }}
+    {{- else }}
+    interval: {{ .Values.kubelet.serviceMonitor.resourceInterval }}
     {{- end }}
     {{- if .Values.kubelet.serviceMonitor.proxyUrl }}
     proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }}

diff --git a/charts/kube-prometheus-stack/values.yaml b/charts/kube-prometheus-stack/values.yaml
@@ -1362,10 +1362,6 @@ kubelet:
     ##
     insecureSkipVerify: true
 
-    ## Enable scraping /metrics/cadvisor from kubelet's service
-    ##
-    cAdvisor: true
-
     ## Enable scraping /metrics/probes from kubelet's service
     ##
     probes: true
@@ -1376,7 +1372,18 @@ kubelet:
     resource: false
     # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource
     resourcePath: "/metrics/resource/v1alpha1"
+    ## Configure the scrape interval for resource metrics. This is configured to the default Kubelet cAdvisor
+    ## minimum housekeeping interval in order to avoid missing samples. Note, this value is ignored
+    ## if kubelet.serviceMonitor.interval is not empty.
+    resourceInterval: 10s
 
+    ## Enable scraping /metrics/cadvisor from kubelet's service
+    ##
+    cAdvisor: true
+    ## Configure the scrape interval for cAdvisor. This is configured to the default Kubelet cAdvisor
+    ## minimum housekeeping interval in order to avoid missing samples. Note, this value is ignored
+    ## if kubelet.serviceMonitor.interval is not empty.
+    cAdvisorInterval: 10s
     ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
     ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig
     ##