[kube-prometheus-stack] migrate custom parts

Signed-off-by: junot <[email protected]>
WizTelemetry · Jul 15, 2024 · 263acc9 · 263acc9
1 parent 7bcd7c9
commit 263acc9
Show file tree

Hide file tree

Showing 19 changed files with 2,447 additions and 904 deletions.
diff --git a/charts/kube-prometheus-stack/Chart.yaml b/charts/kube-prometheus-stack/Chart.yaml
@@ -50,12 +50,8 @@ dependencies:
     condition: kubeStateMetrics.enabled
   - name: prometheus-node-exporter
     version: "4.37.*"
-    repository: https://prometheus-community.github.io/helm-charts
+    repository: file://../prometheus-node-exporter
     condition: nodeExporter.enabled
-  - name: grafana
-    version: "8.3.*"
-    repository: https://grafana.github.io/helm-charts
-    condition: grafana.enabled
   - name: prometheus-windows-exporter
     repository: https://prometheus-community.github.io/helm-charts
     version: "0.3.*"

diff --git a/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kube-prometheus-stack/templates/_helpers.tpl
@@ -36,7 +36,9 @@ The longest name that gets created adds and extra 37 characters, so truncation s
 
 {{/* Prometheus custom resource instance name */}}
 {{- define "kube-prometheus-stack.prometheus.crname" -}}
-{{- if .Values.cleanPrometheusOperatorObjectNames }}
+{{- if .Values.prometheus.crname }}
+{{- .Values.prometheus.crname }}
+{{- else if .Values.cleanPrometheusOperatorObjectNames }}
 {{- include "kube-prometheus-stack.fullname" . }}
 {{- else }}
 {{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }}

diff --git a/charts/kube-prometheus-stack/templates/extra-objects.yaml b/charts/kube-prometheus-stack/templates/extra-objects.yaml
@@ -2,3 +2,8 @@
 ---
 {{ tpl (toYaml .) $ }}
 {{ end }}
+
+{{- if .Values.extraManifestsTpl }}
+---
+{{- tpl .Values.extraManifestsTpl . }}
+{{- end }}
diff --git a/charts/kube-prometheus-stack/templates/prometheus/_rules-whizard-telemetry.tpl b/charts/kube-prometheus-stack/templates/prometheus/_rules-whizard-telemetry.tpl
@@ -0,0 +1,13 @@
+{{- /*
+Generated file. Do not change in-place! In order to change this file first read following link:
+https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
+*/ -}}
+{{- define "rules-whizard-telemetry.names" }}
+rules:
+  - "whizard-telemetry-apiserver.rules"
+  - "whizard-telemetry-cluster.rules"
+  - "whizard-telemetry-namespace.rules"
+  - "whizard-telemetry-node.rules"
+  - "whizard-telemetry-etcd.rules"
+  - "whizard-telemetry-kube-scheduler.rules"
+{{- end }}
diff --git a/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml
@@ -215,8 +215,11 @@ spec:
 {{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }}
 {{- end }}
 {{- end }}
-{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }}
+{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite .Values.prometheus.agentMode) }}
   remoteWrite:
+{{- if and .Values.prometheus.agentMode .Values.prometheus.prometheusSpec.internalRemoteWrite }}
+{{ tpl (toYaml .Values.prometheus.prometheusSpec.internalRemoteWrite | indent 4) . }}
+{{- end }}
 {{- if .Values.prometheus.prometheusSpec.remoteWrite }}
 {{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }}
 {{- end }}

diff --git a/.../templates/prometheus/rules-1.14-whizard-telemetry/whizard-telemetry-apiserver.rules.yaml b/.../templates/prometheus/rules-1.14-whizard-telemetry/whizard-telemetry-apiserver.rules.yaml
@@ -0,0 +1,71 @@
+{{- /*
+Generated from 'whizard-telemetry-apiserver.rules' group from file://manifests/whizard-telemetry-prometheusRule.yaml
+Do not change in-place! In order to change this file first read following link:
+https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
+*/ -}}
+{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
+{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.whizardTelemetry }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "whizard-telemetry-apiserver.rules" | trunc 63 | trimSuffix "-" }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  labels:
+    app: {{ template "kube-prometheus-stack.name" . }}
+{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- if .Values.defaultRules.labels }}
+{{ toYaml .Values.defaultRules.labels | indent 4 }}
+{{- end }}
+{{- if .Values.defaultRules.annotations }}
+  annotations:
+{{ toYaml .Values.defaultRules.annotations | indent 4 }}
+{{- end }}
+spec:
+  groups:
+  - name: whizard-telemetry-apiserver.rules
+    rules:
+    - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(apiserver_request_total{job="apiserver"}[5m]))
+      record: apiserver:apiserver_request_total:sum_irate
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb)(irate(apiserver_request_total{job="apiserver"}[5m]))
+      record: apiserver:apiserver_request_total:sum_verb_irate
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(apiserver_request_duration_seconds_sum{job="apiserver",subresource!="log", verb!~"LIST|WATCH|WATCHLIST|PROXY|CONNECT"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(apiserver_request_duration_seconds_count{job="apiserver", subresource!="log",verb!~"LIST|WATCH|WATCHLIST|PROXY|CONNECT"}[5m]))
+      record: apiserver:apiserver_request_duration:avg
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb)(irate(apiserver_request_duration_seconds_sum{job="apiserver",subresource!="log", verb!~"LIST|WATCH|WATCHLIST|PROXY|CONNECT"}[5m]))  / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb)(irate(apiserver_request_duration_seconds_count{job="apiserver", subresource!="log",verb!~"LIST|WATCH|WATCHLIST|PROXY|CONNECT"}[5m]))
+      record: apiserver:apiserver_request_duration:avg_by_verb
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+{{- end }}
diff --git a/...ck/templates/prometheus/rules-1.14-whizard-telemetry/whizard-telemetry-cluster.rules.yaml b/...ck/templates/prometheus/rules-1.14-whizard-telemetry/whizard-telemetry-cluster.rules.yaml
@@ -0,0 +1,133 @@
+{{- /*
+Generated from 'whizard-telemetry-cluster.rules' group from file://manifests/whizard-telemetry-prometheusRule.yaml
+Do not change in-place! In order to change this file first read following link:
+https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
+*/ -}}
+{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
+{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.whizardTelemetry }}
+{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "whizard-telemetry-cluster.rules" | trunc 63 | trimSuffix "-" }}
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  labels:
+    app: {{ template "kube-prometheus-stack.name" . }}
+{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- if .Values.defaultRules.labels }}
+{{ toYaml .Values.defaultRules.labels | indent 4 }}
+{{- end }}
+{{- if .Values.defaultRules.annotations }}
+  annotations:
+{{ toYaml .Values.defaultRules.annotations | indent 4 }}
+{{- end }}
+spec:
+  groups:
+  - name: whizard-telemetry-cluster.rules
+    rules:
+    - expr: |-
+        max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, workspace, namespace, pod, qos_class, phase, workload, workload_type) (
+                  kube_pod_info{job="{{ $kubeStateMetricsJob }}"}
+                * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left (qos_class)
+                  max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, qos_class) (
+                    kube_pod_status_qos_class{job="{{ $kubeStateMetricsJob }}"} > 0
+                  )
+              * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left (phase)
+                max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, phase) (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}"} > 0)
+            * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left (workload, workload_type)
+              max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, workload, workload_type) (
+                  label_join(
+                    label_join(
+                      kube_pod_owner{job="{{ $kubeStateMetricsJob }}",owner_kind!~"ReplicaSet|DaemonSet|StatefulSet|Job"},
+                      "workload",
+                      "$1",
+                      "owner_name"
+                    ),
+                    "workload_type",
+                    "$1",
+                    "owner_kind"
+                  )
+                or
+                    kube_pod_owner{job="{{ $kubeStateMetricsJob }}",owner_kind=~"ReplicaSet|DaemonSet|StatefulSet|Job"}
+                  * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left (workload_type, workload)
+                    namespace_workload_pod:kube_pod_owner:relabel
+              )
+          * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace) group_left (workspace)
+            max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workspace) (kube_namespace_labels{job="{{ $kubeStateMetricsJob }}"})
+        )
+      record: 'workspace_workload_node:kube_pod_info:'
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    - expr: |-
+        label_replace(
+          max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, role, internal_ip) (
+                kube_node_info{job="{{ $kubeStateMetricsJob }}"}
+              * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) group_left (role)
+                max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, role) (
+                        (
+                            kube_node_role{job="{{ $kubeStateMetricsJob }}",role="worker"}
+                          unless ignoring (role)
+                            kube_node_role{job="{{ $kubeStateMetricsJob }}",role=~"control-plane|edge"}
+                        )
+                      or
+                        (
+                            kube_node_role{job="{{ $kubeStateMetricsJob }}",role="control-plane"}
+                          unless ignoring (role)
+                            kube_node_role{job="{{ $kubeStateMetricsJob }}",role="edge"}
+                        )
+                    or
+                      kube_node_role{job="{{ $kubeStateMetricsJob }}",role="edge"}
+                  or
+                    topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) (
+                      1,
+                        kube_node_role{job="{{ $kubeStateMetricsJob }}"}
+                      unless ignoring (role)
+                        (kube_node_role{job="{{ $kubeStateMetricsJob }}",role=~"edge|control-plane|worker|fakenode"})
+                    )
+                )
+            or
+                kube_node_info{job="{{ $kubeStateMetricsJob }}"}
+              unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node)
+                kube_node_role{job="{{ $kubeStateMetricsJob }}"}
+          ),
+          "host_ip",
+          "$1",
+          "internal_ip",
+          "(.*)"
+        )
+      record: 'node_role_ip:kube_node_info:'
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    - expr: |-
+        max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, pod_ip, host_ip, node) (
+              kube_pod_info{job="{{ $kubeStateMetricsJob }}"}
+            * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, uid) group_left ()
+              kube_pod_start_time{job="{{ $kubeStateMetricsJob }}"}
+          )
+        * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left (workspace, qos_class, phase)
+          workspace_workload_node:kube_pod_info:
+      record: 'pod_start_time:kube_pod_info:'
+      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+      labels:
+        {{- with .Values.defaultRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.defaultRules.additionalRuleGroupLabels.whizardTelemetry }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- end }}
+{{- end }}