[kube-prometheus-stack] Update mixins, switch to kube-prometheus mixi…

…n and fix label aggregation in TargetDown alert

Signed-off-by: Jan-Otto Kröpke <[email protected]>

charts/kube-prometheus-stack/Chart.yaml

@@ -23,7 +23,7 @@ name: kube-prometheus-stack
 sources:
   - https://github.com/prometheus-community/helm-charts
   - https://github.com/prometheus-operator/kube-prometheus
-version: 59.0.0
+version: 59.1.0
 appVersion: v0.74.0
 kubeVersion: ">=1.19.0-0"
 home: https://github.com/prometheus-operator/kube-prometheus

charts/kube-prometheus-stack/hack/sync_prometheus_rules.py

@@ -39,39 +39,38 @@ def new_representer(dumper, data):
 # Source files list
 charts = [
     {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/alertmanager-prometheusRule.yaml' % (refs['ref.kube-prometheus'],),
+        'git': 'https://github.com/prometheus-operator/kube-prometheus.git',
+        'branch': refs['ref.kube-prometheus'],
+        'source': 'main.libsonnet',
+        'cwd': 'jsonnet/kube-prometheus',
         'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/kubePrometheus-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/kubernetesControlPlane-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/kubeStateMetrics-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/nodeExporter-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/prometheus-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
-    },
-    {
-        'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/%s/manifests/prometheusOperator-prometheusRule.yaml'% (refs['ref.kube-prometheus'],),
-        'destination': '../templates/prometheus/rules-1.14',
-        'min_kubernetes': '1.14.0-0'
+        'min_kubernetes': '1.14.0-0',
+        'mixin': """
+        local kp =
+          (import 'main.libsonnet') + {
+            values+:: {
+              common+: {
+                namespace: 'monitoring',
+              },
+              kubernetesControlPlane+: {
+                kubeProxy: true,
+              },
+            },
+            grafana: {},
+          };
+
+        {
+          groups: std.flattenArrays([
+            kp[component][resource].spec.groups
+            for component in std.objectFields(kp)
+            for resource in std.filter(
+              function(resource)
+                kp[component][resource].kind == 'PrometheusRule',
+              std.objectFields(kp[component])
+            )
+          ]),
+        }
+        """
     },
     {
         'git': 'https://github.com/kubernetes-monitoring/kubernetes-mixin.git',
@@ -80,12 +79,18 @@ def new_representer(dumper, data):
         'cwd': 'rules',
         'destination': '../templates/prometheus/rules-1.14',
         'min_kubernetes': '1.14.0-0',
-        'is_mixin': True,
-        'mixin_vars': {'_config': {
-            'clusterLabel': 'cluster',
-            'windowsExporterSelector': 'job="windows-exporter"',
-            'kubeStateMetricsSelector': 'job="kube-state-metrics"',
-        }}
+        'mixin': """
+        local kp =
+            { prometheusAlerts+:: {}, prometheusRules+:: {}} +
+            (import "windows.libsonnet") +
+            {'_config': {
+                'clusterLabel': 'cluster',
+                'windowsExporterSelector': 'job="windows-exporter"',
+                'kubeStateMetricsSelector': 'job="kube-state-metrics"',
+            }};
+
+        kp.prometheusAlerts + kp.prometheusRules
+        """
     },
     {
         'git': 'https://github.com/etcd-io/etcd.git',
@@ -94,8 +99,11 @@ def new_representer(dumper, data):
         'cwd': 'contrib/mixin',
         'destination': '../templates/prometheus/rules-1.14',
         'min_kubernetes': '1.14.0-0',
-        'is_mixin': True,
-        'mixin_vars': {'_config+': {}}
+        'mixin': """
+        local kp = { prometheusAlerts+:: {}, prometheusRules+:: {}} + (import "mixin.libsonnet");
+
+        kp.prometheusAlerts + kp.prometheusRules
+        """
     },
 ]
 
@@ -185,6 +193,9 @@ def new_representer(dumper, data):
         'init': ''},
     '(namespace,service)': {
         'replacement': '(namespace,service,cluster)',
+        'init': ''},
+    '(namespace, job, handler': {
+        'replacement': '(cluster, namespace, job, handler',
         'init': ''}
 }
 
@@ -508,7 +519,8 @@ def write_group_to_file(group, url, destination, min_kubernetes, max_kubernetes)
     lines += re.sub(
         r'\s(by|on) ?\(',
         r' \1 ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}',
-        rules
+        rules,
+        flags=re.IGNORECASE
     )
 
     # footer
@@ -563,7 +575,7 @@ def main():
             subprocess.run(["git", "-C", checkout_dir, "fetch", "--depth", "1", "origin", branch, "--quiet"])
             subprocess.run(["git", "-c", "advice.detachedHead=false", "-C", checkout_dir, "checkout", "FETCH_HEAD", "--quiet"])
 
-            if chart.get('is_mixin'):
+            if chart.get('mixin'):
                 cwd = os.getcwd()
 
                 source_cwd = chart['cwd']
@@ -579,22 +591,12 @@ def main():
                     f.write(chart['content'])
                     f.close()
 
-                mixin_vars = json.dumps(chart['mixin_vars'])
-
                 print("Generating rules from %s" % mixin_file)
                 print("Change cwd to %s" % checkout_dir + '/' + source_cwd)
                 os.chdir(mixin_dir)
 
-                mixin = """
-                local kp =
-                    { prometheusAlerts+:: {}, prometheusRules+:: {}} +
-                    (import "%s") +
-                    %s;
-
-                kp.prometheusAlerts + kp.prometheusRules
-                """
+                alerts = json.loads(_jsonnet.evaluate_snippet(mixin_file, chart['mixin'], import_callback=jsonnet_import_callback))
 
-                alerts = json.loads(_jsonnet.evaluate_snippet(mixin_file, mixin % (mixin_file, mixin_vars), import_callback=jsonnet_import_callback))
                 os.chdir(cwd)
             else:
                 with open(checkout_dir + '/' + chart['source'], "r") as f:
@@ -610,7 +612,7 @@ def main():
                 print('Skipping the file, response code %s not equals 200' % response.status_code)
                 continue
             raw_text = response.text
-            if chart.get('is_mixin'):
+            if chart.get('mixin'):
                 alerts = json.loads(_jsonnet.evaluate_snippet(url, raw_text + '.prometheusAlerts'))
             else:
                 alerts = yaml.full_load(raw_text)

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/alertmanager-prometheusRule.yaml
+Generated from 'alertmanager.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/config-reloaders.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/prometheusOperator-prometheusRule.yaml
+Generated from 'config-reloaders' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubePrometheus-prometheusRule.yaml
+Generated from 'general.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}
@@ -36,7 +36,7 @@ spec:
         description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.'
         runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown
         summary: One or more targets are unreachable.
-      expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10
+      expr: 100 * (count(up == 0) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, job, namespace, service) / count(up) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, job, namespace, service)) > 10
       for: {{ dig "TargetDown" "for" "10m" .Values.customRules }}
       {{- with .Values.defaultRules.keepFiringFor }}
       keep_firing_for: "{{ . }}"

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-memory-cache' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-memory-rss' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-memory-swap' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.container-resource' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'k8s.rules.pod-owner' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'kube-apiserver-availability.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'kube-apiserver-burnrate.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'kube-apiserver-histogram.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubernetesControlPlane-prometheusRule.yaml
+Generated from 'kube-apiserver-slos' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubePrometheus-prometheusRule.yaml
+Generated from 'kube-prometheus-general.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}

charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml

@@ -1,5 +1,5 @@
 {{- /*
-Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/65922b9fd8c3869c06686b44f5f3aa9f96560666/manifests/kubePrometheus-prometheusRule.yaml
+Generated from 'kube-prometheus-node-recording.rules' group from https://github.com/prometheus-operator/kube-prometheus.git
 Do not change in-place! In order to change this file first read following link:
 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
 */ -}}
@@ -24,7 +24,7 @@ spec:
   groups:
   - name: kube-prometheus-node-recording.rules
     rules:
-    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)
+    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance)
       record: instance:node_cpu:rate:sum
       {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }}
       labels:
@@ -35,7 +35,7 @@ spec:
           {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- end }}
-    - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
+    - expr: sum(rate(node_network_receive_bytes_total[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance)
       record: instance:node_network_receive_bytes:rate:sum
       {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }}
       labels:
@@ -46,7 +46,7 @@ spec:
           {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- end }}
-    - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
+    - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance)
       record: instance:node_network_transmit_bytes:rate:sum
       {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }}
       labels:
@@ -57,7 +57,7 @@ spec:
           {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- end }}
-    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)
+    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance, cpu)) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance)
       record: instance:node_cpu:ratio
       {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }}
       labels:
@@ -79,7 +79,7 @@ spec:
           {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- end }}
-    - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
+    - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance, cpu))
       record: cluster:node_cpu:ratio
       {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }}
       labels: