Skip to content

Commit

Permalink
[kube-prometheus-stack] update templates with custom for and serverit…
Browse files Browse the repository at this point in the history
…y rules

Signed-off-by: Christian Körner <[email protected]>
  • Loading branch information
ps-xaf committed Feb 17, 2024
1 parent 7c6f3fe commit 6689703
Show file tree
Hide file tree
Showing 18 changed files with 149 additions and 149 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerFailedReload" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -80,7 +80,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerMembersInconsistent" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -114,7 +114,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerFailedToSendAlerts" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -148,7 +148,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -182,7 +182,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -214,7 +214,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerConfigInconsistent" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -252,7 +252,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerClusterDown" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -290,7 +290,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "AlertmanagerClusterCrashlooping" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "ConfigReloaderSidecarErrors" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdMembersDown" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -76,7 +76,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdInsufficientMembers" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -103,7 +103,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdNoLeader" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -130,7 +130,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighNumberOfLeaderChanges" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -161,7 +161,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -192,7 +192,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -221,7 +221,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdGRPCRequestsSlow" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -250,7 +250,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdMemberCommunicationSlow" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -277,7 +277,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighNumberOfFailedProposals" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -306,7 +306,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighFsyncDurations" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -335,7 +335,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighFsyncDurations" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -364,7 +364,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdHighCommitDurations" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -391,7 +391,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdDatabaseQuotaLowSpace" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand All @@ -418,7 +418,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdExcessiveDatabaseGrowth" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -446,7 +446,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "etcdDatabaseHighFragmentationRatio" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "TargetDown" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -76,7 +76,7 @@ spec:
summary: An alert that should always be firing to certify that Alertmanager is working properly.
expr: vector(1)
labels:
severity: {{ dig "Watchdog" "labelsSeverity" "none" .Values.customRules }}
severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -112,7 +112,7 @@ spec:
summary: Info-level alert inhibition.
expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1
labels:
severity: {{ dig "InfoInhibitor" "labelsSeverity" "none" .Values.customRules }}
severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ spec:
{{- end }}
labels:
long: 1h
severity: {{ dig "KubeAPIErrorBudgetBurn" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }}
short: 5m
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }}
{{- with .Values.defaultRules.additionalRuleLabels }}
Expand Down Expand Up @@ -79,7 +79,7 @@ spec:
{{- end }}
labels:
long: 6h
severity: {{ dig "KubeAPIErrorBudgetBurn" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }}
short: 30m
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }}
{{- with .Values.defaultRules.additionalRuleLabels }}
Expand Down Expand Up @@ -112,7 +112,7 @@ spec:
{{- end }}
labels:
long: 1d
severity: {{ dig "KubeAPIErrorBudgetBurn" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }}
short: 2h
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }}
{{- with .Values.defaultRules.additionalRuleLabels }}
Expand Down Expand Up @@ -145,7 +145,7 @@ spec:
{{- end }}
labels:
long: 3d
severity: {{ dig "KubeAPIErrorBudgetBurn" "labelsSeverity" "warning" .Values.customRules }}
severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }}
short: 6h
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }}
{{- with .Values.defaultRules.additionalRuleLabels }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "KubeStateMetricsListErrors" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -79,7 +79,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "KubeStateMetricsWatchErrors" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -107,7 +107,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "KubeStateMetricsShardingMismatch" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -139,7 +139,7 @@ spec:
keep_firing_for: "{{ . }}"
{{- end }}
labels:
severity: {{ dig "KubeStateMetricsShardsMissing" "labelsSeverity" "critical" .Values.customRules }}
severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }}
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }}
{{- with .Values.defaultRules.additionalRuleLabels }}
{{- toYaml . | nindent 8 }}
Expand Down
Loading

0 comments on commit 6689703

Please sign in to comment.