[kube-prometheus-stack] support custom for and severity rules

Signed-off-by: Christian Körner <[email protected]>

charts/kube-prometheus-stack/ci/03-non-defaults-values.yaml

@@ -33,3 +33,10 @@ prometheus:
       logFormat: json
     additionalConfigString: |-
       logLevel: {{ print "debug" | quote }}
+
+customRules:
+  AlertmanagerFailedReload:
+    for: 3m
+  AlertmanagerMembersInconsistent:
+    for: 5m
+    severity: "warning"

charts/kube-prometheus-stack/hack/sync_prometheus_rules.py

@@ -406,6 +406,63 @@ def add_custom_keep_firing_for(rules, indent=4):
     return rules
 
 
+def add_custom_for(rules, indent=4):
+    """Add custom 'for:' condition in rules"""
+    replace_field = "for:"
+    rules = add_custom_alert_rules(rules, replace_field, indent)
+
+    return rules
+
+
+def add_custom_severity(rules, indent=4):
+    """Add custom 'severity:' condition in rules"""
+    replace_field = "severity:"
+    rules = add_custom_alert_rules(rules, replace_field, indent)
+
+    return rules
+
+
+def add_custom_alert_rules(rules, key_to_replace, indent):
+    """Extend alert field to allow custom values"""
+    key_to_replace_indented = ' ' * indent + key_to_replace
+    alertkey_field = '- alert:'
+    found_alert_key = False
+    alertname = None
+    updated_rules = ''
+
+    # pylint: disable=C0200
+    i = 0
+    while i < len(rules):
+        if rules[i:i + len(alertkey_field)] == alertkey_field:
+            found_alert_key = True
+            start_index_word_after = i + len(alertkey_field) + 1
+            end_index_alertkey_field = start_index_word_after
+            while end_index_alertkey_field < len(rules) and rules[end_index_alertkey_field].isalnum():
+                end_index_alertkey_field += 1
+
+            alertname = rules[start_index_word_after:end_index_alertkey_field]
+
+        if found_alert_key:
+            if rules[i:i + len(key_to_replace_indented)] == key_to_replace_indented:
+                found_alert_key = False
+                start_index_key_value = i + len(key_to_replace_indented) + 1
+                end_index_key_to_replace = start_index_key_value
+                while end_index_key_to_replace < len(rules) and rules[end_index_key_to_replace].isalnum():
+                    end_index_key_to_replace += 1
+
+                word_after_key_to_replace = rules[start_index_key_value:end_index_key_to_replace]
+                new_key = key_to_replace_indented + ' {{ dig "' + alertname + \
+                    '" "' + key_to_replace[:-1] + '" "' + \
+                    word_after_key_to_replace + '" .Values.customRules }}'
+                updated_rules += new_key
+                i = end_index_key_to_replace
+
+        updated_rules += rules[i]
+        i += 1
+
+    return updated_rules
+
+
 def write_group_to_file(group, url, destination, min_kubernetes, max_kubernetes):
     fix_expr(group['rules'])
     group_name = group['name']
@@ -423,6 +480,8 @@ def write_group_to_file(group, url, destination, min_kubernetes, max_kubernetes)
     rules = add_custom_labels(rules, group)
     rules = add_custom_annotations(rules, group)
     rules = add_custom_keep_firing_for(rules)
+    rules = add_custom_for(rules)
+    rules = add_custom_severity(rules)
     rules = add_rules_conditions_from_condition_map(rules)
     rules = add_rules_per_rule_conditions(rules, group)
     # initialize header

charts/kube-prometheus-stack/values.yaml

@@ -40,7 +40,7 @@ customRules: {}
   #   for: 3m
   # AlertmanagerMembersInconsistent:
   #   for: 5m
-  #   labelsSeverity: "warning"
+  #   severity: "warning"
 
 ## Create default rules for monitoring the cluster
 ##