Merge pull request #5 from logzio/dev

v1.0.3
logzio · Dec 20, 2023 · fd98f07 · fd98f07
2 parents e3ffc47 + c10df41
commit fd98f07
Show file tree

Hide file tree

Showing 18 changed files with 2,051 additions and 796 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -12,7 +12,8 @@ jobs:
       LOGZIO_API_URL: https://api.logz.io
       LOGZIO_API_TOKEN: ${{ secrets.LOGZIO_API_TOKEN }}
       RULES_DS: ${{ secrets.RULES_DS }}
-      CONFIGMAP_ANNOTATION: prometheus.io/kube-rules
+      RULES_CONFIGMAP_ANNOTATION: prometheus.io/kube-rules
+      ALERTMANAGER_CONFIGMAP_ANNOTATION: prometheus.io/kube-alertmanager
 
     steps:
       - name: Set up Go
@@ -42,5 +43,9 @@ jobs:
         run: go mod download
 
       - name: Run tests
-        run: go test ./... -cover
+        run: go test -v ./... -coverprofile=coverage.out
+
+      - name: Extract coverage percentage
+        run: go tool cover -func=coverage.out | grep total | awk '{print $3}'
+
 
diff --git a/README.md b/README.md
@@ -8,18 +8,28 @@ Before running this software, ensure you have:
 - Access to a Kubernetes cluster
 - Logz.io account with API access
 
+## Supported contact point types
+- `Email`
+- `Slack`
+- `Pagerduty`
+
+More types will be supported in the future, If you have a specific request please post an issue with your request
+
 ## Configuration
 
 Configure the application using the following environment variables:
 
-| Environment Variable   | Description                                                                        | Default Value              |
-|------------------------|------------------------------------------------------------------------------------|----------------------------|
-| `LOGZIO_API_TOKEN`     | The API token for your Logz.io account.                                            | `None`                     |
-| `LOGZIO_API_URL`       | The URL endpoint for the Logz.io API.                                              | `https://api.logz.io`      |
-| `CONFIGMAP_ANNOTATION` | The specific annotation the controller should look for in Prometheus alert rules.  | `prometheus.io/kube-rules` |
-| `RULES_DS`             | The metrics data source name in logz.io for the Prometheus rules.                  | `None`                     |
-| `ENV_ID`               | Environment identifier, usually cluster name.                                      | `my-env`                   |
-| `WORKER_COUNT`         | The number of workers to process the alerts.                                       | `2`                        |
+| Environment Variable                | Description                                                                                       | Default Value                     |
+|-------------------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------|
+| `LOGZIO_API_TOKEN`                  | The API token for your Logz.io account.                                                           | `None`                            |
+| `LOGZIO_API_URL`                    | The URL endpoint for the Logz.io API.                                                             | `https://api.logz.io`             |
+| `RULES_CONFIGMAP_ANNOTATION`        | The specific annotation the controller should look for in Prometheus alert rules.                 | `prometheus.io/kube-rules`        |
+| `ALERTMANAGER_CONFIGMAP_ANNOTATION` | The specific annotation the controller should look for in Prometheus alert manager configuration. | `prometheus.io/kube-alertmanager` |
+| `RULES_DS`                          | The metrics data source name in logz.io for the Prometheus rules.                                 | `None`                            |
+| `ENV_ID`                            | Environment identifier, usually cluster name.                                                     | `my-env`                          |
+| `WORKER_COUNT`                      | The number of workers to process the alerts.                                                      | `2`                               |
+| `IGNORE_SLACK_TEXT`                 | Ignore slack contact points `text` field.                                                         | `flase`                           |
+| `IGNORE_SLACK_TITLE`                | Ignore slack contact points `title` field.                                                        | `false`                           |
 
 Please ensure to set all necessary environment variables before running the application.
 
@@ -30,12 +40,12 @@ To start using the controller:
 2. Navigate to the project directory.
 3. Run the controller `make run-local`.
 
-### ConfigMap Format
-The controller is designed to process ConfigMaps containing Prometheus alert rules. These ConfigMaps must be annotated with a specific key that matches the value of the `ANNOTATION` environment variable for the controller to process them.
+### ConfigMap format
+The controller is designed to process ConfigMaps containing Prometheus alert rules and promethium alert manager configuration. These ConfigMaps must be annotated with a specific key that matches the value of the `RULES_CONFIGMAP_ANNOTATION` or `ALERTMANAGER_CONFIGMAP_ANNOTATION` environment variables for the controller to process them.
 
-### Example ConfigMap
+### Example rules configMap
 
-Below is an example of how a ConfigMap should be structured:
+Below is an example of how a rules configMap should be structured:
 
 ```yaml
 apiVersion: v1
@@ -60,7 +70,68 @@ data:
 - Replace `prometheus.io/kube-rules` with the actual annotation you use to identify relevant ConfigMaps. The data section should contain your Prometheus alert rules in YAML format.
 - Deploy the configmap to your cluster `kubectl apply -f <configmap-file>.yml`
 
+### Example alert manager configMap
+
+Below is an example of how a alert manager ConfigMap should be structured:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logzio-rules
+  namespace: monitoring
+  annotations:
+    prometheus.io/kube-alertmanager: "true"
+data:
+  all_instances_down_otel_collector: |
+    global:
+      # Global configurations, adjust these to your SMTP server details
+      smtp_smarthost: 'smtp.example.com:587'
+      smtp_from: '[email protected]'
+      smtp_auth_username: 'alertmanager'
+      smtp_auth_password: 'password'
+    # The root route on which each incoming alert enters.
+    route:
+      receiver: 'default-receiver'
+      group_by: ['alertname', 'env']
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 1h
+      # Child routes
+      routes:
+        - match:
+            env: production
+          receiver: 'slack-production'
+          continue: true
+        - match:
+            env: staging
+          receiver: 'slack-staging'
+          continue: true
+    
+    # Receivers defines ways to send notifications about alerts.
+    receivers:
+      - name: 'default-receiver'
+        email_configs:
+          - to: '[email protected]'
+      - name: 'slack-production'
+        slack_configs:
+          - api_url: 'https://hooks.slack.com/services/T00000000/B00000000/'
+            channel: '#prod-alerts'
+      - name: 'slack-staging'
+        slack_configs:
+          - api_url: 'https://hooks.slack.com/services/T00000000/B11111111/'
+            channel: '#staging-alerts'
+
+```
+- Replace `prometheus.io/kube-alertmanager` with the actual annotation you use to identify relevant ConfigMaps. The data section should contain your Prometheus alert rules in YAML format.
+- Deploy the configmap to your cluster `kubectl apply -f <configmap-file>.yml`
+
+
 ## Changelog
+- v1.0.3
+  - Handle Prometheus alert manager configuration file
+  - Add CRUD operations for contact points and notification policies
+  - Add `IGNORE_SLACK_TEXT` and `IGNORE_SLACK_TITLE` flags
 - v1.0.2
   - Add `reduce` query to alerts (grafana alerts can evaluate alerts only from reduced data)
 - v1.0.1

diff --git a/common/common.go b/common/common.go
@@ -0,0 +1,246 @@
+package common
+
+import (
+	"flag"
+	"fmt"
+	"github.com/logzio/logzio_terraform_client/grafana_alerts"
+	"github.com/prometheus/prometheus/model/rulefmt"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+	"k8s.io/klog/v2"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"time"
+)
+
+const (
+	LetterBytes   = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits
+	TypeSlack     = "slack"
+	TypeEmail     = "email"
+	TypePagerDuty = "pagerduty" // # of letter indices fitting in 63 bits
+)
+
+var (
+	helpFlag, ignoreSlackTextFlag, ignoreSlackTitleFlag                                                                     *bool
+	logzioAPITokenFlag, rulesConfigmapAnnotation, alertManagerConfigmapAnnotation, logzioAPIURLFlag, rulesDSFlag, envIDFlag *string
+	workerCountFlag                                                                                                         *int
+)
+
+// NewConfig creates a Config struct, populating it with values from command-line flags and environment variables.
+func NewConfig() *Config {
+	// Define flags
+	if flag.Lookup("help") == nil {
+		helpFlag = flag.Bool("help", false, "Display help")
+	}
+	if flag.Lookup("rules-annotation") == nil {
+		rulesConfigmapAnnotation = flag.String("rules-annotation", "prometheus.io/kube-rules", "Annotation that states that this configmap contains prometheus rules")
+	}
+	if flag.Lookup("alertmanager-annotation") == nil {
+		alertManagerConfigmapAnnotation = flag.String("alertmanager-annotation", "prometheus.io/kube-alertmanager", "Annotation that states that this configmap contains alertmanager configuration")
+	}
+	if flag.Lookup("logzio-api-token") == nil {
+		logzioAPITokenFlag = flag.String("logzio-api-token", "", "LOGZIO API token")
+	}
+	if flag.Lookup("logzio-api-url") == nil {
+		logzioAPIURLFlag = flag.String("logzio-api-url", "https://api.logz.io", "LOGZIO API URL")
+	}
+	if flag.Lookup("rules-ds") == nil {
+		rulesDSFlag = flag.String("rules-ds", "", "name of the data source for the alert rules")
+	}
+	if flag.Lookup("env-id") == nil {
+		envIDFlag = flag.String("env-id", "my-env", "environment identifier, usually cluster name")
+	}
+	if flag.Lookup("workers") == nil {
+		workerCountFlag = flag.Int("workers", 2, "The number of workers to process the alerts")
+	}
+	if flag.Lookup("ignore-slack-text") == nil {
+		ignoreSlackTextFlag = flag.Bool("ignore-slack-text", false, "Ignore slack contact points text field")
+	}
+	if flag.Lookup("ignore-slack-title") == nil {
+		ignoreSlackTitleFlag = flag.Bool("ignore-slack-title", false, "Ignore slack contact points title field")
+	}
+	// Parse the flags
+	flag.Parse()
+
+	if *helpFlag {
+		flag.PrintDefaults()
+		os.Exit(0)
+	}
+
+	// Environment variables have higher precedence than flags
+	logzioAPIURL := getEnvWithFallback("LOGZIO_API_URL", *logzioAPIURLFlag)
+	envID := getEnvWithFallback("ENV_ID", *envIDFlag)
+
+	ignoreSlackText := getEnvWithFallback("IGNORE_SLACK_TEXT", strconv.FormatBool(*ignoreSlackTextFlag))
+	ignoreSlackTextBool, err := strconv.ParseBool(ignoreSlackText)
+	if err != nil {
+		klog.Fatal("Invalid value for IGNORE_SLACK_TEXT")
+	}
+
+	ignoreSlackTitle := getEnvWithFallback("IGNORE_SLACK_TITLE", strconv.FormatBool(*ignoreSlackTitleFlag))
+	ignoreSlackTitleBool, err := strconv.ParseBool(ignoreSlackTitle)
+	if err != nil {
+		klog.Fatal("Invalid value for IGNORE_SLACK_TITLE")
+	}
+
+	// api token is mandatory
+	logzioAPIToken := getEnvWithFallback("LOGZIO_API_TOKEN", *logzioAPITokenFlag)
+	if logzioAPIToken == "" {
+		klog.Fatal("No logzio api token provided")
+	}
+	rulesDS := getEnvWithFallback("RULES_DS", *rulesDSFlag)
+	if rulesDS == "" {
+		klog.Fatal("No rules data source provided")
+	}
+	// Annotation must be provided either by flag or environment variable
+	rulesAnnotation := getEnvWithFallback("RULES_CONFIGMAP_ANNOTATION", *rulesConfigmapAnnotation)
+	if rulesAnnotation == "" {
+		klog.Fatal("No rules configmap annotation provided")
+	}
+	// Annotation must be provided either by flag or environment variable
+	alertManagerAnnotation := getEnvWithFallback("ALERTMANAGER_CONFIGMAP_ANNOTATION", *alertManagerConfigmapAnnotation)
+	if alertManagerAnnotation == "" {
+		klog.Fatal("No alert manager configmap annotation provided")
+	}
+	workerCountStr := getEnvWithFallback("WORKERS_COUNT", strconv.Itoa(*workerCountFlag))
+	workerCount, err := strconv.Atoi(workerCountStr)
+
+	if err != nil {
+		workerCount = 2 // default value
+	}
+
+	return &Config{
+		RulesAnnotation:        rulesAnnotation,
+		AlertManagerAnnotation: alertManagerAnnotation,
+		LogzioAPIToken:         logzioAPIToken,
+		LogzioAPIURL:           logzioAPIURL,
+		RulesDS:                rulesDS,
+		EnvID:                  envID,
+		WorkerCount:            workerCount,
+		IgnoreSlackText:        ignoreSlackTextBool,
+		IgnoreSlackTitle:       ignoreSlackTitleBool,
+	}
+}
+
+// getEnvWithFallback tries to get the value from an environment variable and falls back to the given default value if not found.
+func getEnvWithFallback(envName, defaultValue string) string {
+	if value, exists := os.LookupEnv(envName); exists {
+		return value
+	}
+	return defaultValue
+}
+
+// GenerateRandomString borrowed from here https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go
+func GenerateRandomString(n int) string {
+	if n <= 0 {
+		return "" // Return an empty string for non-positive lengths
+	}
+	b := make([]byte, n)
+	src := rand.NewSource(time.Now().UnixNano())
+	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
+		if remain == 0 {
+			cache, remain = src.Int63(), letterIdxMax
+		}
+		if idx := int(cache & letterIdxMask); idx < len(LetterBytes) {
+			b[i] = LetterBytes[idx]
+			i--
+		}
+		cache >>= letterIdxBits
+		remain--
+	}
+
+	return string(b)
+}
+
+// ParseDuration turns a duration string (example: 5m, 1h) into an int64 value
+func ParseDuration(durationStr string) (int64, error) {
+	// Check if the string is empty
+	if durationStr == "" {
+		return 0, fmt.Errorf("duration string is empty")
+	}
+
+	// Handle the special case where the duration string is just a number (assumed to be seconds)
+	if _, err := strconv.Atoi(durationStr); err == nil {
+		seconds, _ := strconv.ParseInt(durationStr, 10, 64)
+		return seconds * int64(time.Second), nil
+	}
+
+	// Parse the duration string
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		return 0, err
+	}
+
+	// Convert the time.Duration value to an int64
+	return int64(duration), nil
+}
+
+func CreateNameStub(cm *corev1.ConfigMap) string {
+	name := cm.GetObjectMeta().GetName()
+	namespace := cm.GetObjectMeta().GetNamespace()
+
+	return fmt.Sprintf("%s-%s", namespace, name)
+}
+
+// IsAlertEqual compares two AlertRule objects for equality.
+func IsAlertEqual(rule rulefmt.RuleNode, grafanaRule grafana_alerts.GrafanaAlertRule) bool {
+	// Start with name comparison; if these don't match, they're definitely not equal.
+	if rule.Alert.Value != grafanaRule.Title {
+		return false
+	}
+	if !reflect.DeepEqual(rule.Labels, grafanaRule.Labels) {
+		return false
+	}
+	if !reflect.DeepEqual(rule.Annotations, grafanaRule.Annotations) {
+		return false
+	}
+	forAtt, _ := ParseDuration(rule.For.String())
+	if forAtt != grafanaRule.For {
+		return false
+	}
+	if rule.Expr.Value != grafanaRule.Data[0].Model.(map[string]interface{})["expr"] {
+		return false
+	}
+	return true
+}
+
+// GetConfig returns a Kubernetes config
+func GetConfig() (*rest.Config, error) {
+	var config *rest.Config
+
+	kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config")
+	if _, err := os.Stat(kubeconfig); err == nil {
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return config, nil
+}
+
+// Config holds all the configuration needed for the application to run.
+type Config struct {
+	RulesAnnotation        string
+	AlertManagerAnnotation string
+	LogzioAPIToken         string
+	LogzioAPIURL           string
+	RulesDS                string
+	EnvID                  string
+	WorkerCount            int
+	IgnoreSlackText        bool
+	IgnoreSlackTitle       bool
+}