diff --git a/README.md b/README.md index 2b8dfb94..89e7cb14 100644 --- a/README.md +++ b/README.md @@ -607,6 +607,35 @@ prefix: `consul-alerts/config/notifiers/ilert/` | api-key | The API key of the alert source. (mandatory) | | incident-key-template | Format of the incident key. [Default: `{{.Node}}:{{.Service}}:{{.Check}}` | +#### Prometheus + +To enable the Prometheus built-in notifier, set +`consul-alerts/config/notifiers/prometheus/enabled` to `true`. Prometheus details +needs to be configured. + +prefix: `consul-alerts/config/notifiers/prometheus/` + +| key | description | +|--------------|-----------------------------------------------------------------------| +| enabled | Enable the prometheus notifier. [Default: false] | +| cluster-name | The name of the cluster. [Default: "Consul Alerts"] | +| base-urls | Base URLs of the Prometheus cluster (mandatory) | +| endpoint | The endpoint to append to the end of each base-url | +| payload | The payload to send to the Prometheus server (mandatory) | + +The value of 'payload' must be a json map of type string. Value will be rendered as a template. +``` +{ + "alertName": "{{ .Check }}/{{ .Service }}/{{ .Node }}", + "host": "{{ .Node }}", + "service": "{{ .Service }}", + "severity": "{{ .Status }}", + "output": "{{ .Output }}", + "notes": "{{ .Notes }}", + "time": "{{ .Timestamp }}" +} +``` + Health Check via API -------------------- diff --git a/consul-alerts.go b/consul-alerts.go index 91be3598..ddd116a6 100644 --- a/consul-alerts.go +++ b/consul-alerts.go @@ -250,6 +250,7 @@ func builtinNotifiers() map[string]notifier.Notifier { awssnsNotifier := consulClient.AwsSnsNotifier() victoropsNotifier := consulClient.VictorOpsNotifier() httpEndpointNotifier := consulClient.HttpEndpointNotifier() + prometheusNotifier := consulClient.PrometheusNotifier() ilertNotifier := consulClient.ILertNotifier() notifiers := map[string]notifier.Notifier{} @@ -288,7 +289,10 @@ func builtinNotifiers() map[string]notifier.Notifier { } if httpEndpointNotifier.Enabled { notifiers[httpEndpointNotifier.NotifierName()] = httpEndpointNotifier - } + } + if prometheusNotifier.Enabled { + notifiers[prometheusNotifier.NotifierName()] = prometheusNotifier + } if ilertNotifier.Enabled { notifiers[ilertNotifier.NotifierName()] = ilertNotifier } diff --git a/consul/client.go b/consul/client.go index c8ac8989..1e9a930d 100644 --- a/consul/client.go +++ b/consul/client.go @@ -253,6 +253,18 @@ func (c *ConsulAlertClient) LoadConfig() { case "consul-alerts/config/notifiers/http-endpoint/payload": valErr = loadCustomValue(&config.Notifiers.HttpEndpoint.Payload, val, ConfigTypeStrMap) + // Prometheus notifier config + case "consul-alerts/config/notifiers/prometheus/enabled": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Enabled, val, ConfigTypeBool) + case "consul-alerts/config/notifiers/prometheus/cluster-name": + valErr = loadCustomValue(&config.Notifiers.Prometheus.ClusterName, val, ConfigTypeString) + case "consul-alerts/config/notifiers/prometheus/base-urls": + valErr = loadCustomValue(&config.Notifiers.Prometheus.BaseURLs, val, ConfigTypeStrArray) + case "consul-alerts/config/notifiers/prometheus/endpoint": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Endpoint, val, ConfigTypeString) + case "consul-alerts/config/notifiers/prometheus/payload": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Payload, val, ConfigTypeStrMap) + // iLert notfier config case "consul-alerts/config/notifiers/ilert/enabled": valErr = loadCustomValue(&config.Notifiers.ILert.Enabled, val, ConfigTypeBool) @@ -566,6 +578,10 @@ func (c *ConsulAlertClient) HttpEndpointNotifier() *notifier.HttpEndpointNotifie return c.config.Notifiers.HttpEndpoint } +func (c *ConsulAlertClient) PrometheusNotifier() *notifier.PrometheusNotifier { + return c.config.Notifiers.Prometheus +} + func (c *ConsulAlertClient) ILertNotifier() *notifier.ILertNotifier { return c.config.Notifiers.ILert } diff --git a/consul/interface.go b/consul/interface.go index 5d43cf9b..f286451f 100644 --- a/consul/interface.go +++ b/consul/interface.go @@ -81,6 +81,7 @@ type Consul interface { AwsSnsNotifier() *notifier.AwsSnsNotifier VictorOpsNotifier() *notifier.VictorOpsNotifier HttpEndpointNotifier() *notifier.HttpEndpointNotifier + PrometheusNotifier() *notifier.PrometheusNotifier ILertNotifier() *notifier.ILertNotifier CheckChangeThreshold() int @@ -173,7 +174,12 @@ func DefaultAlertConfig() *ConsulAlertConfig { httpEndpoint := ¬ifier.HttpEndpointNotifier{ Enabled: false, ClusterName: "Consul-Alerts", - } + } + + prometheus := ¬ifier.PrometheusNotifier{ + Enabled: false, + ClusterName: "Consul-Alerts", + } ilert := ¬ifier.ILertNotifier{ Enabled: false, @@ -193,6 +199,7 @@ func DefaultAlertConfig() *ConsulAlertConfig { AwsSns: awsSns, VictorOps: victorOps, HttpEndpoint: httpEndpoint, + Prometheus: prometheus, ILert: ilert, Custom: []string{}, } diff --git a/notifier/notifier.go b/notifier/notifier.go index 7d236755..c39c37b4 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -53,6 +53,7 @@ type Notifiers struct { AwsSns *AwsSnsNotifier `json:"awssns"` VictorOps *VictorOpsNotifier `json:"victorops"` HttpEndpoint *HttpEndpointNotifier `json:"http-endpoint"` + Prometheus *PrometheusNotifier `json:"prometheus"` ILert *ILertNotifier `json:"ilert"` Custom []string `json:"custom"` } @@ -83,6 +84,8 @@ func (n Notifiers) GetNotifier(name string) (Notifier, bool) { return n.VictorOps, true case n.HttpEndpoint != nil && n.HttpEndpoint.NotifierName() == name: return n.HttpEndpoint, true + case n.Prometheus != nil && n.Prometheus.NotifierName() == name: + return n.Prometheus, true case n.ILert != nil && n.ILert.NotifierName() == name: return n.ILert, true diff --git a/notifier/prometheus-notifier.go b/notifier/prometheus-notifier.go new file mode 100644 index 00000000..46d04f33 --- /dev/null +++ b/notifier/prometheus-notifier.go @@ -0,0 +1,134 @@ +package notifier + +import ( + "fmt" + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "html/template" + + log "github.com/AcalephStorage/consul-alerts/Godeps/_workspace/src/github.com/Sirupsen/logrus" +) + +type PrometheusNotifier struct { + Enabled bool + ClusterName string `json:"cluster-name"` + BaseURLs []string `json:"base-urls"` + Endpoint string `json:"endpoint"` + Payload map[string]string `json:"payload"` +} + +type TemplatePayloadData struct { + Node string + Service string + Check string + Status string + Output string + Notes string + Timestamp string +} + +// NotifierName provides name for notifier selection +func (notifier *PrometheusNotifier) NotifierName() string { + return "prometheus" +} + +func (notifier *PrometheusNotifier) Copy() Notifier { + n := *notifier + return &n +} + +func renderPayload(t TemplatePayloadData, templateFile string, defaultTemplate string) (string, error) { + var tmpl *template.Template + var err error + if templateFile == "" { + tmpl, err = template.New("base").Parse(defaultTemplate) + } else { + tmpl, err = template.ParseFiles(templateFile) + } + + if err != nil { + return "", err + } + + var body bytes.Buffer + if err := tmpl.Execute(&body, t); err != nil { + return "", err + } + + return body.String(), nil +} + +//Notify sends messages to the endpoint notifier +func (notifier *PrometheusNotifier) Notify(messages Messages) bool { + var values []map[string]map[string]string + + for _, m := range messages { + value := map[string]string{} + t := TemplatePayloadData{ + Node: m.Node, + Service: m.Service, + Check: m.Check, + Status: m.Status, + Output: m.Output, + Notes: m.Notes, + Timestamp: m.Timestamp.Format("2006-01-02T15:04:05-0700"), + } + + for payloadKey, payloadVal := range notifier.Payload { + data, err := renderPayload(t, "", payloadVal) + if err != nil { + log.Println("Error rendering template: ", err) + return false + } + value[payloadKey] = string(data) + } + + values = append(values, map[string]map[string]string{"labels": value}) + } + + requestBody, err := json.Marshal(values) + if err != nil { + log.Println("Unable to encode POST data") + return false + } + + c := make(chan bool) + defer close(c) + for _, bu := range notifier.BaseURLs { + endpoint := fmt.Sprintf("%s%s", bu, notifier.Endpoint) + + // Channel senders. Logging the result where needed, and sending status back + go func() { + if res, err := http.Post(endpoint, "application/json", bytes.NewBuffer(requestBody)); err != nil { + log.Printf(fmt.Sprintf("Unable to send data to Prometheus server (%s): %s", endpoint, err)) + c <- false + } else { + defer res.Body.Close() + statusCode := res.StatusCode + + if statusCode != 200 { + body, _ := ioutil.ReadAll(res.Body) + log.Printf(fmt.Sprintf("Unable to notify Prometheus server (%s): %s", endpoint, string(body))) + c <- false + } else { + log.Printf(fmt.Sprintf("Notification sent to Prometheus server (%s).", endpoint)) + c <- true + } + } + }() + } + + // Channel receiver. Making sure to return the final result in bool + for i := 0; i < len(notifier.BaseURLs); i++ { + select { + case r := <- c: + if (! r) { + return false + } + } + } + + return true +}