Add Alertmanager

* Configure Alertmanager to send webhooks to the `webhook` service.
* Connect Loki and Prometheus to Alertmanager.
* Add the Alertmanager as a datasource to Grafana

Limitations:

* The webhook only processes the first alert

host-nuc.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ./hardware/nuc.nix
+    ./modules/alertmanager.nix
     ./modules/cachix.nix
     ./modules/common.nix
     ./modules/consul/server.nix
@@ -15,6 +16,7 @@
     ./modules/remote-builder
     ./modules/traefik.nix
     ./modules/vpn.nix
+    ./modules/webhook.nix
   ];
 
   system.stateVersion = "22.05";

modules/alertmanager.nix

@@ -0,0 +1,37 @@
+{ config, ... }:
+
+{
+  imports = [ ./consul-catalog.nix ];
+
+  services.prometheus.alertmanager = {
+    enable = true;
+    configuration = {
+      route.receiver = "webhook";
+      receivers = [
+        {
+          name = "webhook";
+          webhook_configs = [
+            {
+              url = "http://localhost:${toString config.services.webhook.port}/hooks/alertmanager";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
+  services.consul.catalog = [
+    {
+      name = "alertmanager";
+      port = config.services.prometheus.alertmanager.port;
+      tags = (import ./lib/traefik.nix).tagsForHost "alertmanager";
+      check = {
+        name = "Health endpoint";
+        http = "http://localhost:${toString config.services.prometheus.alertmanager.port}/-/healthy";
+        interval = "10s";
+      };
+    }
+  ];
+
+  networking.firewall.allowedTCPPorts = [ config.services.prometheus.alertmanager.port ];
+}

modules/grafana/dashboards/nixos.json

@@ -56,7 +56,7 @@
           "showLineNumbers": false,
           "showMiniMap": false
         },
-        "content": "* [Grafana](http://nuc:3000)\n* [Gitolite](http://nuc:8022)\n* [Node Exporter (nuc)](http://nuc:9100)\n* [Ntfy (nuc)](http://nuc:8080)\n* [Prometheus](http://nuc:9090)",
+        "content": "* [Alertmanager](http://nuc:9093)\n* [Grafana](http://nuc:3000)\n* [Gitolite](http://nuc:8022)\n* [Node Exporter (nuc)](http://nuc:9100)\n* [Ntfy (nuc)](http://nuc:8080)\n* [Prometheus](http://nuc:9090)",
         "mode": "markdown"
       },
       "pluginVersion": "9.4.9",

modules/grafana/default.nix

@@ -25,6 +25,13 @@
           type = "loki";
           url = "http://loki.thewagner.home";
         }
+        {
+          name = "Alertmanager";
+          type = "alertmanager";
+          url = "http://alertmanager.thewagner.home";
+          jsonData.implementation = "prometheus";
+          jsonData.handleGrafanaManagedAlerts = true;
+        }
       ];
 
       dashboards.settings.providers = [

modules/loki.nix

@@ -46,6 +46,17 @@ let
       reject_old_samples = true;
       reject_old_samples_max_age = "168h";
     };
+
+    ruler = {
+      storage = {
+        type = "local";
+        local.directory = "/tmp/rules";
+      };
+      rule_path = "/tmp/scratch";
+      alertmanager_url = "http://alertmanager.thewagner.home";
+      ring.kvstore.store = "inmemory";
+      enable_api = true;
+    };
   };
 
 in

modules/prometheus.nix

@@ -56,13 +56,23 @@ let
     }
   ];
 
+  alertmanagers = [
+    {
+      static_configs = [
+        {
+          targets = [ "alertmanager.thewagner.home" ];
+        }
+      ];
+    }
+  ];
+
 in
 {
   imports = [ ./consul-catalog.nix ];
 
   services.prometheus = {
     enable = true;
-    inherit scrapeConfigs;
+    inherit alertmanagers scrapeConfigs;
   };
 
   services.consul.catalog = [

modules/webhook.nix

@@ -0,0 +1,67 @@
+{ config, pkgs, ... }:
+
+let
+
+  command = pkgs.writeShellScriptBin "ntfy" ''
+    externalUrl=$1
+    status=$2
+    summary=$3
+    description=$4
+
+    if [ "$status" = "firing" ]; then
+      icon=rotating_light
+    else
+      icon=tada
+    fi
+
+    ${pkgs.curl}/bin/curl \
+      -H "X-Tags: $icon" \
+      -H "Title: $summary" \
+      -H "Click: $externalUrl" \
+      -d "$description" \
+      "http://nuc:8080/home-thewagner-ec1"
+  '';
+
+in
+
+{
+  imports = [ ./consul-catalog.nix ];
+
+  services.webhook = {
+    enable = true;
+    hooks = {
+      alertmanager = {
+        execute-command = "${command}/bin/ntfy";
+        incoming-payload-content-type = "application/json";
+        pass-arguments-to-command = [
+          {
+            source = "payload";
+            name = "externalURL";
+          }
+          {
+            source = "payload";
+            name = "alerts.0.status";
+          }
+          {
+            source = "payload";
+            name = "alerts.0.annotations.summary";
+          }
+          {
+            source = "payload";
+            name = "alerts.0.annotations.description";
+          }
+        ];
+      };
+    };
+  };
+
+  services.consul.catalog = [
+    {
+      name = "webhook";
+      port = config.services.webhook.port;
+      tags = (import ./lib/traefik.nix).tagsForHost "webhook";
+    }
+  ];
+
+  networking.firewall.allowedTCPPorts = [ config.services.webhook.port ];
+}

router/config

@@ -47,6 +47,10 @@ while uci -q get dhcp.@cname[0]; do
     uci delete dhcp.@cname[0]
 done
 
+uci add dhcp cname
+uci set dhcp.@cname[-1].cname="alertmanager.thewagner.home"
+uci set dhcp.@cname[-1].target="nuc.thewagner.home"
+
 uci add dhcp cname
 uci set dhcp.@cname[-1].cname="git.thewagner.home"
 uci set dhcp.@cname[-1].target="nuc.thewagner.home"