diff --git a/host-nuc.nix b/host-nuc.nix index 31f8564..c33380e 100644 --- a/host-nuc.nix +++ b/host-nuc.nix @@ -3,6 +3,7 @@ { imports = [ ./hardware/nuc.nix + ./modules/alertmanager.nix ./modules/cachix.nix ./modules/common.nix ./modules/consul/server.nix @@ -15,6 +16,7 @@ ./modules/remote-builder ./modules/traefik.nix ./modules/vpn.nix + ./modules/webhook.nix ]; system.stateVersion = "22.05"; diff --git a/modules/alertmanager.nix b/modules/alertmanager.nix new file mode 100644 index 0000000..e0738e0 --- /dev/null +++ b/modules/alertmanager.nix @@ -0,0 +1,37 @@ +{ config, ... }: + +{ + imports = [ ./consul-catalog.nix ]; + + services.prometheus.alertmanager = { + enable = true; + configuration = { + route.receiver = "webhook"; + receivers = [ + { + name = "webhook"; + webhook_configs = [ + { + url = "http://localhost:${toString config.services.webhook.port}/hooks/alertmanager"; + } + ]; + } + ]; + }; + }; + + services.consul.catalog = [ + { + name = "alertmanager"; + port = config.services.prometheus.alertmanager.port; + tags = (import ./lib/traefik.nix).tagsForHost "alertmanager"; + check = { + name = "Health endpoint"; + http = "http://localhost:${toString config.services.prometheus.alertmanager.port}/-/healthy"; + interval = "10s"; + }; + } + ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.alertmanager.port ]; +} diff --git a/modules/grafana/dashboards/nixos.json b/modules/grafana/dashboards/nixos.json index 65e4d80..e43970f 100644 --- a/modules/grafana/dashboards/nixos.json +++ b/modules/grafana/dashboards/nixos.json @@ -56,7 +56,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "* [Grafana](http://nuc:3000)\n* [Gitolite](http://nuc:8022)\n* [Node Exporter (nuc)](http://nuc:9100)\n* [Ntfy (nuc)](http://nuc:8080)\n* [Prometheus](http://nuc:9090)", + "content": "* [Alertmanager](http://nuc:9093)\n* [Grafana](http://nuc:3000)\n* [Gitolite](http://nuc:8022)\n* [Node Exporter (nuc)](http://nuc:9100)\n* [Ntfy (nuc)](http://nuc:8080)\n* [Prometheus](http://nuc:9090)", "mode": "markdown" }, "pluginVersion": "9.4.9", diff --git a/modules/grafana/default.nix b/modules/grafana/default.nix index 92f85bd..e7e16cf 100644 --- a/modules/grafana/default.nix +++ b/modules/grafana/default.nix @@ -25,6 +25,13 @@ type = "loki"; url = "http://loki.thewagner.home"; } + { + name = "Alertmanager"; + type = "alertmanager"; + url = "http://alertmanager.thewagner.home"; + jsonData.implementation = "prometheus"; + jsonData.handleGrafanaManagedAlerts = true; + } ]; dashboards.settings.providers = [ diff --git a/modules/loki.nix b/modules/loki.nix index c74a98d..0f8189e 100644 --- a/modules/loki.nix +++ b/modules/loki.nix @@ -46,6 +46,17 @@ let reject_old_samples = true; reject_old_samples_max_age = "168h"; }; + + ruler = { + storage = { + type = "local"; + local.directory = "/tmp/rules"; + }; + rule_path = "/tmp/scratch"; + alertmanager_url = "http://alertmanager.thewagner.home"; + ring.kvstore.store = "inmemory"; + enable_api = true; + }; }; in diff --git a/modules/prometheus.nix b/modules/prometheus.nix index e838ec4..3d24138 100644 --- a/modules/prometheus.nix +++ b/modules/prometheus.nix @@ -56,13 +56,23 @@ let } ]; + alertmanagers = [ + { + static_configs = [ + { + targets = [ "alertmanager.thewagner.home" ]; + } + ]; + } + ]; + in { imports = [ ./consul-catalog.nix ]; services.prometheus = { enable = true; - inherit scrapeConfigs; + inherit alertmanagers scrapeConfigs; }; services.consul.catalog = [ diff --git a/modules/webhook.nix b/modules/webhook.nix new file mode 100644 index 0000000..274a12d --- /dev/null +++ b/modules/webhook.nix @@ -0,0 +1,67 @@ +{ config, pkgs, ... }: + +let + + command = pkgs.writeShellScriptBin "ntfy" '' + externalUrl=$1 + status=$2 + summary=$3 + description=$4 + + if [ "$status" = "firing" ]; then + icon=rotating_light + else + icon=tada + fi + + ${pkgs.curl}/bin/curl \ + -H "X-Tags: $icon" \ + -H "Title: $summary" \ + -H "Click: $externalUrl" \ + -d "$description" \ + "http://nuc:8080/home-thewagner-ec1" + ''; + +in + +{ + imports = [ ./consul-catalog.nix ]; + + services.webhook = { + enable = true; + hooks = { + alertmanager = { + execute-command = "${command}/bin/ntfy"; + incoming-payload-content-type = "application/json"; + pass-arguments-to-command = [ + { + source = "payload"; + name = "externalURL"; + } + { + source = "payload"; + name = "alerts.0.status"; + } + { + source = "payload"; + name = "alerts.0.annotations.summary"; + } + { + source = "payload"; + name = "alerts.0.annotations.description"; + } + ]; + }; + }; + }; + + services.consul.catalog = [ + { + name = "webhook"; + port = config.services.webhook.port; + tags = (import ./lib/traefik.nix).tagsForHost "webhook"; + } + ]; + + networking.firewall.allowedTCPPorts = [ config.services.webhook.port ]; +} diff --git a/router/config b/router/config index 6034ae3..c015bb1 100644 --- a/router/config +++ b/router/config @@ -47,6 +47,10 @@ while uci -q get dhcp.@cname[0]; do uci delete dhcp.@cname[0] done +uci add dhcp cname +uci set dhcp.@cname[-1].cname="alertmanager.thewagner.home" +uci set dhcp.@cname[-1].target="nuc.thewagner.home" + uci add dhcp cname uci set dhcp.@cname[-1].cname="git.thewagner.home" uci set dhcp.@cname[-1].target="nuc.thewagner.home"