From 41504d5bdfd73784178f55bbd5f91120f174feff Mon Sep 17 00:00:00 2001 From: Ethan Turkeltaub Date: Wed, 6 Nov 2024 15:50:31 -0500 Subject: [PATCH] Make tweaks to Borgmatic backups, monitoring --- flake.lock | 23 +- flake.nix | 2 +- hosts/matrix/configuration.nix | 3 +- hosts/monitor/configuration.nix | 2 +- hosts/monitor/profiles/grafana/default.nix | 151 +-- .../provisioning/borgmatic/backups.json | 937 +++++++++++++++--- hosts/monitor/profiles/prometheus.nix | 1 + hosts/omnibus/configuration.nix | 21 +- modules/nixos/e10/services/backup.nix | 109 -- modules/overlays/default.nix | 8 +- modules/profiles/backups/borgmatic/common.nix | 13 + .../profiles/backups/borgmatic/secrets.yml | 6 +- modules/profiles/backups/borgmatic/system.nix | 6 +- modules/profiles/filesystems/files/backup.nix | 28 +- modules/profiles/media-management/plex.nix | 5 +- modules/profiles/monitoring/loki/default.nix | 3 +- modules/profiles/monitoring/prometheus.nix | 3 +- .../prometheus-borgmatic-exporter.nix | 3 +- 18 files changed, 869 insertions(+), 455 deletions(-) delete mode 100644 modules/nixos/e10/services/backup.nix diff --git a/flake.lock b/flake.lock index a9f42d3..b4be410 100644 --- a/flake.lock +++ b/flake.lock @@ -971,22 +971,6 @@ "type": "github" } }, - "nixpkgs-24-05": { - "locked": { - "lastModified": 1729691686, - "narHash": "sha256-BAuPWW+9fa1moZTU+jFh+1cUtmsuF8asgzFwejM4wac=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "32e940c7c420600ef0d1ef396dc63b04ee9cad37", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", - "type": "github" - } - }, "nixpkgs-lib": { "locked": { "lastModified": 1727825735, @@ -1001,11 +985,11 @@ }, "nixpkgs-master": { "locked": { - "lastModified": 1729999895, - "narHash": "sha256-G+NKsdtdw0nRuthtRyo4w9sjo4fwK61apWTN8Cu4p9c=", + "lastModified": 1730780403, + "narHash": "sha256-+mYcXx+APQgK+erYG7zt0ur1HvV+nXmIxniw8vrwz90=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4c01c31a433a8cb85fba9b2694a47db478ef38e9", + "rev": "dd52e79ae101c3237228e374b2d044d35a75de3a", "type": "github" }, "original": { @@ -1262,7 +1246,6 @@ "nixos-generators": "nixos-generators", "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_4", - "nixpkgs-24-05": "nixpkgs-24-05", "nixpkgs-master": "nixpkgs-master", "sops-nix": "sops-nix", "treefmt": "treefmt" diff --git a/flake.nix b/flake.nix index 863b521..68ba4ba 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - nixpkgs-24-05.url = "github:NixOS/nixpkgs/nixos-24.05"; + nixpkgs-master.url = "github:NixOS/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; diff --git a/hosts/matrix/configuration.nix b/hosts/matrix/configuration.nix index 0a05cda..ce4f8a0 100644 --- a/hosts/matrix/configuration.nix +++ b/hosts/matrix/configuration.nix @@ -53,7 +53,8 @@ }; }; - e10.services.backup.jobs.system.paths = lib.mkAfter [ "/var/www" ]; + services.borgmatic.configurations.system.source_directories = + lib.mkAfter [ "/var/www" ]; system.stateVersion = "24.05"; } diff --git a/hosts/monitor/configuration.nix b/hosts/monitor/configuration.nix index 3570a7b..657459a 100644 --- a/hosts/monitor/configuration.nix +++ b/hosts/monitor/configuration.nix @@ -4,7 +4,7 @@ profiles.monitoring.loki.default profiles.monitoring.rsyslogd profiles.monitoring.thanos.default - profiles.telemetry.prometheus-pve-exporter.default + # profiles.telemetry.prometheus-pve-exporter.default profiles.telemetry.prometheus-redis-exporter profiles.services.uptime-kuma ] ++ [ ./profiles/prometheus.nix ./profiles/grafana/default.nix ]; diff --git a/hosts/monitor/profiles/grafana/default.nix b/hosts/monitor/profiles/grafana/default.nix index 0cc4946..3b7ef25 100644 --- a/hosts/monitor/profiles/grafana/default.nix +++ b/hosts/monitor/profiles/grafana/default.nix @@ -146,7 +146,7 @@ } { name = "Borgmatic Backups"; - options.path = ./provisioning/borgmatic/logs.json; + options.path = ./provisioning/borgmatic/backups.json; } ]; @@ -195,116 +195,27 @@ rules.settings = { apiVersion = 1; - deleteRules = [ ]; + deleteRules = [ + { uid = "5166671F-AEF3-434D-99B4-18C7A32BD708"; } + + { uid = "9FEB48CD-F6ED-4B7A-86DA-A912D67D142C"; } + + { uid = "77EE3CEA-A817-4A48-86D6-62839DE1A713"; } + + { uid = "32B02B4C-282B-447C-91E9-98B196B9390F"; } + + { uid = "E52329A1-393E-47B0-B849-7DB025FA47A0"; } + + { uid = "9814CF38-373F-4251-BE8A-2438D9C4A88C"; } + + { uid = "75AA8FE9-04D2-4502-B59D-4F2F5BD1DFE0"; } + ]; groups = [{ orgId = 1; name = "Default"; interval = "60s"; folder = "Homelab"; - rules = let - mkBackupRule = { host, uuid }: - let hostname = host.config.networking.hostName; - in { - uid = uuid; - title = "System failed to backup on ${hostname}"; - condition = "C"; - data = [ - { - refId = "A"; - queryType = "range"; - relativeTimeRange = { - from = 86400; - to = 0; - }; - datasourceUid = "P8E80F9AEF21F6940"; - model = { - editorMode = "code"; - expr = '' - count_over_time({application="borgmatic", host="${hostname}"} |~ "Failed backup" [$__auto])''; - intervalMs = 1000; - maxDataPoints = 43200; - queryType = "range"; - refId = "A"; - }; - } - { - refId = "B"; - relativeTimeRange = { - from = 86400; - to = 0; - }; - datasourceUid = "__expr__"; - model = { - conditions = [{ - evaluator = { - params = [ ]; - type = "gt"; - }; - operator = { type = "and"; }; - query = { params = [ "B" ]; }; - reducer = { - params = [ ]; - type = "last"; - }; - type = "query"; - }]; - datasource = { - type = "__expr__"; - uid = "__expr__"; - }; - expression = "A"; - intervalMs = 1000; - maxDataPoints = 43200; - reducer = "count"; - refId = "B"; - type = "reduce"; - }; - } - { - refId = "C"; - relativeTimeRange = { - from = 86400; - to = 0; - }; - datasourceUid = "__expr__"; - model = { - conditions = [{ - evaluator = { - params = [ 0 ]; - type = "gt"; - }; - operator = { type = "and"; }; - query = { params = [ "C" ]; }; - reducer = { - params = [ ]; - type = "last"; - }; - type = "query"; - }]; - datasource = { - type = "__expr__"; - uid = "__expr__"; - }; - expression = "B"; - intervalMs = 1000; - maxDataPoints = 43200; - refId = "C"; - type = "threshold"; - }; - } - ]; - noDataState = "OK"; - execErrState = "Error"; - for = "5m"; - annotations = { - description = ""; - runbook_url = ""; - summary = ""; - }; - labels = { }; - isPaused = false; - }; - in [ + rules = [ { uid = "a0513fe1-c679-4411-8512-d52334814942"; title = "Rack UPS is on battery power"; @@ -451,34 +362,6 @@ labels = { severity = "critical"; }; isPaused = false; } - (mkBackupRule { - host = hosts.omnibus; - uuid = "5166671F-AEF3-434D-99B4-18C7A32BD708"; - }) - (mkBackupRule { - host = hosts.monitor; - uuid = "9FEB48CD-F6ED-4B7A-86DA-A912D67D142C"; - }) - (mkBackupRule { - host = hosts.htpc; - uuid = "77EE3CEA-A817-4A48-86D6-62839DE1A713"; - }) - (mkBackupRule { - host = hosts.matrix; - uuid = "32B02B4C-282B-447C-91E9-98B196B9390F"; - }) - (mkBackupRule { - host = hosts.controller; - uuid = "E52329A1-393E-47B0-B849-7DB025FA47A0"; - }) - (mkBackupRule { - host = hosts.builder; - uuid = "9814CF38-373F-4251-BE8A-2438D9C4A88C"; - }) - (mkBackupRule { - host = hosts.gateway; - uuid = "75AA8FE9-04D2-4502-B59D-4F2F5BD1DFE0"; - }) { uid = "e1a38758-5093-49e2-8041-ae81225b1ca5"; title = "Packet loss is greater than 1%"; diff --git a/hosts/monitor/profiles/grafana/provisioning/borgmatic/backups.json b/hosts/monitor/profiles/grafana/provisioning/borgmatic/backups.json index ea81514..e56887f 100644 --- a/hosts/monitor/profiles/grafana/provisioning/borgmatic/backups.json +++ b/hosts/monitor/profiles/grafana/provisioning/borgmatic/backups.json @@ -18,9 +18,142 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 47, + "id": 52, "links": [], "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by () (borg_total_backups{instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Backup Count", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 0 + }, + "id": 101, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by () (borg_total_deduplicated_compressed_size{instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Deduplicated Compressed Size", + "type": "stat" + }, { "datasource": { "default": false, @@ -106,12 +239,12 @@ ] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, + "h": 8, + "w": 20, + "x": 4, "y": 0 }, - "id": 5, + "id": 99, "options": { "cellHeight": "sm", "footer": { @@ -131,7 +264,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "borg_last_backup_timestamp{instance=~\"$instance\"} * 1000", + "expr": "borg_last_backup_timestamp{instance=~\"$instance\", repository=\"$repository\"} * 1000", "format": "table", "instant": false, "legendFormat": "__auto", @@ -280,10 +413,86 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 4 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by () (borg_total_size{instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "panels": [], + "repeat": "repository", + "repeatDirection": "h", + "title": "Details for $repository", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] } @@ -291,17 +500,17 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 6, + "h": 5, + "w": 2, "x": 0, "y": 9 }, - "id": 1, + "id": 2, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", - "orientation": "horizontal", + "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], @@ -317,17 +526,16 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "borg_total_backups{instance=~\"$instance\"}", + "expr": "sum by(repository) (borg_total_backups{instance=~\"$instance\", repository=~\"$repository\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": true, - "legendFormat": "{{repository}}", - "range": false, + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A", "useBackend": false } @@ -339,13 +547,45 @@ "datasource": { "default": false, "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, - "description": "", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -354,6 +594,10 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, @@ -362,17 +606,281 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 6, - "x": 6, + "h": 5, + "w": 7, + "x": 2, "y": 9 }, - "id": 2, + "id": 5, "options": { - "colorMode": "value", - "graphMode": "area", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "editorMode": "code", + "exemplar": false, + "expr": "borg_total_size{instance=~\"$instance\", repository=~\"$repository\"}", + "instant": false, + "legendFormat": "{{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Repository Raw Size", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 9, + "y": 9 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "editorMode": "code", + "exemplar": false, + "expr": "borg_total_compressed_size{instance=~\"$instance\", repository=~\"$repository\"}", + "instant": false, + "legendFormat": "{{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Repository Total Compressed Size", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 16, + "y": 9 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P5DCFC7561CCDE821" + }, + "editorMode": "code", + "exemplar": false, + "expr": "borg_total_deduplicated_compressed_size{instance=~\"$instance\", repository=~\"$repository\"}", + "instant": false, + "legendFormat": "{{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Repository Total Deduplicated and Compressed Size", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 86400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 14 + }, + "id": 21, + "options": { + "colorMode": "background_solid", + "graphMode": "none", "justifyMode": "auto", - "orientation": "horizontal", + "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], @@ -388,34 +896,62 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "borg_total_deduplicated_compressed_size{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "time() - max by(repository) (borg_last_backup_timestamp{instance=~\"$instance\", repository=~\"$repository\"})", + "instant": false, "legendFormat": "{{repository}}", - "range": false, - "refId": "A", - "useBackend": false + "range": true, + "refId": "A" } ], - "title": "Total deduplicated compressed size", + "title": "Time Since Last Backup", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -426,66 +962,95 @@ "value": null }, { - "color": "orange", - "value": 86400 + "color": "red", + "value": 80 } ] }, - "unit": "s" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 6, - "x": 12, - "y": 9 + "h": 5, + "w": 7, + "x": 2, + "y": 14 }, - "id": 4, + "id": 49, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.2.2+security-01", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, "editorMode": "code", "exemplar": false, - "expr": "time() - borg_last_backup_timestamp{instance=~\"$instance\"}", - "instant": true, + "expr": "borg_last_backup_size{instance=~\"$instance\", repository=~\"$repository\"}", + "instant": false, "legendFormat": "{{repository}}", - "range": false, + "range": true, "refId": "A" } ], - "title": "Time since last backup", - "type": "stat" + "title": "Last Backup Raw Size", + "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -494,57 +1059,53 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "bytes" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 9 + "h": 5, + "w": 7, + "x": 9, + "y": 14 }, - "id": 3, + "id": 50, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.2.2+security-01", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "P5DCFC7561CCDE821" }, - "disableTextWrap": false, - "editorMode": "builder", + "editorMode": "code", "exemplar": false, - "expr": "borg_total_size{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, + "expr": "borg_last_backup_compressed_size{instance=~\"$instance\", repository=~\"$repository\"}", + "instant": false, "legendFormat": "{{repository}}", - "range": false, - "refId": "A", - "useBackend": false + "range": true, + "refId": "A" } ], - "title": "Total size", - "type": "stat" + "title": "Last Backup Compressed Size", + "type": "timeseries" }, { "datasource": { @@ -581,7 +1142,7 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": 3600000, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -590,7 +1151,6 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -605,17 +1165,17 @@ } ] }, - "unit": "short" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 19 + "h": 5, + "w": 7, + "x": 16, + "y": 14 }, - "id": 7, + "id": 60, "options": { "legend": { "calcs": [], @@ -635,16 +1195,83 @@ "uid": "P5DCFC7561CCDE821" }, "editorMode": "code", - "expr": "borg_total_backups{instance=~\"$instance\"}", + "exemplar": false, + "expr": "borg_last_backup_deduplicated_compressed_size{instance=~\"$instance\", repository=~\"$repository\"}", "instant": false, "legendFormat": "{{repository}}", "range": true, "refId": "A" } ], - "title": "Total Backups", + "title": "Repository Last Backup Deduplicated and Compressed Size", "type": "timeseries" }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeAsSystem" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by(repository) (borg_last_backup_timestamp{instance=~\"$instance\", repository=~\"$repository\"} * 1000)", + "instant": false, + "legendFormat": "{{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Backup Timestamp", + "type": "stat" + }, { "datasource": { "default": false, @@ -680,7 +1307,7 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": 3600000, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -703,17 +1330,17 @@ } ] }, - "unit": "decbytes" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 + "h": 5, + "w": 7, + "x": 2, + "y": 19 }, - "id": 6, + "id": 61, "options": { "legend": { "calcs": [], @@ -733,14 +1360,15 @@ "uid": "P5DCFC7561CCDE821" }, "editorMode": "code", - "expr": "borg_total_size{instance=~\"$instance\"}", + "exemplar": false, + "expr": "borg_last_backup_files{instance=~\"$instance\", repository=~\"$repository\"}", "instant": false, "legendFormat": "{{repository}}", "range": true, "refId": "A" } ], - "title": "Total Size", + "title": "Last Backup Number of Files", "type": "timeseries" }, { @@ -749,7 +1377,6 @@ "type": "prometheus", "uid": "P5DCFC7561CCDE821" }, - "description": "Size of repository on disk", "fieldConfig": { "defaults": { "color": { @@ -779,7 +1406,7 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": 3600000, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -802,17 +1429,17 @@ } ] }, - "unit": "decbytes" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 + "h": 5, + "w": 7, + "x": 9, + "y": 19 }, - "id": 8, + "id": 62, "options": { "legend": { "calcs": [], @@ -832,15 +1459,15 @@ "uid": "P5DCFC7561CCDE821" }, "editorMode": "code", - "expr": "borg_total_deduplicated_compressed_size{instance=~\"$instance\"}", - "hide": false, + "exemplar": false, + "expr": "borg_last_backup_duration{instance=~\"$instance\", repository=~\"$repository\"}", "instant": false, "legendFormat": "{{repository}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Deduplicated Compressed Size", + "title": "Last Backup Duration", "type": "timeseries" } ], @@ -868,9 +1495,9 @@ }, { "current": { - "selected": true, - "text": ["builder:9996"], - "value": ["builder:9996"] + "selected": false, + "text": "builder:9996", + "value": "builder:9996" }, "datasource": { "type": "prometheus", @@ -893,6 +1520,34 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": { + "selected": false, + "text": "/mnt/files/backup/builder-system", + "value": "/mnt/files/backup/builder-system" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(borg_total_backups{instance=~\"$instance\"},repository)", + "hide": 0, + "includeAll": true, + "label": "Repository", + "multi": true, + "name": "repository", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(borg_total_backups{instance=~\"$instance\"},repository)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -903,7 +1558,7 @@ "timepicker": {}, "timezone": "browser", "title": "Borgmatic Backups", - "uid": "ee2gordu5atc0f", - "version": 7, + "uid": "de31iu9c481z4c", + "version": 15, "weekStart": "" } diff --git a/hosts/monitor/profiles/prometheus.nix b/hosts/monitor/profiles/prometheus.nix index eed1b89..37654f0 100644 --- a/hosts/monitor/profiles/prometheus.nix +++ b/hosts/monitor/profiles/prometheus.nix @@ -446,6 +446,7 @@ } { job_name = "borgmatic_omnibus"; + scrape_timeout = "30s"; static_configs = [{ targets = [ "${hosts.omnibus.config.networking.hostName}:${ diff --git a/hosts/omnibus/configuration.nix b/hosts/omnibus/configuration.nix index 3638654..0e11a22 100644 --- a/hosts/omnibus/configuration.nix +++ b/hosts/omnibus/configuration.nix @@ -85,26 +85,27 @@ "directory mask" = "0755"; "force user" = config.users.users.ethan.name; }; - }; - - environment.systemPackages = with pkgs; [ yt-dlp ]; - e10.services.backup.jobs.files = { - repoName = "${config.networking.hostName}-files"; - paths = [ "/data/files" ]; - pruneKeep = { - monthly = 1; - weekly = 0; - daily = 1; + backup = { + path = "/data/files/backup"; + browseable = "yes"; + "read only" = "no"; + "guest ok" = "no"; + "create mask" = "0644"; + "directory mask" = "0755"; + "force user" = config.users.users.ethan.name; }; }; + environment.systemPackages = with pkgs; [ yt-dlp ]; + programs.fish.shellAliases.iotop = '' bash -c "sudo sysctl kernel.task_delayacct=1 && sudo ${pkgs.iotop}/bin/iotop ; sudo sysctl kernel.task_delayacct=0" ''; services.borgmatic.configurations.files = { source_directories = [ "/data/files" ]; + exclude_patterns = [ "/data/files/backup" ]; repositories = [{ label = "rsync.net"; path = "ssh://de2228@de2228.rsync.net/./omnibus-files"; diff --git a/modules/nixos/e10/services/backup.nix b/modules/nixos/e10/services/backup.nix deleted file mode 100644 index ee86b27..0000000 --- a/modules/nixos/e10/services/backup.nix +++ /dev/null @@ -1,109 +0,0 @@ -{ config, lib, ... }: - -with lib; - -let cfg = config.e10.services.backup; -in { - options.e10.services.backup = { - enable = mkEnableOption "Enable backup to rsync.net via borgbackup"; - - username = mkOption { - type = types.str; - description = "Username for rsync.net"; - }; - - hostname = mkOption { - type = types.str; - description = "Hostname for rsync.net account"; - default = "${cfg.username}.rsync.net"; - }; - - publicKeyFile = mkOption { - type = types.path; - description = "File with public key for rsync.net server"; - }; - - privateKeyFile = mkOption { - type = types.path; - description = "File with private key for rsync.net server"; - }; - - passphraseFile = mkOption { - type = types.path; - description = "File with passphrase for encrypting the backup"; - }; - - jobs = mkOption { - type = types.attrsOf (types.submodule { - options = { - repoName = mkOption { - type = types.str; - description = "Name of the repo on rsync.net"; - }; - - paths = mkOption { - type = types.listOf types.str; - description = "List of paths to back up"; - default = [ ]; - }; - - exclude = mkOption { - type = types.listOf types.str; - description = "List of paths to exclude from the back up"; - default = [ ]; - }; - - pruneKeep = mkOption { - type = types.attrsOf types.anything; - description = - "Options to pass to `services.borgbackup.jobs..prune.keep`"; - default = { - within = "1d"; - daily = 1; - weekly = 4; - monthly = 2; - }; - }; - - extraOptions = mkOption { - type = types.attrs; - description = - "Extra options to pass to `services.borgbackup.jobs.`"; - default = { }; - }; - }; - }); - }; - description = "Borg job configuration settings"; - default = { }; - }; - - config = mkIf cfg.enable { - programs.ssh.knownHosts."${cfg.hostname}" = { - inherit (cfg) publicKeyFile; - }; - - services.borgbackup.jobs = mapAttrs (_: jobConfig: - { - inherit (jobConfig) paths exclude; - - repo = "${cfg.username}@${cfg.hostname}:${jobConfig.repoName}"; - - encryption = { - mode = "repokey-blake2"; - passCommand = "cat ${cfg.passphraseFile}"; - }; - environment = { BORG_REMOTE_PATH = "/usr/local/bin/borg1/borg1"; }; - compression = "auto,lzma"; - startAt = "*-*-* 03:00:00"; - extraArgs = "--remote-path=borg1"; - extraCreateArgs = "--debug --stats"; - prune.keep = jobConfig.pruneKeep; - preHook = '' - set -x - eval $(ssh-agent) - ssh-add ${cfg.privateKeyFile} - ''; - } // jobConfig.extraOptions) cfg.jobs; - }; -} diff --git a/modules/overlays/default.nix b/modules/overlays/default.nix index 48b6161..f224c40 100644 --- a/modules/overlays/default.nix +++ b/modules/overlays/default.nix @@ -8,17 +8,11 @@ config.allowUnfree = true; }; - - nixpkgs-24-05 = import inputs.nixpkgs-24-05 { - inherit system; - - config.allowUnfree = true; - }; in { inherit (nixpkgs-master) prowlarr radarr sabnzbd sonarr netbox; inherit (nixpkgs-master.python312Packages) pymdown-extensions onnx; - inherit (nixpkgs-24-05) prometheus-pve-exporter; + inherit (nixpkgs-master) prometheus-borgmatic-exporter; inherit (self'.packages) overseerr mongodb-ce-6_0; }; diff --git a/modules/profiles/backups/borgmatic/common.nix b/modules/profiles/backups/borgmatic/common.nix index 208f363..b061d73 100644 --- a/modules/profiles/backups/borgmatic/common.nix +++ b/modules/profiles/backups/borgmatic/common.nix @@ -13,6 +13,13 @@ mode = "0400"; owner = "borgmatic"; }; + + apprise_url_ses = { + format = "yaml"; + sopsFile = ./secrets.yml; + mode = "0777"; + owner = "borgmatic"; + }; }; users = { @@ -57,6 +64,12 @@ config = "__config"; }; }; + on_error = ['' + ${lib.getExe pkgs.apprise} \ + --title "[E10] Backup failed for ${config.networking.hostName}" \ + --body "Backup failed for ${config.networking.hostName} on {repository}" \ + $(cat ${config.sops.secrets.apprise_url_ses.path}) + '']; }; }; } diff --git a/modules/profiles/backups/borgmatic/secrets.yml b/modules/profiles/backups/borgmatic/secrets.yml index 2a1bce8..7d1bc57 100644 --- a/modules/profiles/backups/borgmatic/secrets.yml +++ b/modules/profiles/backups/borgmatic/secrets.yml @@ -1,5 +1,7 @@ rsync_net_ssh_key: ENC[AES256_GCM,data:3SP210tvWiodATv9WFqo6t3qyCpt+6pYXZ7EKtUa4LKgPiFdTg10uijaIfblVZ4YXjgp+QeafAznTxgF3bvuFXrhp9ikWrJPPe9FhAPYTsC6nUuloHvvku3sGuDGqI6XOJaP8L1yaXbsIF7qhH429l4BpJLj6NZp2misBG+e6TSWWtGbq1kUe8jjkaNvd/VmCkIX80ECLKzmaN7igJEInkxzgXPKKaNMfUSw0FQtSm3qTdvNiRnmQaOQIKDeJxI8nHhHB9FYCPPSdjIo8xqUhWyyDjo9M4KTWLhtNUIlzAJ+jHX6WYLTVPINAZwAnuBSZQc6FadWqf+MiKzUuA5dnzzcs57OOYyWvlI3Fq5R1zQ59Ihwv1LpW3xUGa4D2Dj+5idj7dWiJjQQ9pGkY7IJ4a/JSjoirLFObxVNCzerT6dWaMPIYHuGvZp5AY8OWA0Ts3I5XQ+EfNt5QVollvqo33LldtrrV15GeMXU/dJTFUa4dNDzMupD5W1LikE2bCLyaMrzesBtvtREz5llhtshiJkgcLuocEaVvryomYOsklLavatKmdEw78rWujXbS0Zwd9lnHei2kegE4WTu//0zweQywkBrUKUHnQ7jMDhDkMenzxdLoHFFsFWaK24fTllGg5AdvwBvQeCIYmtPJUN7kCb722+maaXrE5LPZGq7hJZQA3lp391qmWe0lOCFiN5RufKsD1CLMLHDK8HyCFT++aUvpaIWLmshzgYaTkMegCSEC5lyGBoNfzPzp8gAoxIdPhUHModroOR4ZFi2+TzqVZMT6qg9KvOUGuaXL5yezCjQJMrLjZZtI2tFXKy/K9PPFo4zNq7GCYVL2Qw/Ig4vPfbBf8gLeoNAwseAmBQX4BusDIYchhtOgJSS04OMutRjAvGeR032UlMsGqo8dTMFyoxlOKH+MI15rKtNk+waTDlnWXoNur9Cj2iMZWL0muaHxz3LEsxYS3DLCXCYrVlDYEHengZqcoa+baGMn242Dk+jtygUF5NN+F62ZYjqSOrxuEGqZ6MFWLKRRmD/LssMHBsC68hZn6okFoNWyQm4Oo/HzeMZ06/y+Q4xmyZTkqXXJZdLIhzDyuqilZvlK1l2uXcCZwiBQv0t8qG9n+1FoH6hwZS996ej09grQYFCxQ4SsPaOwlVczxOYKuziGHBxAevcLjnIsd99Rl4cbXbqEfO4pogh6tiLFSQbVY+yhgGRF8uJi9kq31we4GoEmr+km32cWObKjW/A0wIG/QzuIWBDhr1U4fzxdslJOmhygk+CAaS/vp1b7dC2eTmRy4B7+DJDA1UkB5bn76CWOoclVJmp0FbKsXRjVTtS/3CshHGVqJAV1wUQouLLMt8IoV1RhbmNU1PS7h0cfor09/DCe0+TU/efo1Hmwcriex4fLQGEIvSvRE4lbjpKACX+97q21AwnwMayVKfUGz9vylMPBmLTm2lQobHopkPFwOui+va/hHqIK4bO4PKyPQv6kHaFGbCQf/RzwpU+R8g8APKPpDmZA3IxWZWbw6ieV3aMRHXiQ26WEN80HjdUoWIAjK398WJSulsFmTkK9QGOQR5BHCYWSNQ5JAO2xYS/H6wZcNXt9yeqrFcc1NgCsOaHmfSPNyN097w5MBPNzYvScxKEdzrh2PRNCdRJ3WZprE6WAeW2kmXahrkp4eZRYFhBfC4nHjOfRTLAnps/gfMH/rjh41ZL6sBvDJtllSqYpoTmN6trf4XZx1RE2sh0tIL3mrIFfre8lxok0hfu6QyZGMBb7zSV8l4BJoob9L30WFIVoDxmVxKfPhWqW5TLrKDi8qcK4zVaMoMMVl7OsL9aAhM1LUsRyZ9/ZEkL7eHFpd6y4gDVHONBlJ/zbX1RBBEZ14vN36Syzl02SEQqSr0xABBTvLMn1y1E7Oz07LWM0KVzCsRRrCYIgjuiX9aWTjeSR4d+/sb7CxvGX2FqTDvtbJ/K1QzNgyTjOXEpJtwlZ4rRyu97zp2vCw7h1F81AwALLTKQ7s97z1EgzKC5nJfbJJFWWFTy4u578YIwn4zSCyFUovq7sCInOqpvl/uKd9VKLQ7GulMsCnRoHLY92lJ3aTYHNrKuoUXwYN+WUtkITq5jp3tNbr1neRNqYMEvV95Z14gNfZl8C/nDZHsEVdAWq1U9LcnA0m/987y2XBxSRsstCsRMweizYme+TXEeWFoYORCRtSLBbQOBpbyp9A6by0O4xtpzQeab6VaDEKlLZV/BTh4=,iv:WQDeDc0BZOkVcx9LXn/3c8ALcZcl/2gR7S5D9xRjuxU=,tag:uf+CMkfO+hLakeT570f/kA==,type:str] borg_backup_passphrase: ENC[AES256_GCM,data:ThbvpHdWsTR4BbvrSxxlZCSiWFc=,iv:WfQfv5PXoZblh/BqJzoNmlQAAO6Ko6ldEK1LezS1TPQ=,tag:EBeLrU2O2koFGwdbGQGNPw==,type:str] +apprise_url_pushover: ENC[AES256_GCM,data:PpHKTTEm6EPjPELh6v9zc+4t3yLPgV9eh2ZvflHgN/rA+tibI563o08IV65YXMPhAVjP+W9DnqbgOIhYCTUUWswEgfSR,iv:ErKdkaRyi3O85OkTdf8JQPoWQSWVZ1hN7tYp1xjH+ag=,tag:dAeecHfwQZ9VhqZjR6/KSQ==,type:str] +apprise_url_ses: ENC[AES256_GCM,data:XUhNENF8ffVzQtqMJYZLOPxnX9/pc27DUdmt/7ErxpldZAOGopNPYl54rI3t51VBiF0zr6m5BwPxTS61teIYYbuLvUMQpSn0NT/OARMTWylWUXf6xabl+r2ezytINg==,iv:qrWnEr9hiKiFDRcIeuW/e8U8OaTi6jItXR8M5RWw2Xs=,tag:sCMdcOm/BTYQhJ112WfUYg==,type:str] sops: kms: [] gcp_kms: [] @@ -96,8 +98,8 @@ sops: d0swR0VEYVBWVS9ENkovTGRjcmVoencKV84G9GcmSUwJqGEz7uDDTAPKKcQoZ7QN Qg4SskJ4wcyiARjYt2/Z4DcIcdHnW5tGcEy9dlozpDBnx1HkLG0G6Q== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-11-02T20:04:31Z" - mac: ENC[AES256_GCM,data:5IeEqQR1OsQXEPngikxRyQp1chD382fRH5T8gEdCL+WGKduIHhe3WJXcik1JdBYo0ZM++UBK1lyeRPl/Uc1AqMSTTtWVABPZH5xJ6WczCtdWfRn16I6IhlMlwPqxY0aO6sm3GfKnbgluRZEs6gQlISzBLNVGKY/PbmT1j+o6sAw=,iv:0VeI2aQhpW3gP8aIgknc2B09FQp1eqHoSl0e5u3uRaM=,tag:K4InqkP9c2O3IfPhE5oOKA==,type:str] + lastmodified: "2024-11-04T14:39:43Z" + mac: ENC[AES256_GCM,data:GXcbhyREoIydqvEinF5lW9ExgbEv147XORK85tbeec7+b6lyYVmlOnUO16VZG30ExnDEtT1t+zx3R8rbsUhcfnLGRBEErYSKHV2IQOCsgIWSefp6YYmTnZBxosgMdXQ6wlRZv9vg0t14PuJpWA+KO65bWXveukQgEGFIxmHcmaE=,iv:sJVSIs+rWmCU3y4qEQDJho0Za3Fo8rKtr77N7msS/so=,tag:RnXeMC8cNKlf69bg27hQzg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1 diff --git a/modules/profiles/backups/borgmatic/system.nix b/modules/profiles/backups/borgmatic/system.nix index 24ea3b0..25c1c47 100644 --- a/modules/profiles/backups/borgmatic/system.nix +++ b/modules/profiles/backups/borgmatic/system.nix @@ -21,11 +21,7 @@ } { label = "omnibus"; - # If backing up omnibus, just back up to the already-present directory - path = if config.networking.hostName == "omnibus" then - "/data/files/backup/${config.networking.hostName}-system" - else - "/mnt/files/backup/${config.networking.hostName}-system"; + path = "/mnt/files/backup/${config.networking.hostName}-system"; } ]; keep_daily = 7; diff --git a/modules/profiles/filesystems/files/backup.nix b/modules/profiles/filesystems/files/backup.nix index 307faf4..a70e8a4 100644 --- a/modules/profiles/filesystems/files/backup.nix +++ b/modules/profiles/filesystems/files/backup.nix @@ -1,20 +1,16 @@ -{ config, lib, hosts, ... }: { +{ config, hosts, ... }: { imports = [ ./common.nix ]; - # Only mount this if it's *not* omnibus -- it's not possible to conditionally import, and this needs to be imported - # for backups to work elsewhere, so better to add the conditional here - fileSystems = lib.mkIf (config.networking.hostName != "omnibus") { - "/mnt/files/backup" = let - # Use local network if local, otherwise use Tailscale - host = if builtins.elem "@external" config.deployment.tags then - hosts.omnibus.config.networking.hostName - else - hosts.omnibus.config.satan.address; - in { - device = - "${host}:${hosts.omnibus.config.disko.devices.zpool.files.datasets.root.mountpoint}/backup"; - fsType = "nfs"; - options = [ "x-systemd.automount" "exec" ]; - }; + fileSystems."/mnt/files/backup" = let + # Use local network if local, otherwise use Tailscale + host = if builtins.elem "@external" config.deployment.tags then + hosts.omnibus.config.networking.hostName + else + hosts.omnibus.config.satan.address; + in { + device = + "${host}:${hosts.omnibus.config.disko.devices.zpool.files.datasets.root.mountpoint}/backup"; + fsType = "nfs"; + options = [ "x-systemd.automount" "exec" ]; }; } diff --git a/modules/profiles/media-management/plex.nix b/modules/profiles/media-management/plex.nix index bc7df8a..57256c5 100644 --- a/modules/profiles/media-management/plex.nix +++ b/modules/profiles/media-management/plex.nix @@ -1,4 +1,4 @@ -{ config, lib, ... }: { +{ config, ... }: { services.plex = { enable = true; openFirewall = true; @@ -7,7 +7,4 @@ systemd.tmpfiles.rules = [ "d '/data/local/tmp/plex/transcode' 0777 ${config.services.plex.user} ${config.services.plex.group} - -" ]; - - e10.services.backup.jobs.system.exclude = - lib.mkAfter [ "/var/lib/plex/Plex Media Server/Cache" ]; } diff --git a/modules/profiles/monitoring/loki/default.nix b/modules/profiles/monitoring/loki/default.nix index be30361..d0e1b13 100644 --- a/modules/profiles/monitoring/loki/default.nix +++ b/modules/profiles/monitoring/loki/default.nix @@ -93,5 +93,6 @@ [ config.services.loki.configuration.server.http_listen_port ]; }; - e10.services.backup.jobs.system.exclude = [ "/var/lib/prometheus2/data/wal" ]; + services.borgmatic.configurations.system.exclude_patterns = + [ "/var/lib/prometheus2/data/wal" ]; } diff --git a/modules/profiles/monitoring/prometheus.nix b/modules/profiles/monitoring/prometheus.nix index 81de79f..07645ae 100644 --- a/modules/profiles/monitoring/prometheus.nix +++ b/modules/profiles/monitoring/prometheus.nix @@ -14,5 +14,6 @@ globalConfig.external_labels.prometheus = "${config.networking.hostName}"; }; - e10.services.backup.jobs.system.exclude = [ "/var/lib/prometheus2/data/wal" ]; + services.borgmatic.configurations.system.exclude_patterns = + [ "/var/lib/prometheus2/data/wal" ]; } diff --git a/modules/profiles/telemetry/prometheus-borgmatic-exporter.nix b/modules/profiles/telemetry/prometheus-borgmatic-exporter.nix index 0cd2c3a..2100b44 100644 --- a/modules/profiles/telemetry/prometheus-borgmatic-exporter.nix +++ b/modules/profiles/telemetry/prometheus-borgmatic-exporter.nix @@ -3,8 +3,7 @@ enable = true; openFirewall = true; - # TODO: Monitor all configuration files - configFile = "/etc/borgmatic.d/system.yaml"; + configFile = "/etc/borgmatic.d/"; }; systemd.services.prometheus-borgmatic-exporter.serviceConfig = {