Make filesystem space prediction window configurable (prometheus#2844)

Signed-off-by: fitz123 <[email protected]>
grafana · Nov 13, 2023 · 81fc05c · 81fc05c
1 parent 7333465
commit 81fc05c
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet
@@ -10,7 +10,7 @@
               (
                 node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceFillingUpWarningThreshold)d
               and
-                predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 24*60*60) < 0
+                predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[%(fsSpaceFillingUpPredictionWindow)s], 24*60*60) < 0
               and
                 node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0
               )

diff --git a/docs/node-mixin/config.libsonnet b/docs/node-mixin/config.libsonnet
@@ -54,13 +54,19 @@
     // 'NodeFilesystemSpaceFillingUp' alerts. These alerts fire if the disk
     // usage grows in a way that it is predicted to run out in 4h or 1d
     // and if the provided thresholds have been reached right now.
-    // In some cases you'll want to adjust these, e.g. by default Kubernetes
+    // In some cases you'll want to adjust these, e.g., by default, Kubernetes
     // runs the image garbage collection when the disk usage reaches 85%
     // of its available space. In that case, you'll want to reduce the
     // critical threshold below to something like 14 or 15, otherwise
     // the alert could fire under normal node usage.
+    // Additionally, the prediction window for the alert can be configured
+    // to account for environments where disk usage can fluctuate within
+    // a short time frame. By extending the prediction window, you can
+    // reduce false positives caused by temporary spikes, providing a
+    // more accurate prediction of disk space issues.
     fsSpaceFillingUpWarningThreshold: 40,
     fsSpaceFillingUpCriticalThreshold: 20,
+    fsSpaceFillingUpPredictionWindow: '6h',
 
     // Available disk space (%) thresholds on which to trigger the
     // 'NodeFilesystemAlmostOutOfSpace' alerts.