Add alert for WAL remote write falling behind.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>
2025-01-12 18:01:36 +00:00 · 2019-02-12 15:22:58 +00:00 · 2019-02-12 15:22:58 +00:00 · e248ffb220
commit e248ffb220
parent 638204c775
1 changed files with 16 additions and 0 deletions
--- a/documentation/prometheus-mixin/alerts.libsonnet
+++ b/documentation/prometheus-mixin/alerts.libsonnet
@ -156,6 +156,22 @@
              message: 'Prometheus failed to send {{ printf "%.1f" $value }}% samples',
            },
          },
+          {
+            alert: 'PrometheusRemoteWriteBehind',
+            expr: |||
+              prometheus_remote_storage_highest_timestamp_in{%(prometheusSelector)s}
+                - on(job, instance) group_right
+              prometheus_remote_storage_queue_highest_sent_timestamp{%(prometheusSelector)s}
+                > 60
+            ||| % $._config,
+            'for': '15m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              message: 'Prometheus remote write is {{ printf "%.1f" $value }}s behind.',
+            },
+          },
          {
            alert: 'PrometheusRuleFailures',
            'for': '15m',