From 26a7f80aa1c5f8375f5cace7cd8f25da94b5a7a8 Mon Sep 17 00:00:00 2001
From: Haoyu Sun <hasun@redhat.com>
Date: Thu, 23 Jun 2022 14:54:11 +0200
Subject: [PATCH] add alert PrometheusHighQueryLoad.

Signed-off-by: Haoyu Sun <hasun@redhat.com>
---
 documentation/prometheus-mixin/alerts.libsonnet | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/documentation/prometheus-mixin/alerts.libsonnet b/documentation/prometheus-mixin/alerts.libsonnet
index 481bf6039..524e36ec0 100644
--- a/documentation/prometheus-mixin/alerts.libsonnet
+++ b/documentation/prometheus-mixin/alerts.libsonnet
@@ -317,6 +317,20 @@
               description: '{{ printf "%%.0f" $value }} targets in Prometheus %(prometheusName)s have failed to sync because invalid configuration was supplied.' % $._config,
             },
           },
+          {
+            alert: 'PrometheusHighQueryLoad',
+            expr: |||
+              avg_over_time(prometheus_engine_queries{job="prometheus-k8s"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s"}[5m]) > 0.8
+            ||| % $._config,
+            'for': '15m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Prometheus is reaching its maximum capacity serving concurrent requests.',
+              description: 'Prometheus %(prometheusName)s query API has less than 20%% available capacity in its query engine for the last 15 minutes.' % $._config,
+            },
+          },
         ] + if $._config.prometheusHAGroupLabels == '' then self.rulesWithoutHA else self.rulesWithHA,
         rulesWithoutHA:: [
           {