Merge pull request #10908 from raptorsun/queryCapacityAlert

Add Alert PrometheusQueryOverload to mixins
This commit is contained in:
Björn Rabenstein 2022-07-13 18:06:56 +02:00 committed by GitHub
commit b06a3222b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 0 deletions

View File

@ -317,6 +317,20 @@
description: '{{ printf "%%.0f" $value }} targets in Prometheus %(prometheusName)s have failed to sync because invalid configuration was supplied.' % $._config,
},
},
{
alert: 'PrometheusHighQueryLoad',
expr: |||
avg_over_time(prometheus_engine_queries{job="prometheus-k8s"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s"}[5m]) > 0.8
||| % $._config,
'for': '15m',
labels: {
severity: 'warning',
},
annotations: {
summary: 'Prometheus is reaching its maximum capacity serving concurrent requests.',
description: 'Prometheus %(prometheusName)s query API has less than 20%% available capacity in its query engine for the last 15 minutes.' % $._config,
},
},
] + if $._config.prometheusHAGroupLabels == '' then self.rulesWithoutHA else self.rulesWithHA,
rulesWithoutHA:: [
{