2015-01-21 19:07:45 +00:00
|
|
|
// Copyright 2013 The Prometheus Authors
|
2013-02-07 10:49:04 +00:00
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2013-01-07 22:24:26 +00:00
|
|
|
package rules
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"strings"
|
|
|
|
"testing"
|
2013-03-21 17:06:15 +00:00
|
|
|
"time"
|
2013-06-25 12:02:27 +00:00
|
|
|
|
2015-08-20 15:18:46 +00:00
|
|
|
"github.com/prometheus/common/model"
|
2013-06-25 12:02:27 +00:00
|
|
|
|
2015-03-30 17:43:19 +00:00
|
|
|
"github.com/prometheus/prometheus/promql"
|
2013-01-07 22:24:26 +00:00
|
|
|
)
|
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
func TestAlertingRule(t *testing.T) {
|
|
|
|
suite, err := promql.NewTest(t, `
|
|
|
|
load 5m
|
2016-07-12 16:11:31 +00:00
|
|
|
http_requests{job="app-server", instance="0", group="canary", severity="overwrite-me"} 75 85 95 105 105 95 85
|
|
|
|
http_requests{job="app-server", instance="1", group="canary", severity="overwrite-me"} 80 90 100 110 120 130 140
|
2015-06-30 09:51:05 +00:00
|
|
|
`)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
2015-03-30 17:43:19 +00:00
|
|
|
}
|
2015-06-30 09:51:05 +00:00
|
|
|
defer suite.Close()
|
2015-03-30 17:43:19 +00:00
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
if err := suite.Run(); err != nil {
|
|
|
|
t.Fatal(err)
|
2015-03-30 17:43:19 +00:00
|
|
|
}
|
2013-01-07 22:24:26 +00:00
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
expr, err := promql.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Unable to parse alert expression: %s", err)
|
2013-01-07 22:24:26 +00:00
|
|
|
}
|
2013-04-22 22:26:59 +00:00
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
rule := NewAlertingRule(
|
|
|
|
"HTTPRequestRateLow",
|
|
|
|
expr,
|
|
|
|
time.Minute,
|
2016-07-12 16:11:31 +00:00
|
|
|
model.LabelSet{"severity": "{{\"c\"}}ritical"},
|
2015-12-11 16:12:34 +00:00
|
|
|
model.LabelSet{},
|
2015-06-30 09:51:05 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var tests = []struct {
|
|
|
|
time time.Duration
|
|
|
|
result []string
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
time: 0,
|
|
|
|
result: []string{
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
|
2015-02-21 16:45:47 +00:00
|
|
|
},
|
2015-06-30 09:51:05 +00:00
|
|
|
}, {
|
|
|
|
time: 5 * time.Minute,
|
|
|
|
result: []string{
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
|
2015-03-30 17:43:19 +00:00
|
|
|
},
|
2015-06-30 09:51:05 +00:00
|
|
|
}, {
|
|
|
|
time: 10 * time.Minute,
|
|
|
|
result: []string{
|
2016-02-05 04:42:55 +00:00
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
2015-06-30 09:51:05 +00:00
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
|
2015-03-30 17:43:19 +00:00
|
|
|
},
|
|
|
|
},
|
2013-04-24 09:51:40 +00:00
|
|
|
{
|
2016-02-05 04:42:55 +00:00
|
|
|
time: 15 * time.Minute,
|
|
|
|
result: []string{
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
|
|
|
|
},
|
2013-04-24 09:51:40 +00:00
|
|
|
},
|
|
|
|
{
|
2015-06-30 09:51:05 +00:00
|
|
|
time: 20 * time.Minute,
|
2016-02-05 04:42:55 +00:00
|
|
|
result: []string{},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
time: 25 * time.Minute,
|
|
|
|
result: []string{
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
time: 30 * time.Minute,
|
|
|
|
result: []string{
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
|
|
|
|
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
|
|
|
|
},
|
2013-04-24 09:51:40 +00:00
|
|
|
},
|
|
|
|
}
|
2015-03-30 17:43:19 +00:00
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
for i, test := range tests {
|
2015-08-20 15:18:46 +00:00
|
|
|
evalTime := model.Time(0).Add(test.time)
|
2015-03-30 17:43:19 +00:00
|
|
|
|
2016-11-18 16:12:50 +00:00
|
|
|
res, err := rule.Eval(suite.Context(), evalTime, suite.QueryEngine(), "")
|
2013-04-24 09:51:40 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Error during alerting rule evaluation: %s", err)
|
|
|
|
}
|
2015-03-30 17:43:19 +00:00
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
actual := strings.Split(res.String(), "\n")
|
|
|
|
expected := annotateWithTime(test.result, evalTime)
|
|
|
|
if actual[0] == "" {
|
|
|
|
actual = []string{}
|
2013-04-24 09:51:40 +00:00
|
|
|
}
|
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
if len(actual) != len(expected) {
|
|
|
|
t.Errorf("%d. Number of samples in expected and actual output don't match (%d vs. %d)", i, len(expected), len(actual))
|
2013-04-24 09:51:40 +00:00
|
|
|
}
|
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
for j, expectedSample := range expected {
|
2013-04-24 09:51:40 +00:00
|
|
|
found := false
|
2015-06-30 09:51:05 +00:00
|
|
|
for _, actualSample := range actual {
|
2013-04-24 09:51:40 +00:00
|
|
|
if actualSample == expectedSample {
|
|
|
|
found = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !found {
|
|
|
|
t.Errorf("%d.%d. Couldn't find expected sample in output: '%v'", i, j, expectedSample)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-30 09:51:05 +00:00
|
|
|
if t.Failed() {
|
|
|
|
t.Errorf("%d. Expected and actual outputs don't match:", i)
|
|
|
|
t.Fatalf("Expected:\n%v\n----\nActual:\n%v", strings.Join(expected, "\n"), strings.Join(actual, "\n"))
|
2013-04-24 09:51:40 +00:00
|
|
|
}
|
2016-08-01 22:32:01 +00:00
|
|
|
|
|
|
|
for _, aa := range rule.ActiveAlerts() {
|
|
|
|
if _, ok := aa.Labels[model.MetricNameLabel]; ok {
|
|
|
|
t.Fatalf("%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
|
|
|
|
}
|
|
|
|
}
|
2013-04-24 09:51:40 +00:00
|
|
|
}
|
|
|
|
}
|
2015-06-30 09:51:05 +00:00
|
|
|
|
2015-08-20 15:18:46 +00:00
|
|
|
func annotateWithTime(lines []string, timestamp model.Time) []string {
|
2015-06-30 09:51:05 +00:00
|
|
|
annotatedLines := []string{}
|
|
|
|
for _, line := range lines {
|
|
|
|
annotatedLines = append(annotatedLines, fmt.Sprintf(line, timestamp))
|
|
|
|
}
|
|
|
|
return annotatedLines
|
|
|
|
}
|