prometheus/notifier/notifier_test.go

541 lines
13 KiB
Go
Raw Normal View History

// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2016-03-01 11:37:22 +00:00
package notifier
import (
"bytes"
"context"
2018-01-16 11:25:52 +00:00
"crypto/md5"
2015-12-10 15:31:50 +00:00
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
2015-12-10 15:31:50 +00:00
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/prometheus/prometheus/pkg/relabel"
yaml "gopkg.in/yaml.v2"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
Refactor SD configuration to remove `config` dependency (#3629) * refactor: move targetGroup struct and CheckOverflow() to their own package * refactor: move auth and security related structs to a utility package, fix import error in utility package * refactor: Azure SD, remove SD struct from config * refactor: DNS SD, remove SD struct from config into dns package * refactor: ec2 SD, move SD struct from config into the ec2 package * refactor: file SD, move SD struct from config to file discovery package * refactor: gce, move SD struct from config to gce discovery package * refactor: move HTTPClientConfig and URL into util/config, fix import error in httputil * refactor: consul, move SD struct from config into consul discovery package * refactor: marathon, move SD struct from config into marathon discovery package * refactor: triton, move SD struct from config to triton discovery package, fix test * refactor: zookeeper, move SD structs from config to zookeeper discovery package * refactor: openstack, remove SD struct from config, move into openstack discovery package * refactor: kubernetes, move SD struct from config into kubernetes discovery package * refactor: notifier, use targetgroup package instead of config * refactor: tests for file, marathon, triton SD - use targetgroup package instead of config.TargetGroup * refactor: retrieval, use targetgroup package instead of config.TargetGroup * refactor: storage, use config util package * refactor: discovery manager, use targetgroup package instead of config.TargetGroup * refactor: use HTTPClient and TLS config from configUtil instead of config * refactor: tests, use targetgroup package instead of config.TargetGroup * refactor: fix tagetgroup.Group pointers that were removed by mistake * refactor: openstack, kubernetes: drop prefixes * refactor: remove import aliases forced due to vscode bug * refactor: move main SD struct out of config into discovery/config * refactor: rename configUtil to config_util * refactor: rename yamlUtil to yaml_config * refactor: kubernetes, remove prefixes * refactor: move the TargetGroup package to discovery/ * refactor: fix order of imports
2017-12-29 20:01:34 +00:00
"github.com/prometheus/prometheus/discovery/targetgroup"
2016-12-29 15:53:11 +00:00
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/util/testutil"
)
func TestPostPath(t *testing.T) {
2016-02-04 10:56:14 +00:00
var cases = []struct {
in, out string
}{
{
in: "",
out: "/api/v1/alerts",
2016-02-04 10:56:14 +00:00
},
{
in: "/",
out: "/api/v1/alerts",
2016-02-04 10:56:14 +00:00
},
{
in: "/prefix",
out: "/prefix/api/v1/alerts",
2016-02-04 10:56:14 +00:00
},
{
in: "/prefix//",
out: "/prefix/api/v1/alerts",
2016-02-04 10:56:14 +00:00
},
{
in: "prefix//",
out: "/prefix/api/v1/alerts",
2016-02-04 10:56:14 +00:00
},
}
for _, c := range cases {
testutil.Equals(t, c.out, postPath(c.in))
2016-02-04 10:56:14 +00:00
}
}
2015-12-10 15:31:50 +00:00
func TestHandlerNextBatch(t *testing.T) {
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{}, nil)
2015-12-10 15:31:50 +00:00
for i := range make([]struct{}, 2*maxBatchSize+1) {
2016-12-29 15:53:11 +00:00
h.queue = append(h.queue, &Alert{
Labels: labels.FromStrings("alertname", fmt.Sprintf("%d", i)),
2015-12-10 15:31:50 +00:00
})
}
2016-12-29 15:53:11 +00:00
expected := append([]*Alert{}, h.queue...)
2015-12-10 15:31:50 +00:00
b := h.nextBatch()
testutil.Equals(t, maxBatchSize, len(b))
testutil.Assert(t, alertsEqual(expected[0:maxBatchSize], b), "First batch did not match")
2015-12-10 15:31:50 +00:00
b = h.nextBatch()
testutil.Equals(t, maxBatchSize, len(b))
testutil.Assert(t, alertsEqual(expected[maxBatchSize:2*maxBatchSize], b), "Second batch did not match")
2015-12-10 15:31:50 +00:00
b = h.nextBatch()
testutil.Equals(t, 1, len(b))
2015-12-10 15:31:50 +00:00
testutil.Assert(t, alertsEqual(expected[2*maxBatchSize:], b), "Third batch did not match")
testutil.Assert(t, len(h.queue) == 0, "Expected queue to be empty but got %d alerts", len(h.queue))
}
2016-12-29 15:53:11 +00:00
func alertsEqual(a, b []*Alert) bool {
2015-12-10 15:31:50 +00:00
if len(a) != len(b) {
2016-12-29 15:53:11 +00:00
fmt.Println("len mismatch")
2015-12-10 15:31:50 +00:00
return false
}
for i, alert := range a {
2016-12-29 15:53:11 +00:00
if !labels.Equal(alert.Labels, b[i].Labels) {
fmt.Println("mismatch", alert.Labels, b[i].Labels)
2015-12-10 15:31:50 +00:00
return false
}
}
return true
}
func TestHandlerSendAll(t *testing.T) {
2015-12-10 15:31:50 +00:00
var (
2016-12-29 15:53:11 +00:00
expected []*Alert
status1, status2 int
2015-12-10 15:31:50 +00:00
)
f := func(w http.ResponseWriter, r *http.Request) {
2015-12-10 15:31:50 +00:00
defer r.Body.Close()
2016-12-29 15:53:11 +00:00
var alerts []*Alert
testutil.Ok(t, json.NewDecoder(r.Body).Decode(&alerts))
2015-12-10 15:31:50 +00:00
testutil.Assert(t, alertsEqual(alerts, expected), "Unexpected alerts received %v exp %v", alerts, expected)
}
server1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, _ := r.BasicAuth()
testutil.Assert(
t,
user == "prometheus" || pass == "testing_password",
"Incorrect auth details for an alertmanager",
)
f(w, r)
w.WriteHeader(status1)
}))
server2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, pass, _ := r.BasicAuth()
testutil.Assert(
t,
user == "" || pass == "",
"Incorrectly received auth details for an alertmanager",
)
f(w, r)
w.WriteHeader(status2)
2015-12-10 15:31:50 +00:00
}))
defer server1.Close()
defer server2.Close()
2015-12-10 15:31:50 +00:00
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{}, nil)
authClient, _ := config_util.NewClientFromConfig(config_util.HTTPClientConfig{
Refactor SD configuration to remove `config` dependency (#3629) * refactor: move targetGroup struct and CheckOverflow() to their own package * refactor: move auth and security related structs to a utility package, fix import error in utility package * refactor: Azure SD, remove SD struct from config * refactor: DNS SD, remove SD struct from config into dns package * refactor: ec2 SD, move SD struct from config into the ec2 package * refactor: file SD, move SD struct from config to file discovery package * refactor: gce, move SD struct from config to gce discovery package * refactor: move HTTPClientConfig and URL into util/config, fix import error in httputil * refactor: consul, move SD struct from config into consul discovery package * refactor: marathon, move SD struct from config into marathon discovery package * refactor: triton, move SD struct from config to triton discovery package, fix test * refactor: zookeeper, move SD structs from config to zookeeper discovery package * refactor: openstack, remove SD struct from config, move into openstack discovery package * refactor: kubernetes, move SD struct from config into kubernetes discovery package * refactor: notifier, use targetgroup package instead of config * refactor: tests for file, marathon, triton SD - use targetgroup package instead of config.TargetGroup * refactor: retrieval, use targetgroup package instead of config.TargetGroup * refactor: storage, use config util package * refactor: discovery manager, use targetgroup package instead of config.TargetGroup * refactor: use HTTPClient and TLS config from configUtil instead of config * refactor: tests, use targetgroup package instead of config.TargetGroup * refactor: fix tagetgroup.Group pointers that were removed by mistake * refactor: openstack, kubernetes: drop prefixes * refactor: remove import aliases forced due to vscode bug * refactor: move main SD struct out of config into discovery/config * refactor: rename configUtil to config_util * refactor: rename yamlUtil to yaml_config * refactor: kubernetes, remove prefixes * refactor: move the TargetGroup package to discovery/ * refactor: fix order of imports
2017-12-29 20:01:34 +00:00
BasicAuth: &config_util.BasicAuth{
Username: "prometheus",
Password: "testing_password",
},
}, "auth_alertmanager")
h.alertmanagers = make(map[string]*alertmanagerSet)
h.alertmanagers["1"] = &alertmanagerSet{
ams: []alertmanager{
alertmanagerMock{
urlf: func() string { return server1.URL },
},
},
cfg: &config.AlertmanagerConfig{
Timeout: model.Duration(time.Second),
},
client: authClient,
}
h.alertmanagers["2"] = &alertmanagerSet{
ams: []alertmanager{
alertmanagerMock{
urlf: func() string { return server2.URL },
},
},
cfg: &config.AlertmanagerConfig{
Timeout: model.Duration(time.Second),
},
}
2015-12-10 15:31:50 +00:00
for i := range make([]struct{}, maxBatchSize) {
2016-12-29 15:53:11 +00:00
h.queue = append(h.queue, &Alert{
Labels: labels.FromStrings("alertname", fmt.Sprintf("%d", i)),
2015-12-10 15:31:50 +00:00
})
2016-12-29 15:53:11 +00:00
expected = append(expected, &Alert{
Labels: labels.FromStrings("alertname", fmt.Sprintf("%d", i)),
2015-12-10 15:31:50 +00:00
})
}
status1 = http.StatusOK
status2 = http.StatusOK
testutil.Assert(t, h.sendAll(h.queue...), "all sends failed unexpectedly")
2015-12-10 15:31:50 +00:00
status1 = http.StatusNotFound
testutil.Assert(t, h.sendAll(h.queue...), "all sends failed unexpectedly")
status2 = http.StatusInternalServerError
testutil.Assert(t, !h.sendAll(h.queue...), "all sends succeeded unexpectedly")
}
func TestCustomDo(t *testing.T) {
const testURL = "http://testurl.com/"
const testBody = "testbody"
var received bool
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{
Do: func(_ context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
received = true
body, err := ioutil.ReadAll(req.Body)
testutil.Ok(t, err)
testutil.Equals(t, testBody, string(body))
testutil.Equals(t, testURL, req.URL.String())
return &http.Response{
Body: ioutil.NopCloser(bytes.NewBuffer(nil)),
}, nil
},
2017-08-11 18:45:52 +00:00
}, nil)
h.sendOne(context.Background(), nil, testURL, []byte(testBody))
testutil.Assert(t, received, "Expected to receive an alert, but didn't")
}
func TestExternalLabels(t *testing.T) {
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{
QueueCapacity: 3 * maxBatchSize,
ExternalLabels: labels.Labels{{Name: "a", Value: "b"}},
RelabelConfigs: []*relabel.Config{
{
SourceLabels: model.LabelNames{"alertname"},
TargetLabel: "a",
Action: "replace",
Regex: relabel.MustNewRegexp("externalrelabelthis"),
Replacement: "c",
},
},
2017-08-11 18:45:52 +00:00
}, nil)
// This alert should get the external label attached.
2016-12-29 15:53:11 +00:00
h.Send(&Alert{
Labels: labels.FromStrings("alertname", "test"),
})
// This alert should get the external label attached, but then set to "c"
// through relabelling.
2016-12-29 15:53:11 +00:00
h.Send(&Alert{
Labels: labels.FromStrings("alertname", "externalrelabelthis"),
})
2016-12-29 15:53:11 +00:00
expected := []*Alert{
{Labels: labels.FromStrings("alertname", "test", "a", "b")},
{Labels: labels.FromStrings("alertname", "externalrelabelthis", "a", "c")},
}
testutil.Assert(t, alertsEqual(expected, h.queue), "Expected alerts %v, got %v", expected, h.queue)
}
func TestHandlerRelabel(t *testing.T) {
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{
QueueCapacity: 3 * maxBatchSize,
RelabelConfigs: []*relabel.Config{
2016-09-15 03:13:27 +00:00
{
SourceLabels: model.LabelNames{"alertname"},
Action: "drop",
Regex: relabel.MustNewRegexp("drop"),
},
2016-09-15 03:13:27 +00:00
{
SourceLabels: model.LabelNames{"alertname"},
TargetLabel: "alertname",
Action: "replace",
Regex: relabel.MustNewRegexp("rename"),
Replacement: "renamed",
},
},
2017-08-11 18:45:52 +00:00
}, nil)
// This alert should be dropped due to the configuration
2016-12-29 15:53:11 +00:00
h.Send(&Alert{
Labels: labels.FromStrings("alertname", "drop"),
})
// This alert should be replaced due to the configuration
2016-12-29 15:53:11 +00:00
h.Send(&Alert{
Labels: labels.FromStrings("alertname", "rename"),
})
2016-12-29 15:53:11 +00:00
expected := []*Alert{
{Labels: labels.FromStrings("alertname", "renamed")},
}
testutil.Assert(t, alertsEqual(expected, h.queue), "Expected alerts %v, got %v", expected, h.queue)
}
func TestHandlerQueueing(t *testing.T) {
2015-12-10 15:31:50 +00:00
var (
unblock = make(chan struct{})
called = make(chan struct{})
2016-12-29 15:53:11 +00:00
expected []*Alert
2015-12-10 15:31:50 +00:00
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called <- struct{}{}
<-unblock
defer r.Body.Close()
2016-12-29 15:53:11 +00:00
var alerts []*Alert
testutil.Ok(t, json.NewDecoder(r.Body).Decode(&alerts))
2015-12-10 15:31:50 +00:00
testutil.Assert(t, alertsEqual(expected, alerts), "Expected alerts %v, got %v", expected, alerts)
2015-12-10 15:31:50 +00:00
}))
2018-01-30 17:45:37 +00:00
h := NewManager(&Options{
QueueCapacity: 3 * maxBatchSize,
},
2017-08-11 18:45:52 +00:00
nil,
)
h.alertmanagers = make(map[string]*alertmanagerSet)
h.alertmanagers["1"] = &alertmanagerSet{
ams: []alertmanager{
alertmanagerMock{
urlf: func() string { return server.URL },
},
},
cfg: &config.AlertmanagerConfig{
Timeout: model.Duration(time.Second),
},
}
2015-12-10 15:31:50 +00:00
2016-12-29 15:53:11 +00:00
var alerts []*Alert
2015-12-10 15:31:50 +00:00
for i := range make([]struct{}, 20*maxBatchSize) {
2016-12-29 15:53:11 +00:00
alerts = append(alerts, &Alert{
Labels: labels.FromStrings("alertname", fmt.Sprintf("%d", i)),
2015-12-10 15:31:50 +00:00
})
}
c := make(chan map[string][]*targetgroup.Group)
go h.Run(c)
2015-12-10 15:31:50 +00:00
defer h.Stop()
h.Send(alerts[:4*maxBatchSize]...)
// If the batch is larger than the queue size, the front should be truncated
// from the front. Thus, we start at i=1.
for i := 1; i < 4; i++ {
select {
case <-called:
expected = alerts[i*maxBatchSize : (i+1)*maxBatchSize]
unblock <- struct{}{}
2016-01-25 17:31:54 +00:00
case <-time.After(5 * time.Second):
2015-12-10 15:31:50 +00:00
t.Fatalf("Alerts were not pushed")
}
}
2015-12-10 15:31:50 +00:00
// Send one batch, wait for it to arrive and block so the queue fills up.
// Then check whether the queue is truncated in the front once its full.
h.Send(alerts[:maxBatchSize]...)
<-called
// Fill the 3*maxBatchSize queue.
h.Send(alerts[1*maxBatchSize : 2*maxBatchSize]...)
h.Send(alerts[2*maxBatchSize : 3*maxBatchSize]...)
h.Send(alerts[3*maxBatchSize : 4*maxBatchSize]...)
// Send the batch that drops the first one.
h.Send(alerts[4*maxBatchSize : 5*maxBatchSize]...)
expected = alerts[:maxBatchSize]
unblock <- struct{}{}
for i := 2; i < 5; i++ {
2015-12-10 15:31:50 +00:00
select {
case <-called:
expected = alerts[i*maxBatchSize : (i+1)*maxBatchSize]
unblock <- struct{}{}
2016-01-25 17:31:54 +00:00
case <-time.After(5 * time.Second):
2015-12-10 15:31:50 +00:00
t.Fatalf("Alerts were not pushed")
}
}
}
type alertmanagerMock struct {
urlf func() string
}
func (a alertmanagerMock) url() *url.URL {
u, err := url.Parse(a.urlf())
if err != nil {
panic(err)
}
return u
}
func TestLabelSetNotReused(t *testing.T) {
tg := makeInputTargetGroup()
_, _, err := alertmanagerFromGroup(tg, &config.AlertmanagerConfig{})
testutil.Ok(t, err)
// Target modified during alertmanager extraction
testutil.Equals(t, tg, makeInputTargetGroup())
}
func TestReload(t *testing.T) {
var tests = []struct {
in *targetgroup.Group
out string
}{
{
in: &targetgroup.Group{
Targets: []model.LabelSet{
{
"__address__": "alertmanager:9093",
},
},
},
out: "http://alertmanager:9093/api/v1/alerts",
},
}
2018-01-30 17:45:37 +00:00
n := NewManager(&Options{}, nil)
cfg := &config.Config{}
s := `
alerting:
alertmanagers:
- static_configs:
`
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
t.Fatalf("Unable to load YAML config: %s", err)
}
if err := n.ApplyConfig(cfg); err != nil {
t.Fatalf("Error Applying the config:%v", err)
}
tgs := make(map[string][]*targetgroup.Group)
for _, tt := range tests {
2018-01-16 11:25:52 +00:00
b, err := json.Marshal(cfg.AlertingConfig.AlertmanagerConfigs[0])
if err != nil {
t.Fatalf("Error creating config hash:%v", err)
}
tgs[fmt.Sprintf("%x", md5.Sum(b))] = []*targetgroup.Group{
tt.in,
}
n.reload(tgs)
res := n.Alertmanagers()[0].String()
testutil.Equals(t, res, tt.out)
}
}
func TestDroppedAlertmanagers(t *testing.T) {
var tests = []struct {
in *targetgroup.Group
out string
}{
{
in: &targetgroup.Group{
Targets: []model.LabelSet{
{
"__address__": "alertmanager:9093",
},
},
},
out: "http://alertmanager:9093/api/v1/alerts",
},
}
n := NewManager(&Options{}, nil)
cfg := &config.Config{}
s := `
alerting:
alertmanagers:
- static_configs:
relabel_configs:
- source_labels: ['__address__']
regex: 'alertmanager:9093'
action: drop
`
if err := yaml.UnmarshalStrict([]byte(s), cfg); err != nil {
t.Fatalf("Unable to load YAML config: %s", err)
}
if err := n.ApplyConfig(cfg); err != nil {
t.Fatalf("Error Applying the config:%v", err)
}
tgs := make(map[string][]*targetgroup.Group)
for _, tt := range tests {
b, err := json.Marshal(cfg.AlertingConfig.AlertmanagerConfigs[0])
if err != nil {
t.Fatalf("Error creating config hash:%v", err)
}
tgs[fmt.Sprintf("%x", md5.Sum(b))] = []*targetgroup.Group{
tt.in,
}
n.reload(tgs)
res := n.DroppedAlertmanagers()[0].String()
testutil.Equals(t, res, tt.out)
}
}
Refactor SD configuration to remove `config` dependency (#3629) * refactor: move targetGroup struct and CheckOverflow() to their own package * refactor: move auth and security related structs to a utility package, fix import error in utility package * refactor: Azure SD, remove SD struct from config * refactor: DNS SD, remove SD struct from config into dns package * refactor: ec2 SD, move SD struct from config into the ec2 package * refactor: file SD, move SD struct from config to file discovery package * refactor: gce, move SD struct from config to gce discovery package * refactor: move HTTPClientConfig and URL into util/config, fix import error in httputil * refactor: consul, move SD struct from config into consul discovery package * refactor: marathon, move SD struct from config into marathon discovery package * refactor: triton, move SD struct from config to triton discovery package, fix test * refactor: zookeeper, move SD structs from config to zookeeper discovery package * refactor: openstack, remove SD struct from config, move into openstack discovery package * refactor: kubernetes, move SD struct from config into kubernetes discovery package * refactor: notifier, use targetgroup package instead of config * refactor: tests for file, marathon, triton SD - use targetgroup package instead of config.TargetGroup * refactor: retrieval, use targetgroup package instead of config.TargetGroup * refactor: storage, use config util package * refactor: discovery manager, use targetgroup package instead of config.TargetGroup * refactor: use HTTPClient and TLS config from configUtil instead of config * refactor: tests, use targetgroup package instead of config.TargetGroup * refactor: fix tagetgroup.Group pointers that were removed by mistake * refactor: openstack, kubernetes: drop prefixes * refactor: remove import aliases forced due to vscode bug * refactor: move main SD struct out of config into discovery/config * refactor: rename configUtil to config_util * refactor: rename yamlUtil to yaml_config * refactor: kubernetes, remove prefixes * refactor: move the TargetGroup package to discovery/ * refactor: fix order of imports
2017-12-29 20:01:34 +00:00
func makeInputTargetGroup() *targetgroup.Group {
return &targetgroup.Group{
Targets: []model.LabelSet{
{
model.AddressLabel: model.LabelValue("1.1.1.1:9090"),
model.LabelName("notcommon1"): model.LabelValue("label"),
},
},
Labels: model.LabelSet{
model.LabelName("common"): model.LabelValue("label"),
},
Source: "testsource",
}
}