RepoMirrors/alertmanager
1
0
Fork 0
mirror of https://github.com/prometheus/alertmanager synced 2025-02-19 20:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #703 from prometheus/fix-resolve

Browse Source 
Fix resolve notifications
This commit is contained in:
Fabian Reinartz 2017-04-17 14:19:04 +02:00 committed by GitHub
commit 8820ce7827
9 changed files with 449 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -66,5 +66,7 @@ promu:
GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \
$(GO) get -u github.com/prometheus/promu
proto:
scripts/genproto.sh
.PHONY: all style format build test vet assets tarball docker promu
.PHONY: all style format build test vet assets tarball docker promu proto

25
nflog/nflog.go
View File

@ -44,8 +44,7 @@ type Log interface {
// The Log* methods store a notification log entry for
// a fully qualified receiver and a given IDs identifying the
// alert object.
LogActive(r *pb.Receiver, key, hash []byte) error
LogResolved(r *pb.Receiver, key, hash []byte) error
Log(r *pb.Receiver, key []byte, firing, resolved []uint64) error
// Query the log along the given Paramteres.
//
@ -325,23 +324,13 @@ Loop:
}
}
// LogActive implements the Log interface.
func (l *nlog) LogActive(r *pb.Receiver, key, hash []byte) error {
return l.log(r, key, hash, false)
}
// LogResolved implements the Log interface.
func (l *nlog) LogResolved(r *pb.Receiver, key, hash []byte) error {
return l.log(r, key, hash, true)
}
// stateKey returns a string key for a log entry consisting of the group key
// and receiver.
func stateKey(k []byte, r *pb.Receiver) string {
return fmt.Sprintf("%s:%s", k, r)
}
func (l *nlog) log(r *pb.Receiver, gkey, ghash []byte, resolved bool) error {
func (l *nlog) Log(r *pb.Receiver, gkey []byte, firingAlerts, resolvedAlerts []uint64) error {
// Write all st with the same timestamp.
now := l.now()
key := stateKey(gkey, r)
@ -372,11 +361,11 @@ func (l *nlog) log(r *pb.Receiver, gkey, ghash []byte, resolved bool) error {
e := &pb.MeshEntry{
Entry: &pb.Entry{
Receiver: r,
GroupKey: gkey,
GroupHash: ghash,
Resolved: resolved,
Timestamp: ts,
Receiver: r,
GroupKey: gkey,
Timestamp: ts,
FiringAlerts: firingAlerts,
ResolvedAlerts: resolvedAlerts,
},
ExpiresAt: expts,
}

105
nflog/nflogpb/nflog.pb.go
View File

@ -46,6 +46,27 @@ func (m *Receiver) String() string { return proto.CompactTextString(m
func (*Receiver) ProtoMessage() {}
func (*Receiver) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *Receiver) GetGroupName() string {
if m != nil {
return m.GroupName
}
return ""
}
func (m *Receiver) GetIntegration() string {
if m != nil {
return m.Integration
}
return ""
}
func (m *Receiver) GetIdx() uint32 {
if m != nil {
return m.Idx
}
return 0
}
// Entry holds information about a successful notification
// sent to a receiver.
type Entry struct {
@ -54,11 +75,17 @@ type Entry struct {
// The receiver that was notified.
Receiver *Receiver `protobuf:"bytes,2,opt,name=receiver" json:"receiver,omitempty"`
// Hash over the state of the group at notification time.
// Deprecated in favor of FiringAlerts field, but kept for compatibility.
GroupHash []byte `protobuf:"bytes,3,opt,name=group_hash,json=groupHash,proto3" json:"group_hash,omitempty"`
// Whether the notification was about a resolved alert.
// Deprecated in favor of ResolvedAlerts field, but kept for compatibility.
Resolved bool `protobuf:"varint,4,opt,name=resolved" json:"resolved,omitempty"`
// Timestamp of the succeeding notification.
Timestamp *google_protobuf.Timestamp `protobuf:"bytes,5,opt,name=timestamp" json:"timestamp,omitempty"`
// FiringAlerts list of hashes of firing alerts at last notification.
FiringAlerts []uint64 `protobuf:"varint,6,rep,packed,name=firing_alerts,json=firingAlerts" json:"firing_alerts,omitempty"`
// ResolvedAlerts list hashes of resolved alerts at last notification.
ResolvedAlerts []uint64 `protobuf:"varint,7,rep,packed,name=resolved_alerts,json=resolvedAlerts" json:"resolved_alerts,omitempty"`
}
func (m *Entry) Reset() { *m = Entry{} }
@ -66,6 +93,13 @@ func (m *Entry) String() string { return proto.CompactTextString(m) }
func (*Entry) ProtoMessage() {}
func (*Entry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *Entry) GetGroupKey() []byte {
if m != nil {
return m.GroupKey
}
return nil
}
func (m *Entry) GetReceiver() *Receiver {
if m != nil {
return m.Receiver
@ -73,6 +107,20 @@ func (m *Entry) GetReceiver() *Receiver {
return nil
}
func (m *Entry) GetGroupHash() []byte {
if m != nil {
return m.GroupHash
}
return nil
}
func (m *Entry) GetResolved() bool {
if m != nil {
return m.Resolved
}
return false
}
func (m *Entry) GetTimestamp() *google_protobuf.Timestamp {
if m != nil {
return m.Timestamp
@ -80,6 +128,20 @@ func (m *Entry) GetTimestamp() *google_protobuf.Timestamp {
return nil
}
func (m *Entry) GetFiringAlerts() []uint64 {
if m != nil {
return m.FiringAlerts
}
return nil
}
func (m *Entry) GetResolvedAlerts() []uint64 {
if m != nil {
return m.ResolvedAlerts
}
return nil
}
// MeshEntry is a wrapper message to communicate a notify log
// entry through a mesh network.
type MeshEntry struct {
@ -118,24 +180,27 @@ func init() {
func init() { proto.RegisterFile("nflog/nflogpb/nflog.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 299 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x7c, 0x50, 0xc1, 0x4e, 0x83, 0x40,
0x10, 0x4d, 0xad, 0x55, 0x98, 0x56, 0xa3, 0x7b, 0xc2, 0x1a, 0x63, 0xd3, 0x78, 0xf0, 0xe2, 0x36,
0xa9, 0x17, 0x3d, 0x7a, 0x30, 0x31, 0x31, 0x7a, 0xd8, 0x78, 0x35, 0x04, 0xec, 0x14, 0x36, 0x02,
0x4b, 0x96, 0x6d, 0x53, 0xfe, 0xd0, 0xcf, 0x12, 0x66, 0x17, 0xea, 0xc9, 0x0b, 0xcc, 0xbe, 0xf7,
0x32, 0xef, 0xbd, 0x81, 0x8b, 0x62, 0x9d, 0xa9, 0x64, 0x41, 0xdf, 0x32, 0xb6, 0x7f, 0x5e, 0x6a,
0x65, 0x14, 0x3b, 0x76, 0xe0, 0xf4, 0x3a, 0x51, 0x2a, 0xc9, 0x70, 0x41, 0x70, 0xbc, 0x59, 0x2f,
0x8c, 0xcc, 0xb1, 0x32, 0x51, 0x5e, 0x5a, 0xe5, 0xfc, 0x13, 0x3c, 0x81, 0x5f, 0x28, 0xb7, 0xa8,
0xd9, 0x15, 0x40, 0xa2, 0xd5, 0xa6, 0x0c, 0x8b, 0x28, 0xc7, 0x60, 0x30, 0x1b, 0xdc, 0xfa, 0xc2,
0x27, 0xe4, 0xbd, 0x01, 0xd8, 0x0c, 0xc6, 0xb2, 0x30, 0x98, 0xe8, 0xc8, 0x48, 0x55, 0x04, 0x07,
0xc4, 0xff, 0x85, 0xd8, 0x19, 0x0c, 0xe5, 0x6a, 0x17, 0x0c, 0x1b, 0xe6, 0x44, 0xb4, 0xe3, 0xfc,
0x67, 0x00, 0xa3, 0xe7, 0xc2, 0xe8, 0x9a, 0x5d, 0x82, 0x5d, 0x15, 0x7e, 0x63, 0x4d, 0xbb, 0x27,
0xc2, 0x23, 0xe0, 0x15, 0x6b, 0x76, 0x07, 0x9e, 0x76, 0x29, 0x68, 0xef, 0x78, 0x79, 0xce, 0x5d,
0x05, 0xde, 0xc5, 0x13, 0xbd, 0x64, 0x1f, 0x34, 0x8d, 0xaa, 0x94, 0xec, 0x26, 0x2e, 0xe8, 0x4b,
0x03, 0xb0, 0x69, 0xbb, 0xad, 0x52, 0xd9, 0x16, 0x57, 0xc1, 0x61, 0x43, 0x7a, 0xa2, 0x7f, 0xb3,
0x07, 0xf0, 0xfb, 0x13, 0x04, 0x23, 0xb2, 0x9a, 0x72, 0x7b, 0x24, 0xde, 0x1d, 0x89, 0x7f, 0x74,
0x0a, 0xb1, 0x17, 0xcf, 0x33, 0xf0, 0xdf, 0xb0, 0x4a, 0x6d, 0x9b, 0x1b, 0x18, 0x61, 0x3b, 0x50,
0x93, 0xf1, 0xf2, 0xb4, 0x4f, 0x4b, 0xb4, 0xb0, 0x24, 0x7b, 0x04, 0xc0, 0x5d, 0x29, 0x1b, 0xf3,
0x30, 0x32, 0xae, 0xd8, 0xbf, 0x6e, 0x4e, 0xfd, 0x64, 0xe2, 0x23, 0xa2, 0xef, 0x7f, 0x03, 0x00,
0x00, 0xff, 0xff, 0xd7, 0x8e, 0x51, 0xb4, 0xe5, 0x01, 0x00, 0x00,
// 341 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0x4f, 0x4f, 0xe3, 0x30,
0x10, 0xc5, 0x95, 0xfe, 0x4d, 0xa6, 0x7f, 0x76, 0xd7, 0xa7, 0x6c, 0x57, 0x2b, 0xa2, 0x82, 0x44,
0x2e, 0xa4, 0x52, 0xb9, 0xc0, 0xb1, 0x07, 0x24, 0x24, 0x04, 0x07, 0x8b, 0x2b, 0x8a, 0x5c, 0x3a,
0x4d, 0x2c, 0x92, 0x38, 0x72, 0xdc, 0xaa, 0xfd, 0x22, 0x7c, 0x5e, 0xd4, 0x71, 0x92, 0x72, 0xe2,
0x92, 0xd8, 0x6f, 0x7e, 0x7a, 0xf3, 0xfc, 0xe0, 0x6f, 0xb1, 0xcd, 0x54, 0xb2, 0xa0, 0x6f, 0xb9,
0xb6, 0xff, 0xa8, 0xd4, 0xca, 0x28, 0x36, 0xac, 0xc5, 0xd9, 0x45, 0xa2, 0x54, 0x92, 0xe1, 0x82,
0xe4, 0xf5, 0x6e, 0xbb, 0x30, 0x32, 0xc7, 0xca, 0x88, 0xbc, 0xb4, 0xe4, 0xfc, 0x0d, 0x5c, 0x8e,
0xef, 0x28, 0xf7, 0xa8, 0xd9, 0x7f, 0x80, 0x44, 0xab, 0x5d, 0x19, 0x17, 0x22, 0x47, 0xdf, 0x09,
0x9c, 0xd0, 0xe3, 0x1e, 0x29, 0x2f, 0x22, 0x47, 0x16, 0xc0, 0x48, 0x16, 0x06, 0x13, 0x2d, 0x8c,
0x54, 0x85, 0xdf, 0xa1, 0xf9, 0x77, 0x89, 0xfd, 0x86, 0xae, 0xdc, 0x1c, 0xfc, 0x6e, 0xe0, 0x84,
0x13, 0x7e, 0x3a, 0xce, 0x3f, 0x3b, 0xd0, 0x7f, 0x28, 0x8c, 0x3e, 0xb2, 0x7f, 0x60, 0xad, 0xe2,
0x0f, 0x3c, 0x92, 0xf7, 0x98, 0xbb, 0x24, 0x3c, 0xe1, 0x91, 0xdd, 0x80, 0xab, 0xeb, 0x14, 0xe4,
0x3b, 0x5a, 0xfe, 0x89, 0xea, 0x27, 0x44, 0x4d, 0x3c, 0xde, 0x22, 0xe7, 0xa0, 0xa9, 0xa8, 0x52,
0x5a, 0x37, 0xae, 0x83, 0x3e, 0x8a, 0x2a, 0x65, 0xb3, 0x93, 0x5b, 0xa5, 0xb2, 0x3d, 0x6e, 0xfc,
0x5e, 0xe0, 0x84, 0x2e, 0x6f, 0xef, 0xec, 0x0e, 0xbc, 0xb6, 0x02, 0xbf, 0x4f, 0xab, 0x66, 0x91,
0x2d, 0x29, 0x6a, 0x4a, 0x8a, 0x5e, 0x1b, 0x82, 0x9f, 0x61, 0x76, 0x09, 0x93, 0xad, 0xd4, 0xb2,
0x48, 0x62, 0x91, 0xa1, 0x36, 0x95, 0x3f, 0x08, 0xba, 0x61, 0x8f, 0x8f, 0xad, 0xb8, 0x22, 0x8d,
0x5d, 0xc3, 0xaf, 0x66, 0x55, 0x83, 0x0d, 0x09, 0x9b, 0x36, 0xb2, 0x05, 0xe7, 0x19, 0x78, 0xcf,
0x58, 0xa5, 0xb6, 0x9b, 0x2b, 0xe8, 0xe3, 0xe9, 0x40, 0xbd, 0x8c, 0x96, 0xd3, 0xf6, 0xed, 0x34,
0xe6, 0x76, 0xc8, 0xee, 0x01, 0xf0, 0x50, 0x4a, 0x8d, 0x55, 0x2c, 0x4c, 0x5d, 0xd3, 0x8f, 0xd9,
0x6b, 0x7a, 0x65, 0xd6, 0x03, 0x1a, 0xdf, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0xe7, 0x4a, 0xda,
0xe6, 0x33, 0x02, 0x00, 0x00,
}

6
nflog/nflogpb/nflog.proto
View File

@ -22,11 +22,17 @@ message Entry {
// The receiver that was notified.
Receiver receiver = 2;
// Hash over the state of the group at notification time.
// Deprecated in favor of FiringAlerts field, but kept for compatibility.
bytes group_hash = 3;
// Whether the notification was about a resolved alert.
// Deprecated in favor of ResolvedAlerts field, but kept for compatibility.
bool resolved = 4;
// Timestamp of the succeeding notification.
google.protobuf.Timestamp timestamp = 5;
// FiringAlerts list of hashes of firing alerts at the last notification time.
repeated uint64 firing_alerts = 6;
// ResolvedAlerts list of hashes of resolved alerts at the last notification time.
repeated uint64 resolved_alerts = 7;
}
// MeshEntry is a wrapper message to communicate a notify log

47
nflog/nflogpb/set.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright 2017 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nflogpb
// IsFiringSubset returns whether the given subset is a subset of the alerts
// that were firing at the time of the last notification.
func (m *Entry) IsFiringSubset(subset map[uint64]struct{}) bool {
set := map[uint64]struct{}{}
for i := range m.FiringAlerts {
set[m.FiringAlerts[i]] = struct{}{}
}
return isSubset(set, subset)
}
// IsFiringSubset returns whether the given subset is a subset of the alerts
// that were resolved at the time of the last notification.
func (m *Entry) IsResolvedSubset(subset map[uint64]struct{}) bool {
set := map[uint64]struct{}{}
for i := range m.ResolvedAlerts {
set[m.ResolvedAlerts[i]] = struct{}{}
}
return isSubset(set, subset)
}
func isSubset(set, subset map[uint64]struct{}) bool {
for k, _ := range subset {
_, exists := set[k]
if !exists {
return false
}
}
return true
}

175
notify/notify.go
View File

@ -14,14 +14,14 @@
package notify
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"fmt"
"sort"
"sync"
"time"
"github.com/cenkalti/backoff"
"github.com/cespare/xxhash"
"github.com/golang/protobuf/ptypes"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
@ -69,7 +69,8 @@ const (
keyRepeatInterval
keyGroupLabels
keyGroupKey
keyNotificationHash
keyFiringAlerts
keyResolvedAlerts
keyNow
)
@ -83,9 +84,14 @@ func WithGroupKey(ctx context.Context, fp model.Fingerprint) context.Context {
return context.WithValue(ctx, keyGroupKey, fp)
}
// WithNotificationHash populates a context with a notification hash.
func WithNotificationHash(ctx context.Context, hash []byte) context.Context {
return context.WithValue(ctx, keyNotificationHash, hash)
// WithFiringAlerts populates a context with a slice of firing alerts.
func WithFiringAlerts(ctx context.Context, alerts []uint64) context.Context {
return context.WithValue(ctx, keyFiringAlerts, alerts)
}
// WithResolvedAlerts populates a context with a slice of resolved alerts.
func WithResolvedAlerts(ctx context.Context, alerts []uint64) context.Context {
return context.WithValue(ctx, keyResolvedAlerts, alerts)
}
// WithGroupLabels populates a context with grouping labels.
@ -154,10 +160,17 @@ func Now(ctx context.Context) (time.Time, bool) {
return v, ok
}
// NotificationHash extracts a notification hash from the context. Iff none exists,
// the second argument is false.
func NotificationHash(ctx context.Context) ([]byte, bool) {
v, ok := ctx.Value(keyNotificationHash).([]byte)
// FiringAlerts extracts a slice of firing alerts from the context.
// Iff none exists, the second argument is false.
func FiringAlerts(ctx context.Context) ([]uint64, bool) {
v, ok := ctx.Value(keyFiringAlerts).([]uint64)
return v, ok
}
// ResolvedAlerts extracts a slice of firing alerts from the context.
// Iff none exists, the second argument is false.
func ResolvedAlerts(ctx context.Context) ([]uint64, bool) {
v, ok := ctx.Value(keyResolvedAlerts).([]uint64)
return v, ok
}
@ -206,7 +219,7 @@ func createStage(rc *config.Receiver, tmpl *template.Template, wait func() time.
}
var s MultiStage
s = append(s, NewWaitStage(wait))
s = append(s, NewDedupStage(notificationLog, recv))
s = append(s, NewDedupStage(notificationLog, recv, i.conf.SendResolved()))
s = append(s, NewRetryStage(i))
s = append(s, NewSetNotifiesStage(notificationLog, recv))
@ -382,26 +395,22 @@ func (ws *WaitStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.
// DedupStage filters alerts.
// Filtering happens based on a notification log.
type DedupStage struct {
nflog nflog.Log
recv *nflogpb.Receiver
nflog nflog.Log
recv *nflogpb.Receiver
sendResolved bool
// TODO(fabxc): consider creating an AlertBatch type received
// by stages that implements these functions.
// This can then also handle caching so we can skip passing
// the hash around as a context.
hash func([]*types.Alert) []byte
resolved func([]*types.Alert) bool
now func() time.Time
now func() time.Time
hash func(*types.Alert) uint64
}
// NewDedupStage wraps a DedupStage that runs against the given notification log.
func NewDedupStage(l nflog.Log, recv *nflogpb.Receiver) *DedupStage {
func NewDedupStage(l nflog.Log, recv *nflogpb.Receiver, sendResolved bool) *DedupStage {
return &DedupStage{
nflog: l,
recv: recv,
hash: hashAlerts,
resolved: allAlertsResolved,
now: utcNow,
nflog: l,
recv: recv,
now: utcNow,
sendResolved: sendResolved,
hash: hashAlert,
}
}
@ -409,28 +418,42 @@ func utcNow() time.Time {
return time.Now().UTC()
}
// TODO(fabxc): this could get slow, but is fine for now. We may want to
// have something mor sophisticated at some point.
// Alternatives are FNV64a as in fingerprints or xxhash.
func hashAlerts(alerts []*types.Alert) []byte {
// The xor'd sum so we don't have to sort the alerts.
// XXX(fabxc): this approach caused collision issues with FNV64a in
// the past. However, sha256 should not suffer from the bit cancelation
// in in small input changes.
xsum := [sha256.Size]byte{}
var hashBuffers = sync.Pool{}
for _, a := range alerts {
b := make([]byte, 9)
binary.BigEndian.PutUint64(b, uint64(a.Fingerprint()))
// Resolved status is part of the identity.
if a.Resolved() {
b[8] = 1
}
for i, b := range sha256.Sum256(b) {
xsum[i] ^= b
}
func getHashBuffer() []byte {
b := hashBuffers.Get()
if b == nil {
return make([]byte, 0, 1024)
}
return xsum[:]
return b.([]byte)
}
func putHashBuffer(b []byte) {
b = b[:0]
hashBuffers.Put(b)
}
func hashAlert(a *types.Alert) uint64 {
const sep = '\xff'
b := getHashBuffer()
labelNames := make(model.LabelNames, 0, len(a.Labels))
for labelName, _ := range a.Labels {
labelNames = append(labelNames, labelName)
}
sort.Sort(labelNames)
for _, labelName := range labelNames {
b = append(b, string(labelName)...)
b = append(b, sep)
b = append(b, string(a.Labels[labelName])...)
b = append(b, sep)
}
hash := xxhash.Sum64(b)
putHashBuffer(b)
return hash
}
func allAlertsResolved(alerts []*types.Alert) bool {
@ -442,14 +465,18 @@ func allAlertsResolved(alerts []*types.Alert) bool {
return true
}
func (n *DedupStage) needsUpdate(entry *nflogpb.Entry, hash []byte, resolved bool, repeat time.Duration) (bool, error) {
func (n *DedupStage) needsUpdate(entry *nflogpb.Entry, firing, resolved map[uint64]struct{}, repeat time.Duration) (bool, error) {
// If we haven't notified about the alert group before, notify right away
// unless we only have resolved alerts.
if entry == nil {
return !resolved, nil
return ((len(firing) > 0) || (n.sendResolved && len(resolved) > 0)), nil
}
// Check whether the contents have changed.
if !bytes.Equal(entry.GroupHash, hash) {
if !entry.IsFiringSubset(firing) {
return true, nil
}
if n.sendResolved && !entry.IsResolvedSubset(resolved) {
return true, nil
}
@ -476,10 +503,25 @@ func (n *DedupStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.
return ctx, nil, fmt.Errorf("repeat interval missing")
}
hash := n.hash(alerts)
resolved := n.resolved(alerts)
firingSet := map[uint64]struct{}{}
resolvedSet := map[uint64]struct{}{}
firing := []uint64{}
resolved := []uint64{}
ctx = WithNotificationHash(ctx, hash)
var hash uint64
for _, a := range alerts {
hash = n.hash(a)
if a.Resolved() {
resolved = append(resolved, hash)
resolvedSet[hash] = struct{}{}
} else {
firing = append(firing, hash)
firingSet[hash] = struct{}{}
}
}
ctx = WithFiringAlerts(ctx, firing)
ctx = WithResolvedAlerts(ctx, resolved)
entries, err := n.nflog.Query(nflog.QGroupKey(gkeyb), nflog.QReceiver(n.recv))
@ -494,13 +536,12 @@ func (n *DedupStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.
case 2:
return ctx, nil, fmt.Errorf("Unexpected entry result size %d", len(entries))
}
if ok, err := n.needsUpdate(entry, hash, resolved, repeatInterval); err != nil {
if ok, err := n.needsUpdate(entry, firingSet, resolvedSet, repeatInterval); err != nil {
return ctx, nil, err
} else if ok {
return ctx, alerts, nil
}
return ctx, nil, nil
}
// RetryStage notifies via passed integration with exponential backoff until it
@ -570,25 +611,18 @@ func (r RetryStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.C
type SetNotifiesStage struct {
nflog nflog.Log
recv *nflogpb.Receiver
resolved func([]*types.Alert) bool
}
// NewSetNotifiesStage returns a new instance of a SetNotifiesStage.
func NewSetNotifiesStage(l nflog.Log, recv *nflogpb.Receiver) *SetNotifiesStage {
return &SetNotifiesStage{
nflog: l,
recv: recv,
resolved: allAlertsResolved,
nflog: l,
recv: recv,
}
}
// Exec implements the Stage interface.
func (n SetNotifiesStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {
hash, ok := NotificationHash(ctx)
if !ok {
return ctx, nil, fmt.Errorf("notification hash missing")
}
gkey, ok := GroupKey(ctx)
if !ok {
return ctx, nil, fmt.Errorf("group key missing")
@ -596,8 +630,15 @@ func (n SetNotifiesStage) Exec(ctx context.Context, alerts ...*types.Alert) (con
gkeyb := make([]byte, 8)
binary.BigEndian.PutUint64(gkeyb, uint64(gkey))
if n.resolved(alerts) {
return ctx, alerts, n.nflog.LogResolved(n.recv, gkeyb, hash)
firing, ok := FiringAlerts(ctx)
if !ok {
return ctx, nil, fmt.Errorf("firing alerts missing")
}
return ctx, alerts, n.nflog.LogActive(n.recv, gkeyb, hash)
resolved, ok := ResolvedAlerts(ctx)
if !ok {
return ctx, nil, fmt.Errorf("resolved alerts missing")
}
return ctx, alerts, n.nflog.Log(n.recv, gkeyb, firing, resolved)
}

147
notify/notify_test.go
View File

@ -55,20 +55,15 @@ type testNflog struct {
qres []*nflogpb.Entry
qerr error
logActiveFunc func(r *nflogpb.Receiver, gkey, hash []byte) error
logResolvedFunc func(r *nflogpb.Receiver, gkey, hash []byte) error
logFunc func(r *nflogpb.Receiver, gkey []byte, firingAlerts, resolvedAlerts []uint64) error
}
func (l *testNflog) Query(p ...nflog.QueryParam) ([]*nflogpb.Entry, error) {
return l.qres, l.qerr
}
func (l *testNflog) LogActive(r *nflogpb.Receiver, gkey, hash []byte) error {
return l.logActiveFunc(r, gkey, hash)
}
func (l *testNflog) LogResolved(r *nflogpb.Receiver, gkey, hash []byte) error {
return l.logResolvedFunc(r, gkey, hash)
func (l *testNflog) Log(r *nflogpb.Receiver, gkey []byte, firingAlerts, resolvedAlerts []uint64) error {
return l.logFunc(r, gkey, firingAlerts, resolvedAlerts)
}
func (l *testNflog) GC() (int, error) {
@ -87,48 +82,58 @@ func mustTimestampProto(ts time.Time) *timestamp.Timestamp {
return tspb
}
func alertHashSet(hashes ...uint64) map[uint64]struct{} {
res := map[uint64]struct{}{}
for _, h := range hashes {
res[h] = struct{}{}
}
return res
}
func TestDedupStageNeedsUpdate(t *testing.T) {
now := utcNow()
cases := []struct {
entry *nflogpb.Entry
hash []byte
resolved bool
repeat time.Duration
entry *nflogpb.Entry
firingAlerts map[uint64]struct{}
repeat time.Duration
res bool
resErr bool
}{
{
entry: nil,
res: true,
entry: nil,
firingAlerts: alertHashSet(2, 3, 4),
res: true,
}, {
entry: &nflogpb.Entry{GroupHash: []byte{1, 2, 3}},
hash: []byte{2, 3, 4},
res: true,
entry: &nflogpb.Entry{FiringAlerts: []uint64{1, 2, 3}},
firingAlerts: alertHashSet(2, 3, 4),
res: true,
}, {
entry: &nflogpb.Entry{
GroupHash: []byte{1, 2, 3},
Timestamp: nil, // parsing will error
FiringAlerts: []uint64{1, 2, 3},
Timestamp: nil, // parsing will error
},
hash: []byte{1, 2, 3},
resErr: true,
firingAlerts: alertHashSet(1, 2, 3),
resErr: true,
}, {
entry: &nflogpb.Entry{
GroupHash: []byte{1, 2, 3},
Timestamp: mustTimestampProto(now.Add(-9 * time.Minute)),
FiringAlerts: []uint64{1, 2, 3},
Timestamp: mustTimestampProto(now.Add(-9 * time.Minute)),
},
repeat: 10 * time.Minute,
hash: []byte{1, 2, 3},
res: false,
repeat: 10 * time.Minute,
firingAlerts: alertHashSet(1, 2, 3),
res: false,
}, {
entry: &nflogpb.Entry{
GroupHash: []byte{1, 2, 3},
Timestamp: mustTimestampProto(now.Add(-11 * time.Minute)),
FiringAlerts: []uint64{1, 2, 3},
Timestamp: mustTimestampProto(now.Add(-11 * time.Minute)),
},
repeat: 10 * time.Minute,
hash: []byte{1, 2, 3},
res: true,
repeat: 10 * time.Minute,
firingAlerts: alertHashSet(1, 2, 3),
res: true,
},
}
for i, c := range cases {
@ -137,7 +142,7 @@ func TestDedupStageNeedsUpdate(t *testing.T) {
s := &DedupStage{
now: func() time.Time { return now },
}
ok, err := s.needsUpdate(c.entry, c.hash, c.resolved, c.repeat)
ok, err := s.needsUpdate(c.entry, c.firingAlerts, nil, c.repeat)
if c.resErr {
require.Error(t, err)
} else {
@ -148,9 +153,17 @@ func TestDedupStageNeedsUpdate(t *testing.T) {
}
func TestDedupStage(t *testing.T) {
i := 0
now := utcNow()
s := &DedupStage{
hash: func([]*types.Alert) []byte { return []byte{1, 2, 3} },
resolved: func([]*types.Alert) bool { return false },
hash: func(a *types.Alert) uint64 {
res := uint64(i)
i++
return res
},
now: func() time.Time {
return now
},
}
ctx := context.Background()
@ -181,35 +194,41 @@ func TestDedupStage(t *testing.T) {
ctx, res, err = s.Exec(ctx, alerts...)
require.NoError(t, err, "unexpected error on not found log entry")
require.Equal(t, alerts, res, "input alerts differ from result alerts")
// The hash must be added to the context.
hash, ok := NotificationHash(ctx)
require.True(t, ok, "notification has missing in context")
require.Equal(t, []byte{1, 2, 3}, hash, "notification hash does not match")
s.nflog = &testNflog{
qerr: nil,
qres: []*nflogpb.Entry{
{GroupHash: []byte{1, 2, 3}},
{GroupHash: []byte{2, 3, 4}},
{FiringAlerts: []uint64{0, 1, 2}},
{FiringAlerts: []uint64{1, 2, 3}},
},
}
ctx, res, err = s.Exec(ctx, alerts...)
require.Contains(t, err.Error(), "result size")
// Must return no error and no alerts no need to update.
i = 0
s.nflog = &testNflog{
qerr: nflog.ErrNotFound,
qres: []*nflogpb.Entry{
{
FiringAlerts: []uint64{0, 1, 2},
Timestamp: mustTimestampProto(now),
},
},
}
s.resolved = func([]*types.Alert) bool { return true }
ctx, res, err = s.Exec(ctx, alerts...)
require.NoError(t, err)
require.Nil(t, res, "unexpected alerts returned")
// Must return no error and all input alerts on changes.
i = 0
s.nflog = &testNflog{
qerr: nil,
qres: []*nflogpb.Entry{
{GroupHash: []byte{1, 2, 3, 4}},
{
FiringAlerts: []uint64{1, 2, 3, 4},
Timestamp: mustTimestampProto(now),
},
},
}
ctx, res, err = s.Exec(ctx, alerts...)
@ -361,28 +380,31 @@ func TestSetNotifiesStage(t *testing.T) {
ctx := context.Background()
resctx, res, err := s.Exec(ctx, alerts...)
require.EqualError(t, err, "notification hash missing")
require.Nil(t, res)
require.NotNil(t, resctx)
ctx = WithNotificationHash(ctx, []byte{1, 2, 3})
_, res, err = s.Exec(ctx, alerts...)
require.EqualError(t, err, "group key missing")
require.Nil(t, res)
require.NotNil(t, resctx)
ctx = WithGroupKey(ctx, 1)
s.resolved = func([]*types.Alert) bool { return false }
tnflog.logActiveFunc = func(r *nflogpb.Receiver, gkey, hash []byte) error {
resctx, res, err = s.Exec(ctx, alerts...)
require.EqualError(t, err, "firing alerts missing")
require.Nil(t, res)
require.NotNil(t, resctx)
ctx = WithFiringAlerts(ctx, []uint64{0, 1, 2})
resctx, res, err = s.Exec(ctx, alerts...)
require.EqualError(t, err, "resolved alerts missing")
require.Nil(t, res)
require.NotNil(t, resctx)
ctx = WithResolvedAlerts(ctx, []uint64{})
tnflog.logFunc = func(r *nflogpb.Receiver, gkey []byte, firingAlerts, resolvedAlerts []uint64) error {
require.Equal(t, s.recv, r)
require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 1}, gkey)
require.Equal(t, []byte{1, 2, 3}, hash)
return nil
}
tnflog.logResolvedFunc = func(r *nflogpb.Receiver, gkey, hash []byte) error {
t.Fatalf("LogResolved called unexpectedly")
require.Equal(t, []uint64{0, 1, 2}, firingAlerts)
require.Equal(t, []uint64{}, resolvedAlerts)
return nil
}
resctx, res, err = s.Exec(ctx, alerts...)
@ -390,15 +412,14 @@ func TestSetNotifiesStage(t *testing.T) {
require.Equal(t, alerts, res)
require.NotNil(t, resctx)
s.resolved = func([]*types.Alert) bool { return true }
tnflog.logActiveFunc = func(r *nflogpb.Receiver, gkey, hash []byte) error {
t.Fatalf("LogActive called unexpectedly")
return nil
}
tnflog.logResolvedFunc = func(r *nflogpb.Receiver, gkey, hash []byte) error {
ctx = WithFiringAlerts(ctx, []uint64{})
ctx = WithResolvedAlerts(ctx, []uint64{0, 1, 2})
tnflog.logFunc = func(r *nflogpb.Receiver, gkey []byte, firingAlerts, resolvedAlerts []uint64) error {
require.Equal(t, s.recv, r)
require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 1}, gkey)
require.Equal(t, []byte{1, 2, 3}, hash)
require.Equal(t, []uint64{}, firingAlerts)
require.Equal(t, []uint64{0, 1, 2}, resolvedAlerts)
return nil
}
resctx, res, err = s.Exec(ctx, alerts...)

92
silence/silencepb/silence.pb.go
View File

@ -70,6 +70,27 @@ func (m *Matcher) String() string { return proto.CompactTextString(m)
func (*Matcher) ProtoMessage() {}
func (*Matcher) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *Matcher) GetType() Matcher_Type {
if m != nil {
return m.Type
}
return Matcher_EQUAL
}
func (m *Matcher) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *Matcher) GetPattern() string {
if m != nil {
return m.Pattern
}
return ""
}
// A comment can be attached to a silence.
type Comment struct {
Author string `protobuf:"bytes,1,opt,name=author" json:"author,omitempty"`
@ -82,6 +103,20 @@ func (m *Comment) String() string { return proto.CompactTextString(m)
func (*Comment) ProtoMessage() {}
func (*Comment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *Comment) GetAuthor() string {
if m != nil {
return m.Author
}
return ""
}
func (m *Comment) GetComment() string {
if m != nil {
return m.Comment
}
return ""
}
func (m *Comment) GetTimestamp() *google_protobuf.Timestamp {
if m != nil {
return m.Timestamp
@ -111,6 +146,13 @@ func (m *Silence) String() string { return proto.CompactTextString(m)
func (*Silence) ProtoMessage() {}
func (*Silence) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *Silence) GetId() string {
if m != nil {
return m.Id
}
return ""
}
func (m *Silence) GetMatchers() []*Matcher {
if m != nil {
return m.Matchers
@ -183,29 +225,29 @@ func init() {
func init() { proto.RegisterFile("silence/silencepb/silence.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 372 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x91, 0xcf, 0x4b, 0xeb, 0x40,
0x10, 0xc7, 0x5f, 0xd2, 0x34, 0x69, 0xa6, 0x50, 0xca, 0x1e, 0xde, 0x0b, 0x85, 0x47, 0x25, 0x27,
0x41, 0xd9, 0x42, 0x7b, 0x50, 0x8f, 0x45, 0x8a, 0x17, 0x0b, 0xba, 0x56, 0xf0, 0x26, 0x69, 0x33,
0xb6, 0x81, 0xe6, 0x07, 0xc9, 0x44, 0xf4, 0xec, 0x7f, 0xe2, 0x5f, 0xea, 0x66, 0xb3, 0x89, 0x48,
0x0f, 0xf5, 0x94, 0x99, 0xcc, 0xe7, 0x3b, 0x33, 0xdf, 0x59, 0x18, 0x17, 0xd1, 0x1e, 0x93, 0x0d,
0x4e, 0xf4, 0x37, 0x5b, 0x37, 0x11, 0xcf, 0xf2, 0x94, 0x52, 0xe6, 0xb6, 0x85, 0xd1, 0x78, 0x9b,
0xa6, 0xdb, 0x3d, 0x4e, 0x54, 0x61, 0x5d, 0xbe, 0x4c, 0x28, 0x8a, 0xb1, 0xa0, 0x20, 0xce, 0x6a,
0xd6, 0xff, 0x30, 0xc0, 0x59, 0x06, 0xb4, 0xd9, 0x61, 0xce, 0xce, 0xc0, 0xa2, 0xf7, 0x0c, 0x3d,
0xe3, 0xc4, 0x38, 0x1d, 0x4c, 0xff, 0xf1, 0xb6, 0x0d, 0xd7, 0x04, 0x5f, 0xc9, 0xb2, 0x50, 0x10,
0x63, 0x60, 0x25, 0x41, 0x8c, 0x9e, 0x29, 0x61, 0x57, 0xa8, 0x98, 0x79, 0xe0, 0x64, 0x01, 0x11,
0xe6, 0x89, 0xd7, 0x51, 0xbf, 0x9b, 0xd4, 0xff, 0x0f, 0x56, 0xa5, 0x65, 0x2e, 0x74, 0x17, 0xf7,
0x8f, 0xf3, 0xdb, 0xe1, 0x1f, 0x06, 0x60, 0x8b, 0xc5, 0xcd, 0xe2, 0xe9, 0x6e, 0x68, 0xf8, 0x25,
0x38, 0xd7, 0x69, 0x1c, 0x63, 0x42, 0xec, 0x2f, 0xd8, 0x41, 0x49, 0xbb, 0x34, 0x57, 0x6b, 0xb8,
0x42, 0x67, 0x55, 0xef, 0x4d, 0x8d, 0xe8, 0x91, 0x4d, 0xca, 0x2e, 0xc1, 0x6d, 0x5d, 0xa9, 0xb9,
0xfd, 0xe9, 0x88, 0xd7, 0xbe, 0x79, 0xe3, 0x9b, 0xaf, 0x1a, 0x42, 0x7c, 0xc3, 0xfe, 0xa7, 0x09,
0xce, 0x43, 0x6d, 0x92, 0x0d, 0xc0, 0x8c, 0x42, 0x3d, 0x53, 0x46, 0x8c, 0x43, 0x2f, 0xae, 0x5d,
0x17, 0x72, 0x60, 0x47, 0x36, 0x65, 0x87, 0x07, 0x11, 0x2d, 0xc3, 0x2e, 0xc0, 0x95, 0x4d, 0x73,
0x2a, 0x9e, 0x03, 0xfa, 0xc5, 0x16, 0xbd, 0x1a, 0x9e, 0x13, 0x9b, 0x81, 0x83, 0x49, 0xa8, 0x64,
0xd6, 0x51, 0x99, 0x5d, 0xa1, 0x52, 0x74, 0x05, 0x50, 0x66, 0x61, 0x40, 0x18, 0x56, 0xba, 0xee,
0x71, 0xd3, 0x9a, 0x96, 0x52, 0x69, 0x4c, 0x5f, 0xae, 0xf0, 0x9c, 0x03, 0x63, 0xfa, 0x19, 0x44,
0xcb, 0xf8, 0xaf, 0xd0, 0x5f, 0x62, 0xb1, 0x6b, 0xee, 0x74, 0x0e, 0x8e, 0xa6, 0xd5, 0xb1, 0x7e,
0xaa, 0x35, 0x24, 0x1a, 0xa4, 0xda, 0x13, 0xdf, 0xb2, 0x28, 0x47, 0xe5, 0xcf, 0x3c, 0xbe, 0xa7,
0xa6, 0xe7, 0xb4, 0xb6, 0x55, 0x79, 0xf6, 0x15, 0x00, 0x00, 0xff, 0xff, 0x7d, 0xe9, 0xc3, 0x5f,
0xef, 0x02, 0x00, 0x00,
// 376 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x91, 0xc1, 0x6a, 0xdb, 0x40,
0x10, 0x40, 0x2b, 0x59, 0x96, 0xac, 0x31, 0x18, 0xb3, 0x87, 0x56, 0x18, 0x8a, 0x8d, 0x4e, 0x86,
0x96, 0x35, 0xd8, 0x87, 0xb6, 0x47, 0x51, 0x4c, 0x2f, 0x35, 0xb4, 0x1b, 0x07, 0x72, 0x0b, 0x6b,
0x6b, 0x62, 0x0b, 0x2c, 0x69, 0x91, 0x46, 0x21, 0x3e, 0xe7, 0x4f, 0xf2, 0xa5, 0x41, 0xab, 0x95,
0x42, 0xf0, 0xc1, 0x39, 0x69, 0x47, 0xf3, 0x66, 0x67, 0xde, 0x2c, 0x4c, 0xcb, 0xe4, 0x84, 0xd9,
0x1e, 0x17, 0xe6, 0xab, 0x76, 0xed, 0x89, 0xab, 0x22, 0xa7, 0x9c, 0xf9, 0x5d, 0x62, 0x32, 0x3d,
0xe4, 0xf9, 0xe1, 0x84, 0x0b, 0x9d, 0xd8, 0x55, 0x0f, 0x0b, 0x4a, 0x52, 0x2c, 0x49, 0xa6, 0xaa,
0x61, 0xc3, 0x67, 0x0b, 0xbc, 0x8d, 0xa4, 0xfd, 0x11, 0x0b, 0xf6, 0x0d, 0x1c, 0x3a, 0x2b, 0x0c,
0xac, 0x99, 0x35, 0x1f, 0x2d, 0xbf, 0xf0, 0xee, 0x1a, 0x6e, 0x08, 0xbe, 0x3d, 0x2b, 0x14, 0x1a,
0x62, 0x0c, 0x9c, 0x4c, 0xa6, 0x18, 0xd8, 0x33, 0x6b, 0xee, 0x0b, 0x7d, 0x66, 0x01, 0x78, 0x4a,
0x12, 0x61, 0x91, 0x05, 0x3d, 0xfd, 0xbb, 0x0d, 0xc3, 0xaf, 0xe0, 0xd4, 0xb5, 0xcc, 0x87, 0xfe,
0xfa, 0xff, 0x6d, 0xf4, 0x77, 0xfc, 0x89, 0x01, 0xb8, 0x62, 0xfd, 0x67, 0x7d, 0xf7, 0x6f, 0x6c,
0x85, 0x15, 0x78, 0xbf, 0xf3, 0x34, 0xc5, 0x8c, 0xd8, 0x67, 0x70, 0x65, 0x45, 0xc7, 0xbc, 0xd0,
0x63, 0xf8, 0xc2, 0x44, 0xf5, 0xdd, 0xfb, 0x06, 0x31, 0x2d, 0xdb, 0x90, 0xfd, 0x04, 0xbf, 0xb3,
0xd2, 0x7d, 0x87, 0xcb, 0x09, 0x6f, 0xbc, 0x79, 0xeb, 0xcd, 0xb7, 0x2d, 0x21, 0xde, 0xe0, 0xf0,
0xc5, 0x06, 0xef, 0xa6, 0x91, 0x64, 0x23, 0xb0, 0x93, 0xd8, 0xf4, 0xb4, 0x93, 0x98, 0x71, 0x18,
0xa4, 0x8d, 0x75, 0x19, 0xd8, 0xb3, 0xde, 0x7c, 0xb8, 0x64, 0x97, 0x0b, 0x11, 0x1d, 0xc3, 0x7e,
0x80, 0x5f, 0x92, 0x2c, 0xa8, 0xbc, 0x97, 0xf4, 0x81, 0x29, 0x06, 0x0d, 0x1c, 0x11, 0x5b, 0x81,
0x87, 0x59, 0xac, 0xcb, 0x9c, 0xab, 0x65, 0x6e, 0x8d, 0x46, 0xc4, 0x7e, 0x01, 0x54, 0x2a, 0x96,
0x84, 0x71, 0x5d, 0xd7, 0xbf, 0x2e, 0x6d, 0xe8, 0x88, 0x6a, 0x31, 0xb3, 0xb9, 0x32, 0xf0, 0x2e,
0xc4, 0xcc, 0x33, 0x88, 0x8e, 0x09, 0x1f, 0x61, 0xb8, 0xc1, 0xf2, 0xd8, 0xee, 0xe9, 0x3b, 0x78,
0x86, 0xd6, 0xcb, 0x7a, 0x5f, 0x6d, 0x20, 0xd1, 0x22, 0xf5, 0x9c, 0xf8, 0xa4, 0x92, 0x02, 0xb5,
0x9f, 0x7d, 0x7d, 0x4e, 0x43, 0x47, 0xb4, 0x73, 0x75, 0x7a, 0xf5, 0x1a, 0x00, 0x00, 0xff, 0xff,
0x7d, 0xe9, 0xc3, 0x5f, 0xef, 0x02, 0x00, 0x00,
}

85
test/acceptance/send_test.go
View File

@ -334,12 +334,7 @@ func TestResolvedFilter(t *testing.T) {
// filtered, have to end up in the SetNotifiesStage, otherwise when an alert
// fires again it is ambiguous whether it was resolved in between or not.
var wg sync.WaitGroup
wg.Add(10)
for i := 0; i < 10; i++ {
go func() {
conf := `
conf := `
global:
resolve_timeout: 10s
@ -358,48 +353,52 @@ receivers:
send_resolved: false
`
at := NewAcceptanceTest(t, &AcceptanceOpts{
Tolerance: 150 * time.Millisecond,
})
at := NewAcceptanceTest(t, &AcceptanceOpts{
Tolerance: 150 * time.Millisecond,
})
co1 := at.Collector("webhook1")
wh1 := NewWebhook(co1)
co1 := at.Collector("webhook1")
wh1 := NewWebhook(co1)
co2 := at.Collector("webhook2")
wh2 := NewWebhook(co2)
co2 := at.Collector("webhook2")
wh2 := NewWebhook(co2)
am := at.Alertmanager(fmt.Sprintf(conf, wh1.Address(), wh2.Address()))
am := at.Alertmanager(fmt.Sprintf(conf, wh1.Address(), wh2.Address()))
am.Push(At(1),
Alert("alertname", "test", "lbl", "v1"),
Alert("alertname", "test", "lbl", "v2"),
)
am.Push(At(1),
Alert("alertname", "test", "lbl", "v1"),
Alert("alertname", "test", "lbl", "v2"),
)
am.Push(At(3),
Alert("alertname", "test", "lbl", "v1").Active(1, 4),
Alert("alertname", "test", "lbl", "v3"),
)
am.Push(At(8),
Alert("alertname", "test", "lbl", "v3").Active(3),
)
am.Push(At(7),
Alert("alertname", "test", "lbl", "v1"),
)
co1.Want(Between(2, 2.5),
Alert("alertname", "test", "lbl", "v1").Active(1),
Alert("alertname", "test", "lbl", "v2").Active(1),
)
co1.Want(Between(7, 7.5),
Alert("alertname", "test", "lbl", "v1").Active(1, 4),
Alert("alertname", "test", "lbl", "v2").Active(1),
Alert("alertname", "test", "lbl", "v3").Active(3),
)
co1.Want(Between(12, 12.5),
Alert("alertname", "test", "lbl", "v2").Active(1, 11),
Alert("alertname", "test", "lbl", "v3").Active(3),
)
co1.Want(Between(2, 2.5),
Alert("alertname", "test", "lbl", "v1").Active(1),
Alert("alertname", "test", "lbl", "v2").Active(1),
)
co1.Want(Between(12, 13),
Alert("alertname", "test", "lbl", "v1").Active(1),
Alert("alertname", "test", "lbl", "v2").Active(1, 11),
)
co2.Want(Between(2, 2.5),
Alert("alertname", "test", "lbl", "v1").Active(1),
Alert("alertname", "test", "lbl", "v2").Active(1),
)
co2.Want(Between(7, 7.5),
Alert("alertname", "test", "lbl", "v2").Active(1),
Alert("alertname", "test", "lbl", "v3").Active(3),
)
co2.Want(Between(2, 2.5),
Alert("alertname", "test", "lbl", "v1").Active(1),
Alert("alertname", "test", "lbl", "v2").Active(1),
)
co2.Want(Between(12, 13),
Alert("alertname", "test", "lbl", "v1").Active(1),
)
at.Run()
wg.Done()
}()
}
wg.Wait()
at.Run()
}