2015-09-25 16:14:46 +00:00
|
|
|
package main
|
2015-07-02 16:38:05 +00:00
|
|
|
|
|
|
|
import (
|
2015-09-27 11:09:02 +00:00
|
|
|
"fmt"
|
2015-09-27 10:27:32 +00:00
|
|
|
"sync"
|
2015-09-27 17:50:41 +00:00
|
|
|
"time"
|
2015-09-27 10:27:32 +00:00
|
|
|
|
2015-09-28 10:12:27 +00:00
|
|
|
"github.com/prometheus/common/log"
|
2015-09-27 17:50:41 +00:00
|
|
|
"github.com/prometheus/common/model"
|
2015-09-24 22:14:41 +00:00
|
|
|
"golang.org/x/net/context"
|
2015-09-25 16:14:46 +00:00
|
|
|
|
2015-09-26 16:13:15 +00:00
|
|
|
"github.com/prometheus/alertmanager/config"
|
2015-09-27 17:50:41 +00:00
|
|
|
"github.com/prometheus/alertmanager/provider"
|
2015-09-25 16:14:46 +00:00
|
|
|
"github.com/prometheus/alertmanager/types"
|
2015-07-02 16:38:05 +00:00
|
|
|
)
|
|
|
|
|
2015-09-27 11:09:02 +00:00
|
|
|
type notifyKey int
|
|
|
|
|
|
|
|
const (
|
|
|
|
notifyName notifyKey = iota
|
2015-09-27 17:50:41 +00:00
|
|
|
notifyRepeatInterval
|
|
|
|
notifySendResolved
|
2015-09-27 11:09:02 +00:00
|
|
|
)
|
|
|
|
|
2015-07-02 16:38:05 +00:00
|
|
|
type Notifier interface {
|
2015-09-26 12:12:55 +00:00
|
|
|
Notify(context.Context, ...*types.Alert) error
|
2015-07-02 16:38:05 +00:00
|
|
|
}
|
|
|
|
|
2015-09-27 17:50:41 +00:00
|
|
|
type dedupingNotifier struct {
|
|
|
|
notifies provider.Notifies
|
|
|
|
notifier Notifier
|
|
|
|
}
|
|
|
|
|
2015-09-27 19:41:30 +00:00
|
|
|
func newDedupingNotifier(notifies provider.Notifies, n Notifier) Notifier {
|
|
|
|
return &dedupingNotifier{
|
|
|
|
notifies: notifies,
|
|
|
|
notifier: n,
|
2015-09-27 18:17:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-27 17:50:41 +00:00
|
|
|
func (n *dedupingNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
|
|
|
name, ok := ctx.Value(notifyName).(string)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("notifier name missing")
|
|
|
|
}
|
|
|
|
|
|
|
|
repeatInterval, ok := ctx.Value(notifyRepeatInterval).(time.Duration)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("repeat interval missing")
|
|
|
|
}
|
|
|
|
|
|
|
|
sendResolved, ok := ctx.Value(notifySendResolved).(bool)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("send resolved missing")
|
|
|
|
}
|
|
|
|
|
|
|
|
var fps []model.Fingerprint
|
|
|
|
for _, a := range alerts {
|
|
|
|
fps = append(fps, a.Fingerprint())
|
|
|
|
}
|
|
|
|
|
|
|
|
notifies, err := n.notifies.Get(name, fps...)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
now := time.Now()
|
|
|
|
|
2015-09-27 17:54:40 +00:00
|
|
|
var newNotifies []*types.Notify
|
|
|
|
|
2015-09-27 17:50:41 +00:00
|
|
|
var filtered []*types.Alert
|
|
|
|
for i, a := range alerts {
|
|
|
|
last := notifies[i]
|
|
|
|
|
2015-09-27 18:17:09 +00:00
|
|
|
if last != nil {
|
|
|
|
// If the initial alert was not delivered successfully,
|
|
|
|
// there is no point in sending a resolved notification.
|
|
|
|
if a.Resolved() && (!last.Delivered || !sendResolved) {
|
2015-09-27 17:50:41 +00:00
|
|
|
continue
|
|
|
|
}
|
2015-09-27 18:17:09 +00:00
|
|
|
|
|
|
|
// Always send if the alert went from resolved to unresolved.
|
|
|
|
if last.Resolved && !a.Resolved() {
|
|
|
|
// Do not send again if last was delivered unless
|
|
|
|
// the repeat interval has already passed.
|
|
|
|
if last.Delivered && !now.After(last.Timestamp.Add(repeatInterval)) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
2015-09-27 17:50:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
filtered = append(filtered, a)
|
2015-09-27 18:17:09 +00:00
|
|
|
|
|
|
|
newNotifies = append(newNotifies, &types.Notify{
|
|
|
|
Alert: a.Fingerprint(),
|
|
|
|
SendTo: name,
|
|
|
|
Delivered: true,
|
|
|
|
Resolved: a.Resolved(),
|
|
|
|
Timestamp: now,
|
|
|
|
})
|
2015-09-27 17:50:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := n.notifier.Notify(ctx, filtered...); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2015-09-27 18:17:09 +00:00
|
|
|
return n.notifies.Set(name, newNotifies...)
|
2015-09-27 17:50:41 +00:00
|
|
|
}
|
|
|
|
|
2015-09-27 11:09:02 +00:00
|
|
|
// routedNotifier dispatches the alerts to one of a set of
|
|
|
|
// named notifiers based on the name value provided in the context.
|
|
|
|
type routedNotifier struct {
|
2015-09-27 17:50:41 +00:00
|
|
|
mtx sync.RWMutex
|
|
|
|
notifiers map[string]Notifier
|
|
|
|
notifierOpts map[string]*config.NotificationConfig
|
2015-09-27 12:37:34 +00:00
|
|
|
|
|
|
|
// build creates a new set of named notifiers based on a config.
|
|
|
|
build func(*config.Config) map[string]Notifier
|
|
|
|
}
|
|
|
|
|
2015-09-27 19:41:30 +00:00
|
|
|
func newRoutedNotifier(build func(*config.Config) map[string]Notifier) *routedNotifier {
|
|
|
|
return &routedNotifier{build: build}
|
2015-09-27 11:09:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (n *routedNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
|
|
|
name, ok := ctx.Value(notifyName).(string)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("notifier name missing")
|
|
|
|
}
|
|
|
|
|
|
|
|
n.mtx.RLock()
|
|
|
|
defer n.mtx.RUnlock()
|
|
|
|
|
|
|
|
notifier, ok := n.notifiers[name]
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("notifier %q does not exist", name)
|
|
|
|
}
|
2015-09-27 17:50:41 +00:00
|
|
|
opts := n.notifierOpts[name]
|
|
|
|
|
|
|
|
// Populate the context with the the filtering options
|
|
|
|
// of the notifier.
|
2015-09-27 18:21:29 +00:00
|
|
|
ctx = context.WithValue(ctx, notifyRepeatInterval, time.Duration(opts.RepeatInterval))
|
2015-09-27 17:50:41 +00:00
|
|
|
ctx = context.WithValue(ctx, notifySendResolved, opts.SendResolved)
|
2015-09-27 11:09:02 +00:00
|
|
|
|
|
|
|
return notifier.Notify(ctx, alerts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *routedNotifier) ApplyConfig(conf *config.Config) {
|
|
|
|
n.mtx.Lock()
|
|
|
|
defer n.mtx.Unlock()
|
|
|
|
|
2015-09-27 12:37:34 +00:00
|
|
|
n.notifiers = n.build(conf)
|
2015-09-27 18:21:29 +00:00
|
|
|
n.notifierOpts = map[string]*config.NotificationConfig{}
|
|
|
|
|
|
|
|
for _, opts := range conf.NotificationConfigs {
|
|
|
|
n.notifierOpts[opts.Name] = opts
|
|
|
|
}
|
2015-09-27 11:09:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// mutingNotifier wraps a notifier and applies a Silencer
|
2015-09-27 10:27:32 +00:00
|
|
|
// before sending out an alert.
|
2015-09-27 11:09:02 +00:00
|
|
|
type mutingNotifier struct {
|
2015-09-27 17:50:41 +00:00
|
|
|
notifier Notifier
|
2015-09-27 10:27:32 +00:00
|
|
|
silencer types.Silencer
|
|
|
|
}
|
|
|
|
|
2015-09-27 11:09:02 +00:00
|
|
|
func (n *mutingNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
2015-09-27 10:27:32 +00:00
|
|
|
var filtered []*types.Alert
|
|
|
|
for _, a := range alerts {
|
|
|
|
// TODO(fabxc): increment total alerts counter.
|
|
|
|
// Do not send the alert if the silencer mutes it.
|
|
|
|
if !n.silencer.Mutes(a.Labels) {
|
|
|
|
// TODO(fabxc): increment muted alerts counter.
|
|
|
|
filtered = append(filtered, a)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-27 17:50:41 +00:00
|
|
|
return n.notifier.Notify(ctx, filtered...)
|
2015-09-27 10:27:32 +00:00
|
|
|
}
|
2015-09-27 12:07:04 +00:00
|
|
|
|
|
|
|
type LogNotifier struct {
|
|
|
|
name string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ln *LogNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
|
|
|
log.Infof("notify %q", ln.name)
|
|
|
|
|
|
|
|
for _, a := range alerts {
|
|
|
|
log.Infof("- %v", a)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|