2015-09-25 16:14:46 +00:00
|
|
|
package main
|
2015-07-02 16:38:05 +00:00
|
|
|
|
|
|
|
import (
|
2015-09-27 10:27:32 +00:00
|
|
|
"sync"
|
|
|
|
|
2015-09-25 16:14:46 +00:00
|
|
|
"github.com/prometheus/common/model"
|
2015-07-02 16:38:05 +00:00
|
|
|
"github.com/prometheus/log"
|
2015-09-24 22:14:41 +00:00
|
|
|
"golang.org/x/net/context"
|
2015-09-25 16:14:46 +00:00
|
|
|
|
2015-09-26 16:13:15 +00:00
|
|
|
"github.com/prometheus/alertmanager/config"
|
2015-09-27 10:27:32 +00:00
|
|
|
"github.com/prometheus/alertmanager/provider"
|
2015-09-25 16:14:46 +00:00
|
|
|
"github.com/prometheus/alertmanager/types"
|
2015-07-02 16:38:05 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type Notifier interface {
|
2015-09-26 12:12:55 +00:00
|
|
|
Notify(context.Context, ...*types.Alert) error
|
2015-07-02 16:38:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type LogNotifier struct {
|
2015-07-04 12:41:10 +00:00
|
|
|
name string
|
2015-07-02 16:38:05 +00:00
|
|
|
}
|
|
|
|
|
2015-09-26 12:12:55 +00:00
|
|
|
func (ln *LogNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
2015-07-04 12:41:10 +00:00
|
|
|
log.Infof("notify %q", ln.name)
|
|
|
|
|
2015-09-26 12:12:55 +00:00
|
|
|
for _, a := range alerts {
|
2015-09-26 15:54:49 +00:00
|
|
|
log.Infof("- %v", a)
|
2015-09-26 12:12:55 +00:00
|
|
|
}
|
2015-07-04 12:41:10 +00:00
|
|
|
return nil
|
2015-07-02 16:38:05 +00:00
|
|
|
}
|
2015-09-24 22:14:41 +00:00
|
|
|
|
2015-09-27 10:27:32 +00:00
|
|
|
// silencedNotifier wraps a notifier and applies a Silencer
|
|
|
|
// before sending out an alert.
|
|
|
|
type silencedNotifier struct {
|
|
|
|
Notifier
|
|
|
|
|
|
|
|
silencer types.Silencer
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *silencedNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
|
|
|
|
var filtered []*types.Alert
|
|
|
|
for _, a := range alerts {
|
|
|
|
// TODO(fabxc): increment total alerts counter.
|
|
|
|
// Do not send the alert if the silencer mutes it.
|
|
|
|
if !n.silencer.Mutes(a.Labels) {
|
|
|
|
// TODO(fabxc): increment muted alerts counter.
|
|
|
|
filtered = append(filtered, a)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return n.Notifier.Notify(ctx, filtered...)
|
|
|
|
}
|
|
|
|
|
|
|
|
type Inhibitor struct {
|
|
|
|
alerts provider.Alerts
|
|
|
|
rules []*InhibitRule
|
|
|
|
|
|
|
|
mtx sync.RWMutex
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
|
|
|
|
ih.mtx.RLock()
|
|
|
|
defer ih.mtx.RUnlock()
|
|
|
|
|
|
|
|
alerts, err := ih.alerts.All()
|
|
|
|
if err != nil {
|
|
|
|
// TODO(fabxc): log error.
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, alert := range alerts {
|
|
|
|
if alert.Resolved() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, rule := range ih.rules {
|
|
|
|
if rule.Mutes(alert.Labels, lset) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ih *Inhibitor) ApplyConfig(conf *config.Config) {
|
|
|
|
ih.mtx.Lock()
|
|
|
|
defer ih.mtx.Unlock()
|
|
|
|
|
|
|
|
ih.rules = []*InhibitRule{}
|
|
|
|
for _, cr := range conf.InhibitRules {
|
|
|
|
ih.rules = append(ih.rules, NewInhibitRule(cr))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-25 16:14:46 +00:00
|
|
|
// An InhibitRule specifies that a class of (source) alerts should inhibit
|
|
|
|
// notifications for another class of (target) alerts if all specified matching
|
|
|
|
// labels are equal between the two alerts. This may be used to inhibit alerts
|
|
|
|
// from sending notifications if their meaning is logically a subset of a
|
|
|
|
// higher-level alert.
|
|
|
|
type InhibitRule struct {
|
|
|
|
// The set of Filters which define the group of source alerts (which inhibit
|
|
|
|
// the target alerts).
|
|
|
|
SourceMatchers types.Matchers
|
|
|
|
// The set of Filters which define the group of target alerts (which are
|
|
|
|
// inhibited by the source alerts).
|
|
|
|
TargetMatchers types.Matchers
|
|
|
|
// A set of label names whose label values need to be identical in source and
|
|
|
|
// target alerts in order for the inhibition to take effect.
|
2015-09-26 16:13:15 +00:00
|
|
|
Equal map[model.LabelName]struct{}
|
|
|
|
}
|
|
|
|
|
2015-09-27 10:27:32 +00:00
|
|
|
func NewInhibitRule(cr *config.InhibitRule) *InhibitRule {
|
2015-09-26 16:13:15 +00:00
|
|
|
var (
|
|
|
|
sourcem types.Matchers
|
|
|
|
targetm types.Matchers
|
|
|
|
equal map[model.LabelName]struct{}
|
|
|
|
)
|
|
|
|
|
|
|
|
for ln, lv := range cr.SourceMatch {
|
|
|
|
sourcem = append(sourcem, types.NewMatcher(model.LabelName(ln), lv))
|
|
|
|
}
|
|
|
|
for ln, lv := range cr.SourceMatchRE {
|
|
|
|
m, err := types.NewRegexMatcher(model.LabelName(ln), lv.String())
|
|
|
|
if err != nil {
|
|
|
|
// Must have been sanitized during config validation.
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
sourcem = append(sourcem, m)
|
|
|
|
}
|
|
|
|
|
|
|
|
for ln, lv := range cr.TargetMatch {
|
|
|
|
targetm = append(targetm, types.NewMatcher(model.LabelName(ln), lv))
|
|
|
|
}
|
|
|
|
for ln, lv := range cr.TargetMatchRE {
|
|
|
|
m, err := types.NewRegexMatcher(model.LabelName(ln), lv.String())
|
|
|
|
if err != nil {
|
|
|
|
// Must have been sanitized during config validation.
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
targetm = append(targetm, m)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, ln := range cr.Equal {
|
|
|
|
equal[ln] = struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
return &InhibitRule{
|
|
|
|
SourceMatchers: sourcem,
|
|
|
|
TargetMatchers: targetm,
|
|
|
|
Equal: equal,
|
|
|
|
}
|
2015-09-24 22:14:41 +00:00
|
|
|
}
|
|
|
|
|
2015-09-27 10:27:32 +00:00
|
|
|
func (r *InhibitRule) Mutes(source, target model.LabelSet) bool {
|
|
|
|
return r.TargetMatchers.Match(target) && r.SourceMatchers.Match(source)
|
2015-09-24 22:14:41 +00:00
|
|
|
}
|