2015-10-11 15:24:49 +00:00
|
|
|
// Copyright 2015 Prometheus Team
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2015-09-27 11:20:54 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sync"
|
|
|
|
|
2015-09-29 08:45:58 +00:00
|
|
|
"github.com/prometheus/common/log"
|
2015-09-27 11:20:54 +00:00
|
|
|
"github.com/prometheus/common/model"
|
|
|
|
|
|
|
|
"github.com/prometheus/alertmanager/config"
|
|
|
|
"github.com/prometheus/alertmanager/provider"
|
|
|
|
"github.com/prometheus/alertmanager/types"
|
|
|
|
)
|
|
|
|
|
2015-11-05 09:49:32 +00:00
|
|
|
// An Inhibitor determines whether a given label set is muted
|
|
|
|
// based on the currently active alerts and a set of inhibition rules.
|
2015-09-27 11:20:54 +00:00
|
|
|
type Inhibitor struct {
|
|
|
|
alerts provider.Alerts
|
|
|
|
rules []*InhibitRule
|
2015-11-09 13:34:57 +00:00
|
|
|
marker types.Marker
|
2015-09-27 11:20:54 +00:00
|
|
|
|
|
|
|
mtx sync.RWMutex
|
|
|
|
}
|
|
|
|
|
2015-11-05 09:49:32 +00:00
|
|
|
// NewInhibitor returns a new Inhibitor.
|
2015-11-09 13:34:57 +00:00
|
|
|
func NewInhibitor(ap provider.Alerts, rs []*config.InhibitRule, mk types.Marker) *Inhibitor {
|
2015-10-11 14:54:39 +00:00
|
|
|
ih := &Inhibitor{
|
|
|
|
alerts: ap,
|
2015-11-09 13:34:57 +00:00
|
|
|
marker: mk,
|
2015-10-11 14:54:39 +00:00
|
|
|
}
|
|
|
|
for _, cr := range rs {
|
|
|
|
ih.rules = append(ih.rules, NewInhibitRule(cr))
|
|
|
|
}
|
|
|
|
return ih
|
|
|
|
}
|
2015-09-27 11:20:54 +00:00
|
|
|
|
2015-11-05 09:49:32 +00:00
|
|
|
// Mutes returns true iff the given label set is muted.
|
2015-10-11 14:54:39 +00:00
|
|
|
func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
|
2015-09-29 08:45:58 +00:00
|
|
|
alerts := ih.alerts.GetPending()
|
|
|
|
defer alerts.Close()
|
2015-09-27 11:20:54 +00:00
|
|
|
|
2015-10-07 08:59:20 +00:00
|
|
|
// TODO(fabxc): improve erroring for iterators so it does not
|
|
|
|
// go silenced here.
|
|
|
|
|
2015-09-29 08:45:58 +00:00
|
|
|
for alert := range alerts.Next() {
|
|
|
|
if err := alerts.Err(); err != nil {
|
|
|
|
log.Errorf("Error iterating alerts: %s", err)
|
|
|
|
continue
|
|
|
|
}
|
2015-09-27 11:20:54 +00:00
|
|
|
if alert.Resolved() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, rule := range ih.rules {
|
|
|
|
if rule.Mutes(alert.Labels, lset) {
|
2015-11-09 13:34:57 +00:00
|
|
|
ih.marker.SetInhibited(lset.Fingerprint(), true)
|
2015-09-27 11:20:54 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2015-11-09 13:34:57 +00:00
|
|
|
|
|
|
|
ih.marker.SetInhibited(lset.Fingerprint(), false)
|
|
|
|
|
2015-09-27 11:20:54 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// An InhibitRule specifies that a class of (source) alerts should inhibit
|
|
|
|
// notifications for another class of (target) alerts if all specified matching
|
|
|
|
// labels are equal between the two alerts. This may be used to inhibit alerts
|
|
|
|
// from sending notifications if their meaning is logically a subset of a
|
|
|
|
// higher-level alert.
|
|
|
|
type InhibitRule struct {
|
|
|
|
// The set of Filters which define the group of source alerts (which inhibit
|
|
|
|
// the target alerts).
|
|
|
|
SourceMatchers types.Matchers
|
|
|
|
// The set of Filters which define the group of target alerts (which are
|
|
|
|
// inhibited by the source alerts).
|
|
|
|
TargetMatchers types.Matchers
|
|
|
|
// A set of label names whose label values need to be identical in source and
|
|
|
|
// target alerts in order for the inhibition to take effect.
|
|
|
|
Equal map[model.LabelName]struct{}
|
|
|
|
}
|
|
|
|
|
2015-11-05 09:49:32 +00:00
|
|
|
// NewInhibitRule returns a new InihibtRule based on a configuration definition.
|
2015-09-27 11:20:54 +00:00
|
|
|
func NewInhibitRule(cr *config.InhibitRule) *InhibitRule {
|
|
|
|
var (
|
|
|
|
sourcem types.Matchers
|
|
|
|
targetm types.Matchers
|
|
|
|
)
|
|
|
|
|
|
|
|
for ln, lv := range cr.SourceMatch {
|
|
|
|
sourcem = append(sourcem, types.NewMatcher(model.LabelName(ln), lv))
|
|
|
|
}
|
|
|
|
for ln, lv := range cr.SourceMatchRE {
|
|
|
|
m, err := types.NewRegexMatcher(model.LabelName(ln), lv.String())
|
|
|
|
if err != nil {
|
|
|
|
// Must have been sanitized during config validation.
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
sourcem = append(sourcem, m)
|
|
|
|
}
|
|
|
|
|
|
|
|
for ln, lv := range cr.TargetMatch {
|
|
|
|
targetm = append(targetm, types.NewMatcher(model.LabelName(ln), lv))
|
|
|
|
}
|
|
|
|
for ln, lv := range cr.TargetMatchRE {
|
|
|
|
m, err := types.NewRegexMatcher(model.LabelName(ln), lv.String())
|
|
|
|
if err != nil {
|
|
|
|
// Must have been sanitized during config validation.
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
targetm = append(targetm, m)
|
|
|
|
}
|
|
|
|
|
2015-10-05 11:23:40 +00:00
|
|
|
equal := map[model.LabelName]struct{}{}
|
2015-09-27 11:20:54 +00:00
|
|
|
for _, ln := range cr.Equal {
|
|
|
|
equal[ln] = struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
return &InhibitRule{
|
|
|
|
SourceMatchers: sourcem,
|
|
|
|
TargetMatchers: targetm,
|
|
|
|
Equal: equal,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-05 09:49:32 +00:00
|
|
|
// Mutes returns true iff the Inhibition rule applies for the given
|
|
|
|
// source and target label set.
|
2015-09-27 11:20:54 +00:00
|
|
|
func (r *InhibitRule) Mutes(source, target model.LabelSet) bool {
|
2015-09-28 20:08:47 +00:00
|
|
|
if !r.TargetMatchers.Match(target) || !r.SourceMatchers.Match(source) {
|
2015-09-27 12:41:18 +00:00
|
|
|
return false
|
|
|
|
}
|
2015-09-28 20:08:47 +00:00
|
|
|
for ln := range r.Equal {
|
|
|
|
if source[ln] != target[ln] {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
2015-09-27 12:41:18 +00:00
|
|
|
|
|
|
|
return true
|
2015-09-27 11:20:54 +00:00
|
|
|
}
|