RepoMirrors
/
alertmanager
mirror of https://github.com/prometheus/alertmanager
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add negative matchers for inhibitors (#2460) 

Add new matchers for inhibition rules

Signed-off-by: aSquare14 <atibhi.a@gmail.com>
This commit is contained in:
Atibhi Agrawal 2021-01-22 20:24:11 +05:30 committed by GitHub
parent 3f46b62d75
commit 4ffba65ff5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 146 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/config.go
View File

@ -645,17 +645,21 @@ func (r *Route) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Both alerts have to have a set of labels being equal.
type InhibitRule struct {
// SourceMatch defines a set of labels that have to equal the given
// value for source alerts.
// value for source alerts. Deprecated. Remove before v1.0 release.
SourceMatch map[string]string `yaml:"source_match,omitempty" json:"source_match,omitempty"`
// SourceMatchRE defines pairs like SourceMatch but does regular expression
// matching.
// matching. Deprecated. Remove before v1.0 release.
SourceMatchRE MatchRegexps `yaml:"source_match_re,omitempty" json:"source_match_re,omitempty"`
// SourceMatchers defines a set of label matchers that have to be fulfilled for source alerts.
SourceMatchers Matchers `yaml:"source_matchers,omitempty" json:"source_matchers,omitempty"`
// TargetMatch defines a set of labels that have to equal the given
// value for target alerts.
// value for target alerts. Deprecated. Remove before v1.0 release.
TargetMatch map[string]string `yaml:"target_match,omitempty" json:"target_match,omitempty"`
// TargetMatchRE defines pairs like TargetMatch but does regular expression
// matching.
// matching. Deprecated. Remove before v1.0 release.
TargetMatchRE MatchRegexps `yaml:"target_match_re,omitempty" json:"target_match_re,omitempty"`
// TargetMatchers defines a set of label matchers that have to be fulfilled for target alerts.
TargetMatchers Matchers `yaml:"target_matchers,omitempty" json:"target_matchers,omitempty"`
// A set of labels that must be equal between the source and target alert
// for them to be a match.
Equal model.LabelNames `yaml:"equal,omitempty" json:"equal,omitempty"`

54
inhibit/inhibit.go
View File

@ -24,6 +24,7 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/alertmanager/provider"
"github.com/prometheus/alertmanager/store"
"github.com/prometheus/alertmanager/types"
@ -71,7 +72,7 @@ func (ih *Inhibitor) run(ctx context.Context) {
}
// Update the inhibition rules' cache.
for _, r := range ih.rules {
if r.SourceMatchers.Match(a.Labels) {
if r.SourceMatchers.Matches(a.Labels) {
if err := r.scache.Set(a); err != nil {
level.Error(ih.logger).Log("msg", "error on set alert", "err", err)
}
@ -128,13 +129,13 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
fp := lset.Fingerprint()
for _, r := range ih.rules {
if !r.TargetMatchers.Match(lset) {
if !r.TargetMatchers.Matches(lset) {
// If target side of rule doesn't match, we don't need to look any further.
continue
}
// If we are here, the target side matches. If the source side matches, too, we
// need to exclude inhibiting alerts for which the same is true.
if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Match(lset)); eq {
if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset)); eq {
ih.marker.SetInhibited(fp, inhibitedByFP.String())
return true
}
@ -152,10 +153,10 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
type InhibitRule struct {
// The set of Filters which define the group of source alerts (which inhibit
// the target alerts).
SourceMatchers types.Matchers
SourceMatchers labels.Matchers
// The set of Filters which define the group of target alerts (which are
// inhibited by the source alerts).
TargetMatchers types.Matchers
TargetMatchers labels.Matchers
// A set of label names whose label values need to be identical in source and
// target alerts in order for the inhibition to take effect.
Equal map[model.LabelName]struct{}
@ -167,23 +168,50 @@ type InhibitRule struct {
// NewInhibitRule returns a new InhibitRule based on a configuration definition.
func NewInhibitRule(cr *config.InhibitRule) *InhibitRule {
var (
sourcem types.Matchers
targetm types.Matchers
sourcem labels.Matchers
targetm labels.Matchers
)
// cr.SourceMatch will be deprecated. This for loop appends regex matchers.
for ln, lv := range cr.SourceMatch {
sourcem = append(sourcem, types.NewMatcher(model.LabelName(ln), lv))
matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
if err != nil {
// This error must not happen because the config already validates the yaml.
panic(err)
}
sourcem = append(sourcem, matcher)
}
// cr.SourceMatchRE will be deprecated. This for loop appends regex matchers.
for ln, lv := range cr.SourceMatchRE {
sourcem = append(sourcem, types.NewRegexMatcher(model.LabelName(ln), lv.Regexp))
matcher, err := labels.NewMatcher(labels.MatchRegexp, ln, lv.String())
if err != nil {
// This error must not happen because the config already validates the yaml.
panic(err)
}
sourcem = append(sourcem, matcher)
}
// We append the new-style matchers. This can be simplified once the deprecated matcher syntax is removed.
sourcem = append(sourcem, cr.SourceMatchers...)
// cr.TargetMatch will be deprecated. This for loop appends regex matchers.
for ln, lv := range cr.TargetMatch {
targetm = append(targetm, types.NewMatcher(model.LabelName(ln), lv))
matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
if err != nil {
// This error must not happen because the config already validates the yaml.
panic(err)
}
targetm = append(targetm, matcher)
}
// cr.TargetMatchRE will be deprecated. This for loop appends regex matchers.
for ln, lv := range cr.TargetMatchRE {
targetm = append(targetm, types.NewRegexMatcher(model.LabelName(ln), lv.Regexp))
matcher, err := labels.NewMatcher(labels.MatchRegexp, ln, lv.String())
if err != nil {
// This error must not happen because the config already validates the yaml.
panic(err)
}
targetm = append(targetm, matcher)
}
// We append the new-style matchers. This can be simplified once the deprecated matcher syntax is removed.
targetm = append(targetm, cr.TargetMatchers...)
equal := map[model.LabelName]struct{}{}
for _, ln := range cr.Equal {
@ -214,7 +242,7 @@ Outer:
continue Outer
}
}
if excludeTwoSidedMatch && r.TargetMatchers.Match(a.Labels) {
if excludeTwoSidedMatch && r.TargetMatchers.Matches(a.Labels) {
continue Outer
}
return a.Fingerprint(), true

97
inhibit/inhibit_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/alertmanager/provider"
"github.com/prometheus/alertmanager/store"
"github.com/prometheus/alertmanager/types"
@ -150,6 +151,7 @@ func TestInhibitRuleMatches(t *testing.T) {
TargetMatch: map[string]string{"t2": "1"},
Equal: model.LabelNames{"e"},
}
m := types.NewMarker(prometheus.NewRegistry())
ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
now := time.Now()
@ -231,6 +233,101 @@ func TestInhibitRuleMatches(t *testing.T) {
}
}
}
func TestInhibitRuleMatchers(t *testing.T) {
t.Parallel()
rule1 := config.InhibitRule{
SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s1", Value: "1"}},
TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchNotEqual, Name: "t1", Value: "1"}},
Equal: model.LabelNames{"e"},
}
rule2 := config.InhibitRule{
SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s2", Value: "1"}},
TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "t2", Value: "1"}},
Equal: model.LabelNames{"e"},
}
m := types.NewMarker(prometheus.NewRegistry())
ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
now := time.Now()
// Active alert that matches the source filter of rule1.
sourceAlert1 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
StartsAt: now.Add(-time.Minute),
EndsAt: now.Add(time.Hour),
},
}
// Active alert that matches the source filter _and_ the target filter of rule2.
sourceAlert2 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
StartsAt: now.Add(-time.Minute),
EndsAt: now.Add(time.Hour),
},
}
ih.rules[0].scache = store.NewAlerts()
ih.rules[0].scache.Set(sourceAlert1)
ih.rules[1].scache = store.NewAlerts()
ih.rules[1].scache.Set(sourceAlert2)
cases := []struct {
target model.LabelSet
expected bool
}{
{
// Matches target filter of rule1, inhibited.
target: model.LabelSet{"t1": "1", "e": "1"},
expected: false,
},
{
// Matches target filter of rule2, inhibited.
target: model.LabelSet{"t2": "1", "e": "1"},
expected: true,
},
{
// Matches target filter of rule1 (plus noise), inhibited.
target: model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
expected: false,
},
{
// Matches target filter of rule1 plus rule2, inhibited.
target: model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
expected: true,
},
{
// Doesn't match target filter, not inhibited.
target: model.LabelSet{"t1": "0", "e": "1"},
expected: true,
},
{
// Matches both source and target filters of rule1,
// inhibited because sourceAlert1 matches only the
// source filter of rule1.
target: model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
expected: false,
},
{
// Matches both source and target filters of rule2,
// not inhibited because sourceAlert2 matches also both the
// source and target filter of rule2.
target: model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
expected: true,
},
{
// Matches target filter, equal label doesn't match, not inhibited
target: model.LabelSet{"t1": "1", "e": "0"},
expected: false,
},
}
for _, c := range cases {
if actual := ih.Mutes(c.target); actual != c.expected {
t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
}
}
}
type fakeAlerts struct {
alerts []*types.Alert