Merge pull request #176 from prometheus/silinh

Set previous inhibition/silence state

main.go

@@ -113,9 +113,9 @@ func main() {
 		n := notify.Notifier(router)
 
 		n = notify.Log(n, log.With("step", "route"))
-		n = notify.Mute(silences, n)
+		n = notify.Silence(silences, n, marker)
 		n = notify.Log(n, log.With("step", "silence"))
-		n = notify.Mute(inhibitor, n)
+		n = notify.Inhibit(inhibitor, n, marker)
 		n = notify.Log(n, log.With("step", "inhibit"))
 
 		return n

notify/notify.go

@@ -344,27 +344,70 @@ func (rs Router) Notify(ctx context.Context, alerts ...*types.Alert) error {
 	return notifier.Notify(ctx, alerts...)
 }
 
-// MutingNotifier wraps a notifier and applies a Silencer
-// before sending out an alert.
-type MutingNotifier struct {
-	types.Muter
+// SilenceNotifier filters alerts through a silence muter before
+// passing it on to the next Notifier
+type SilenceNotifier struct {
 	notifier Notifier
+	muter    types.Muter
+	marker   types.Marker
 }
 
-// Mute wraps a notifier in a MutingNotifier with the given Muter.
-func Mute(m types.Muter, n Notifier) *MutingNotifier {
-	return &MutingNotifier{Muter: m, notifier: n}
+// Silence returns a new SilenceNotifier.
+func Silence(m types.Muter, n Notifier, mk types.Marker) *SilenceNotifier {
+	return &SilenceNotifier{
+		notifier: n,
+		muter:    m,
+		marker:   mk,
+	}
 }
 
-// Notify implements Notifier.
-func (n *MutingNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
+// Notify implements the Notifier interface.
+func (n *SilenceNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
 	var filtered []*types.Alert
 	for _, a := range alerts {
+		_, ok := n.marker.Silenced(a.Fingerprint())
 		// TODO(fabxc): increment total alerts counter.
 		// Do not send the alert if the silencer mutes it.
-		if !n.Mutes(a.Labels) {
+		if !n.muter.Mutes(a.Labels) {
 			// TODO(fabxc): increment muted alerts counter.
 			filtered = append(filtered, a)
+			// Store whether a previously silenced alert is firing again.
+			a.WasSilenced = ok
+		}
+	}
+
+	return n.notifier.Notify(ctx, filtered...)
+}
+
+// InhibitNotifier filters alerts through an inhibition muter before
+// passing it on to the next Notifier
+type InhibitNotifier struct {
+	notifier Notifier
+	muter    types.Muter
+	marker   types.Marker
+}
+
+// Inhibit return a new InhibitNotifier.
+func Inhibit(m types.Muter, n Notifier, mk types.Marker) *InhibitNotifier {
+	return &InhibitNotifier{
+		notifier: n,
+		muter:    m,
+		marker:   mk,
+	}
+}
+
+// Notify implements the Notifier interface.
+func (n *InhibitNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
+	var filtered []*types.Alert
+	for _, a := range alerts {
+		ok := n.marker.Inhibited(a.Fingerprint())
+		// TODO(fabxc): increment total alerts counter.
+		// Do not send the alert if the silencer mutes it.
+		if !n.muter.Mutes(a.Labels) {
+			// TODO(fabxc): increment muted alerts counter.
+			filtered = append(filtered, a)
+			// Store whether a previously inhibited alert is firing again.
+			a.WasInhibited = ok
 		}
 	}
 

notify/notify_test.go

@@ -239,15 +239,16 @@ func TestRoutedNotifier(t *testing.T) {
 	}
 }
 
-func TestMutingNotifier(t *testing.T) {
+func TestSilenceNotifier(t *testing.T) {
 	// Mute all label sets that have a "mute" key.
 	muter := types.MuteFunc(func(lset model.LabelSet) bool {
 		_, ok := lset["mute"]
 		return ok
 	})
 
+	marker := types.NewMarker()
 	record := &recordNotifier{}
-	muteNotifer := Mute(muter, record)
+	silenceNotifer := Silence(muter, record, marker)
 
 	in := []model.LabelSet{
 		{},
@@ -273,13 +274,76 @@ func TestMutingNotifier(t *testing.T) {
 		})
 	}
 
-	if err := muteNotifer.Notify(nil, inAlerts...); err != nil {
+	// Set the second alert als previously silenced. It is expected to have
+	// the WasSilenced flag set to true afterwards.
+	marker.SetSilenced(inAlerts[1].Fingerprint(), 123)
+
+	if err := silenceNotifer.Notify(nil, inAlerts...); err != nil {
 		t.Fatalf("Notifying failed: %s", err)
 	}
 
 	var got []model.LabelSet
-	for _, a := range record.alerts {
+	for i, a := range record.alerts {
 		got = append(got, a.Labels)
+		if a.WasSilenced != (i == 1) {
+			t.Errorf("Expected WasSilenced to be %v for %d, was %v", i == 1, i, a.WasSilenced)
+		}
+	}
+
+	if !reflect.DeepEqual(got, out) {
+		t.Fatalf("Muting failed, expected: %v\ngot %v", out, got)
+	}
+}
+
+func TestInhibitNotifier(t *testing.T) {
+	// Mute all label sets that have a "mute" key.
+	muter := types.MuteFunc(func(lset model.LabelSet) bool {
+		_, ok := lset["mute"]
+		return ok
+	})
+
+	marker := types.NewMarker()
+	record := &recordNotifier{}
+	inhibitNotifer := Inhibit(muter, record, marker)
+
+	in := []model.LabelSet{
+		{},
+		{"test": "set"},
+		{"mute": "me"},
+		{"foo": "bar", "test": "set"},
+		{"foo": "bar", "mute": "me"},
+		{},
+		{"not": "muted"},
+	}
+	out := []model.LabelSet{
+		{},
+		{"test": "set"},
+		{"foo": "bar", "test": "set"},
+		{},
+		{"not": "muted"},
+	}
+
+	var inAlerts []*types.Alert
+	for _, lset := range in {
+		inAlerts = append(inAlerts, &types.Alert{
+			Alert: model.Alert{Labels: lset},
+		})
+	}
+
+	// Set the second alert as previously inhibited. It is expected to have
+	// the WasInhibited flag set to true afterwards.
+	marker.SetInhibited(inAlerts[1].Fingerprint(), true)
+
+	if err := inhibitNotifer.Notify(nil, inAlerts...); err != nil {
+		t.Fatalf("Notifying failed: %s", err)
+	}
+
+	var got []model.LabelSet
+	for i, a := range record.alerts {
+		got = append(got, a.Labels)
+		if a.WasInhibited != (i == 1) {
+			t.Errorf("Expected WasInhibited to be %v for %d, was %v", i == 1, i, a.WasInhibited)
+		}
 	}
 
 	if !reflect.DeepEqual(got, out) {

template/template.go

@@ -209,9 +209,11 @@ type Data struct {
 
 // Alert holds one alert for notification templates.
 type Alert struct {
-	Status      string
-	Labels      KV
-	Annotations KV
+	Status       string
+	Labels       KV
+	Annotations  KV
+	WasSilenced  bool
+	WasInhibited bool
 }
 
 // Alerts is a list of Alert objects.
@@ -240,12 +242,10 @@ func (as Alerts) Resolved() []Alert {
 }
 
 // Data assembles data for template expansion.
-func (t *Template) Data(recv string, groupLabels model.LabelSet, as ...*types.Alert) *Data {
-	alerts := types.Alerts(as...)
-
+func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data {
 	data := &Data{
 		Receiver:          strings.SplitN(recv, "/", 2)[0],
-		Status:            string(alerts.Status()),
+		Status:            string(types.Alerts(alerts...).Status()),
 		Alerts:            make(Alerts, 0, len(alerts)),
 		GroupLabels:       KV{},
 		CommonLabels:      KV{},
@@ -255,9 +255,11 @@ func (t *Template) Data(recv string, groupLabels model.LabelSet, as ...*types.Al
 
 	for _, a := range alerts {
 		alert := Alert{
-			Status:      string(a.Status()),
-			Labels:      make(KV, len(a.Labels)),
-			Annotations: make(KV, len(a.Annotations)),
+			Status:       string(a.Status()),
+			Labels:       make(KV, len(a.Labels)),
+			Annotations:  make(KV, len(a.Annotations)),
+			WasSilenced:  a.WasSilenced,
+			WasInhibited: a.WasInhibited,
 		}
 		for k, v := range a.Labels {
 			alert.Labels[string(k)] = string(v)

types/types.go

@@ -138,8 +138,10 @@ type Alert struct {
 	model.Alert
 
 	// The authoritative timestamp.
-	UpdatedAt time.Time `json:"-"`
-	Timeout   bool      `json:"-"`
+	UpdatedAt    time.Time `json:"-"`
+	Timeout      bool      `json:"-"`
+	WasSilenced  bool      `json:"-"`
+	WasInhibited bool      `json:"-"`
 }
 
 // AlertSlice is a sortable slice of Alerts.