Track and expose silenced/inhibited status of alerts

dispatch.go

@@ -25,6 +25,8 @@ type Dispatcher struct {
 	alerts   provider.Alerts
 	notifier notify.Notifier
 
+	marker types.Marker
+
 	aggrGroups map[*Route]map[model.Fingerprint]*aggrGroup
 
 	done   chan struct{}
@@ -35,11 +37,12 @@ type Dispatcher struct {
 }
 
 // NewDispatcher returns a new Dispatcher.
-func NewDispatcher(ap provider.Alerts, r *Route, n notify.Notifier) *Dispatcher {
+func NewDispatcher(ap provider.Alerts, r *Route, n notify.Notifier, mk types.Marker) *Dispatcher {
 	disp := &Dispatcher{
 		alerts:   ap,
 		notifier: n,
 		route:    r,
+		marker:   mk,
 		log:      log.With("component", "dispatcher"),
 	}
 	return disp
@@ -59,8 +62,8 @@ func (d *Dispatcher) Run() {
 // UIGroup is the representation of a group of alerts as provided by
 // the API.
 type UIGroup struct {
-	RouteOpts *RouteOpts   `json:"routeOpts"`
-	Alerts    model.Alerts `json:"alerts"`
+	RouteOpts *RouteOpts `json:"routeOpts"`
+	Alerts    []*UIAlert `json:"alerts"`
 }
 
 type UIGroups struct {
@@ -68,6 +71,13 @@ type UIGroups struct {
 	Groups []*UIGroup     `json:"groups"`
 }
 
+type UIAlert struct {
+	*model.Alert
+
+	Inhibited bool   `json:"inhibited"`
+	Silenced  uint64 `json:"silenced,omitempty"`
+}
+
 func (d *Dispatcher) Groups() []*UIGroups {
 	var groups []*UIGroups
 
@@ -88,9 +98,20 @@ func (d *Dispatcher) Groups() []*UIGroups {
 				groups = append(groups, uig)
 			}
 
+			var uiAlerts []*UIAlert
+			for _, a := range types.Alerts(alerts...) {
+				sid, _ := d.marker.Silenced(a.Fingerprint())
+
+				uiAlerts = append(uiAlerts, &UIAlert{
+					Alert:     a,
+					Inhibited: d.marker.Inhibited(a.Fingerprint()),
+					Silenced:  sid,
+				})
+			}
+
 			uig.Groups = append(uig.Groups, &UIGroup{
 				RouteOpts: &route.RouteOpts,
-				Alerts:    types.Alerts(alerts...),
+				Alerts:    uiAlerts,
 			})
 		}
 	}

inhibit.go

@@ -29,14 +29,16 @@ import (
 type Inhibitor struct {
 	alerts provider.Alerts
 	rules  []*InhibitRule
+	marker types.Marker
 
 	mtx sync.RWMutex
 }
 
 // NewInhibitor returns a new Inhibitor.
-func NewInhibitor(ap provider.Alerts, rs []*config.InhibitRule) *Inhibitor {
+func NewInhibitor(ap provider.Alerts, rs []*config.InhibitRule, mk types.Marker) *Inhibitor {
 	ih := &Inhibitor{
 		alerts: ap,
+		marker: mk,
 	}
 	for _, cr := range rs {
 		ih.rules = append(ih.rules, NewInhibitRule(cr))
@@ -62,10 +64,14 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
 		}
 		for _, rule := range ih.rules {
 			if rule.Mutes(alert.Labels, lset) {
+				ih.marker.SetInhibited(lset.Fingerprint(), true)
 				return true
 			}
 		}
 	}
+
+	ih.marker.SetInhibited(lset.Fingerprint(), false)
+
 	return false
 }
 

main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/provider"
 	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
 )
 
 var (
@@ -46,6 +47,8 @@ func main() {
 	}
 	defer db.Close()
 
+	marker := types.NewMarker()
+
 	alerts, err := provider.NewSQLAlerts(db)
 	if err != nil {
 		log.Fatal(err)
@@ -54,7 +57,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	silences, err := provider.NewSQLSilences(db)
+	silences, err := provider.NewSQLSilences(db, marker)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -119,8 +122,8 @@ func main() {
 
 		disp.Stop()
 
-		inhibitor = NewInhibitor(alerts, conf.InhibitRules)
-		disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.NotificationConfigs))
+		inhibitor = NewInhibitor(alerts, conf.InhibitRules, marker)
+		disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.NotificationConfigs), marker)
 
 		go disp.Run()
 

provider/sql.go

@@ -472,10 +472,11 @@ func (a *SQLAlerts) Put(alerts ...*types.Alert) error {
 }
 
 type SQLSilences struct {
-	db *sql.DB
+	db     *sql.DB
+	marker types.Marker
 }
 
-func NewSQLSilences(db *sql.DB) (*SQLSilences, error) {
+func NewSQLSilences(db *sql.DB, mk types.Marker) (*SQLSilences, error) {
 	tx, err := db.Begin()
 	if err != nil {
 		return nil, err
@@ -486,7 +487,7 @@ func NewSQLSilences(db *sql.DB) (*SQLSilences, error) {
 	}
 	tx.Commit()
 
-	return &SQLSilences{db: db}, nil
+	return &SQLSilences{db: db, marker: mk}, nil
 }
 
 const createSilencesTable = `
@@ -513,9 +514,12 @@ func (s *SQLSilences) Mutes(lset model.LabelSet) bool {
 
 	for _, sil := range sils {
 		if sil.Mutes(lset) {
+			s.marker.SetSilenced(lset.Fingerprint(), sil.ID)
 			return true
 		}
 	}
+
+	s.marker.SetSilenced(lset.Fingerprint())
 	return false
 }
 

types/types.go

@@ -17,11 +17,72 @@ import (
 	"fmt"
 	"hash/fnv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/common/model"
 )
 
+type Marker interface {
+	SetInhibited(alert model.Fingerprint, b bool)
+	SetSilenced(alert model.Fingerprint, sil ...uint64)
+
+	Silenced(alert model.Fingerprint) (uint64, bool)
+	Inhibited(alert model.Fingerprint) bool
+}
+
+func NewMarker() Marker {
+	return &memMarker{
+		inhibited: map[model.Fingerprint]struct{}{},
+		silenced:  map[model.Fingerprint]uint64{},
+	}
+}
+
+type memMarker struct {
+	inhibited map[model.Fingerprint]struct{}
+	silenced  map[model.Fingerprint]uint64
+
+	mtx sync.RWMutex
+}
+
+func (m *memMarker) Inhibited(alert model.Fingerprint) bool {
+	m.mtx.RLock()
+	defer m.mtx.RUnlock()
+
+	_, ok := m.inhibited[alert]
+	return ok
+}
+
+func (m *memMarker) Silenced(alert model.Fingerprint) (uint64, bool) {
+	m.mtx.RLock()
+	defer m.mtx.RUnlock()
+
+	sid, ok := m.silenced[alert]
+	return sid, ok
+}
+
+func (m *memMarker) SetInhibited(alert model.Fingerprint, b bool) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	if !b {
+		delete(m.inhibited, alert)
+	} else {
+		m.inhibited[alert] = struct{}{}
+	}
+}
+
+func (m *memMarker) SetSilenced(alert model.Fingerprint, sil ...uint64) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	if len(sil) == 0 {
+		delete(m.silenced, alert)
+	} else {
+		m.silenced[alert] = sil[0]
+	}
+}
+
 type MultiError []error
 
 func (e MultiError) Error() string {