diff --git a/dispatch.go b/dispatch.go index e0edd547..ac805ec7 100644 --- a/dispatch.go +++ b/dispatch.go @@ -25,6 +25,8 @@ type Dispatcher struct { alerts provider.Alerts notifier notify.Notifier + marker types.Marker + aggrGroups map[*Route]map[model.Fingerprint]*aggrGroup done chan struct{} @@ -35,11 +37,12 @@ type Dispatcher struct { } // NewDispatcher returns a new Dispatcher. -func NewDispatcher(ap provider.Alerts, r *Route, n notify.Notifier) *Dispatcher { +func NewDispatcher(ap provider.Alerts, r *Route, n notify.Notifier, mk types.Marker) *Dispatcher { disp := &Dispatcher{ alerts: ap, notifier: n, route: r, + marker: mk, log: log.With("component", "dispatcher"), } return disp @@ -59,8 +62,8 @@ func (d *Dispatcher) Run() { // UIGroup is the representation of a group of alerts as provided by // the API. type UIGroup struct { - RouteOpts *RouteOpts `json:"routeOpts"` - Alerts model.Alerts `json:"alerts"` + RouteOpts *RouteOpts `json:"routeOpts"` + Alerts []*UIAlert `json:"alerts"` } type UIGroups struct { @@ -68,6 +71,13 @@ type UIGroups struct { Groups []*UIGroup `json:"groups"` } +type UIAlert struct { + *model.Alert + + Inhibited bool `json:"inhibited"` + Silenced uint64 `json:"silenced,omitempty"` +} + func (d *Dispatcher) Groups() []*UIGroups { var groups []*UIGroups @@ -88,9 +98,20 @@ func (d *Dispatcher) Groups() []*UIGroups { groups = append(groups, uig) } + var uiAlerts []*UIAlert + for _, a := range types.Alerts(alerts...) { + sid, _ := d.marker.Silenced(a.Fingerprint()) + + uiAlerts = append(uiAlerts, &UIAlert{ + Alert: a, + Inhibited: d.marker.Inhibited(a.Fingerprint()), + Silenced: sid, + }) + } + uig.Groups = append(uig.Groups, &UIGroup{ RouteOpts: &route.RouteOpts, - Alerts: types.Alerts(alerts...), + Alerts: uiAlerts, }) } } diff --git a/inhibit.go b/inhibit.go index 6fcf8bf1..b1b177e5 100644 --- a/inhibit.go +++ b/inhibit.go @@ -29,14 +29,16 @@ import ( type Inhibitor struct { alerts provider.Alerts rules []*InhibitRule + marker types.Marker mtx sync.RWMutex } // NewInhibitor returns a new Inhibitor. -func NewInhibitor(ap provider.Alerts, rs []*config.InhibitRule) *Inhibitor { +func NewInhibitor(ap provider.Alerts, rs []*config.InhibitRule, mk types.Marker) *Inhibitor { ih := &Inhibitor{ alerts: ap, + marker: mk, } for _, cr := range rs { ih.rules = append(ih.rules, NewInhibitRule(cr)) @@ -62,10 +64,14 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool { } for _, rule := range ih.rules { if rule.Mutes(alert.Labels, lset) { + ih.marker.SetInhibited(lset.Fingerprint(), true) return true } } } + + ih.marker.SetInhibited(lset.Fingerprint(), false) + return false } diff --git a/main.go b/main.go index 2d97b4a7..5406d5b4 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/provider" "github.com/prometheus/alertmanager/template" + "github.com/prometheus/alertmanager/types" ) var ( @@ -46,6 +47,8 @@ func main() { } defer db.Close() + marker := types.NewMarker() + alerts, err := provider.NewSQLAlerts(db) if err != nil { log.Fatal(err) @@ -54,7 +57,7 @@ func main() { if err != nil { log.Fatal(err) } - silences, err := provider.NewSQLSilences(db) + silences, err := provider.NewSQLSilences(db, marker) if err != nil { log.Fatal(err) } @@ -119,8 +122,8 @@ func main() { disp.Stop() - inhibitor = NewInhibitor(alerts, conf.InhibitRules) - disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.NotificationConfigs)) + inhibitor = NewInhibitor(alerts, conf.InhibitRules, marker) + disp = NewDispatcher(alerts, NewRoute(conf.Route, nil), build(conf.NotificationConfigs), marker) go disp.Run() diff --git a/provider/sql.go b/provider/sql.go index 2b185008..65a645db 100644 --- a/provider/sql.go +++ b/provider/sql.go @@ -472,10 +472,11 @@ func (a *SQLAlerts) Put(alerts ...*types.Alert) error { } type SQLSilences struct { - db *sql.DB + db *sql.DB + marker types.Marker } -func NewSQLSilences(db *sql.DB) (*SQLSilences, error) { +func NewSQLSilences(db *sql.DB, mk types.Marker) (*SQLSilences, error) { tx, err := db.Begin() if err != nil { return nil, err @@ -486,7 +487,7 @@ func NewSQLSilences(db *sql.DB) (*SQLSilences, error) { } tx.Commit() - return &SQLSilences{db: db}, nil + return &SQLSilences{db: db, marker: mk}, nil } const createSilencesTable = ` @@ -513,9 +514,12 @@ func (s *SQLSilences) Mutes(lset model.LabelSet) bool { for _, sil := range sils { if sil.Mutes(lset) { + s.marker.SetSilenced(lset.Fingerprint(), sil.ID) return true } } + + s.marker.SetSilenced(lset.Fingerprint()) return false } diff --git a/types/types.go b/types/types.go index 1ddb9f3d..c684f41a 100644 --- a/types/types.go +++ b/types/types.go @@ -17,11 +17,72 @@ import ( "fmt" "hash/fnv" "strings" + "sync" "time" "github.com/prometheus/common/model" ) +type Marker interface { + SetInhibited(alert model.Fingerprint, b bool) + SetSilenced(alert model.Fingerprint, sil ...uint64) + + Silenced(alert model.Fingerprint) (uint64, bool) + Inhibited(alert model.Fingerprint) bool +} + +func NewMarker() Marker { + return &memMarker{ + inhibited: map[model.Fingerprint]struct{}{}, + silenced: map[model.Fingerprint]uint64{}, + } +} + +type memMarker struct { + inhibited map[model.Fingerprint]struct{} + silenced map[model.Fingerprint]uint64 + + mtx sync.RWMutex +} + +func (m *memMarker) Inhibited(alert model.Fingerprint) bool { + m.mtx.RLock() + defer m.mtx.RUnlock() + + _, ok := m.inhibited[alert] + return ok +} + +func (m *memMarker) Silenced(alert model.Fingerprint) (uint64, bool) { + m.mtx.RLock() + defer m.mtx.RUnlock() + + sid, ok := m.silenced[alert] + return sid, ok +} + +func (m *memMarker) SetInhibited(alert model.Fingerprint, b bool) { + m.mtx.Lock() + defer m.mtx.Unlock() + + if !b { + delete(m.inhibited, alert) + } else { + m.inhibited[alert] = struct{}{} + } +} + +func (m *memMarker) SetSilenced(alert model.Fingerprint, sil ...uint64) { + m.mtx.Lock() + defer m.mtx.Unlock() + + if len(sil) == 0 { + delete(m.silenced, alert) + } else { + m.silenced[alert] = sil[0] + } +} + type MultiError []error func (e MultiError) Error() string {