Apply inhibition during notification filtering

manager/config.go

@@ -148,10 +148,11 @@ func (r *InhibitRule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		TargetMatch   map[string]string `yaml:"target_match"`
 		TargetMatchRE map[string]string `yaml:"target_match_re"`
 		Equal         model.LabelNames  `yaml:"equal"`
+
 		// Catches all undefined fields and must be empty after parsing.
 		XXX map[string]interface{} `yaml:",inline"`
 	}{}
-	if err := unmarshal(v); err != nil {
+	if err := unmarshal(&v); err != nil {
 		return err
 	}
 
@@ -210,26 +211,13 @@ type NotificationConfig struct {
 	// Notify when resolved.
 	SendResolved bool `yaml:"send_resolved"`
 
-	// Zero or more PagerDuty notification configurations.
 	PagerdutyConfigs []*PagerdutyConfig `yaml:"pagerduty_configs"`
-
-	// Zero or more email notification configurations.
-	EmailConfigs []*EmailConfig `yaml:"email_configs"`
-
-	// Zero or more pushover notification configurations.
-	PushoverConfigs []*PushoverConfig `yaml:"pushover_configs"`
-
-	// Zero or more hipchat notification configurations.
-	HipchatConfigs []*HipchatConfig `yaml:"hipchat_configs"`
-
-	// Zero or more slack notification configurations.
-	SlackConfigs []*SlackConfig `yaml:"slack_config"`
-
-	// Zero or more Flowdock notification configurations.
-	FlowdockConfigs []*FlowdockConfig `yaml:"flowdock_config"`
-
-	// Zero or more generic web hook notification configurations.
-	WebhookConfigs []*WebhookConfig `yaml:"webhook_config"`
+	EmailConfigs     []*EmailConfig     `yaml:"email_configs"`
+	PushoverConfigs  []*PushoverConfig  `yaml:"pushover_configs"`
+	HipchatConfigs   []*HipchatConfig   `yaml:"hipchat_configs"`
+	SlackConfigs     []*SlackConfig     `yaml:"slack_config"`
+	FlowdockConfigs  []*FlowdockConfig  `yaml:"flowdock_config"`
+	WebhookConfigs   []*WebhookConfig   `yaml:"webhook_config"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`

manager/dispatch.go

@@ -39,6 +39,45 @@ func NewDispatcher(state State, notifiers []Notifier) *Dispatcher {
 
 func (d *Dispatcher) filter(alerts ...*Alert) ([]*Alert, error) {
 
+	conf, err := d.state.Config().Get()
+	if err != nil {
+		return nil, err
+	}
+	notifies, err := d.state.Notify().List()
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infoln("check inhibit")
+
+	var blaAlerts []*Alert
+
+	for _, alert := range alerts {
+		inhibited := false
+		for _, ir := range conf.InhibitRules {
+			log.Infoln(ir, "against", alert)
+			if !ir.TargetMatchers.Match(alert.Labels) {
+				continue
+			}
+
+			for _, n := range notifies {
+				if !n.LastResolved && ir.SourceMatchers.Match(n.Labels) {
+					inhibited = true
+					break
+				}
+			}
+			if inhibited {
+				break
+			}
+		}
+		if !inhibited {
+			log.Infoln("not inhibited", alert)
+			blaAlerts = append(blaAlerts, alert)
+		} else {
+			log.Infoln("inhibited", alert)
+		}
+	}
+
 	silences, err := d.state.Silence().List()
 	if err != nil {
 		return nil, err
@@ -98,6 +137,7 @@ func (d *Dispatcher) notify(name string, alerts ...*Alert) error {
 		_ = d.state.Notify().Set(alert.Fingerprint(), &NotifyInfo{
 			LastSent:     time.Now(),
 			LastResolved: alert.Resolved(),
+			Labels:       alert.Labels,
 		})
 	}
 

manager/state.go

@@ -38,6 +38,7 @@ type ConfigState interface {
 type NotifyState interface {
 	Get(model.Fingerprint) (*NotifyInfo, error)
 	Set(model.Fingerprint, *NotifyInfo) error
+	List() ([]*NotifyInfo, error)
 }
 
 type SilenceState interface {
@@ -126,6 +127,7 @@ func (s *simpleState) Notify() NotifyState {
 type NotifyInfo struct {
 	LastSent     time.Time
 	LastResolved bool
+	Labels       model.LabelSet
 }
 
 type memNotify struct {
@@ -144,6 +146,14 @@ func (s *memNotify) Set(fp model.Fingerprint, info *NotifyInfo) error {
 	return nil
 }
 
+func (s *memNotify) List() ([]*NotifyInfo, error) {
+	var res []*NotifyInfo
+	for _, ni := range s.m {
+		res = append(res, ni)
+	}
+	return res, nil
+}
+
 type memConfig struct {
 	config *Config
 	mtx    sync.RWMutex