Merge pull request #51 from prometheus/add-resolved-notifications

Add support to notify for resolved alerts

config/config.proto

@@ -24,14 +24,18 @@ message PagerDutyConfig {
 message EmailConfig {
   // Email address to notify.
   optional string email = 1;
+  // Notify when resolved.
+  optional bool send_resolved = 2 [default = false];
 }
 
 // Configuration for notification via pushover.net.
 message PushoverConfig {
-  // Pushover token
+  // Pushover token.
   optional string token = 1;
-  // Pushover user_key
+  // Pushover user_key.
   optional string user_key = 2;
+  // Notify when resolved.
+  optional bool send_resolved = 3 [default = false];
 }
 
 // Configuration for notification via HipChat.
@@ -42,10 +46,14 @@ message HipChatConfig {
   optional string auth_token = 1;
   // HipChat room id, (https://www.hipchat.com/rooms/ids).
   optional int32 room_id = 2;
-  // Color of message.
+  // Color of message when triggered.
   optional string color = 3 [default = "purple"];
+  // Color of message when resolved.
+  optional string color_resolved = 5 [default = "green"];
   // Should this message notify or not.
   optional bool notify = 4 [default = false];
+  // Notify when resolved.
+  optional bool send_resolved = 6 [default = false];
 }
 
 // Notification configuration definition.

config/fixtures/sample.conf.input

@@ -13,6 +13,7 @@ notification_config {
   hipchat_config {
     auth_token: "hipchatauthtoken"
     room_id: 123456
+    send_resolved: true
   }
 }
 

config/generated/config.pb.go

@@ -50,14 +50,18 @@ func (m *PagerDutyConfig) GetServiceKey() string {
 // Configuration for notification via mail.
 type EmailConfig struct {
 	// Email address to notify.
-	Email            *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"`
+	Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"`
-	XXX_unrecognized []byte  `json:"-"`
+	// Notify when resolved.
+	SendResolved     *bool  `protobuf:"varint,2,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
+	XXX_unrecognized []byte `json:"-"`
 }
 
 func (m *EmailConfig) Reset()         { *m = EmailConfig{} }
 func (m *EmailConfig) String() string { return proto.CompactTextString(m) }
 func (*EmailConfig) ProtoMessage()    {}
 
+const Default_EmailConfig_SendResolved bool = false
+
 func (m *EmailConfig) GetEmail() string {
 	if m != nil && m.Email != nil {
 		return *m.Email
@@ -65,19 +69,30 @@ func (m *EmailConfig) GetEmail() string {
 	return ""
 }
 
+func (m *EmailConfig) GetSendResolved() bool {
+	if m != nil && m.SendResolved != nil {
+		return *m.SendResolved
+	}
+	return Default_EmailConfig_SendResolved
+}
+
 // Configuration for notification via pushover.net.
 type PushoverConfig struct {
-	// Pushover token
+	// Pushover token.
 	Token *string `protobuf:"bytes,1,opt,name=token" json:"token,omitempty"`
-	// Pushover user_key
+	// Pushover user_key.
-	UserKey          *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"`
+	UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"`
-	XXX_unrecognized []byte  `json:"-"`
+	// Notify when resolved.
+	SendResolved     *bool  `protobuf:"varint,3,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
+	XXX_unrecognized []byte `json:"-"`
 }
 
 func (m *PushoverConfig) Reset()         { *m = PushoverConfig{} }
 func (m *PushoverConfig) String() string { return proto.CompactTextString(m) }
 func (*PushoverConfig) ProtoMessage()    {}
 
+const Default_PushoverConfig_SendResolved bool = false
+
 func (m *PushoverConfig) GetToken() string {
 	if m != nil && m.Token != nil {
 		return *m.Token
@@ -92,15 +107,27 @@ func (m *PushoverConfig) GetUserKey() string {
 	return ""
 }
 
+func (m *PushoverConfig) GetSendResolved() bool {
+	if m != nil && m.SendResolved != nil {
+		return *m.SendResolved
+	}
+	return Default_PushoverConfig_SendResolved
+}
+
+// Configuration for notification via HipChat.
 type HipChatConfig struct {
-	// Hipchat auth token, https://www.hipchat.com/docs/api/auth
+	// HipChat auth token, (https://www.hipchat.com/docs/api/auth).
 	AuthToken *string `protobuf:"bytes,1,opt,name=auth_token" json:"auth_token,omitempty"`
-	// Hipchat room id, https://www.hipchat.com/rooms/ids
+	// HipChat room id, (https://www.hipchat.com/rooms/ids).
 	RoomId *int32 `protobuf:"varint,2,opt,name=room_id" json:"room_id,omitempty"`
-	// color of message
+	// Color of message when triggered.
 	Color *string `protobuf:"bytes,3,opt,name=color,def=purple" json:"color,omitempty"`
-	// should this message notify or not
+	// Color of message when resolved.
-	Notify           *bool  `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"`
+	ColorResolved *string `protobuf:"bytes,5,opt,name=color_resolved,def=green" json:"color_resolved,omitempty"`
+	// Should this message notify or not.
+	Notify *bool `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"`
+	// Notify when resolved.
+	SendResolved     *bool  `protobuf:"varint,6,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
 	XXX_unrecognized []byte `json:"-"`
 }
 
@@ -109,7 +136,9 @@ func (m *HipChatConfig) String() string { return proto.CompactTextString(m) }
 func (*HipChatConfig) ProtoMessage()    {}
 
 const Default_HipChatConfig_Color string = "purple"
+const Default_HipChatConfig_ColorResolved string = "green"
 const Default_HipChatConfig_Notify bool = false
+const Default_HipChatConfig_SendResolved bool = false
 
 func (m *HipChatConfig) GetAuthToken() string {
 	if m != nil && m.AuthToken != nil {
@@ -132,6 +161,13 @@ func (m *HipChatConfig) GetColor() string {
 	return Default_HipChatConfig_Color
 }
 
+func (m *HipChatConfig) GetColorResolved() string {
+	if m != nil && m.ColorResolved != nil {
+		return *m.ColorResolved
+	}
+	return Default_HipChatConfig_ColorResolved
+}
+
 func (m *HipChatConfig) GetNotify() bool {
 	if m != nil && m.Notify != nil {
 		return *m.Notify
@@ -139,6 +175,13 @@ func (m *HipChatConfig) GetNotify() bool {
 	return Default_HipChatConfig_Notify
 }
 
+func (m *HipChatConfig) GetSendResolved() bool {
+	if m != nil && m.SendResolved != nil {
+		return *m.SendResolved
+	}
+	return Default_HipChatConfig_SendResolved
+}
+
 // Notification configuration definition.
 type NotificationConfig struct {
 	// Name of this NotificationConfig. Referenced from AggregationRule.
@@ -149,7 +192,7 @@ type NotificationConfig struct {
 	EmailConfig []*EmailConfig `protobuf:"bytes,3,rep,name=email_config" json:"email_config,omitempty"`
 	// Zero or more pushover notification configurations.
 	PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"`
-	// Zero or more hipchat notification configuration.
+	// Zero or more hipchat notification configurations.
 	HipchatConfig    []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"`
 	XXX_unrecognized []byte           `json:"-"`
 }

manager/manager.go

@@ -279,6 +279,7 @@ func (s *memoryAlertManager) removeExpiredAggregates() {
 
 		if time.Since(agg.LastRefreshed) > s.minRefreshInterval {
 			delete(s.aggregates, agg.Alert.Fingerprint())
+			s.notifier.QueueNotification(agg.Alert, notificationOpResolve, agg.Rule.NotificationConfigName)
 			s.needsNotificationRefresh = true
 		} else {
 			heap.Push(&s.aggregatesByLastRefreshed, agg)
@@ -342,7 +343,7 @@ func (s *memoryAlertManager) refreshNotifications() {
 			continue
 		}
 		if agg.Rule != nil {
-			s.notifier.QueueNotification(agg.Alert, agg.Rule.NotificationConfigName)
+			s.notifier.QueueNotification(agg.Alert, notificationOpTrigger, agg.Rule.NotificationConfigName)
 			agg.LastNotification = time.Now()
 			agg.NextNotification = agg.LastNotification.Add(agg.Rule.RepeatRate)
 			numSent++

manager/notifier.go

@@ -37,12 +37,17 @@ import (
 	pb "github.com/prometheus/alertmanager/config/generated"
 )
 
-const contentTypeJson = "application/json"
+const (
+	contentTypeJson = "application/json"
+
+	notificationOpTrigger notificationOp = iota
+	notificationOpResolve
+)
 
 var bodyTmpl = template.Must(template.New("message").Parse(`From: Prometheus Alertmanager <{{.From}}>
 To: {{.To}}
 Date: {{.Date}}
-Subject: [ALERT] {{.Alert.Labels.alertname}}: {{.Alert.Summary}}
+Subject: [{{ .Status }}] {{.Alert.Labels.alertname}}: {{.Alert.Summary}}
 
 {{.Alert.Description}}
 
@@ -62,11 +67,13 @@ var (
 	hipchatUrl             = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.")
 )
 
+type notificationOp int
+
 // A Notifier is responsible for sending notifications for alerts according to
 // a provided notification configuration.
 type Notifier interface {
 	// Queue a notification for asynchronous dispatching.
-	QueueNotification(a *Alert, configName string) error
+	QueueNotification(a *Alert, op notificationOp, configName string) error
 	// Replace current notification configs. Already enqueued messages will remain
 	// unaffected.
 	SetNotificationConfigs([]*pb.NotificationConfig)
@@ -80,6 +87,7 @@ type Notifier interface {
 type notificationReq struct {
 	alert              *Alert
 	notificationConfig *pb.NotificationConfig
+	op                 notificationOp
 }
 
 // Alert notification multiplexer and dispatcher.
@@ -112,7 +120,7 @@ func (n *notifier) SetNotificationConfigs(configs []*pb.NotificationConfig) {
 	}
 }
 
-func (n *notifier) QueueNotification(a *Alert, configName string) error {
+func (n *notifier) QueueNotification(a *Alert, op notificationOp, configName string) error {
 	n.mu.Lock()
 	nc, ok := n.notificationConfigs[configName]
 	n.mu.Unlock()
@@ -127,16 +135,24 @@ func (n *notifier) QueueNotification(a *Alert, configName string) error {
 	n.pendingNotifications <- &notificationReq{
 		alert:              a,
 		notificationConfig: nc,
+		op:                 op,
 	}
 	return nil
 }
 
-func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error {
+func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationOp, a *Alert) error {
 	// http://developer.pagerduty.com/documentation/integration/events/trigger
+	eventType := ""
+	switch op {
+	case notificationOpTrigger:
+		eventType = "trigger"
+	case notificationOpResolve:
+		eventType = "resolve"
+	}
 	incidentKey := a.Fingerprint()
 	buf, err := json.Marshal(map[string]interface{}{
 		"service_key":  serviceKey,
-		"event_type":   "trigger",
+		"event_type":   eventType,
 		"description":  a.Description,
 		"incident_key": incidentKey,
 		"details": map[string]interface{}{
@@ -168,13 +184,23 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error
 	return nil
 }
 
-func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color string, notify bool, a *Alert) error {
+func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChatConfig, a *Alert) error {
 	// https://www.hipchat.com/docs/apiv2/method/send_room_notification
 	incidentKey := a.Fingerprint()
+	color := ""
+	status := ""
+	switch op {
+	case notificationOpTrigger:
+		color = config.GetColor()
+		status = "firing"
+	case notificationOpResolve:
+		color = config.GetColorResolved()
+		status = "resolved"
+	}
 	buf, err := json.Marshal(map[string]interface{}{
 		"color":          color,
-		"message":        fmt.Sprintf("<b>%s</b>: %s (<a href='%s'>view</a>)", html.EscapeString(a.Labels["alertname"]), html.EscapeString(a.Summary), a.Payload["GeneratorURL"]),
+		"message":        fmt.Sprintf("<b>%s %s</b>: %s (<a href='%s'>view</a>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]),
-		"notify":         notify,
+		"notify":         config.GetNotify(),
 		"message_format": "html",
 	})
 	if err != nil {
@@ -186,7 +212,7 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color
 		Timeout: timeout,
 	}
 	resp, err := client.Post(
-		*hipchatUrl+fmt.Sprintf("/room/%d/notification?auth_token=%s", roomId, authToken),
+		fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()),
 		contentTypeJson,
 		bytes.NewBuffer(buf),
 	)
@@ -205,21 +231,23 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color
 	return nil
 }
 
-func writeEmailBody(w io.Writer, from string, to string, a *Alert) error {
+func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error {
-	return writeEmailBodyWithTime(w, from, to, a, time.Now())
+	return writeEmailBodyWithTime(w, from, to, status, a, time.Now())
 }
 
-func writeEmailBodyWithTime(w io.Writer, from string, to string, a *Alert, moment time.Time) error {
+func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time) error {
 	err := bodyTmpl.Execute(w, struct {
-		From  string
+		From   string
-		To    string
+		To     string
-		Date  string
+		Date   string
-		Alert *Alert
+		Alert  *Alert
+		Status string
 	}{
-		From:  from,
+		From:   from,
-		To:    to,
+		To:     to,
-		Date:  moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
+		Date:   moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
-		Alert: a,
+		Alert:  a,
+		Status: status,
 	})
 	if err != nil {
 		return err
@@ -263,7 +291,14 @@ func getSMTPAuth(hasAuth bool, mechs string) (smtp.Auth, *tls.Config, error) {
 	return nil, nil, nil
 }
 
-func (n *notifier) sendEmailNotification(to string, a *Alert) error {
+func (n *notifier) sendEmailNotification(to string, op notificationOp, a *Alert) error {
+	status := ""
+	switch op {
+	case notificationOpTrigger:
+		status = "ALERT"
+	case notificationOpResolve:
+		status = "RESOLVED"
+	}
 	// Connect to the SMTP smarthost.
 	c, err := smtp.Dial(*smtpSmartHost)
 	if err != nil {
@@ -300,10 +335,10 @@ func (n *notifier) sendEmailNotification(to string, a *Alert) error {
 	}
 	defer wc.Close()
 
-	return writeEmailBody(wc, *smtpSender, to, a)
+	return writeEmailBody(wc, *smtpSender, status, to, a)
 }
 
-func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) error {
+func (n *notifier) sendPushoverNotification(token string, op notificationOp, userKey string, a *Alert) error {
 	po, err := pushover.NewPushover(token, userKey)
 	if err != nil {
 		return err
@@ -323,28 +358,37 @@ func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) err
 	return err
 }
 
-func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) {
+func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) {
 	for _, pdConfig := range config.PagerdutyConfig {
-		if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), a); err != nil {
+		if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil {
 			glog.Error("Error sending PagerDuty notification: ", err)
 		}
 	}
 	for _, emailConfig := range config.EmailConfig {
+		if op == notificationOpResolve && !emailConfig.GetSendResolved() {
+			return
+		}
 		if *smtpSmartHost == "" {
 			glog.Warning("No SMTP smarthost configured, not sending email notification.")
 			continue
 		}
-		if err := n.sendEmailNotification(emailConfig.GetEmail(), a); err != nil {
+		if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil {
 			glog.Error("Error sending email notification: ", err)
 		}
 	}
 	for _, poConfig := range config.PushoverConfig {
-		if err := n.sendPushoverNotification(poConfig.GetToken(), poConfig.GetUserKey(), a); err != nil {
+		if op == notificationOpResolve && !poConfig.GetSendResolved() {
+			return
+		}
+		if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil {
 			glog.Error("Error sending Pushover notification: ", err)
 		}
 	}
 	for _, hcConfig := range config.HipchatConfig {
-		if err := n.sendHipChatNotification(hcConfig.GetAuthToken(), hcConfig.GetRoomId(), hcConfig.GetColor(), hcConfig.GetNotify(), a); err != nil {
+		if op == notificationOpResolve && !hcConfig.GetSendResolved() {
+			return
+		}
+		if err := n.sendHipChatNotification(op, hcConfig, a); err != nil {
 			glog.Error("Error sending HipChat notification: ", err)
 		}
 	}
@@ -352,7 +396,7 @@ func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) {
 
 func (n *notifier) Dispatch() {
 	for req := range n.pendingNotifications {
-		n.handleNotification(req.alert, req.notificationConfig)
+		n.handleNotification(req.alert, req.op, req.notificationConfig)
 	}
 }
 

manager/notifier_test.go

@@ -39,7 +39,7 @@ func TestWriteEmailBody(t *testing.T) {
 	buf := &bytes.Buffer{}
 	location, _ := time.LoadLocation("Europe/Amsterdam")
 	moment := time.Date(1918, 11, 11, 11, 0, 3, 0, location)
-	writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", event, moment)
+	writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", "ALERT", event, moment)
 
 	expected := `From: Prometheus Alertmanager <from@prometheus.io>
 To: to@prometheus.io