From 735e8075d2fe7a948547e9e2a96d2a0cb494f93d Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Fri, 24 Apr 2015 15:17:49 +0200 Subject: [PATCH] Add support to notify for resolved alerts Change-Id: I31fc51d2a47d92e9d7ac2ba224c7fce02b28444e --- config/config.proto | 14 +++- config/fixtures/sample.conf.input | 1 + config/generated/config.pb.go | 67 +++++++++++++++---- manager/manager.go | 3 +- manager/notifier.go | 104 +++++++++++++++++++++--------- manager/notifier_test.go | 2 +- 6 files changed, 144 insertions(+), 47 deletions(-) diff --git a/config/config.proto b/config/config.proto index e53cf148..15fbc668 100644 --- a/config/config.proto +++ b/config/config.proto @@ -24,14 +24,18 @@ message PagerDutyConfig { message EmailConfig { // Email address to notify. optional string email = 1; + // Notify when resolved. + optional bool send_resolved = 2 [default = false]; } // Configuration for notification via pushover.net. message PushoverConfig { - // Pushover token + // Pushover token. optional string token = 1; - // Pushover user_key + // Pushover user_key. optional string user_key = 2; + // Notify when resolved. + optional bool send_resolved = 3 [default = false]; } // Configuration for notification via HipChat. @@ -42,10 +46,14 @@ message HipChatConfig { optional string auth_token = 1; // HipChat room id, (https://www.hipchat.com/rooms/ids). optional int32 room_id = 2; - // Color of message. + // Color of message when triggered. optional string color = 3 [default = "purple"]; + // Color of message when resolved. + optional string color_resolved = 5 [default = "green"]; // Should this message notify or not. optional bool notify = 4 [default = false]; + // Notify when resolved. + optional bool send_resolved = 6 [default = false]; } // Notification configuration definition. diff --git a/config/fixtures/sample.conf.input b/config/fixtures/sample.conf.input index d9c29e9c..89b5d648 100644 --- a/config/fixtures/sample.conf.input +++ b/config/fixtures/sample.conf.input @@ -13,6 +13,7 @@ notification_config { hipchat_config { auth_token: "hipchatauthtoken" room_id: 123456 + send_resolved: true } } diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index e837fd4f..5d34b619 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -50,14 +50,18 @@ func (m *PagerDutyConfig) GetServiceKey() string { // Configuration for notification via mail. type EmailConfig struct { // Email address to notify. - Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"` - XXX_unrecognized []byte `json:"-"` + Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,2,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *EmailConfig) Reset() { *m = EmailConfig{} } func (m *EmailConfig) String() string { return proto.CompactTextString(m) } func (*EmailConfig) ProtoMessage() {} +const Default_EmailConfig_SendResolved bool = false + func (m *EmailConfig) GetEmail() string { if m != nil && m.Email != nil { return *m.Email @@ -65,19 +69,30 @@ func (m *EmailConfig) GetEmail() string { return "" } +func (m *EmailConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_EmailConfig_SendResolved +} + // Configuration for notification via pushover.net. type PushoverConfig struct { - // Pushover token + // Pushover token. Token *string `protobuf:"bytes,1,opt,name=token" json:"token,omitempty"` - // Pushover user_key - UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"` - XXX_unrecognized []byte `json:"-"` + // Pushover user_key. + UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,3,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *PushoverConfig) Reset() { *m = PushoverConfig{} } func (m *PushoverConfig) String() string { return proto.CompactTextString(m) } func (*PushoverConfig) ProtoMessage() {} +const Default_PushoverConfig_SendResolved bool = false + func (m *PushoverConfig) GetToken() string { if m != nil && m.Token != nil { return *m.Token @@ -92,15 +107,27 @@ func (m *PushoverConfig) GetUserKey() string { return "" } +func (m *PushoverConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_PushoverConfig_SendResolved +} + +// Configuration for notification via HipChat. type HipChatConfig struct { - // Hipchat auth token, https://www.hipchat.com/docs/api/auth + // HipChat auth token, (https://www.hipchat.com/docs/api/auth). AuthToken *string `protobuf:"bytes,1,opt,name=auth_token" json:"auth_token,omitempty"` - // Hipchat room id, https://www.hipchat.com/rooms/ids + // HipChat room id, (https://www.hipchat.com/rooms/ids). RoomId *int32 `protobuf:"varint,2,opt,name=room_id" json:"room_id,omitempty"` - // color of message + // Color of message when triggered. Color *string `protobuf:"bytes,3,opt,name=color,def=purple" json:"color,omitempty"` - // should this message notify or not - Notify *bool `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"` + // Color of message when resolved. + ColorResolved *string `protobuf:"bytes,5,opt,name=color_resolved,def=green" json:"color_resolved,omitempty"` + // Should this message notify or not. + Notify *bool `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,6,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -109,7 +136,9 @@ func (m *HipChatConfig) String() string { return proto.CompactTextString(m) } func (*HipChatConfig) ProtoMessage() {} const Default_HipChatConfig_Color string = "purple" +const Default_HipChatConfig_ColorResolved string = "green" const Default_HipChatConfig_Notify bool = false +const Default_HipChatConfig_SendResolved bool = false func (m *HipChatConfig) GetAuthToken() string { if m != nil && m.AuthToken != nil { @@ -132,6 +161,13 @@ func (m *HipChatConfig) GetColor() string { return Default_HipChatConfig_Color } +func (m *HipChatConfig) GetColorResolved() string { + if m != nil && m.ColorResolved != nil { + return *m.ColorResolved + } + return Default_HipChatConfig_ColorResolved +} + func (m *HipChatConfig) GetNotify() bool { if m != nil && m.Notify != nil { return *m.Notify @@ -139,6 +175,13 @@ func (m *HipChatConfig) GetNotify() bool { return Default_HipChatConfig_Notify } +func (m *HipChatConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_HipChatConfig_SendResolved +} + // Notification configuration definition. type NotificationConfig struct { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -149,7 +192,7 @@ type NotificationConfig struct { EmailConfig []*EmailConfig `protobuf:"bytes,3,rep,name=email_config" json:"email_config,omitempty"` // Zero or more pushover notification configurations. PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"` - // Zero or more hipchat notification configuration. + // Zero or more hipchat notification configurations. HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"` XXX_unrecognized []byte `json:"-"` } diff --git a/manager/manager.go b/manager/manager.go index 63f67759..3f1a0be2 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -279,6 +279,7 @@ func (s *memoryAlertManager) removeExpiredAggregates() { if time.Since(agg.LastRefreshed) > s.minRefreshInterval { delete(s.aggregates, agg.Alert.Fingerprint()) + s.notifier.QueueNotification(agg.Alert, notificationOpResolve, agg.Rule.NotificationConfigName) s.needsNotificationRefresh = true } else { heap.Push(&s.aggregatesByLastRefreshed, agg) @@ -342,7 +343,7 @@ func (s *memoryAlertManager) refreshNotifications() { continue } if agg.Rule != nil { - s.notifier.QueueNotification(agg.Alert, agg.Rule.NotificationConfigName) + s.notifier.QueueNotification(agg.Alert, notificationOpTrigger, agg.Rule.NotificationConfigName) agg.LastNotification = time.Now() agg.NextNotification = agg.LastNotification.Add(agg.Rule.RepeatRate) numSent++ diff --git a/manager/notifier.go b/manager/notifier.go index 83dc749d..6f103a8d 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -37,12 +37,17 @@ import ( pb "github.com/prometheus/alertmanager/config/generated" ) -const contentTypeJson = "application/json" +const ( + contentTypeJson = "application/json" + + notificationOpTrigger notificationOp = iota + notificationOpResolve +) var bodyTmpl = template.Must(template.New("message").Parse(`From: Prometheus Alertmanager <{{.From}}> To: {{.To}} Date: {{.Date}} -Subject: [ALERT] {{.Alert.Labels.alertname}}: {{.Alert.Summary}} +Subject: [{{ .Status }}] {{.Alert.Labels.alertname}}: {{.Alert.Summary}} {{.Alert.Description}} @@ -62,11 +67,13 @@ var ( hipchatUrl = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.") ) +type notificationOp int + // A Notifier is responsible for sending notifications for alerts according to // a provided notification configuration. type Notifier interface { // Queue a notification for asynchronous dispatching. - QueueNotification(a *Alert, configName string) error + QueueNotification(a *Alert, op notificationOp, configName string) error // Replace current notification configs. Already enqueued messages will remain // unaffected. SetNotificationConfigs([]*pb.NotificationConfig) @@ -80,6 +87,7 @@ type Notifier interface { type notificationReq struct { alert *Alert notificationConfig *pb.NotificationConfig + op notificationOp } // Alert notification multiplexer and dispatcher. @@ -112,7 +120,7 @@ func (n *notifier) SetNotificationConfigs(configs []*pb.NotificationConfig) { } } -func (n *notifier) QueueNotification(a *Alert, configName string) error { +func (n *notifier) QueueNotification(a *Alert, op notificationOp, configName string) error { n.mu.Lock() nc, ok := n.notificationConfigs[configName] n.mu.Unlock() @@ -127,16 +135,24 @@ func (n *notifier) QueueNotification(a *Alert, configName string) error { n.pendingNotifications <- ¬ificationReq{ alert: a, notificationConfig: nc, + op: op, } return nil } -func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error { +func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationOp, a *Alert) error { // http://developer.pagerduty.com/documentation/integration/events/trigger + eventType := "" + switch op { + case notificationOpTrigger: + eventType = "trigger" + case notificationOpResolve: + eventType = "resolve" + } incidentKey := a.Fingerprint() buf, err := json.Marshal(map[string]interface{}{ "service_key": serviceKey, - "event_type": "trigger", + "event_type": eventType, "description": a.Description, "incident_key": incidentKey, "details": map[string]interface{}{ @@ -168,13 +184,23 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error return nil } -func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color string, notify bool, a *Alert) error { +func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChatConfig, a *Alert) error { // https://www.hipchat.com/docs/apiv2/method/send_room_notification incidentKey := a.Fingerprint() + color := "" + status := "" + switch op { + case notificationOpTrigger: + color = config.GetColor() + status = "firing" + case notificationOpResolve: + color = config.GetColorResolved() + status = "resolved" + } buf, err := json.Marshal(map[string]interface{}{ "color": color, - "message": fmt.Sprintf("%s: %s (view)", html.EscapeString(a.Labels["alertname"]), html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), - "notify": notify, + "message": fmt.Sprintf("%s %s: %s (view)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), + "notify": config.GetNotify(), "message_format": "html", }) if err != nil { @@ -186,7 +212,7 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color Timeout: timeout, } resp, err := client.Post( - *hipchatUrl+fmt.Sprintf("/room/%d/notification?auth_token=%s", roomId, authToken), + fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()), contentTypeJson, bytes.NewBuffer(buf), ) @@ -205,21 +231,23 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color return nil } -func writeEmailBody(w io.Writer, from string, to string, a *Alert) error { - return writeEmailBodyWithTime(w, from, to, a, time.Now()) +func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error { + return writeEmailBodyWithTime(w, from, to, status, a, time.Now()) } -func writeEmailBodyWithTime(w io.Writer, from string, to string, a *Alert, moment time.Time) error { +func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time) error { err := bodyTmpl.Execute(w, struct { - From string - To string - Date string - Alert *Alert + From string + To string + Date string + Alert *Alert + Status string }{ - From: from, - To: to, - Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"), - Alert: a, + From: from, + To: to, + Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"), + Alert: a, + Status: status, }) if err != nil { return err @@ -263,7 +291,14 @@ func getSMTPAuth(hasAuth bool, mechs string) (smtp.Auth, *tls.Config, error) { return nil, nil, nil } -func (n *notifier) sendEmailNotification(to string, a *Alert) error { +func (n *notifier) sendEmailNotification(to string, op notificationOp, a *Alert) error { + status := "" + switch op { + case notificationOpTrigger: + status = "ALERT" + case notificationOpResolve: + status = "RESOLVED" + } // Connect to the SMTP smarthost. c, err := smtp.Dial(*smtpSmartHost) if err != nil { @@ -300,10 +335,10 @@ func (n *notifier) sendEmailNotification(to string, a *Alert) error { } defer wc.Close() - return writeEmailBody(wc, *smtpSender, to, a) + return writeEmailBody(wc, *smtpSender, status, to, a) } -func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) error { +func (n *notifier) sendPushoverNotification(token string, op notificationOp, userKey string, a *Alert) error { po, err := pushover.NewPushover(token, userKey) if err != nil { return err @@ -323,28 +358,37 @@ func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) err return err } -func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) { +func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { for _, pdConfig := range config.PagerdutyConfig { - if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), a); err != nil { + if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil { glog.Error("Error sending PagerDuty notification: ", err) } } for _, emailConfig := range config.EmailConfig { + if op == notificationOpResolve && !emailConfig.GetSendResolved() { + return + } if *smtpSmartHost == "" { glog.Warning("No SMTP smarthost configured, not sending email notification.") continue } - if err := n.sendEmailNotification(emailConfig.GetEmail(), a); err != nil { + if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil { glog.Error("Error sending email notification: ", err) } } for _, poConfig := range config.PushoverConfig { - if err := n.sendPushoverNotification(poConfig.GetToken(), poConfig.GetUserKey(), a); err != nil { + if op == notificationOpResolve && !poConfig.GetSendResolved() { + return + } + if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil { glog.Error("Error sending Pushover notification: ", err) } } for _, hcConfig := range config.HipchatConfig { - if err := n.sendHipChatNotification(hcConfig.GetAuthToken(), hcConfig.GetRoomId(), hcConfig.GetColor(), hcConfig.GetNotify(), a); err != nil { + if op == notificationOpResolve && !hcConfig.GetSendResolved() { + return + } + if err := n.sendHipChatNotification(op, hcConfig, a); err != nil { glog.Error("Error sending HipChat notification: ", err) } } @@ -352,7 +396,7 @@ func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) { func (n *notifier) Dispatch() { for req := range n.pendingNotifications { - n.handleNotification(req.alert, req.notificationConfig) + n.handleNotification(req.alert, req.op, req.notificationConfig) } } diff --git a/manager/notifier_test.go b/manager/notifier_test.go index 96344e68..e73f6505 100644 --- a/manager/notifier_test.go +++ b/manager/notifier_test.go @@ -39,7 +39,7 @@ func TestWriteEmailBody(t *testing.T) { buf := &bytes.Buffer{} location, _ := time.LoadLocation("Europe/Amsterdam") moment := time.Date(1918, 11, 11, 11, 0, 3, 0, location) - writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", event, moment) + writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", "ALERT", event, moment) expected := `From: Prometheus Alertmanager To: to@prometheus.io