RepoMirrors
/
alertmanager
mirror of https://github.com/prometheus/alertmanager
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support to notify for resolved alerts 

Change-Id: I31fc51d2a47d92e9d7ac2ba224c7fce02b28444e
This commit is contained in:
Johannes 'fish' Ziemke 2015-04-24 15:17:49 +02:00
parent ffff50027a
commit 735e8075d2
6 changed files with 144 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
config/config.proto
View File

@ -24,14 +24,18 @@ message PagerDutyConfig {
message EmailConfig {
// Email address to notify.
optional string email = 1;
// Notify when resolved.
optional bool send_resolved = 2 [default = false];
}
// Configuration for notification via pushover.net.
message PushoverConfig {
// Pushover token
// Pushover token.
optional string token = 1;
// Pushover user_key
// Pushover user_key.
optional string user_key = 2;
// Notify when resolved.
optional bool send_resolved = 3 [default = false];
}
// Configuration for notification via HipChat.
@ -42,10 +46,14 @@ message HipChatConfig {
optional string auth_token = 1;
// HipChat room id, (https://www.hipchat.com/rooms/ids).
optional int32 room_id = 2;
// Color of message.
// Color of message when triggered.
optional string color = 3 [default = "purple"];
// Color of message when resolved.
optional string color_resolved = 5 [default = "green"];
// Should this message notify or not.
optional bool notify = 4 [default = false];
// Notify when resolved.
optional bool send_resolved = 6 [default = false];
}
// Notification configuration definition.

1
config/fixtures/sample.conf.input
View File

@ -13,6 +13,7 @@ notification_config {
hipchat_config {
auth_token: "hipchatauthtoken"
room_id: 123456
send_resolved: true
}
}

67
config/generated/config.pb.go
View File

@ -50,14 +50,18 @@ func (m *PagerDutyConfig) GetServiceKey() string {
// Configuration for notification via mail.
type EmailConfig struct {
// Email address to notify.
Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"`
XXX_unrecognized []byte `json:"-"`
Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"`
// Notify when resolved.
SendResolved *bool `protobuf:"varint,2,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *EmailConfig) Reset() { *m = EmailConfig{} }
func (m *EmailConfig) String() string { return proto.CompactTextString(m) }
func (*EmailConfig) ProtoMessage() {}
const Default_EmailConfig_SendResolved bool = false
func (m *EmailConfig) GetEmail() string {
if m != nil && m.Email != nil {
return *m.Email
@ -65,19 +69,30 @@ func (m *EmailConfig) GetEmail() string {
return ""
}
func (m *EmailConfig) GetSendResolved() bool {
if m != nil && m.SendResolved != nil {
return *m.SendResolved
}
return Default_EmailConfig_SendResolved
}
// Configuration for notification via pushover.net.
type PushoverConfig struct {
// Pushover token
// Pushover token.
Token *string `protobuf:"bytes,1,opt,name=token" json:"token,omitempty"`
// Pushover user_key
UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"`
XXX_unrecognized []byte `json:"-"`
// Pushover user_key.
UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"`
// Notify when resolved.
SendResolved *bool `protobuf:"varint,3,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *PushoverConfig) Reset() { *m = PushoverConfig{} }
func (m *PushoverConfig) String() string { return proto.CompactTextString(m) }
func (*PushoverConfig) ProtoMessage() {}
const Default_PushoverConfig_SendResolved bool = false
func (m *PushoverConfig) GetToken() string {
if m != nil && m.Token != nil {
return *m.Token
@ -92,15 +107,27 @@ func (m *PushoverConfig) GetUserKey() string {
return ""
}
func (m *PushoverConfig) GetSendResolved() bool {
if m != nil && m.SendResolved != nil {
return *m.SendResolved
}
return Default_PushoverConfig_SendResolved
}
// Configuration for notification via HipChat.
type HipChatConfig struct {
// Hipchat auth token, https://www.hipchat.com/docs/api/auth
// HipChat auth token, (https://www.hipchat.com/docs/api/auth).
AuthToken *string `protobuf:"bytes,1,opt,name=auth_token" json:"auth_token,omitempty"`
// Hipchat room id, https://www.hipchat.com/rooms/ids
// HipChat room id, (https://www.hipchat.com/rooms/ids).
RoomId *int32 `protobuf:"varint,2,opt,name=room_id" json:"room_id,omitempty"`
// color of message
// Color of message when triggered.
Color *string `protobuf:"bytes,3,opt,name=color,def=purple" json:"color,omitempty"`
// should this message notify or not
Notify *bool `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"`
// Color of message when resolved.
ColorResolved *string `protobuf:"bytes,5,opt,name=color_resolved,def=green" json:"color_resolved,omitempty"`
// Should this message notify or not.
Notify *bool `protobuf:"varint,4,opt,name=notify,def=0" json:"notify,omitempty"`
// Notify when resolved.
SendResolved *bool `protobuf:"varint,6,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -109,7 +136,9 @@ func (m *HipChatConfig) String() string { return proto.CompactTextString(m) }
func (*HipChatConfig) ProtoMessage() {}
const Default_HipChatConfig_Color string = "purple"
const Default_HipChatConfig_ColorResolved string = "green"
const Default_HipChatConfig_Notify bool = false
const Default_HipChatConfig_SendResolved bool = false
func (m *HipChatConfig) GetAuthToken() string {
if m != nil && m.AuthToken != nil {
@ -132,6 +161,13 @@ func (m *HipChatConfig) GetColor() string {
return Default_HipChatConfig_Color
}
func (m *HipChatConfig) GetColorResolved() string {
if m != nil && m.ColorResolved != nil {
return *m.ColorResolved
}
return Default_HipChatConfig_ColorResolved
}
func (m *HipChatConfig) GetNotify() bool {
if m != nil && m.Notify != nil {
return *m.Notify
@ -139,6 +175,13 @@ func (m *HipChatConfig) GetNotify() bool {
return Default_HipChatConfig_Notify
}
func (m *HipChatConfig) GetSendResolved() bool {
if m != nil && m.SendResolved != nil {
return *m.SendResolved
}
return Default_HipChatConfig_SendResolved
}
// Notification configuration definition.
type NotificationConfig struct {
// Name of this NotificationConfig. Referenced from AggregationRule.
@ -149,7 +192,7 @@ type NotificationConfig struct {
EmailConfig []*EmailConfig `protobuf:"bytes,3,rep,name=email_config" json:"email_config,omitempty"`
// Zero or more pushover notification configurations.
PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"`
// Zero or more hipchat notification configuration.
// Zero or more hipchat notification configurations.
HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"`
XXX_unrecognized []byte `json:"-"`
}

3
manager/manager.go
View File

@ -279,6 +279,7 @@ func (s *memoryAlertManager) removeExpiredAggregates() {
if time.Since(agg.LastRefreshed) > s.minRefreshInterval {
delete(s.aggregates, agg.Alert.Fingerprint())
s.notifier.QueueNotification(agg.Alert, notificationOpResolve, agg.Rule.NotificationConfigName)
s.needsNotificationRefresh = true
} else {
heap.Push(&s.aggregatesByLastRefreshed, agg)
@ -342,7 +343,7 @@ func (s *memoryAlertManager) refreshNotifications() {
continue
}
if agg.Rule != nil {
s.notifier.QueueNotification(agg.Alert, agg.Rule.NotificationConfigName)
s.notifier.QueueNotification(agg.Alert, notificationOpTrigger, agg.Rule.NotificationConfigName)
agg.LastNotification = time.Now()
agg.NextNotification = agg.LastNotification.Add(agg.Rule.RepeatRate)
numSent++

104
manager/notifier.go
View File

@ -37,12 +37,17 @@ import (
pb "github.com/prometheus/alertmanager/config/generated"
)
const contentTypeJson = "application/json"
const (
contentTypeJson = "application/json"
notificationOpTrigger notificationOp = iota
notificationOpResolve
)
var bodyTmpl = template.Must(template.New("message").Parse(`From: Prometheus Alertmanager <{{.From}}>
To: {{.To}}
Date: {{.Date}}
Subject: [ALERT] {{.Alert.Labels.alertname}}: {{.Alert.Summary}}
Subject: [{{ .Status }}] {{.Alert.Labels.alertname}}: {{.Alert.Summary}}
{{.Alert.Description}}
@ -62,11 +67,13 @@ var (
hipchatUrl = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.")
)
type notificationOp int
// A Notifier is responsible for sending notifications for alerts according to
// a provided notification configuration.
type Notifier interface {
// Queue a notification for asynchronous dispatching.
QueueNotification(a *Alert, configName string) error
QueueNotification(a *Alert, op notificationOp, configName string) error
// Replace current notification configs. Already enqueued messages will remain
// unaffected.
SetNotificationConfigs([]*pb.NotificationConfig)
@ -80,6 +87,7 @@ type Notifier interface {
type notificationReq struct {
alert *Alert
notificationConfig *pb.NotificationConfig
op notificationOp
}
// Alert notification multiplexer and dispatcher.
@ -112,7 +120,7 @@ func (n *notifier) SetNotificationConfigs(configs []*pb.NotificationConfig) {
}
}
func (n *notifier) QueueNotification(a *Alert, configName string) error {
func (n *notifier) QueueNotification(a *Alert, op notificationOp, configName string) error {
n.mu.Lock()
nc, ok := n.notificationConfigs[configName]
n.mu.Unlock()
@ -127,16 +135,24 @@ func (n *notifier) QueueNotification(a *Alert, configName string) error {
n.pendingNotifications <- &notificationReq{
alert: a,
notificationConfig: nc,
op: op,
}
return nil
}
func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error {
func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationOp, a *Alert) error {
// http://developer.pagerduty.com/documentation/integration/events/trigger
eventType := ""
switch op {
case notificationOpTrigger:
eventType = "trigger"
case notificationOpResolve:
eventType = "resolve"
}
incidentKey := a.Fingerprint()
buf, err := json.Marshal(map[string]interface{}{
"service_key": serviceKey,
"event_type": "trigger",
"event_type": eventType,
"description": a.Description,
"incident_key": incidentKey,
"details": map[string]interface{}{
@ -168,13 +184,23 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, a *Alert) error
return nil
}
func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color string, notify bool, a *Alert) error {
func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChatConfig, a *Alert) error {
// https://www.hipchat.com/docs/apiv2/method/send_room_notification
incidentKey := a.Fingerprint()
color := ""
status := ""
switch op {
case notificationOpTrigger:
color = config.GetColor()
status = "firing"
case notificationOpResolve:
color = config.GetColorResolved()
status = "resolved"
}
buf, err := json.Marshal(map[string]interface{}{
"color": color,
"message": fmt.Sprintf("<b>%s</b>: %s (<a href='%s'>view</a>)", html.EscapeString(a.Labels["alertname"]), html.EscapeString(a.Summary), a.Payload["GeneratorURL"]),
"notify": notify,
"message": fmt.Sprintf("<b>%s %s</b>: %s (<a href='%s'>view</a>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]),
"notify": config.GetNotify(),
"message_format": "html",
})
if err != nil {
@ -186,7 +212,7 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color
Timeout: timeout,
}
resp, err := client.Post(
*hipchatUrl+fmt.Sprintf("/room/%d/notification?auth_token=%s", roomId, authToken),
fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()),
contentTypeJson,
bytes.NewBuffer(buf),
)
@ -205,21 +231,23 @@ func (n *notifier) sendHipChatNotification(authToken string, roomId int32, color
return nil
}
func writeEmailBody(w io.Writer, from string, to string, a *Alert) error {
return writeEmailBodyWithTime(w, from, to, a, time.Now())
func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error {
return writeEmailBodyWithTime(w, from, to, status, a, time.Now())
}
func writeEmailBodyWithTime(w io.Writer, from string, to string, a *Alert, moment time.Time) error {
func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time) error {
err := bodyTmpl.Execute(w, struct {
From string
To string
Date string
Alert *Alert
From string
To string
Date string
Alert *Alert
Status string
}{
From: from,
To: to,
Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
Alert: a,
From: from,
To: to,
Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
Alert: a,
Status: status,
})
if err != nil {
return err
@ -263,7 +291,14 @@ func getSMTPAuth(hasAuth bool, mechs string) (smtp.Auth, *tls.Config, error) {
return nil, nil, nil
}
func (n *notifier) sendEmailNotification(to string, a *Alert) error {
func (n *notifier) sendEmailNotification(to string, op notificationOp, a *Alert) error {
status := ""
switch op {
case notificationOpTrigger:
status = "ALERT"
case notificationOpResolve:
status = "RESOLVED"
}
// Connect to the SMTP smarthost.
c, err := smtp.Dial(*smtpSmartHost)
if err != nil {
@ -300,10 +335,10 @@ func (n *notifier) sendEmailNotification(to string, a *Alert) error {
}
defer wc.Close()
return writeEmailBody(wc, *smtpSender, to, a)
return writeEmailBody(wc, *smtpSender, status, to, a)
}
func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) error {
func (n *notifier) sendPushoverNotification(token string, op notificationOp, userKey string, a *Alert) error {
po, err := pushover.NewPushover(token, userKey)
if err != nil {
return err
@ -323,28 +358,37 @@ func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) err
return err
}
func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) {
func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) {
for _, pdConfig := range config.PagerdutyConfig {
if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), a); err != nil {
if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil {
glog.Error("Error sending PagerDuty notification: ", err)
}
}
for _, emailConfig := range config.EmailConfig {
if op == notificationOpResolve && !emailConfig.GetSendResolved() {
return
}
if *smtpSmartHost == "" {
glog.Warning("No SMTP smarthost configured, not sending email notification.")
continue
}
if err := n.sendEmailNotification(emailConfig.GetEmail(), a); err != nil {
if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil {
glog.Error("Error sending email notification: ", err)
}
}
for _, poConfig := range config.PushoverConfig {
if err := n.sendPushoverNotification(poConfig.GetToken(), poConfig.GetUserKey(), a); err != nil {
if op == notificationOpResolve && !poConfig.GetSendResolved() {
return
}
if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil {
glog.Error("Error sending Pushover notification: ", err)
}
}
for _, hcConfig := range config.HipchatConfig {
if err := n.sendHipChatNotification(hcConfig.GetAuthToken(), hcConfig.GetRoomId(), hcConfig.GetColor(), hcConfig.GetNotify(), a); err != nil {
if op == notificationOpResolve && !hcConfig.GetSendResolved() {
return
}
if err := n.sendHipChatNotification(op, hcConfig, a); err != nil {
glog.Error("Error sending HipChat notification: ", err)
}
}
@ -352,7 +396,7 @@ func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) {
func (n *notifier) Dispatch() {
for req := range n.pendingNotifications {
n.handleNotification(req.alert, req.notificationConfig)
n.handleNotification(req.alert, req.op, req.notificationConfig)
}
}

2
manager/notifier_test.go
View File

@ -39,7 +39,7 @@ func TestWriteEmailBody(t *testing.T) {
buf := &bytes.Buffer{}
location, _ := time.LoadLocation("Europe/Amsterdam")
moment := time.Date(1918, 11, 11, 11, 0, 3, 0, location)
writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", event, moment)
writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", "ALERT", event, moment)
expected := `From: Prometheus Alertmanager <from@prometheus.io>
To: to@prometheus.io