diff --git a/.gitignore b/.gitignore index 8f11fae6..c64020dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.build .deps/ alertmanager *-stamp diff --git a/README.md b/README.md index 46c14327..998c3119 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ following aspects: * handling notification repeats * sending alert notifications via external services (currently email, [PagerDuty](http://www.pagerduty.com/), -[HipChat](http://www.hipchat.com/), or +[HipChat](http://www.hipchat.com/), +[Slack](http://www.slack.com/), or [Pushover](https://www.pushover.net/)) See [config/fixtures/sample.conf.input](config/fixtures/sample.conf.input) for diff --git a/config/config.go b/config/config.go index cffeae95..5ccf68eb 100644 --- a/config/config.go +++ b/config/config.go @@ -72,6 +72,11 @@ func (c Config) Validate() error { return fmt.Errorf("Missing room in HipChat config: %s", proto.MarshalTextString(hcc)) } } + for _, sc := range nc.SlackConfig { + if sc.WebhookUrl == nil { + return fmt.Errorf("Missing webhook URL in Slack config: %s", proto.MarshalTextString(sc)) + } + } if _, ok := ncNames[nc.GetName()]; ok { return fmt.Errorf("Notification config name not unique: %s", nc.GetName()) diff --git a/config/config.proto b/config/config.proto index 15fbc668..1948e5fe 100644 --- a/config/config.proto +++ b/config/config.proto @@ -56,6 +56,20 @@ message HipChatConfig { optional bool send_resolved = 6 [default = false]; } +// Configuration for notification via Slack. +message SlackConfig { + // Slack webhook url, (https://api.slack.com/incoming-webhooks). + optional string webhook_url = 1; + // Slack channel override, (like #other-channel or @username). + optional string channel = 2; + // Color of message when triggered. + optional string color = 3 [default = "warning"]; + // Color of message when resolved. + optional string color_resolved = 4 [default = "good"]; + // Notify when resolved. + optional bool send_resolved = 5 [default = false]; +} + // Notification configuration definition. message NotificationConfig { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -68,6 +82,8 @@ message NotificationConfig { repeated PushoverConfig pushover_config = 4; // Zero or more hipchat notification configurations. repeated HipChatConfig hipchat_config = 5; + // Zero or more slack notification configurations. + repeated SlackConfig slack_config = 6; } // A regex-based label filter used in aggregations. diff --git a/config/fixtures/sample.conf.input b/config/fixtures/sample.conf.input index 89b5d648..58edb0b9 100644 --- a/config/fixtures/sample.conf.input +++ b/config/fixtures/sample.conf.input @@ -15,6 +15,10 @@ notification_config { room_id: 123456 send_resolved: true } + slack_config { + webhook_url: "webhookurl" + send_resolved: true + } } aggregation_rule { diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index 5d34b619..2536c09d 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -13,6 +13,7 @@ It has these top-level messages: EmailConfig PushoverConfig HipChatConfig + SlackConfig NotificationConfig Filter AggregationRule @@ -182,6 +183,64 @@ func (m *HipChatConfig) GetSendResolved() bool { return Default_HipChatConfig_SendResolved } +// Configuration for notification via Slack. +type SlackConfig struct { + // Slack webhook url, (https://api.slack.com/incoming-webhooks). + WebhookUrl *string `protobuf:"bytes,1,opt,name=webhook_url" json:"webhook_url,omitempty"` + // Slack channel override, (like #other-channel or @username). + Channel *string `protobuf:"bytes,2,opt,name=channel" json:"channel,omitempty"` + // Color of message when triggered. + Color *string `protobuf:"bytes,3,opt,name=color,def=warning" json:"color,omitempty"` + // Color of message when resolved. + ColorResolved *string `protobuf:"bytes,4,opt,name=color_resolved,def=good" json:"color_resolved,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,5,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *SlackConfig) Reset() { *m = SlackConfig{} } +func (m *SlackConfig) String() string { return proto.CompactTextString(m) } +func (*SlackConfig) ProtoMessage() {} + +const Default_SlackConfig_Color string = "warning" +const Default_SlackConfig_ColorResolved string = "good" +const Default_SlackConfig_SendResolved bool = false + +func (m *SlackConfig) GetWebhookUrl() string { + if m != nil && m.WebhookUrl != nil { + return *m.WebhookUrl + } + return "" +} + +func (m *SlackConfig) GetChannel() string { + if m != nil && m.Channel != nil { + return *m.Channel + } + return "" +} + +func (m *SlackConfig) GetColor() string { + if m != nil && m.Color != nil { + return *m.Color + } + return Default_SlackConfig_Color +} + +func (m *SlackConfig) GetColorResolved() string { + if m != nil && m.ColorResolved != nil { + return *m.ColorResolved + } + return Default_SlackConfig_ColorResolved +} + +func (m *SlackConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_SlackConfig_SendResolved +} + // Notification configuration definition. type NotificationConfig struct { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -193,8 +252,10 @@ type NotificationConfig struct { // Zero or more pushover notification configurations. PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"` // Zero or more hipchat notification configurations. - HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"` - XXX_unrecognized []byte `json:"-"` + HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"` + // Zero or more slack notification configurations. + SlackConfig []*SlackConfig `protobuf:"bytes,6,rep,name=slack_config" json:"slack_config,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *NotificationConfig) Reset() { *m = NotificationConfig{} } @@ -236,6 +297,13 @@ func (m *NotificationConfig) GetHipchatConfig() []*HipChatConfig { return nil } +func (m *NotificationConfig) GetSlackConfig() []*SlackConfig { + if m != nil { + return m.SlackConfig + } + return nil +} + // A regex-based label filter used in aggregations. type Filter struct { // The regex matching the label name. diff --git a/manager/notifier.go b/manager/notifier.go index d763970d..0879ce9d 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -38,7 +38,7 @@ import ( ) const ( - contentTypeJson = "application/json" + contentTypeJSON = "application/json" notificationOpTrigger notificationOp = iota notificationOpResolve @@ -61,10 +61,10 @@ Payload labels: var ( notificationBufferSize = flag.Int("notification.buffer-size", 1000, "Size of buffer for pending notifications.") - pagerdutyApiUrl = flag.String("notification.pagerduty.url", "https://events.pagerduty.com/generic/2010-04-15/create_event.json", "PagerDuty API URL.") + pagerdutyAPIURL = flag.String("notification.pagerduty.url", "https://events.pagerduty.com/generic/2010-04-15/create_event.json", "PagerDuty API URL.") smtpSmartHost = flag.String("notification.smtp.smarthost", "", "Address of the smarthost to send all email notifications to.") smtpSender = flag.String("notification.smtp.sender", "alertmanager@example.org", "Sender email address to use in email notifications.") - hipchatUrl = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.") + hipchatURL = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.") ) type notificationOp int @@ -101,7 +101,7 @@ type notifier struct { notificationConfigs map[string]*pb.NotificationConfig } -// Construct a new notifier. +// NewNotifier construct a new notifier. func NewNotifier(configs []*pb.NotificationConfig) *notifier { notifier := ¬ifier{ pendingNotifications: make(chan *notificationReq, *notificationBufferSize), @@ -165,8 +165,8 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationO } resp, err := http.Post( - *pagerdutyApiUrl, - contentTypeJson, + *pagerdutyAPIURL, + contentTypeJSON, bytes.NewBuffer(buf), ) if err != nil { @@ -212,8 +212,8 @@ func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChat Timeout: timeout, } resp, err := client.Post( - fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()), - contentTypeJson, + fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatURL, config.GetRoomId(), config.GetAuthToken()), + contentTypeJSON, bytes.NewBuffer(buf), ) if err != nil { @@ -231,6 +231,100 @@ func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChat return nil } +// slackReq is the request for sending a slack notification. +type slackReq struct { + Channel string `json:"channel,omitempty"` + Attachments []slackAttachment `json:"attachments"` +} + +// slackAttachment is used to display a richly-formatted message block. +type slackAttachment struct { + Fallback string `json:"fallback"` + Pretext string `json:"pretext,omitempty"` + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Text string `json:"text"` + Color string `json:"color,omitempty"` + MrkdwnIn []string `json:"mrkdwn_in,omitempty"` + Fields []slackAttachmentField `json:"fields,omitempty"` +} + +// slackAttachmentField is displayed in a table inside the message attachment. +type slackAttachmentField struct { + Title string `json:"title"` + Value string `json:"value"` + Short bool `json:"short,omitempty"` +} + +func (n *notifier) sendSlackNotification(op notificationOp, config *pb.SlackConfig, a *Alert) error { + // https://api.slack.com/incoming-webhooks + incidentKey := a.Fingerprint() + color := "" + status := "" + switch op { + case notificationOpTrigger: + color = config.GetColor() + status = "firing" + case notificationOpResolve: + color = config.GetColorResolved() + status = "resolved" + } + + statusField := &slackAttachmentField{ + Title: "Status", + Value: status, + Short: true, + } + + attachment := &slackAttachment{ + Fallback: fmt.Sprintf("*%s %s*: %s (<%s|view>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), + Pretext: fmt.Sprintf("*%s*", html.EscapeString(a.Labels["alertname"])), + Title: html.EscapeString(a.Summary), + TitleLink: a.Payload["GeneratorURL"], + Text: html.EscapeString(a.Description), + Color: color, + MrkdwnIn: []string{"fallback", "pretext"}, + Fields: []slackAttachmentField{ + *statusField, + }, + } + + req := &slackReq{ + Channel: config.GetChannel(), + Attachments: []slackAttachment{ + *attachment, + }, + } + + buf, err := json.Marshal(req) + if err != nil { + return err + } + + timeout := time.Duration(5 * time.Second) + client := http.Client{ + Timeout: timeout, + } + resp, err := client.Post( + config.GetWebhookUrl(), + contentTypeJSON, + bytes.NewBuffer(buf), + ) + if err != nil { + return err + } + defer resp.Body.Close() + + respBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + glog.Infof("Sent Slack notification: %v: HTTP %d: %s", incidentKey, resp.StatusCode, respBuf) + // BUG: Check response for result of operation. + return nil +} + func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error { return writeEmailBodyWithTime(w, from, to, status, a, time.Now()) } @@ -361,7 +455,7 @@ func (n *notifier) sendPushoverNotification(token string, op notificationOp, use func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { for _, pdConfig := range config.PagerdutyConfig { if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil { - glog.Error("Error sending PagerDuty notification: ", err) + glog.Errorln("Error sending PagerDuty notification:", err) } } for _, emailConfig := range config.EmailConfig { @@ -373,7 +467,7 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No continue } if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil { - glog.Error("Error sending email notification: ", err) + glog.Errorln("Error sending email notification:", err) } } for _, poConfig := range config.PushoverConfig { @@ -381,7 +475,7 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No continue } if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil { - glog.Error("Error sending Pushover notification: ", err) + glog.Errorln("Error sending Pushover notification:", err) } } for _, hcConfig := range config.HipchatConfig { @@ -389,7 +483,15 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No continue } if err := n.sendHipChatNotification(op, hcConfig, a); err != nil { - glog.Error("Error sending HipChat notification: ", err) + glog.Errorln("Error sending HipChat notification:", err) + } + } + for _, scConfig := range config.SlackConfig { + if op == notificationOpResolve && !scConfig.GetSendResolved() { + continue + } + if err := n.sendSlackNotification(op, scConfig, a); err != nil { + glog.Errorln("Error sending Slack notification:", err) } } }