Add simple support for Slack notifications

.gitignore

@@ -1,3 +1,4 @@
+.build
 .deps/
 alertmanager
 *-stamp

README.md

@@ -11,7 +11,8 @@ following aspects:
 * handling notification repeats
 * sending alert notifications via external services (currently email,
 [PagerDuty](http://www.pagerduty.com/),
-[HipChat](http://www.hipchat.com/), or
+[HipChat](http://www.hipchat.com/),
+[Slack](http://www.slack.com/), or
 [Pushover](https://www.pushover.net/))
 
 See [config/fixtures/sample.conf.input](config/fixtures/sample.conf.input) for

config/config.go

@@ -72,6 +72,11 @@ func (c Config) Validate() error {
 				return fmt.Errorf("Missing room in HipChat config: %s", proto.MarshalTextString(hcc))
 			}
 		}
+		for _, sc := range nc.SlackConfig {
+			if sc.WebhookUrl == nil {
+				return fmt.Errorf("Missing webhook URL in Slack config: %s", proto.MarshalTextString(sc))
+			}
+		}
 
 		if _, ok := ncNames[nc.GetName()]; ok {
 			return fmt.Errorf("Notification config name not unique: %s", nc.GetName())

config/config.proto

@@ -56,6 +56,20 @@ message HipChatConfig {
   optional bool send_resolved = 6 [default = false];
 }
 
+// Configuration for notification via Slack.
+message SlackConfig {
+  // Slack webhook url, (https://api.slack.com/incoming-webhooks).
+  optional string webhook_url = 1;
+  // Slack channel override, (like #other-channel or @username).
+  optional string channel = 2;
+  // Color of message when triggered.
+  optional string color = 3 [default = "warning"];
+  // Color of message when resolved.
+  optional string color_resolved = 4 [default = "good"];
+  // Notify when resolved.
+  optional bool send_resolved = 5 [default = false];
+}
+
 // Notification configuration definition.
 message NotificationConfig {
   // Name of this NotificationConfig. Referenced from AggregationRule.
@@ -68,6 +82,8 @@ message NotificationConfig {
   repeated PushoverConfig pushover_config = 4;
   // Zero or more hipchat notification configurations.
   repeated HipChatConfig hipchat_config = 5;
+  // Zero or more slack notification configurations.
+  repeated SlackConfig slack_config = 6;
 }
 
 // A regex-based label filter used in aggregations.

config/fixtures/sample.conf.input

@@ -15,6 +15,10 @@ notification_config {
     room_id: 123456
     send_resolved: true
   }
+  slack_config {
+    webhook_url: "webhookurl"
+    send_resolved: true
+  }
 }
 
 aggregation_rule {

config/generated/config.pb.go

@@ -13,6 +13,7 @@ It has these top-level messages:
 	EmailConfig
 	PushoverConfig
 	HipChatConfig
+	SlackConfig
 	NotificationConfig
 	Filter
 	AggregationRule
@@ -182,6 +183,64 @@ func (m *HipChatConfig) GetSendResolved() bool {
 	return Default_HipChatConfig_SendResolved
 }
 
+// Configuration for notification via Slack.
+type SlackConfig struct {
+	// Slack webhook url, (https://api.slack.com/incoming-webhooks).
+	WebhookUrl *string `protobuf:"bytes,1,opt,name=webhook_url" json:"webhook_url,omitempty"`
+	// Slack channel override, (like #other-channel or @username).
+	Channel *string `protobuf:"bytes,2,opt,name=channel" json:"channel,omitempty"`
+	// Color of message when triggered.
+	Color *string `protobuf:"bytes,3,opt,name=color,def=warning" json:"color,omitempty"`
+	// Color of message when resolved.
+	ColorResolved *string `protobuf:"bytes,4,opt,name=color_resolved,def=good" json:"color_resolved,omitempty"`
+	// Notify when resolved.
+	SendResolved     *bool  `protobuf:"varint,5,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"`
+	XXX_unrecognized []byte `json:"-"`
+}
+
+func (m *SlackConfig) Reset()         { *m = SlackConfig{} }
+func (m *SlackConfig) String() string { return proto.CompactTextString(m) }
+func (*SlackConfig) ProtoMessage()    {}
+
+const Default_SlackConfig_Color string = "warning"
+const Default_SlackConfig_ColorResolved string = "good"
+const Default_SlackConfig_SendResolved bool = false
+
+func (m *SlackConfig) GetWebhookUrl() string {
+	if m != nil && m.WebhookUrl != nil {
+		return *m.WebhookUrl
+	}
+	return ""
+}
+
+func (m *SlackConfig) GetChannel() string {
+	if m != nil && m.Channel != nil {
+		return *m.Channel
+	}
+	return ""
+}
+
+func (m *SlackConfig) GetColor() string {
+	if m != nil && m.Color != nil {
+		return *m.Color
+	}
+	return Default_SlackConfig_Color
+}
+
+func (m *SlackConfig) GetColorResolved() string {
+	if m != nil && m.ColorResolved != nil {
+		return *m.ColorResolved
+	}
+	return Default_SlackConfig_ColorResolved
+}
+
+func (m *SlackConfig) GetSendResolved() bool {
+	if m != nil && m.SendResolved != nil {
+		return *m.SendResolved
+	}
+	return Default_SlackConfig_SendResolved
+}
+
 // Notification configuration definition.
 type NotificationConfig struct {
 	// Name of this NotificationConfig. Referenced from AggregationRule.
@@ -193,8 +252,10 @@ type NotificationConfig struct {
 	// Zero or more pushover notification configurations.
 	PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"`
 	// Zero or more hipchat notification configurations.
-	HipchatConfig    []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"`
+	HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"`
-	XXX_unrecognized []byte           `json:"-"`
+	// Zero or more slack notification configurations.
+	SlackConfig      []*SlackConfig `protobuf:"bytes,6,rep,name=slack_config" json:"slack_config,omitempty"`
+	XXX_unrecognized []byte         `json:"-"`
 }
 
 func (m *NotificationConfig) Reset()         { *m = NotificationConfig{} }
@@ -236,6 +297,13 @@ func (m *NotificationConfig) GetHipchatConfig() []*HipChatConfig {
 	return nil
 }
 
+func (m *NotificationConfig) GetSlackConfig() []*SlackConfig {
+	if m != nil {
+		return m.SlackConfig
+	}
+	return nil
+}
+
 // A regex-based label filter used in aggregations.
 type Filter struct {
 	// The regex matching the label name.

manager/notifier.go

@@ -38,7 +38,7 @@ import (
 )
 
 const (
-	contentTypeJson = "application/json"
+	contentTypeJSON = "application/json"
 
 	notificationOpTrigger notificationOp = iota
 	notificationOpResolve
@@ -61,10 +61,10 @@ Payload labels:
 
 var (
 	notificationBufferSize = flag.Int("notification.buffer-size", 1000, "Size of buffer for pending notifications.")
-	pagerdutyApiUrl        = flag.String("notification.pagerduty.url", "https://events.pagerduty.com/generic/2010-04-15/create_event.json", "PagerDuty API URL.")
+	pagerdutyAPIURL        = flag.String("notification.pagerduty.url", "https://events.pagerduty.com/generic/2010-04-15/create_event.json", "PagerDuty API URL.")
 	smtpSmartHost          = flag.String("notification.smtp.smarthost", "", "Address of the smarthost to send all email notifications to.")
 	smtpSender             = flag.String("notification.smtp.sender", "alertmanager@example.org", "Sender email address to use in email notifications.")
-	hipchatUrl             = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.")
+	hipchatURL             = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.")
 )
 
 type notificationOp int
@@ -101,7 +101,7 @@ type notifier struct {
 	notificationConfigs map[string]*pb.NotificationConfig
 }
 
-// Construct a new notifier.
+// NewNotifier construct a new notifier.
 func NewNotifier(configs []*pb.NotificationConfig) *notifier {
 	notifier := &notifier{
 		pendingNotifications: make(chan *notificationReq, *notificationBufferSize),
@@ -165,8 +165,8 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationO
 	}
 
 	resp, err := http.Post(
-		*pagerdutyApiUrl,
+		*pagerdutyAPIURL,
-		contentTypeJson,
+		contentTypeJSON,
 		bytes.NewBuffer(buf),
 	)
 	if err != nil {
@@ -212,8 +212,8 @@ func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChat
 		Timeout: timeout,
 	}
 	resp, err := client.Post(
-		fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()),
+		fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatURL, config.GetRoomId(), config.GetAuthToken()),
-		contentTypeJson,
+		contentTypeJSON,
 		bytes.NewBuffer(buf),
 	)
 	if err != nil {
@@ -231,6 +231,100 @@ func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChat
 	return nil
 }
 
+// slackReq is the request for sending a slack notification.
+type slackReq struct {
+	Channel     string            `json:"channel,omitempty"`
+	Attachments []slackAttachment `json:"attachments"`
+}
+
+// slackAttachment is used to display a richly-formatted message block.
+type slackAttachment struct {
+	Fallback  string                 `json:"fallback"`
+	Pretext   string                 `json:"pretext,omitempty"`
+	Title     string                 `json:"title,omitempty"`
+	TitleLink string                 `json:"title_link,omitempty"`
+	Text      string                 `json:"text"`
+	Color     string                 `json:"color,omitempty"`
+	MrkdwnIn  []string               `json:"mrkdwn_in,omitempty"`
+	Fields    []slackAttachmentField `json:"fields,omitempty"`
+}
+
+// slackAttachmentField is displayed in a table inside the message attachment.
+type slackAttachmentField struct {
+	Title string `json:"title"`
+	Value string `json:"value"`
+	Short bool   `json:"short,omitempty"`
+}
+
+func (n *notifier) sendSlackNotification(op notificationOp, config *pb.SlackConfig, a *Alert) error {
+	// https://api.slack.com/incoming-webhooks
+	incidentKey := a.Fingerprint()
+	color := ""
+	status := ""
+	switch op {
+	case notificationOpTrigger:
+		color = config.GetColor()
+		status = "firing"
+	case notificationOpResolve:
+		color = config.GetColorResolved()
+		status = "resolved"
+	}
+
+	statusField := &slackAttachmentField{
+		Title: "Status",
+		Value: status,
+		Short: true,
+	}
+
+	attachment := &slackAttachment{
+		Fallback:  fmt.Sprintf("*%s %s*: %s (<%s|view>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]),
+		Pretext:   fmt.Sprintf("*%s*", html.EscapeString(a.Labels["alertname"])),
+		Title:     html.EscapeString(a.Summary),
+		TitleLink: a.Payload["GeneratorURL"],
+		Text:      html.EscapeString(a.Description),
+		Color:     color,
+		MrkdwnIn:  []string{"fallback", "pretext"},
+		Fields: []slackAttachmentField{
+			*statusField,
+		},
+	}
+
+	req := &slackReq{
+		Channel: config.GetChannel(),
+		Attachments: []slackAttachment{
+			*attachment,
+		},
+	}
+
+	buf, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	timeout := time.Duration(5 * time.Second)
+	client := http.Client{
+		Timeout: timeout,
+	}
+	resp, err := client.Post(
+		config.GetWebhookUrl(),
+		contentTypeJSON,
+		bytes.NewBuffer(buf),
+	)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	respBuf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	glog.Infof("Sent Slack notification: %v: HTTP %d: %s", incidentKey, resp.StatusCode, respBuf)
+	// BUG: Check response for result of operation.
+	return nil
+}
+
 func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error {
 	return writeEmailBodyWithTime(w, from, to, status, a, time.Now())
 }
@@ -361,7 +455,7 @@ func (n *notifier) sendPushoverNotification(token string, op notificationOp, use
 func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) {
 	for _, pdConfig := range config.PagerdutyConfig {
 		if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil {
-			glog.Error("Error sending PagerDuty notification: ", err)
+			glog.Errorln("Error sending PagerDuty notification:", err)
 		}
 	}
 	for _, emailConfig := range config.EmailConfig {
@@ -373,7 +467,7 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No
 			continue
 		}
 		if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil {
-			glog.Error("Error sending email notification: ", err)
+			glog.Errorln("Error sending email notification:", err)
 		}
 	}
 	for _, poConfig := range config.PushoverConfig {
@@ -381,7 +475,7 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No
 			continue
 		}
 		if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil {
-			glog.Error("Error sending Pushover notification: ", err)
+			glog.Errorln("Error sending Pushover notification:", err)
 		}
 	}
 	for _, hcConfig := range config.HipchatConfig {
@@ -389,7 +483,15 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No
 			continue
 		}
 		if err := n.sendHipChatNotification(op, hcConfig, a); err != nil {
-			glog.Error("Error sending HipChat notification: ", err)
+			glog.Errorln("Error sending HipChat notification:", err)
+		}
+	}
+	for _, scConfig := range config.SlackConfig {
+		if op == notificationOpResolve && !scConfig.GetSendResolved() {
+			continue
+		}
+		if err := n.sendSlackNotification(op, scConfig, a); err != nil {
+			glog.Errorln("Error sending Slack notification:", err)
 		}
 	}
 }