// Copyright 2013 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package manager import ( "bytes" "crypto/tls" "encoding/json" "flag" "fmt" "html" "io" "io/ioutil" "net" "net/http" "net/smtp" "os" "strings" "sync" "text/template" "time" "github.com/golang/glog" "github.com/thorduri/pushover" pb "github.com/prometheus/alertmanager/config/generated" ) const ( contentTypeJson = "application/json" notificationOpTrigger notificationOp = iota notificationOpResolve ) var bodyTmpl = template.Must(template.New("message").Parse(`From: Prometheus Alertmanager <{{.From}}> To: {{.To}} Date: {{.Date}} Subject: [{{ .Status }}] {{.Alert.Labels.alertname}}: {{.Alert.Summary}} {{.Alert.Description}} Grouping labels: {{range $label, $value := .Alert.Labels}} {{$label}} = "{{$value}}"{{end}} Payload labels: {{range $label, $value := .Alert.Payload}} {{$label}} = "{{$value}}"{{end}}`)) var ( notificationBufferSize = flag.Int("notification.buffer-size", 1000, "Size of buffer for pending notifications.") pagerdutyApiUrl = flag.String("notification.pagerduty.url", "https://events.pagerduty.com/generic/2010-04-15/create_event.json", "PagerDuty API URL.") smtpSmartHost = flag.String("notification.smtp.smarthost", "", "Address of the smarthost to send all email notifications to.") smtpSender = flag.String("notification.smtp.sender", "alertmanager@example.org", "Sender email address to use in email notifications.") hipchatUrl = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.") ) type notificationOp int // A Notifier is responsible for sending notifications for alerts according to // a provided notification configuration. type Notifier interface { // Queue a notification for asynchronous dispatching. QueueNotification(a *Alert, op notificationOp, configName string) error // Replace current notification configs. Already enqueued messages will remain // unaffected. SetNotificationConfigs([]*pb.NotificationConfig) // Start alert notification dispatch loop. Dispatch() // Stop the alert notification dispatch loop. Close() } // Request for sending a notification. type notificationReq struct { alert *Alert notificationConfig *pb.NotificationConfig op notificationOp } // Alert notification multiplexer and dispatcher. type notifier struct { // Notifications that are queued to be sent. pendingNotifications chan *notificationReq // Mutex to protect the fields below. mu sync.Mutex // Map of notification configs by name. notificationConfigs map[string]*pb.NotificationConfig } // Construct a new notifier. func NewNotifier(configs []*pb.NotificationConfig) *notifier { notifier := ¬ifier{ pendingNotifications: make(chan *notificationReq, *notificationBufferSize), } notifier.SetNotificationConfigs(configs) return notifier } func (n *notifier) SetNotificationConfigs(configs []*pb.NotificationConfig) { n.mu.Lock() defer n.mu.Unlock() n.notificationConfigs = map[string]*pb.NotificationConfig{} for _, c := range configs { n.notificationConfigs[c.GetName()] = c } } func (n *notifier) QueueNotification(a *Alert, op notificationOp, configName string) error { n.mu.Lock() nc, ok := n.notificationConfigs[configName] n.mu.Unlock() if !ok { return fmt.Errorf("No such notification configuration %s", configName) } // We need to save a reference to the notification config in the // notificationReq since the config might be replaced or gone at the time the // message gets dispatched. n.pendingNotifications <- ¬ificationReq{ alert: a, notificationConfig: nc, op: op, } return nil } func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationOp, a *Alert) error { // http://developer.pagerduty.com/documentation/integration/events/trigger eventType := "" switch op { case notificationOpTrigger: eventType = "trigger" case notificationOpResolve: eventType = "resolve" } incidentKey := a.Fingerprint() buf, err := json.Marshal(map[string]interface{}{ "service_key": serviceKey, "event_type": eventType, "description": a.Description, "incident_key": incidentKey, "details": map[string]interface{}{ "grouping_labels": a.Labels, "extra_labels": a.Payload, }, }) if err != nil { return err } resp, err := http.Post( *pagerdutyApiUrl, contentTypeJson, bytes.NewBuffer(buf), ) if err != nil { return err } defer resp.Body.Close() respBuf, err := ioutil.ReadAll(resp.Body) if err != nil { return err } glog.Infof("Sent PagerDuty notification: %v: HTTP %d: %s", incidentKey, resp.StatusCode, respBuf) // BUG: Check response for result of operation. return nil } func (n *notifier) sendHipChatNotification(op notificationOp, config *pb.HipChatConfig, a *Alert) error { // https://www.hipchat.com/docs/apiv2/method/send_room_notification incidentKey := a.Fingerprint() color := "" status := "" switch op { case notificationOpTrigger: color = config.GetColor() status = "firing" case notificationOpResolve: color = config.GetColorResolved() status = "resolved" } buf, err := json.Marshal(map[string]interface{}{ "color": color, "message": fmt.Sprintf("%s %s: %s (view)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), "notify": config.GetNotify(), "message_format": "html", }) if err != nil { return err } timeout := time.Duration(5 * time.Second) client := http.Client{ Timeout: timeout, } resp, err := client.Post( fmt.Sprintf("%s/room/%d/notification?auth_token=%s", *hipchatUrl, config.GetRoomId(), config.GetAuthToken()), contentTypeJson, bytes.NewBuffer(buf), ) if err != nil { return err } defer resp.Body.Close() respBuf, err := ioutil.ReadAll(resp.Body) if err != nil { return err } glog.Infof("Sent HipChat notification: %v: HTTP %d: %s", incidentKey, resp.StatusCode, respBuf) // BUG: Check response for result of operation. return nil } func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error { return writeEmailBodyWithTime(w, from, to, status, a, time.Now()) } func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time) error { err := bodyTmpl.Execute(w, struct { From string To string Date string Alert *Alert Status string }{ From: from, To: to, Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"), Alert: a, Status: status, }) if err != nil { return err } return nil } func getSMTPAuth(hasAuth bool, mechs string) (smtp.Auth, *tls.Config, error) { if !hasAuth { return nil, nil, nil } username := os.Getenv("SMTP_AUTH_USERNAME") for _, mech := range strings.Split(mechs, " ") { switch mech { case "CRAM-MD5": secret := os.Getenv("SMTP_AUTH_SECRET") if secret == "" { continue } return smtp.CRAMMD5Auth(username, secret), nil, nil case "PLAIN": password := os.Getenv("SMTP_AUTH_PASSWORD") if password == "" { continue } identity := os.Getenv("SMTP_AUTH_IDENTITY") // We need to know the hostname for both auth and TLS. host, _, err := net.SplitHostPort(*smtpSmartHost) if err != nil { return nil, nil, fmt.Errorf("invalid address: %s", err) } auth := smtp.PlainAuth(identity, username, password, host) cfg := &tls.Config{ServerName: host} return auth, cfg, nil } } return nil, nil, nil } func (n *notifier) sendEmailNotification(to string, op notificationOp, a *Alert) error { status := "" switch op { case notificationOpTrigger: status = "ALERT" case notificationOpResolve: status = "RESOLVED" } // Connect to the SMTP smarthost. c, err := smtp.Dial(*smtpSmartHost) if err != nil { return err } defer c.Quit() // Authenticate if we and the server are both configured for it. auth, tlsConfig, err := getSMTPAuth(c.Extension("AUTH")) if err != nil { return err } if tlsConfig != nil { if err := c.StartTLS(tlsConfig); err != nil { return fmt.Errorf("starttls failed: %s", err) } } if auth != nil { if err := c.Auth(auth); err != nil { return fmt.Errorf("%T failed: %s", auth, err) } } // Set the sender and recipient. c.Mail(*smtpSender) c.Rcpt(to) // Send the email body. wc, err := c.Data() if err != nil { return err } defer wc.Close() return writeEmailBody(wc, *smtpSender, status, to, a) } func (n *notifier) sendPushoverNotification(token string, op notificationOp, userKey string, a *Alert) error { po, err := pushover.NewPushover(token, userKey) if err != nil { return err } // Validate credentials err = po.Validate() if err != nil { return err } // Send pushover message _, _, err = po.Push(&pushover.Message{ Title: a.Summary, Message: a.Description, }) return err } func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { for _, pdConfig := range config.PagerdutyConfig { if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil { glog.Error("Error sending PagerDuty notification: ", err) } } for _, emailConfig := range config.EmailConfig { if op == notificationOpResolve && !emailConfig.GetSendResolved() { continue } if *smtpSmartHost == "" { glog.Warning("No SMTP smarthost configured, not sending email notification.") continue } if err := n.sendEmailNotification(emailConfig.GetEmail(), op, a); err != nil { glog.Error("Error sending email notification: ", err) } } for _, poConfig := range config.PushoverConfig { if op == notificationOpResolve && !poConfig.GetSendResolved() { continue } if err := n.sendPushoverNotification(poConfig.GetToken(), op, poConfig.GetUserKey(), a); err != nil { glog.Error("Error sending Pushover notification: ", err) } } for _, hcConfig := range config.HipchatConfig { if op == notificationOpResolve && !hcConfig.GetSendResolved() { continue } if err := n.sendHipChatNotification(op, hcConfig, a); err != nil { glog.Error("Error sending HipChat notification: ", err) } } } func (n *notifier) Dispatch() { for req := range n.pendingNotifications { n.handleNotification(req.alert, req.op, req.notificationConfig) } } func (n *notifier) Close() { close(n.pendingNotifications) }