alertmanager/notify/impl.go

1600 lines
44 KiB
Go

// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package notify
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"mime/quotedprintable"
"net"
"net/http"
"net/mail"
"net/smtp"
"net/textproto"
"net/url"
"strings"
"time"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/version"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
)
// A Notifier notifies about alerts under constraints of the given context.
// It returns an error if unsuccessful and a flag whether the error is
// recoverable. This information is useful for a retry logic.
type Notifier interface {
Notify(context.Context, ...*types.Alert) (bool, error)
}
// An Integration wraps a notifier and its config to be uniquely identified by
// name and index from its origin in the configuration.
type Integration struct {
notifier Notifier
conf notifierConfig
name string
idx int
}
// Notify implements the Notifier interface.
func (i *Integration) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
return i.notifier.Notify(ctx, alerts...)
}
// BuildReceiverIntegrations builds a list of integration notifiers off of a
// receivers config.
func BuildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []Integration {
var (
integrations []Integration
add = func(name string, i int, n Notifier, nc notifierConfig) {
integrations = append(integrations, Integration{
notifier: n,
conf: nc,
name: name,
idx: i,
})
}
)
for i, c := range nc.WebhookConfigs {
n := NewWebhook(c, tmpl, logger)
add("webhook", i, n, c)
}
for i, c := range nc.EmailConfigs {
n := NewEmail(c, tmpl, logger)
add("email", i, n, c)
}
for i, c := range nc.PagerdutyConfigs {
n := NewPagerDuty(c, tmpl, logger)
add("pagerduty", i, n, c)
}
for i, c := range nc.OpsGenieConfigs {
n := NewOpsGenie(c, tmpl, logger)
add("opsgenie", i, n, c)
}
for i, c := range nc.WechatConfigs {
n := NewWechat(c, tmpl, logger)
add("wechat", i, n, c)
}
for i, c := range nc.SlackConfigs {
n := NewSlack(c, tmpl, logger)
add("slack", i, n, c)
}
for i, c := range nc.HipchatConfigs {
n := NewHipchat(c, tmpl, logger)
add("hipchat", i, n, c)
}
for i, c := range nc.VictorOpsConfigs {
n := NewVictorOps(c, tmpl, logger)
add("victorops", i, n, c)
}
for i, c := range nc.PushoverConfigs {
n := NewPushover(c, tmpl, logger)
add("pushover", i, n, c)
}
return integrations
}
const contentTypeJSON = "application/json"
var userAgentHeader = fmt.Sprintf("Alertmanager/%s", version.Version)
// Webhook implements a Notifier for generic webhooks.
type Webhook struct {
conf *config.WebhookConfig
tmpl *template.Template
logger log.Logger
}
// NewWebhook returns a new Webhook.
func NewWebhook(conf *config.WebhookConfig, t *template.Template, l log.Logger) *Webhook {
return &Webhook{conf: conf, tmpl: t, logger: l}
}
// WebhookMessage defines the JSON object send to webhook endpoints.
type WebhookMessage struct {
*template.Data
// The protocol version.
Version string `json:"version"`
GroupKey string `json:"groupKey"`
}
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
data := w.tmpl.Data(receiverName(ctx, w.logger), groupLabels(ctx, w.logger), alerts...)
groupKey, ok := GroupKey(ctx)
if !ok {
level.Error(w.logger).Log("msg", "group key missing")
}
msg := &WebhookMessage{
Version: "4",
Data: data,
GroupKey: groupKey,
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
req, err := http.NewRequest("POST", w.conf.URL.String(), &buf)
if err != nil {
return true, err
}
req.Header.Set("Content-Type", contentTypeJSON)
req.Header.Set("User-Agent", userAgentHeader)
c, err := commoncfg.NewClientFromConfig(*w.conf.HTTPConfig, "webhook")
if err != nil {
return false, err
}
resp, err := c.Do(req.WithContext(ctx))
if err != nil {
return true, err
}
resp.Body.Close()
return w.retry(resp.StatusCode)
}
func (w *Webhook) retry(statusCode int) (bool, error) {
// Webhooks are assumed to respond with 2xx response codes on a successful
// request and 5xx response codes are assumed to be recoverable.
if statusCode/100 != 2 {
return (statusCode/100 == 5), fmt.Errorf("unexpected status code %v from %s", statusCode, w.conf.URL)
}
return false, nil
}
// Email implements a Notifier for email notifications.
type Email struct {
conf *config.EmailConfig
tmpl *template.Template
logger log.Logger
}
// NewEmail returns a new Email notifier.
func NewEmail(c *config.EmailConfig, t *template.Template, l log.Logger) *Email {
if _, ok := c.Headers["Subject"]; !ok {
c.Headers["Subject"] = config.DefaultEmailSubject
}
if _, ok := c.Headers["To"]; !ok {
c.Headers["To"] = c.To
}
if _, ok := c.Headers["From"]; !ok {
c.Headers["From"] = c.From
}
return &Email{conf: c, tmpl: t, logger: l}
}
// auth resolves a string of authentication mechanisms.
func (n *Email) auth(mechs string) (smtp.Auth, error) {
username := n.conf.AuthUsername
// If no username is set, keep going without authentication.
if n.conf.AuthUsername == "" {
level.Debug(n.logger).Log("msg", "smtp_auth_username is not configured. Attempting to send email without authenticating")
return nil, nil
}
err := &types.MultiError{}
for _, mech := range strings.Split(mechs, " ") {
switch mech {
case "CRAM-MD5":
secret := string(n.conf.AuthSecret)
if secret == "" {
err.Add(errors.New("missing secret for CRAM-MD5 auth mechanism"))
continue
}
return smtp.CRAMMD5Auth(username, secret), nil
case "PLAIN":
password := string(n.conf.AuthPassword)
if password == "" {
err.Add(errors.New("missing password for PLAIN auth mechanism"))
continue
}
identity := n.conf.AuthIdentity
// We need to know the hostname for both auth and TLS.
host, _, err := net.SplitHostPort(n.conf.Smarthost)
if err != nil {
return nil, fmt.Errorf("invalid address: %s", err)
}
return smtp.PlainAuth(identity, username, password, host), nil
case "LOGIN":
password := string(n.conf.AuthPassword)
if password == "" {
err.Add(errors.New("missing password for LOGIN auth mechanism"))
continue
}
return LoginAuth(username, password), nil
}
}
if err.Len() == 0 {
err.Add(errors.New("unknown auth mechanism: " + mechs))
}
return nil, err
}
// Notify implements the Notifier interface.
func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
// We need to know the hostname for both auth and TLS.
var c *smtp.Client
host, port, err := net.SplitHostPort(n.conf.Smarthost)
if err != nil {
return false, fmt.Errorf("invalid address: %s", err)
}
if port == "465" {
tlsConfig, err := commoncfg.NewTLSConfig(&n.conf.TLSConfig)
if err != nil {
return false, err
}
if tlsConfig.ServerName == "" {
tlsConfig.ServerName = host
}
conn, err := tls.Dial("tcp", n.conf.Smarthost, tlsConfig)
if err != nil {
return true, err
}
c, err = smtp.NewClient(conn, host)
if err != nil {
return true, err
}
} else {
// Connect to the SMTP smarthost.
c, err = smtp.Dial(n.conf.Smarthost)
if err != nil {
return true, err
}
}
defer func() {
if err := c.Quit(); err != nil {
level.Error(n.logger).Log("msg", "failed to close SMTP connection", "err", err)
}
}()
if n.conf.Hello != "" {
err := c.Hello(n.conf.Hello)
if err != nil {
return true, err
}
}
// Global Config guarantees RequireTLS is not nil
if *n.conf.RequireTLS {
if ok, _ := c.Extension("STARTTLS"); !ok {
return true, fmt.Errorf("require_tls: true (default), but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
}
tlsConf, err := commoncfg.NewTLSConfig(&n.conf.TLSConfig)
if err != nil {
return false, err
}
if tlsConf.ServerName == "" {
tlsConf.ServerName = host
}
if err := c.StartTLS(tlsConf); err != nil {
return true, fmt.Errorf("starttls failed: %s", err)
}
}
if ok, mech := c.Extension("AUTH"); ok {
auth, err := n.auth(mech)
if err != nil {
return true, err
}
if auth != nil {
if err := c.Auth(auth); err != nil {
return true, fmt.Errorf("%T failed: %s", auth, err)
}
}
}
var (
tmplErr error
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
tmpl = tmplText(n.tmpl, data, &tmplErr)
from = tmpl(n.conf.From)
to = tmpl(n.conf.To)
)
if tmplErr != nil {
return false, fmt.Errorf("failed to template 'from' or 'to': %v", tmplErr)
}
addrs, err := mail.ParseAddressList(from)
if err != nil {
return false, fmt.Errorf("parsing from addresses: %s", err)
}
if len(addrs) != 1 {
return false, fmt.Errorf("must be exactly one from address")
}
if err := c.Mail(addrs[0].Address); err != nil {
return true, fmt.Errorf("sending mail from: %s", err)
}
addrs, err = mail.ParseAddressList(to)
if err != nil {
return false, fmt.Errorf("parsing to addresses: %s", err)
}
for _, addr := range addrs {
if err := c.Rcpt(addr.Address); err != nil {
return true, fmt.Errorf("sending rcpt to: %s", err)
}
}
// Send the email body.
wc, err := c.Data()
if err != nil {
return true, err
}
defer wc.Close()
for header, t := range n.conf.Headers {
value, err := n.tmpl.ExecuteTextString(t, data)
if err != nil {
return false, fmt.Errorf("executing %q header template: %s", header, err)
}
fmt.Fprintf(wc, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
}
buffer := &bytes.Buffer{}
multipartWriter := multipart.NewWriter(buffer)
fmt.Fprintf(wc, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
fmt.Fprintf(wc, "Content-Type: multipart/alternative; boundary=%s\r\n", multipartWriter.Boundary())
fmt.Fprintf(wc, "MIME-Version: 1.0\r\n")
// TODO: Add some useful headers here, such as URL of the alertmanager
// and active/resolved.
fmt.Fprintf(wc, "\r\n")
if len(n.conf.Text) > 0 {
// Text template
w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
"Content-Transfer-Encoding": {"quoted-printable"},
"Content-Type": {"text/plain; charset=UTF-8"},
})
if err != nil {
return false, fmt.Errorf("creating part for text template: %s", err)
}
body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
if err != nil {
return false, fmt.Errorf("executing email text template: %s", err)
}
qw := quotedprintable.NewWriter(w)
_, err = qw.Write([]byte(body))
if err != nil {
return true, err
}
err = qw.Close()
if err != nil {
return true, err
}
}
if len(n.conf.HTML) > 0 {
// Html template
// Preferred alternative placed last per section 5.1.4 of RFC 2046
// https://www.ietf.org/rfc/rfc2046.txt
w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
"Content-Transfer-Encoding": {"quoted-printable"},
"Content-Type": {"text/html; charset=UTF-8"},
})
if err != nil {
return false, fmt.Errorf("creating part for html template: %s", err)
}
body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
if err != nil {
return false, fmt.Errorf("executing email html template: %s", err)
}
qw := quotedprintable.NewWriter(w)
_, err = qw.Write([]byte(body))
if err != nil {
return true, err
}
err = qw.Close()
if err != nil {
return true, err
}
}
err = multipartWriter.Close()
if err != nil {
return false, fmt.Errorf("failed to close multipartWriter: %v", err)
}
_, err = wc.Write(buffer.Bytes())
if err != nil {
return false, fmt.Errorf("failed to write body buffer: %v", err)
}
return false, nil
}
// PagerDuty implements a Notifier for PagerDuty notifications.
type PagerDuty struct {
conf *config.PagerdutyConfig
tmpl *template.Template
logger log.Logger
}
// NewPagerDuty returns a new PagerDuty notifier.
func NewPagerDuty(c *config.PagerdutyConfig, t *template.Template, l log.Logger) *PagerDuty {
return &PagerDuty{conf: c, tmpl: t, logger: l}
}
const (
pagerDutyEventTrigger = "trigger"
pagerDutyEventResolve = "resolve"
)
type pagerDutyMessage struct {
RoutingKey string `json:"routing_key,omitempty"`
ServiceKey string `json:"service_key,omitempty"`
DedupKey string `json:"dedup_key,omitempty"`
IncidentKey string `json:"incident_key,omitempty"`
EventType string `json:"event_type,omitempty"`
Description string `json:"description,omitempty"`
EventAction string `json:"event_action"`
Payload *pagerDutyPayload `json:"payload"`
Client string `json:"client,omitempty"`
ClientURL string `json:"client_url,omitempty"`
Details map[string]string `json:"details,omitempty"`
Images []pagerDutyImage `json:"images,omitempty"`
Links []pagerDutyLink `json:"links,omitempty"`
}
type pagerDutyLink struct {
HRef string `json:"href"`
Text string `json:"text"`
}
type pagerDutyImage struct {
Src string `json:"src"`
Alt string `json:"alt"`
Text string `json:"text"`
}
type pagerDutyPayload struct {
Summary string `json:"summary"`
Source string `json:"source"`
Severity string `json:"severity"`
Timestamp string `json:"timestamp,omitempty"`
Class string `json:"class,omitempty"`
Component string `json:"component,omitempty"`
Group string `json:"group,omitempty"`
CustomDetails map[string]string `json:"custom_details,omitempty"`
}
func (n *PagerDuty) notifyV1(
ctx context.Context,
c *http.Client,
eventType, key string,
data *template.Data,
details map[string]string,
as ...*types.Alert,
) (bool, error) {
var tmplErr error
tmpl := tmplText(n.tmpl, data, &tmplErr)
msg := &pagerDutyMessage{
ServiceKey: tmpl(string(n.conf.ServiceKey)),
EventType: eventType,
IncidentKey: hashKey(key),
Description: tmpl(n.conf.Description),
Details: details,
}
apiURL, err := url.Parse("https://events.pagerduty.com/generic/2010-04-15/create_event.json")
if err != nil {
return false, err
}
n.conf.URL = &config.URL{apiURL}
if eventType == pagerDutyEventTrigger {
msg.Client = tmpl(n.conf.Client)
msg.ClientURL = tmpl(n.conf.ClientURL)
}
if tmplErr != nil {
return false, fmt.Errorf("failed to template PagerDuty v1 message: %v", tmplErr)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
resp, err := post(ctx, c, n.conf.URL.String(), contentTypeJSON, &buf)
if err != nil {
return true, err
}
defer resp.Body.Close()
return n.retryV1(resp)
}
func (n *PagerDuty) notifyV2(
ctx context.Context,
c *http.Client,
eventType, key string,
data *template.Data,
details map[string]string,
as ...*types.Alert,
) (bool, error) {
var tmplErr error
tmpl := tmplText(n.tmpl, data, &tmplErr)
if n.conf.Severity == "" {
n.conf.Severity = "error"
}
summary := tmpl(n.conf.Description)
summaryRunes := []rune(summary)
if len(summaryRunes) > 1024 {
summary = string(summaryRunes[:1018]) + " [...]"
}
msg := &pagerDutyMessage{
Client: tmpl(n.conf.Client),
ClientURL: tmpl(n.conf.ClientURL),
RoutingKey: tmpl(string(n.conf.RoutingKey)),
EventAction: eventType,
DedupKey: hashKey(key),
Images: make([]pagerDutyImage, len(n.conf.Images)),
Links: make([]pagerDutyLink, len(n.conf.Links)),
Payload: &pagerDutyPayload{
Summary: summary,
Source: tmpl(n.conf.Client),
Severity: tmpl(n.conf.Severity),
CustomDetails: details,
Class: tmpl(n.conf.Class),
Component: tmpl(n.conf.Component),
Group: tmpl(n.conf.Group),
},
}
for index, item := range n.conf.Images {
msg.Images[index].Src = tmpl(item.Src)
msg.Images[index].Alt = tmpl(item.Alt)
msg.Images[index].Text = tmpl(item.Text)
}
for index, item := range n.conf.Links {
msg.Links[index].HRef = tmpl(item.HRef)
msg.Links[index].Text = tmpl(item.Text)
}
if tmplErr != nil {
return false, fmt.Errorf("failed to template PagerDuty v2 message: %v", tmplErr)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, fmt.Errorf("failed to encode PagerDuty v2 message: %v", err)
}
resp, err := post(ctx, c, n.conf.URL.String(), contentTypeJSON, &buf)
if err != nil {
return true, fmt.Errorf("failed to post message to PagerDuty: %v", err)
}
defer resp.Body.Close()
// See: https://v2.developer.pagerduty.com/docs/events-api-v2#api-response-codes--retry-logic
if resp.StatusCode == http.StatusBadRequest {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, fmt.Errorf("failed to read error response from PagerDuty (status: %d): %v", resp.StatusCode, err)
}
level.Debug(n.logger).Log("msg", "Received error response from PagerDuty", "incident", key, "code", resp.StatusCode, "body", string(body))
}
return n.retryV2(resp.StatusCode)
}
// Notify implements the Notifier interface.
//
// https://v2.developer.pagerduty.com/docs/events-api-v2
func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
key, ok := GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
eventType = pagerDutyEventTrigger
)
if alerts.Status() == model.AlertResolved {
eventType = pagerDutyEventResolve
}
level.Debug(n.logger).Log("msg", "Notifying PagerDuty", "incident", key, "eventType", eventType)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
detail, err := n.tmpl.ExecuteTextString(v, data)
if err != nil {
return false, fmt.Errorf("failed to template %q: %v", v, err)
}
details[k] = detail
}
if err != nil {
return false, err
}
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "pagerduty")
if err != nil {
return false, err
}
if n.conf.ServiceKey != "" {
return n.notifyV1(ctx, c, eventType, key, data, details, as...)
}
return n.notifyV2(ctx, c, eventType, key, data, details, as...)
}
func (n *PagerDuty) retryV1(resp *http.Response) (bool, error) {
// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
// 2xx response codes indicate a successful request.
// https://v2.developer.pagerduty.com/docs/trigger-events
statusCode := resp.StatusCode
if statusCode == 400 && resp.Body != nil {
bs, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, fmt.Errorf("unexpected status code %v : problem reading response: %v", statusCode, err)
}
return false, fmt.Errorf("bad request (status code %v): %v", statusCode, string(bs))
}
if statusCode/100 != 2 {
return (statusCode == 403 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
func (n *PagerDuty) retryV2(statusCode int) (bool, error) {
// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
// 2xx response codes indicate a successful request.
// https://v2.developer.pagerduty.com/docs/events-api-v2#api-response-codes--retry-logic
if statusCode/100 != 2 {
return (statusCode == 429 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// Slack implements a Notifier for Slack notifications.
type Slack struct {
conf *config.SlackConfig
tmpl *template.Template
logger log.Logger
}
// NewSlack returns a new Slack notification handler.
func NewSlack(c *config.SlackConfig, t *template.Template, l log.Logger) *Slack {
return &Slack{
conf: c,
tmpl: t,
logger: l,
}
}
// slackReq is the request for sending a slack notification.
type slackReq struct {
Channel string `json:"channel,omitempty"`
Username string `json:"username,omitempty"`
IconEmoji string `json:"icon_emoji,omitempty"`
IconURL string `json:"icon_url,omitempty"`
LinkNames bool `json:"link_names,omitempty"`
Attachments []slackAttachment `json:"attachments"`
}
// slackAttachment is used to display a richly-formatted message block.
type slackAttachment struct {
Title string `json:"title,omitempty"`
TitleLink string `json:"title_link,omitempty"`
Pretext string `json:"pretext,omitempty"`
Text string `json:"text"`
Fallback string `json:"fallback"`
CallbackID string `json:"callback_id"`
Fields []config.SlackField `json:"fields,omitempty"`
Actions []config.SlackAction `json:"actions,omitempty"`
ImageURL string `json:"image_url,omitempty"`
ThumbURL string `json:"thumb_url,omitempty"`
Footer string `json:"footer"`
Color string `json:"color,omitempty"`
MrkdwnIn []string `json:"mrkdwn_in,omitempty"`
}
// Notify implements the Notifier interface.
func (n *Slack) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
var err error
var (
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
tmplText = tmplText(n.tmpl, data, &err)
)
attachment := &slackAttachment{
Title: tmplText(n.conf.Title),
TitleLink: tmplText(n.conf.TitleLink),
Pretext: tmplText(n.conf.Pretext),
Text: tmplText(n.conf.Text),
Fallback: tmplText(n.conf.Fallback),
CallbackID: tmplText(n.conf.CallbackID),
ImageURL: tmplText(n.conf.ImageURL),
ThumbURL: tmplText(n.conf.ThumbURL),
Footer: tmplText(n.conf.Footer),
Color: tmplText(n.conf.Color),
MrkdwnIn: []string{"fallback", "pretext", "text"},
}
var numFields = len(n.conf.Fields)
if numFields > 0 {
var fields = make([]config.SlackField, numFields)
for index, field := range n.conf.Fields {
// Check if short was defined for the field otherwise fallback to the global setting
var short bool
if field.Short != nil {
short = *field.Short
} else {
short = n.conf.ShortFields
}
// Rebuild the field by executing any templates and setting the new value for short
fields[index] = config.SlackField{
Title: tmplText(field.Title),
Value: tmplText(field.Value),
Short: &short,
}
}
attachment.Fields = fields
}
var numActions = len(n.conf.Actions)
if numActions > 0 {
var actions = make([]config.SlackAction, numActions)
for index, action := range n.conf.Actions {
slackAction := config.SlackAction{
Type: tmplText(action.Type),
Text: tmplText(action.Text),
URL: tmplText(action.URL),
Style: tmplText(action.Style),
Name: tmplText(action.Name),
Value: tmplText(action.Value),
}
if action.ConfirmField != nil {
slackAction.ConfirmField = &config.SlackConfirmationField{
Title: tmplText(action.ConfirmField.Title),
Text: tmplText(action.ConfirmField.Text),
OkText: tmplText(action.ConfirmField.OkText),
DismissText: tmplText(action.ConfirmField.DismissText),
}
}
actions[index] = slackAction
}
attachment.Actions = actions
}
req := &slackReq{
Channel: tmplText(n.conf.Channel),
Username: tmplText(n.conf.Username),
IconEmoji: tmplText(n.conf.IconEmoji),
IconURL: tmplText(n.conf.IconURL),
LinkNames: n.conf.LinkNames,
Attachments: []slackAttachment{*attachment},
}
if err != nil {
return false, err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(req); err != nil {
return false, err
}
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "slack")
if err != nil {
return false, err
}
resp, err := post(ctx, c, n.conf.APIURL.String(), contentTypeJSON, &buf)
if err != nil {
return true, err
}
resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *Slack) retry(statusCode int) (bool, error) {
// Only 5xx response codes are recoverable and 2xx codes are successful.
// https://api.slack.com/incoming-webhooks#handling_errors
// https://api.slack.com/changelog/2016-05-17-changes-to-errors-for-incoming-webhooks
if statusCode/100 != 2 {
return (statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// Hipchat implements a Notifier for Hipchat notifications.
type Hipchat struct {
conf *config.HipchatConfig
tmpl *template.Template
logger log.Logger
}
// NewHipchat returns a new Hipchat notification handler.
func NewHipchat(c *config.HipchatConfig, t *template.Template, l log.Logger) *Hipchat {
return &Hipchat{
conf: c,
tmpl: t,
logger: l,
}
}
type hipchatReq struct {
From string `json:"from"`
Notify bool `json:"notify"`
Message string `json:"message"`
MessageFormat string `json:"message_format"`
Color string `json:"color"`
}
// Notify implements the Notifier interface.
func (n *Hipchat) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
var err error
var msg string
var (
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
tmplText = tmplText(n.tmpl, data, &err)
tmplHTML = tmplHTML(n.tmpl, data, &err)
roomid = tmplText(n.conf.RoomID)
apiURL = n.conf.APIURL.Copy()
)
apiURL.Path += fmt.Sprintf("v2/room/%s/notification", roomid)
q := apiURL.Query()
q.Set("auth_token", string(n.conf.AuthToken))
apiURL.RawQuery = q.Encode()
if n.conf.MessageFormat == "html" {
msg = tmplHTML(n.conf.Message)
} else {
msg = tmplText(n.conf.Message)
}
req := &hipchatReq{
From: tmplText(n.conf.From),
Notify: n.conf.Notify,
Message: msg,
MessageFormat: n.conf.MessageFormat,
Color: tmplText(n.conf.Color),
}
if err != nil {
return false, err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(req); err != nil {
return false, err
}
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "hipchat")
if err != nil {
return false, err
}
resp, err := post(ctx, c, apiURL.String(), contentTypeJSON, &buf)
if err != nil {
return true, err
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *Hipchat) retry(statusCode int) (bool, error) {
// Response codes 429 (rate limiting) and 5xx can potentially recover. 2xx
// responce codes indicate successful requests.
// https://developer.atlassian.com/hipchat/guide/hipchat-rest-api/api-response-codes
if statusCode/100 != 2 {
return (statusCode == 429 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// Wechat implements a Notfier for wechat notifications
type Wechat struct {
conf *config.WechatConfig
tmpl *template.Template
logger log.Logger
accessToken string
accessTokenAt time.Time
}
// Wechat AccessToken with corpid and corpsecret.
type WechatToken struct {
AccessToken string `json:"access_token"`
}
type weChatMessage struct {
Text weChatMessageContent `yaml:"text,omitempty" json:"text,omitempty"`
ToUser string `yaml:"touser,omitempty" json:"touser,omitempty"`
ToParty string `yaml:"toparty,omitempty" json:"toparty,omitempty"`
Totag string `yaml:"totag,omitempty" json:"totag,omitempty"`
AgentID string `yaml:"agentid,omitempty" json:"agentid,omitempty"`
Safe string `yaml:"safe,omitempty" json:"safe,omitempty"`
Type string `yaml:"msgtype,omitempty" json:"msgtype,omitempty"`
}
type weChatMessageContent struct {
Content string `json:"content"`
}
type weChatResponse struct {
Code int `json:"code"`
Error string `json:"error"`
}
// NewWechat returns a new Wechat notifier.
func NewWechat(c *config.WechatConfig, t *template.Template, l log.Logger) *Wechat {
return &Wechat{conf: c, tmpl: t, logger: l}
}
// Notify implements the Notifier interface.
func (n *Wechat) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
key, ok := GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}
level.Debug(n.logger).Log("msg", "Notifying Wechat", "incident", key)
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
var err error
tmpl := tmplText(n.tmpl, data, &err)
if err != nil {
return false, err
}
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "wechat")
if err != nil {
return false, err
}
// Refresh AccessToken over 2 hours
if n.accessToken == "" || time.Since(n.accessTokenAt) > 2*time.Hour {
parameters := url.Values{}
parameters.Add("corpsecret", tmpl(string(n.conf.APISecret)))
parameters.Add("corpid", tmpl(string(n.conf.CorpID)))
if err != nil {
return false, fmt.Errorf("templating error: %s", err)
}
u := n.conf.APIURL.Copy()
u.Path += "gettoken"
u.RawQuery = parameters.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return true, err
}
req.Header.Set("Content-Type", contentTypeJSON)
resp, err := c.Do(req.WithContext(ctx))
if err != nil {
return true, err
}
defer resp.Body.Close()
var wechatToken WechatToken
if err := json.NewDecoder(resp.Body).Decode(&wechatToken); err != nil {
return false, err
}
if wechatToken.AccessToken == "" {
return false, fmt.Errorf("invalid APISecret for CorpID: %s", n.conf.CorpID)
}
// Cache accessToken
n.accessToken = wechatToken.AccessToken
n.accessTokenAt = time.Now()
}
msg := &weChatMessage{
Text: weChatMessageContent{
Content: tmpl(n.conf.Message),
},
ToUser: tmpl(n.conf.ToUser),
ToParty: tmpl(n.conf.ToParty),
Totag: tmpl(n.conf.ToTag),
AgentID: tmpl(n.conf.AgentID),
Type: "text",
Safe: "0",
}
if err != nil {
return false, fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
postMessageURL := n.conf.APIURL.Copy()
postMessageURL.Path += "message/send"
q := postMessageURL.Query()
q.Set("access_token", n.accessToken)
postMessageURL.RawQuery = q.Encode()
req, err := http.NewRequest(http.MethodPost, postMessageURL.String(), &buf)
if err != nil {
return true, err
}
resp, err := c.Do(req.WithContext(ctx))
if err != nil {
return true, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return true, err
}
level.Debug(n.logger).Log("msg", "response: "+string(body), "incident", key)
if resp.StatusCode != 200 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
} else {
var weResp weChatResponse
if err := json.Unmarshal(body, &weResp); err != nil {
return true, err
}
// https://work.weixin.qq.com/api/doc#10649
if weResp.Code == 0 {
return false, nil
}
// AccessToken is expired
if weResp.Code == 42001 {
n.accessToken = ""
return true, errors.New(weResp.Error)
}
return false, errors.New(weResp.Error)
}
}
// OpsGenie implements a Notifier for OpsGenie notifications.
type OpsGenie struct {
conf *config.OpsGenieConfig
tmpl *template.Template
logger log.Logger
}
// NewOpsGenie returns a new OpsGenie notifier.
func NewOpsGenie(c *config.OpsGenieConfig, t *template.Template, l log.Logger) *OpsGenie {
return &OpsGenie{conf: c, tmpl: t, logger: l}
}
type opsGenieCreateMessage struct {
Alias string `json:"alias"`
Message string `json:"message"`
Description string `json:"description,omitempty"`
Details map[string]string `json:"details"`
Source string `json:"source"`
Teams []map[string]string `json:"teams,omitempty"`
Tags []string `json:"tags,omitempty"`
Note string `json:"note,omitempty"`
Priority string `json:"priority,omitempty"`
}
type opsGenieCloseMessage struct {
Source string `json:"source"`
}
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
req, retry, err := n.createRequest(ctx, as...)
if err != nil {
return retry, err
}
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "opsgenie")
if err != nil {
return false, err
}
resp, err := c.Do(req.WithContext(ctx))
if err != nil {
return true, err
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
// Like Split but filter out empty strings.
func safeSplit(s string, sep string) []string {
a := strings.Split(strings.TrimSpace(s), sep)
b := a[:0]
for _, x := range a {
if x != "" {
b = append(b, x)
}
}
return b
}
// Create requests for a list of alerts.
func (n *OpsGenie) createRequest(ctx context.Context, as ...*types.Alert) (*http.Request, bool, error) {
key, ok := GroupKey(ctx)
if !ok {
return nil, false, fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
level.Debug(n.logger).Log("msg", "Notifying OpsGenie", "incident", key)
var err error
tmpl := tmplText(n.tmpl, data, &err)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
var (
msg interface{}
apiURL = n.conf.APIURL.Copy()
alias = hashKey(key)
alerts = types.Alerts(as...)
)
switch alerts.Status() {
case model.AlertResolved:
apiURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
q := apiURL.Query()
q.Set("identifierType", "alias")
apiURL.RawQuery = q.Encode()
msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
default:
message, truncated := truncate(tmpl(n.conf.Message), 130)
if truncated {
level.Debug(n.logger).Log("msg", "truncated message due to OpsGenie message limit", "truncated_message", message, "incident", key)
}
apiURL.Path += "v2/alerts"
var teams []map[string]string
for _, t := range safeSplit(string(tmpl(n.conf.Teams)), ",") {
teams = append(teams, map[string]string{"name": t})
}
tags := safeSplit(string(tmpl(n.conf.Tags)), ",")
msg = &opsGenieCreateMessage{
Alias: alias,
Message: message,
Description: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
Teams: teams,
Tags: tags,
Note: tmpl(n.conf.Note),
Priority: tmpl(n.conf.Priority),
}
}
apiKey := tmpl(string(n.conf.APIKey))
if err != nil {
return nil, false, fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return nil, false, err
}
req, err := http.NewRequest("POST", apiURL.String(), &buf)
if err != nil {
return nil, true, err
}
req.Header.Set("Content-Type", contentTypeJSON)
req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
return req, true, nil
}
func (n *OpsGenie) retry(statusCode int) (bool, error) {
// https://docs.opsgenie.com/docs/response#section-response-codes
// Response codes 429 (rate limiting) and 5xx are potentially recoverable
if statusCode/100 == 5 || statusCode == 429 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// VictorOps implements a Notifier for VictorOps notifications.
type VictorOps struct {
conf *config.VictorOpsConfig
tmpl *template.Template
logger log.Logger
}
// NewVictorOps returns a new VictorOps notifier.
func NewVictorOps(c *config.VictorOpsConfig, t *template.Template, l log.Logger) *VictorOps {
return &VictorOps{
conf: c,
tmpl: t,
logger: l,
}
}
const (
victorOpsEventTrigger = "CRITICAL"
victorOpsEventResolve = "RECOVERY"
)
// Notify implements the Notifier interface.
func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
var err error
var (
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
tmpl = tmplText(n.tmpl, data, &err)
apiURL = n.conf.APIURL.Copy()
)
apiURL.Path += fmt.Sprintf("%s/%s", n.conf.APIKey, tmpl(n.conf.RoutingKey))
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "victorops")
if err != nil {
return false, err
}
buf, err := n.createVictorOpsPayload(ctx, as...)
if err != nil {
return true, err
}
resp, err := post(ctx, c, apiURL.String(), contentTypeJSON, buf)
if err != nil {
return true, err
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
// Create the JSON payload to be sent to the VictorOps API.
func (n *VictorOps) createVictorOpsPayload(ctx context.Context, as ...*types.Alert) (*bytes.Buffer, error) {
victorOpsAllowedEvents := map[string]bool{
"INFO": true,
"WARNING": true,
"CRITICAL": true,
}
key, ok := GroupKey(ctx)
if !ok {
return nil, fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
tmpl = tmplText(n.tmpl, data, &err)
messageType = tmpl(n.conf.MessageType)
stateMessage = tmpl(n.conf.StateMessage)
)
if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] {
messageType = victorOpsEventTrigger
}
if alerts.Status() == model.AlertResolved {
messageType = victorOpsEventResolve
}
stateMessage, truncated := truncate(stateMessage, 20480)
if truncated {
level.Debug(n.logger).Log("msg", "truncated stateMessage due to VictorOps stateMessage limit", "truncated_state_message", stateMessage, "incident", key)
}
msg := map[string]string{
"message_type": messageType,
"entity_id": hashKey(key),
"entity_display_name": tmpl(n.conf.EntityDisplayName),
"state_message": stateMessage,
"monitoring_tool": tmpl(n.conf.MonitoringTool),
}
if err != nil {
return nil, fmt.Errorf("templating error: %s", err)
}
// Add custom fields to the payload.
for k, v := range n.conf.CustomFields {
msg[k] = tmpl(v)
if err != nil {
return nil, fmt.Errorf("templating error: %s", err)
}
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return nil, err
}
return &buf, nil
}
func (n *VictorOps) retry(statusCode int) (bool, error) {
// Missing documentation therefore assuming only 5xx response codes are
// recoverable.
if statusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// Pushover implements a Notifier for Pushover notifications.
type Pushover struct {
conf *config.PushoverConfig
tmpl *template.Template
logger log.Logger
}
// NewPushover returns a new Pushover notifier.
func NewPushover(c *config.PushoverConfig, t *template.Template, l log.Logger) *Pushover {
return &Pushover{conf: c, tmpl: t, logger: l}
}
// Notify implements the Notifier interface.
func (n *Pushover) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
key, ok := GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
level.Debug(n.logger).Log("msg", "Notifying Pushover", "incident", key)
var (
err error
message string
)
tmpl := tmplText(n.tmpl, data, &err)
tmplHTML := tmplHTML(n.tmpl, data, &err)
parameters := url.Values{}
parameters.Add("token", tmpl(string(n.conf.Token)))
parameters.Add("user", tmpl(string(n.conf.UserKey)))
title, truncated := truncate(tmpl(n.conf.Title), 250)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated title due to Pushover title limit", "truncated_title", title, "incident", key)
}
parameters.Add("title", title)
if n.conf.HTML {
parameters.Add("html", "1")
message = tmplHTML(n.conf.Message)
} else {
message = tmpl(n.conf.Message)
}
message, truncated = truncate(message, 1024)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated message due to Pushover message limit", "truncated_message", message, "incident", key)
}
message = strings.TrimSpace(message)
if message == "" {
// Pushover rejects empty messages.
message = "(no details)"
}
parameters.Add("message", message)
supplementaryURL, truncated := truncate(tmpl(n.conf.URL), 512)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated URL due to Pushover url limit", "truncated_url", supplementaryURL, "incident", key)
}
parameters.Add("url", supplementaryURL)
parameters.Add("url_title", tmpl(n.conf.URLTitle))
parameters.Add("priority", tmpl(n.conf.Priority))
parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
parameters.Add("sound", tmpl(n.conf.Sound))
if err != nil {
return false, err
}
apiURL := "https://api.pushover.net/1/messages.json"
u, err := url.Parse(apiURL)
if err != nil {
return false, err
}
u.RawQuery = parameters.Encode()
level.Debug(n.logger).Log("msg", "Sending Pushover message", "incident", key, "url", u.String())
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "pushover")
if err != nil {
return false, err
}
resp, err := post(ctx, c, u.String(), "text/plain", nil)
if err != nil {
return true, err
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *Pushover) retry(statusCode int) (bool, error) {
// Only documented behaviour is that 2xx response codes are successful and
// 4xx are unsuccessful, therefore assuming only 5xx are recoverable.
// https://pushover.net/api#response
if statusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
// tmplText is using monadic error handling in order to make string templating
// less verbose. Use with care as the final error checking is easily missed.
func tmplText(tmpl *template.Template, data *template.Data, err *error) func(string) string {
return func(name string) (s string) {
if *err != nil {
return
}
s, *err = tmpl.ExecuteTextString(name, data)
return s
}
}
// tmplHTML is using monadic error handling in order to make string templating
// less verbose. Use with care as the final error checking is easily missed.
func tmplHTML(tmpl *template.Template, data *template.Data, err *error) func(string) string {
return func(name string) (s string) {
if *err != nil {
return
}
s, *err = tmpl.ExecuteHTMLString(name, data)
return s
}
}
type loginAuth struct {
username, password string
}
func LoginAuth(username, password string) smtp.Auth {
return &loginAuth{username, password}
}
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
return "LOGIN", []byte{}, nil
}
// Used for AUTH LOGIN. (Maybe password should be encrypted)
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
if more {
switch strings.ToLower(string(fromServer)) {
case "username:":
return []byte(a.username), nil
case "password:":
return []byte(a.password), nil
default:
return nil, errors.New("unexpected server challenge")
}
}
return nil, nil
}
// hashKey returns the sha256 for a group key as integrations may have
// maximum length requirements on deduplication keys.
func hashKey(s string) string {
h := sha256.New()
h.Write([]byte(s))
return fmt.Sprintf("%x", h.Sum(nil))
}
func post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return client.Do(req.WithContext(ctx))
}
func truncate(s string, n int) (string, bool) {
r := []rune(s)
if len(r) <= n {
return s, false
}
if n <= 3 {
return string(r[:n]), true
}
return string(r[:n-3]) + "...", true
}