1600 lines
44 KiB
Go
1600 lines
44 KiB
Go
// Copyright 2015 Prometheus Team
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package notify
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"mime"
|
|
"mime/multipart"
|
|
"mime/quotedprintable"
|
|
"net"
|
|
"net/http"
|
|
"net/mail"
|
|
"net/smtp"
|
|
"net/textproto"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-kit/kit/log"
|
|
"github.com/go-kit/kit/log/level"
|
|
commoncfg "github.com/prometheus/common/config"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/prometheus/common/version"
|
|
|
|
"github.com/prometheus/alertmanager/config"
|
|
"github.com/prometheus/alertmanager/template"
|
|
"github.com/prometheus/alertmanager/types"
|
|
)
|
|
|
|
// A Notifier notifies about alerts under constraints of the given context.
|
|
// It returns an error if unsuccessful and a flag whether the error is
|
|
// recoverable. This information is useful for a retry logic.
|
|
type Notifier interface {
|
|
Notify(context.Context, ...*types.Alert) (bool, error)
|
|
}
|
|
|
|
// An Integration wraps a notifier and its config to be uniquely identified by
|
|
// name and index from its origin in the configuration.
|
|
type Integration struct {
|
|
notifier Notifier
|
|
conf notifierConfig
|
|
name string
|
|
idx int
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (i *Integration) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
|
|
return i.notifier.Notify(ctx, alerts...)
|
|
}
|
|
|
|
// BuildReceiverIntegrations builds a list of integration notifiers off of a
|
|
// receivers config.
|
|
func BuildReceiverIntegrations(nc *config.Receiver, tmpl *template.Template, logger log.Logger) []Integration {
|
|
var (
|
|
integrations []Integration
|
|
add = func(name string, i int, n Notifier, nc notifierConfig) {
|
|
integrations = append(integrations, Integration{
|
|
notifier: n,
|
|
conf: nc,
|
|
name: name,
|
|
idx: i,
|
|
})
|
|
}
|
|
)
|
|
|
|
for i, c := range nc.WebhookConfigs {
|
|
n := NewWebhook(c, tmpl, logger)
|
|
add("webhook", i, n, c)
|
|
}
|
|
for i, c := range nc.EmailConfigs {
|
|
n := NewEmail(c, tmpl, logger)
|
|
add("email", i, n, c)
|
|
}
|
|
for i, c := range nc.PagerdutyConfigs {
|
|
n := NewPagerDuty(c, tmpl, logger)
|
|
add("pagerduty", i, n, c)
|
|
}
|
|
for i, c := range nc.OpsGenieConfigs {
|
|
n := NewOpsGenie(c, tmpl, logger)
|
|
add("opsgenie", i, n, c)
|
|
}
|
|
for i, c := range nc.WechatConfigs {
|
|
n := NewWechat(c, tmpl, logger)
|
|
add("wechat", i, n, c)
|
|
}
|
|
for i, c := range nc.SlackConfigs {
|
|
n := NewSlack(c, tmpl, logger)
|
|
add("slack", i, n, c)
|
|
}
|
|
for i, c := range nc.HipchatConfigs {
|
|
n := NewHipchat(c, tmpl, logger)
|
|
add("hipchat", i, n, c)
|
|
}
|
|
for i, c := range nc.VictorOpsConfigs {
|
|
n := NewVictorOps(c, tmpl, logger)
|
|
add("victorops", i, n, c)
|
|
}
|
|
for i, c := range nc.PushoverConfigs {
|
|
n := NewPushover(c, tmpl, logger)
|
|
add("pushover", i, n, c)
|
|
}
|
|
return integrations
|
|
}
|
|
|
|
const contentTypeJSON = "application/json"
|
|
|
|
var userAgentHeader = fmt.Sprintf("Alertmanager/%s", version.Version)
|
|
|
|
// Webhook implements a Notifier for generic webhooks.
|
|
type Webhook struct {
|
|
conf *config.WebhookConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewWebhook returns a new Webhook.
|
|
func NewWebhook(conf *config.WebhookConfig, t *template.Template, l log.Logger) *Webhook {
|
|
return &Webhook{conf: conf, tmpl: t, logger: l}
|
|
}
|
|
|
|
// WebhookMessage defines the JSON object send to webhook endpoints.
|
|
type WebhookMessage struct {
|
|
*template.Data
|
|
|
|
// The protocol version.
|
|
Version string `json:"version"`
|
|
GroupKey string `json:"groupKey"`
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
|
|
data := w.tmpl.Data(receiverName(ctx, w.logger), groupLabels(ctx, w.logger), alerts...)
|
|
|
|
groupKey, ok := GroupKey(ctx)
|
|
if !ok {
|
|
level.Error(w.logger).Log("msg", "group key missing")
|
|
}
|
|
|
|
msg := &WebhookMessage{
|
|
Version: "4",
|
|
Data: data,
|
|
GroupKey: groupKey,
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
req, err := http.NewRequest("POST", w.conf.URL.String(), &buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
req.Header.Set("Content-Type", contentTypeJSON)
|
|
req.Header.Set("User-Agent", userAgentHeader)
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*w.conf.HTTPConfig, "webhook")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := c.Do(req.WithContext(ctx))
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
resp.Body.Close()
|
|
|
|
return w.retry(resp.StatusCode)
|
|
}
|
|
|
|
func (w *Webhook) retry(statusCode int) (bool, error) {
|
|
// Webhooks are assumed to respond with 2xx response codes on a successful
|
|
// request and 5xx response codes are assumed to be recoverable.
|
|
if statusCode/100 != 2 {
|
|
return (statusCode/100 == 5), fmt.Errorf("unexpected status code %v from %s", statusCode, w.conf.URL)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// Email implements a Notifier for email notifications.
|
|
type Email struct {
|
|
conf *config.EmailConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewEmail returns a new Email notifier.
|
|
func NewEmail(c *config.EmailConfig, t *template.Template, l log.Logger) *Email {
|
|
if _, ok := c.Headers["Subject"]; !ok {
|
|
c.Headers["Subject"] = config.DefaultEmailSubject
|
|
}
|
|
if _, ok := c.Headers["To"]; !ok {
|
|
c.Headers["To"] = c.To
|
|
}
|
|
if _, ok := c.Headers["From"]; !ok {
|
|
c.Headers["From"] = c.From
|
|
}
|
|
return &Email{conf: c, tmpl: t, logger: l}
|
|
}
|
|
|
|
// auth resolves a string of authentication mechanisms.
|
|
func (n *Email) auth(mechs string) (smtp.Auth, error) {
|
|
username := n.conf.AuthUsername
|
|
|
|
// If no username is set, keep going without authentication.
|
|
if n.conf.AuthUsername == "" {
|
|
level.Debug(n.logger).Log("msg", "smtp_auth_username is not configured. Attempting to send email without authenticating")
|
|
return nil, nil
|
|
}
|
|
|
|
err := &types.MultiError{}
|
|
for _, mech := range strings.Split(mechs, " ") {
|
|
switch mech {
|
|
case "CRAM-MD5":
|
|
secret := string(n.conf.AuthSecret)
|
|
if secret == "" {
|
|
err.Add(errors.New("missing secret for CRAM-MD5 auth mechanism"))
|
|
continue
|
|
}
|
|
return smtp.CRAMMD5Auth(username, secret), nil
|
|
|
|
case "PLAIN":
|
|
password := string(n.conf.AuthPassword)
|
|
if password == "" {
|
|
err.Add(errors.New("missing password for PLAIN auth mechanism"))
|
|
continue
|
|
}
|
|
identity := n.conf.AuthIdentity
|
|
|
|
// We need to know the hostname for both auth and TLS.
|
|
host, _, err := net.SplitHostPort(n.conf.Smarthost)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid address: %s", err)
|
|
}
|
|
return smtp.PlainAuth(identity, username, password, host), nil
|
|
case "LOGIN":
|
|
password := string(n.conf.AuthPassword)
|
|
if password == "" {
|
|
err.Add(errors.New("missing password for LOGIN auth mechanism"))
|
|
continue
|
|
}
|
|
return LoginAuth(username, password), nil
|
|
}
|
|
}
|
|
if err.Len() == 0 {
|
|
err.Add(errors.New("unknown auth mechanism: " + mechs))
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
// We need to know the hostname for both auth and TLS.
|
|
var c *smtp.Client
|
|
host, port, err := net.SplitHostPort(n.conf.Smarthost)
|
|
if err != nil {
|
|
return false, fmt.Errorf("invalid address: %s", err)
|
|
}
|
|
|
|
if port == "465" {
|
|
tlsConfig, err := commoncfg.NewTLSConfig(&n.conf.TLSConfig)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if tlsConfig.ServerName == "" {
|
|
tlsConfig.ServerName = host
|
|
}
|
|
|
|
conn, err := tls.Dial("tcp", n.conf.Smarthost, tlsConfig)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
c, err = smtp.NewClient(conn, host)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
} else {
|
|
// Connect to the SMTP smarthost.
|
|
c, err = smtp.Dial(n.conf.Smarthost)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
defer func() {
|
|
if err := c.Quit(); err != nil {
|
|
level.Error(n.logger).Log("msg", "failed to close SMTP connection", "err", err)
|
|
}
|
|
}()
|
|
|
|
if n.conf.Hello != "" {
|
|
err := c.Hello(n.conf.Hello)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
|
|
// Global Config guarantees RequireTLS is not nil
|
|
if *n.conf.RequireTLS {
|
|
if ok, _ := c.Extension("STARTTLS"); !ok {
|
|
return true, fmt.Errorf("require_tls: true (default), but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
|
|
}
|
|
|
|
tlsConf, err := commoncfg.NewTLSConfig(&n.conf.TLSConfig)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if tlsConf.ServerName == "" {
|
|
tlsConf.ServerName = host
|
|
}
|
|
|
|
if err := c.StartTLS(tlsConf); err != nil {
|
|
return true, fmt.Errorf("starttls failed: %s", err)
|
|
}
|
|
}
|
|
|
|
if ok, mech := c.Extension("AUTH"); ok {
|
|
auth, err := n.auth(mech)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
if auth != nil {
|
|
if err := c.Auth(auth); err != nil {
|
|
return true, fmt.Errorf("%T failed: %s", auth, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
var (
|
|
tmplErr error
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
tmpl = tmplText(n.tmpl, data, &tmplErr)
|
|
from = tmpl(n.conf.From)
|
|
to = tmpl(n.conf.To)
|
|
)
|
|
if tmplErr != nil {
|
|
return false, fmt.Errorf("failed to template 'from' or 'to': %v", tmplErr)
|
|
}
|
|
|
|
addrs, err := mail.ParseAddressList(from)
|
|
if err != nil {
|
|
return false, fmt.Errorf("parsing from addresses: %s", err)
|
|
}
|
|
if len(addrs) != 1 {
|
|
return false, fmt.Errorf("must be exactly one from address")
|
|
}
|
|
if err := c.Mail(addrs[0].Address); err != nil {
|
|
return true, fmt.Errorf("sending mail from: %s", err)
|
|
}
|
|
addrs, err = mail.ParseAddressList(to)
|
|
if err != nil {
|
|
return false, fmt.Errorf("parsing to addresses: %s", err)
|
|
}
|
|
for _, addr := range addrs {
|
|
if err := c.Rcpt(addr.Address); err != nil {
|
|
return true, fmt.Errorf("sending rcpt to: %s", err)
|
|
}
|
|
}
|
|
|
|
// Send the email body.
|
|
wc, err := c.Data()
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer wc.Close()
|
|
|
|
for header, t := range n.conf.Headers {
|
|
value, err := n.tmpl.ExecuteTextString(t, data)
|
|
if err != nil {
|
|
return false, fmt.Errorf("executing %q header template: %s", header, err)
|
|
}
|
|
fmt.Fprintf(wc, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
|
|
}
|
|
|
|
buffer := &bytes.Buffer{}
|
|
multipartWriter := multipart.NewWriter(buffer)
|
|
|
|
fmt.Fprintf(wc, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
|
|
fmt.Fprintf(wc, "Content-Type: multipart/alternative; boundary=%s\r\n", multipartWriter.Boundary())
|
|
fmt.Fprintf(wc, "MIME-Version: 1.0\r\n")
|
|
|
|
// TODO: Add some useful headers here, such as URL of the alertmanager
|
|
// and active/resolved.
|
|
fmt.Fprintf(wc, "\r\n")
|
|
|
|
if len(n.conf.Text) > 0 {
|
|
// Text template
|
|
w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
|
|
"Content-Transfer-Encoding": {"quoted-printable"},
|
|
"Content-Type": {"text/plain; charset=UTF-8"},
|
|
})
|
|
if err != nil {
|
|
return false, fmt.Errorf("creating part for text template: %s", err)
|
|
}
|
|
body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
|
|
if err != nil {
|
|
return false, fmt.Errorf("executing email text template: %s", err)
|
|
}
|
|
qw := quotedprintable.NewWriter(w)
|
|
_, err = qw.Write([]byte(body))
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
err = qw.Close()
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
|
|
if len(n.conf.HTML) > 0 {
|
|
// Html template
|
|
// Preferred alternative placed last per section 5.1.4 of RFC 2046
|
|
// https://www.ietf.org/rfc/rfc2046.txt
|
|
w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
|
|
"Content-Transfer-Encoding": {"quoted-printable"},
|
|
"Content-Type": {"text/html; charset=UTF-8"},
|
|
})
|
|
if err != nil {
|
|
return false, fmt.Errorf("creating part for html template: %s", err)
|
|
}
|
|
body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
|
|
if err != nil {
|
|
return false, fmt.Errorf("executing email html template: %s", err)
|
|
}
|
|
qw := quotedprintable.NewWriter(w)
|
|
_, err = qw.Write([]byte(body))
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
err = qw.Close()
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
|
|
err = multipartWriter.Close()
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to close multipartWriter: %v", err)
|
|
}
|
|
|
|
_, err = wc.Write(buffer.Bytes())
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to write body buffer: %v", err)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// PagerDuty implements a Notifier for PagerDuty notifications.
|
|
type PagerDuty struct {
|
|
conf *config.PagerdutyConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewPagerDuty returns a new PagerDuty notifier.
|
|
func NewPagerDuty(c *config.PagerdutyConfig, t *template.Template, l log.Logger) *PagerDuty {
|
|
return &PagerDuty{conf: c, tmpl: t, logger: l}
|
|
}
|
|
|
|
const (
|
|
pagerDutyEventTrigger = "trigger"
|
|
pagerDutyEventResolve = "resolve"
|
|
)
|
|
|
|
type pagerDutyMessage struct {
|
|
RoutingKey string `json:"routing_key,omitempty"`
|
|
ServiceKey string `json:"service_key,omitempty"`
|
|
DedupKey string `json:"dedup_key,omitempty"`
|
|
IncidentKey string `json:"incident_key,omitempty"`
|
|
EventType string `json:"event_type,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
EventAction string `json:"event_action"`
|
|
Payload *pagerDutyPayload `json:"payload"`
|
|
Client string `json:"client,omitempty"`
|
|
ClientURL string `json:"client_url,omitempty"`
|
|
Details map[string]string `json:"details,omitempty"`
|
|
Images []pagerDutyImage `json:"images,omitempty"`
|
|
Links []pagerDutyLink `json:"links,omitempty"`
|
|
}
|
|
|
|
type pagerDutyLink struct {
|
|
HRef string `json:"href"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
type pagerDutyImage struct {
|
|
Src string `json:"src"`
|
|
Alt string `json:"alt"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
type pagerDutyPayload struct {
|
|
Summary string `json:"summary"`
|
|
Source string `json:"source"`
|
|
Severity string `json:"severity"`
|
|
Timestamp string `json:"timestamp,omitempty"`
|
|
Class string `json:"class,omitempty"`
|
|
Component string `json:"component,omitempty"`
|
|
Group string `json:"group,omitempty"`
|
|
CustomDetails map[string]string `json:"custom_details,omitempty"`
|
|
}
|
|
|
|
func (n *PagerDuty) notifyV1(
|
|
ctx context.Context,
|
|
c *http.Client,
|
|
eventType, key string,
|
|
data *template.Data,
|
|
details map[string]string,
|
|
as ...*types.Alert,
|
|
) (bool, error) {
|
|
var tmplErr error
|
|
tmpl := tmplText(n.tmpl, data, &tmplErr)
|
|
|
|
msg := &pagerDutyMessage{
|
|
ServiceKey: tmpl(string(n.conf.ServiceKey)),
|
|
EventType: eventType,
|
|
IncidentKey: hashKey(key),
|
|
Description: tmpl(n.conf.Description),
|
|
Details: details,
|
|
}
|
|
|
|
apiURL, err := url.Parse("https://events.pagerduty.com/generic/2010-04-15/create_event.json")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n.conf.URL = &config.URL{apiURL}
|
|
|
|
if eventType == pagerDutyEventTrigger {
|
|
msg.Client = tmpl(n.conf.Client)
|
|
msg.ClientURL = tmpl(n.conf.ClientURL)
|
|
}
|
|
|
|
if tmplErr != nil {
|
|
return false, fmt.Errorf("failed to template PagerDuty v1 message: %v", tmplErr)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := post(ctx, c, n.conf.URL.String(), contentTypeJSON, &buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return n.retryV1(resp)
|
|
}
|
|
|
|
func (n *PagerDuty) notifyV2(
|
|
ctx context.Context,
|
|
c *http.Client,
|
|
eventType, key string,
|
|
data *template.Data,
|
|
details map[string]string,
|
|
as ...*types.Alert,
|
|
) (bool, error) {
|
|
var tmplErr error
|
|
tmpl := tmplText(n.tmpl, data, &tmplErr)
|
|
|
|
if n.conf.Severity == "" {
|
|
n.conf.Severity = "error"
|
|
}
|
|
|
|
summary := tmpl(n.conf.Description)
|
|
summaryRunes := []rune(summary)
|
|
if len(summaryRunes) > 1024 {
|
|
summary = string(summaryRunes[:1018]) + " [...]"
|
|
}
|
|
|
|
msg := &pagerDutyMessage{
|
|
Client: tmpl(n.conf.Client),
|
|
ClientURL: tmpl(n.conf.ClientURL),
|
|
RoutingKey: tmpl(string(n.conf.RoutingKey)),
|
|
EventAction: eventType,
|
|
DedupKey: hashKey(key),
|
|
Images: make([]pagerDutyImage, len(n.conf.Images)),
|
|
Links: make([]pagerDutyLink, len(n.conf.Links)),
|
|
Payload: &pagerDutyPayload{
|
|
Summary: summary,
|
|
Source: tmpl(n.conf.Client),
|
|
Severity: tmpl(n.conf.Severity),
|
|
CustomDetails: details,
|
|
Class: tmpl(n.conf.Class),
|
|
Component: tmpl(n.conf.Component),
|
|
Group: tmpl(n.conf.Group),
|
|
},
|
|
}
|
|
|
|
for index, item := range n.conf.Images {
|
|
msg.Images[index].Src = tmpl(item.Src)
|
|
msg.Images[index].Alt = tmpl(item.Alt)
|
|
msg.Images[index].Text = tmpl(item.Text)
|
|
}
|
|
|
|
for index, item := range n.conf.Links {
|
|
msg.Links[index].HRef = tmpl(item.HRef)
|
|
msg.Links[index].Text = tmpl(item.Text)
|
|
}
|
|
|
|
if tmplErr != nil {
|
|
return false, fmt.Errorf("failed to template PagerDuty v2 message: %v", tmplErr)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return false, fmt.Errorf("failed to encode PagerDuty v2 message: %v", err)
|
|
}
|
|
|
|
resp, err := post(ctx, c, n.conf.URL.String(), contentTypeJSON, &buf)
|
|
if err != nil {
|
|
return true, fmt.Errorf("failed to post message to PagerDuty: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// See: https://v2.developer.pagerduty.com/docs/events-api-v2#api-response-codes--retry-logic
|
|
if resp.StatusCode == http.StatusBadRequest {
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to read error response from PagerDuty (status: %d): %v", resp.StatusCode, err)
|
|
}
|
|
level.Debug(n.logger).Log("msg", "Received error response from PagerDuty", "incident", key, "code", resp.StatusCode, "body", string(body))
|
|
}
|
|
|
|
return n.retryV2(resp.StatusCode)
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
//
|
|
// https://v2.developer.pagerduty.com/docs/events-api-v2
|
|
func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
key, ok := GroupKey(ctx)
|
|
if !ok {
|
|
return false, fmt.Errorf("group key missing")
|
|
}
|
|
|
|
var err error
|
|
var (
|
|
alerts = types.Alerts(as...)
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
eventType = pagerDutyEventTrigger
|
|
)
|
|
if alerts.Status() == model.AlertResolved {
|
|
eventType = pagerDutyEventResolve
|
|
}
|
|
|
|
level.Debug(n.logger).Log("msg", "Notifying PagerDuty", "incident", key, "eventType", eventType)
|
|
|
|
details := make(map[string]string, len(n.conf.Details))
|
|
for k, v := range n.conf.Details {
|
|
detail, err := n.tmpl.ExecuteTextString(v, data)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to template %q: %v", v, err)
|
|
}
|
|
details[k] = detail
|
|
}
|
|
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "pagerduty")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if n.conf.ServiceKey != "" {
|
|
return n.notifyV1(ctx, c, eventType, key, data, details, as...)
|
|
}
|
|
return n.notifyV2(ctx, c, eventType, key, data, details, as...)
|
|
}
|
|
|
|
func (n *PagerDuty) retryV1(resp *http.Response) (bool, error) {
|
|
// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
|
|
// 2xx response codes indicate a successful request.
|
|
// https://v2.developer.pagerduty.com/docs/trigger-events
|
|
statusCode := resp.StatusCode
|
|
|
|
if statusCode == 400 && resp.Body != nil {
|
|
bs, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return false, fmt.Errorf("unexpected status code %v : problem reading response: %v", statusCode, err)
|
|
}
|
|
return false, fmt.Errorf("bad request (status code %v): %v", statusCode, string(bs))
|
|
}
|
|
|
|
if statusCode/100 != 2 {
|
|
return (statusCode == 403 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func (n *PagerDuty) retryV2(statusCode int) (bool, error) {
|
|
// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
|
|
// 2xx response codes indicate a successful request.
|
|
// https://v2.developer.pagerduty.com/docs/events-api-v2#api-response-codes--retry-logic
|
|
if statusCode/100 != 2 {
|
|
return (statusCode == 429 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// Slack implements a Notifier for Slack notifications.
|
|
type Slack struct {
|
|
conf *config.SlackConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewSlack returns a new Slack notification handler.
|
|
func NewSlack(c *config.SlackConfig, t *template.Template, l log.Logger) *Slack {
|
|
return &Slack{
|
|
conf: c,
|
|
tmpl: t,
|
|
logger: l,
|
|
}
|
|
}
|
|
|
|
// slackReq is the request for sending a slack notification.
|
|
type slackReq struct {
|
|
Channel string `json:"channel,omitempty"`
|
|
Username string `json:"username,omitempty"`
|
|
IconEmoji string `json:"icon_emoji,omitempty"`
|
|
IconURL string `json:"icon_url,omitempty"`
|
|
LinkNames bool `json:"link_names,omitempty"`
|
|
Attachments []slackAttachment `json:"attachments"`
|
|
}
|
|
|
|
// slackAttachment is used to display a richly-formatted message block.
|
|
type slackAttachment struct {
|
|
Title string `json:"title,omitempty"`
|
|
TitleLink string `json:"title_link,omitempty"`
|
|
Pretext string `json:"pretext,omitempty"`
|
|
Text string `json:"text"`
|
|
Fallback string `json:"fallback"`
|
|
CallbackID string `json:"callback_id"`
|
|
Fields []config.SlackField `json:"fields,omitempty"`
|
|
Actions []config.SlackAction `json:"actions,omitempty"`
|
|
ImageURL string `json:"image_url,omitempty"`
|
|
ThumbURL string `json:"thumb_url,omitempty"`
|
|
Footer string `json:"footer"`
|
|
|
|
Color string `json:"color,omitempty"`
|
|
MrkdwnIn []string `json:"mrkdwn_in,omitempty"`
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *Slack) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
var err error
|
|
var (
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
tmplText = tmplText(n.tmpl, data, &err)
|
|
)
|
|
|
|
attachment := &slackAttachment{
|
|
Title: tmplText(n.conf.Title),
|
|
TitleLink: tmplText(n.conf.TitleLink),
|
|
Pretext: tmplText(n.conf.Pretext),
|
|
Text: tmplText(n.conf.Text),
|
|
Fallback: tmplText(n.conf.Fallback),
|
|
CallbackID: tmplText(n.conf.CallbackID),
|
|
ImageURL: tmplText(n.conf.ImageURL),
|
|
ThumbURL: tmplText(n.conf.ThumbURL),
|
|
Footer: tmplText(n.conf.Footer),
|
|
Color: tmplText(n.conf.Color),
|
|
MrkdwnIn: []string{"fallback", "pretext", "text"},
|
|
}
|
|
|
|
var numFields = len(n.conf.Fields)
|
|
if numFields > 0 {
|
|
var fields = make([]config.SlackField, numFields)
|
|
for index, field := range n.conf.Fields {
|
|
// Check if short was defined for the field otherwise fallback to the global setting
|
|
var short bool
|
|
if field.Short != nil {
|
|
short = *field.Short
|
|
} else {
|
|
short = n.conf.ShortFields
|
|
}
|
|
|
|
// Rebuild the field by executing any templates and setting the new value for short
|
|
fields[index] = config.SlackField{
|
|
Title: tmplText(field.Title),
|
|
Value: tmplText(field.Value),
|
|
Short: &short,
|
|
}
|
|
}
|
|
attachment.Fields = fields
|
|
}
|
|
|
|
var numActions = len(n.conf.Actions)
|
|
if numActions > 0 {
|
|
var actions = make([]config.SlackAction, numActions)
|
|
for index, action := range n.conf.Actions {
|
|
slackAction := config.SlackAction{
|
|
Type: tmplText(action.Type),
|
|
Text: tmplText(action.Text),
|
|
URL: tmplText(action.URL),
|
|
Style: tmplText(action.Style),
|
|
Name: tmplText(action.Name),
|
|
Value: tmplText(action.Value),
|
|
}
|
|
|
|
if action.ConfirmField != nil {
|
|
slackAction.ConfirmField = &config.SlackConfirmationField{
|
|
Title: tmplText(action.ConfirmField.Title),
|
|
Text: tmplText(action.ConfirmField.Text),
|
|
OkText: tmplText(action.ConfirmField.OkText),
|
|
DismissText: tmplText(action.ConfirmField.DismissText),
|
|
}
|
|
}
|
|
|
|
actions[index] = slackAction
|
|
}
|
|
attachment.Actions = actions
|
|
}
|
|
|
|
req := &slackReq{
|
|
Channel: tmplText(n.conf.Channel),
|
|
Username: tmplText(n.conf.Username),
|
|
IconEmoji: tmplText(n.conf.IconEmoji),
|
|
IconURL: tmplText(n.conf.IconURL),
|
|
LinkNames: n.conf.LinkNames,
|
|
Attachments: []slackAttachment{*attachment},
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(req); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "slack")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := post(ctx, c, n.conf.APIURL.String(), contentTypeJSON, &buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
resp.Body.Close()
|
|
|
|
return n.retry(resp.StatusCode)
|
|
}
|
|
|
|
func (n *Slack) retry(statusCode int) (bool, error) {
|
|
// Only 5xx response codes are recoverable and 2xx codes are successful.
|
|
// https://api.slack.com/incoming-webhooks#handling_errors
|
|
// https://api.slack.com/changelog/2016-05-17-changes-to-errors-for-incoming-webhooks
|
|
if statusCode/100 != 2 {
|
|
return (statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// Hipchat implements a Notifier for Hipchat notifications.
|
|
type Hipchat struct {
|
|
conf *config.HipchatConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewHipchat returns a new Hipchat notification handler.
|
|
func NewHipchat(c *config.HipchatConfig, t *template.Template, l log.Logger) *Hipchat {
|
|
return &Hipchat{
|
|
conf: c,
|
|
tmpl: t,
|
|
logger: l,
|
|
}
|
|
}
|
|
|
|
type hipchatReq struct {
|
|
From string `json:"from"`
|
|
Notify bool `json:"notify"`
|
|
Message string `json:"message"`
|
|
MessageFormat string `json:"message_format"`
|
|
Color string `json:"color"`
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *Hipchat) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
var err error
|
|
var msg string
|
|
var (
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
tmplText = tmplText(n.tmpl, data, &err)
|
|
tmplHTML = tmplHTML(n.tmpl, data, &err)
|
|
roomid = tmplText(n.conf.RoomID)
|
|
apiURL = n.conf.APIURL.Copy()
|
|
)
|
|
apiURL.Path += fmt.Sprintf("v2/room/%s/notification", roomid)
|
|
q := apiURL.Query()
|
|
q.Set("auth_token", string(n.conf.AuthToken))
|
|
apiURL.RawQuery = q.Encode()
|
|
|
|
if n.conf.MessageFormat == "html" {
|
|
msg = tmplHTML(n.conf.Message)
|
|
} else {
|
|
msg = tmplText(n.conf.Message)
|
|
}
|
|
|
|
req := &hipchatReq{
|
|
From: tmplText(n.conf.From),
|
|
Notify: n.conf.Notify,
|
|
Message: msg,
|
|
MessageFormat: n.conf.MessageFormat,
|
|
Color: tmplText(n.conf.Color),
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(req); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "hipchat")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := post(ctx, c, apiURL.String(), contentTypeJSON, &buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
return n.retry(resp.StatusCode)
|
|
}
|
|
|
|
func (n *Hipchat) retry(statusCode int) (bool, error) {
|
|
// Response codes 429 (rate limiting) and 5xx can potentially recover. 2xx
|
|
// responce codes indicate successful requests.
|
|
// https://developer.atlassian.com/hipchat/guide/hipchat-rest-api/api-response-codes
|
|
if statusCode/100 != 2 {
|
|
return (statusCode == 429 || statusCode/100 == 5), fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// Wechat implements a Notfier for wechat notifications
|
|
type Wechat struct {
|
|
conf *config.WechatConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
|
|
accessToken string
|
|
accessTokenAt time.Time
|
|
}
|
|
|
|
// Wechat AccessToken with corpid and corpsecret.
|
|
type WechatToken struct {
|
|
AccessToken string `json:"access_token"`
|
|
}
|
|
|
|
type weChatMessage struct {
|
|
Text weChatMessageContent `yaml:"text,omitempty" json:"text,omitempty"`
|
|
ToUser string `yaml:"touser,omitempty" json:"touser,omitempty"`
|
|
ToParty string `yaml:"toparty,omitempty" json:"toparty,omitempty"`
|
|
Totag string `yaml:"totag,omitempty" json:"totag,omitempty"`
|
|
AgentID string `yaml:"agentid,omitempty" json:"agentid,omitempty"`
|
|
Safe string `yaml:"safe,omitempty" json:"safe,omitempty"`
|
|
Type string `yaml:"msgtype,omitempty" json:"msgtype,omitempty"`
|
|
}
|
|
|
|
type weChatMessageContent struct {
|
|
Content string `json:"content"`
|
|
}
|
|
|
|
type weChatResponse struct {
|
|
Code int `json:"code"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
// NewWechat returns a new Wechat notifier.
|
|
func NewWechat(c *config.WechatConfig, t *template.Template, l log.Logger) *Wechat {
|
|
return &Wechat{conf: c, tmpl: t, logger: l}
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *Wechat) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
key, ok := GroupKey(ctx)
|
|
if !ok {
|
|
return false, fmt.Errorf("group key missing")
|
|
}
|
|
|
|
level.Debug(n.logger).Log("msg", "Notifying Wechat", "incident", key)
|
|
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
|
|
var err error
|
|
tmpl := tmplText(n.tmpl, data, &err)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "wechat")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
// Refresh AccessToken over 2 hours
|
|
if n.accessToken == "" || time.Since(n.accessTokenAt) > 2*time.Hour {
|
|
parameters := url.Values{}
|
|
parameters.Add("corpsecret", tmpl(string(n.conf.APISecret)))
|
|
parameters.Add("corpid", tmpl(string(n.conf.CorpID)))
|
|
if err != nil {
|
|
return false, fmt.Errorf("templating error: %s", err)
|
|
}
|
|
|
|
u := n.conf.APIURL.Copy()
|
|
u.Path += "gettoken"
|
|
u.RawQuery = parameters.Encode()
|
|
|
|
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
req.Header.Set("Content-Type", contentTypeJSON)
|
|
|
|
resp, err := c.Do(req.WithContext(ctx))
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var wechatToken WechatToken
|
|
if err := json.NewDecoder(resp.Body).Decode(&wechatToken); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if wechatToken.AccessToken == "" {
|
|
return false, fmt.Errorf("invalid APISecret for CorpID: %s", n.conf.CorpID)
|
|
}
|
|
|
|
// Cache accessToken
|
|
n.accessToken = wechatToken.AccessToken
|
|
n.accessTokenAt = time.Now()
|
|
}
|
|
|
|
msg := &weChatMessage{
|
|
Text: weChatMessageContent{
|
|
Content: tmpl(n.conf.Message),
|
|
},
|
|
ToUser: tmpl(n.conf.ToUser),
|
|
ToParty: tmpl(n.conf.ToParty),
|
|
Totag: tmpl(n.conf.ToTag),
|
|
AgentID: tmpl(n.conf.AgentID),
|
|
Type: "text",
|
|
Safe: "0",
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("templating error: %s", err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
postMessageURL := n.conf.APIURL.Copy()
|
|
postMessageURL.Path += "message/send"
|
|
q := postMessageURL.Query()
|
|
q.Set("access_token", n.accessToken)
|
|
postMessageURL.RawQuery = q.Encode()
|
|
|
|
req, err := http.NewRequest(http.MethodPost, postMessageURL.String(), &buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
resp, err := c.Do(req.WithContext(ctx))
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
level.Debug(n.logger).Log("msg", "response: "+string(body), "incident", key)
|
|
|
|
if resp.StatusCode != 200 {
|
|
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
|
|
} else {
|
|
var weResp weChatResponse
|
|
if err := json.Unmarshal(body, &weResp); err != nil {
|
|
return true, err
|
|
}
|
|
|
|
// https://work.weixin.qq.com/api/doc#10649
|
|
if weResp.Code == 0 {
|
|
return false, nil
|
|
}
|
|
|
|
// AccessToken is expired
|
|
if weResp.Code == 42001 {
|
|
n.accessToken = ""
|
|
return true, errors.New(weResp.Error)
|
|
}
|
|
|
|
return false, errors.New(weResp.Error)
|
|
}
|
|
}
|
|
|
|
// OpsGenie implements a Notifier for OpsGenie notifications.
|
|
type OpsGenie struct {
|
|
conf *config.OpsGenieConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewOpsGenie returns a new OpsGenie notifier.
|
|
func NewOpsGenie(c *config.OpsGenieConfig, t *template.Template, l log.Logger) *OpsGenie {
|
|
return &OpsGenie{conf: c, tmpl: t, logger: l}
|
|
}
|
|
|
|
type opsGenieCreateMessage struct {
|
|
Alias string `json:"alias"`
|
|
Message string `json:"message"`
|
|
Description string `json:"description,omitempty"`
|
|
Details map[string]string `json:"details"`
|
|
Source string `json:"source"`
|
|
Teams []map[string]string `json:"teams,omitempty"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
Priority string `json:"priority,omitempty"`
|
|
}
|
|
|
|
type opsGenieCloseMessage struct {
|
|
Source string `json:"source"`
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
req, retry, err := n.createRequest(ctx, as...)
|
|
if err != nil {
|
|
return retry, err
|
|
}
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "opsgenie")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := c.Do(req.WithContext(ctx))
|
|
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return n.retry(resp.StatusCode)
|
|
}
|
|
|
|
// Like Split but filter out empty strings.
|
|
func safeSplit(s string, sep string) []string {
|
|
a := strings.Split(strings.TrimSpace(s), sep)
|
|
b := a[:0]
|
|
for _, x := range a {
|
|
if x != "" {
|
|
b = append(b, x)
|
|
}
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Create requests for a list of alerts.
|
|
func (n *OpsGenie) createRequest(ctx context.Context, as ...*types.Alert) (*http.Request, bool, error) {
|
|
key, ok := GroupKey(ctx)
|
|
if !ok {
|
|
return nil, false, fmt.Errorf("group key missing")
|
|
}
|
|
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
|
|
level.Debug(n.logger).Log("msg", "Notifying OpsGenie", "incident", key)
|
|
|
|
var err error
|
|
tmpl := tmplText(n.tmpl, data, &err)
|
|
|
|
details := make(map[string]string, len(n.conf.Details))
|
|
for k, v := range n.conf.Details {
|
|
details[k] = tmpl(v)
|
|
}
|
|
|
|
var (
|
|
msg interface{}
|
|
apiURL = n.conf.APIURL.Copy()
|
|
alias = hashKey(key)
|
|
alerts = types.Alerts(as...)
|
|
)
|
|
switch alerts.Status() {
|
|
case model.AlertResolved:
|
|
apiURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
|
|
q := apiURL.Query()
|
|
q.Set("identifierType", "alias")
|
|
apiURL.RawQuery = q.Encode()
|
|
msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
|
|
default:
|
|
message, truncated := truncate(tmpl(n.conf.Message), 130)
|
|
if truncated {
|
|
level.Debug(n.logger).Log("msg", "truncated message due to OpsGenie message limit", "truncated_message", message, "incident", key)
|
|
}
|
|
|
|
apiURL.Path += "v2/alerts"
|
|
var teams []map[string]string
|
|
for _, t := range safeSplit(string(tmpl(n.conf.Teams)), ",") {
|
|
teams = append(teams, map[string]string{"name": t})
|
|
}
|
|
tags := safeSplit(string(tmpl(n.conf.Tags)), ",")
|
|
|
|
msg = &opsGenieCreateMessage{
|
|
Alias: alias,
|
|
Message: message,
|
|
Description: tmpl(n.conf.Description),
|
|
Details: details,
|
|
Source: tmpl(n.conf.Source),
|
|
Teams: teams,
|
|
Tags: tags,
|
|
Note: tmpl(n.conf.Note),
|
|
Priority: tmpl(n.conf.Priority),
|
|
}
|
|
}
|
|
|
|
apiKey := tmpl(string(n.conf.APIKey))
|
|
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("templating error: %s", err)
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
req, err := http.NewRequest("POST", apiURL.String(), &buf)
|
|
if err != nil {
|
|
return nil, true, err
|
|
}
|
|
req.Header.Set("Content-Type", contentTypeJSON)
|
|
req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
|
|
return req, true, nil
|
|
}
|
|
|
|
func (n *OpsGenie) retry(statusCode int) (bool, error) {
|
|
// https://docs.opsgenie.com/docs/response#section-response-codes
|
|
// Response codes 429 (rate limiting) and 5xx are potentially recoverable
|
|
if statusCode/100 == 5 || statusCode == 429 {
|
|
return true, fmt.Errorf("unexpected status code %v", statusCode)
|
|
} else if statusCode/100 != 2 {
|
|
return false, fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// VictorOps implements a Notifier for VictorOps notifications.
|
|
type VictorOps struct {
|
|
conf *config.VictorOpsConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewVictorOps returns a new VictorOps notifier.
|
|
func NewVictorOps(c *config.VictorOpsConfig, t *template.Template, l log.Logger) *VictorOps {
|
|
return &VictorOps{
|
|
conf: c,
|
|
tmpl: t,
|
|
logger: l,
|
|
}
|
|
}
|
|
|
|
const (
|
|
victorOpsEventTrigger = "CRITICAL"
|
|
victorOpsEventResolve = "RECOVERY"
|
|
)
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
|
|
var err error
|
|
var (
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
tmpl = tmplText(n.tmpl, data, &err)
|
|
apiURL = n.conf.APIURL.Copy()
|
|
)
|
|
apiURL.Path += fmt.Sprintf("%s/%s", n.conf.APIKey, tmpl(n.conf.RoutingKey))
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "victorops")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
buf, err := n.createVictorOpsPayload(ctx, as...)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
resp, err := post(ctx, c, apiURL.String(), contentTypeJSON, buf)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
return n.retry(resp.StatusCode)
|
|
}
|
|
|
|
// Create the JSON payload to be sent to the VictorOps API.
|
|
func (n *VictorOps) createVictorOpsPayload(ctx context.Context, as ...*types.Alert) (*bytes.Buffer, error) {
|
|
victorOpsAllowedEvents := map[string]bool{
|
|
"INFO": true,
|
|
"WARNING": true,
|
|
"CRITICAL": true,
|
|
}
|
|
|
|
key, ok := GroupKey(ctx)
|
|
if !ok {
|
|
return nil, fmt.Errorf("group key missing")
|
|
}
|
|
|
|
var err error
|
|
var (
|
|
alerts = types.Alerts(as...)
|
|
data = n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
tmpl = tmplText(n.tmpl, data, &err)
|
|
|
|
messageType = tmpl(n.conf.MessageType)
|
|
stateMessage = tmpl(n.conf.StateMessage)
|
|
)
|
|
|
|
if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] {
|
|
messageType = victorOpsEventTrigger
|
|
}
|
|
|
|
if alerts.Status() == model.AlertResolved {
|
|
messageType = victorOpsEventResolve
|
|
}
|
|
|
|
stateMessage, truncated := truncate(stateMessage, 20480)
|
|
if truncated {
|
|
level.Debug(n.logger).Log("msg", "truncated stateMessage due to VictorOps stateMessage limit", "truncated_state_message", stateMessage, "incident", key)
|
|
}
|
|
|
|
msg := map[string]string{
|
|
"message_type": messageType,
|
|
"entity_id": hashKey(key),
|
|
"entity_display_name": tmpl(n.conf.EntityDisplayName),
|
|
"state_message": stateMessage,
|
|
"monitoring_tool": tmpl(n.conf.MonitoringTool),
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("templating error: %s", err)
|
|
}
|
|
|
|
// Add custom fields to the payload.
|
|
for k, v := range n.conf.CustomFields {
|
|
msg[k] = tmpl(v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("templating error: %s", err)
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
|
|
return nil, err
|
|
}
|
|
return &buf, nil
|
|
}
|
|
|
|
func (n *VictorOps) retry(statusCode int) (bool, error) {
|
|
// Missing documentation therefore assuming only 5xx response codes are
|
|
// recoverable.
|
|
if statusCode/100 == 5 {
|
|
return true, fmt.Errorf("unexpected status code %v", statusCode)
|
|
} else if statusCode/100 != 2 {
|
|
return false, fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// Pushover implements a Notifier for Pushover notifications.
|
|
type Pushover struct {
|
|
conf *config.PushoverConfig
|
|
tmpl *template.Template
|
|
logger log.Logger
|
|
}
|
|
|
|
// NewPushover returns a new Pushover notifier.
|
|
func NewPushover(c *config.PushoverConfig, t *template.Template, l log.Logger) *Pushover {
|
|
return &Pushover{conf: c, tmpl: t, logger: l}
|
|
}
|
|
|
|
// Notify implements the Notifier interface.
|
|
func (n *Pushover) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
|
key, ok := GroupKey(ctx)
|
|
if !ok {
|
|
return false, fmt.Errorf("group key missing")
|
|
}
|
|
data := n.tmpl.Data(receiverName(ctx, n.logger), groupLabels(ctx, n.logger), as...)
|
|
|
|
level.Debug(n.logger).Log("msg", "Notifying Pushover", "incident", key)
|
|
|
|
var (
|
|
err error
|
|
message string
|
|
)
|
|
tmpl := tmplText(n.tmpl, data, &err)
|
|
tmplHTML := tmplHTML(n.tmpl, data, &err)
|
|
|
|
parameters := url.Values{}
|
|
parameters.Add("token", tmpl(string(n.conf.Token)))
|
|
parameters.Add("user", tmpl(string(n.conf.UserKey)))
|
|
|
|
title, truncated := truncate(tmpl(n.conf.Title), 250)
|
|
if truncated {
|
|
level.Debug(n.logger).Log("msg", "Truncated title due to Pushover title limit", "truncated_title", title, "incident", key)
|
|
}
|
|
parameters.Add("title", title)
|
|
|
|
if n.conf.HTML {
|
|
parameters.Add("html", "1")
|
|
message = tmplHTML(n.conf.Message)
|
|
} else {
|
|
message = tmpl(n.conf.Message)
|
|
}
|
|
|
|
message, truncated = truncate(message, 1024)
|
|
if truncated {
|
|
level.Debug(n.logger).Log("msg", "Truncated message due to Pushover message limit", "truncated_message", message, "incident", key)
|
|
}
|
|
message = strings.TrimSpace(message)
|
|
if message == "" {
|
|
// Pushover rejects empty messages.
|
|
message = "(no details)"
|
|
}
|
|
parameters.Add("message", message)
|
|
|
|
supplementaryURL, truncated := truncate(tmpl(n.conf.URL), 512)
|
|
if truncated {
|
|
level.Debug(n.logger).Log("msg", "Truncated URL due to Pushover url limit", "truncated_url", supplementaryURL, "incident", key)
|
|
}
|
|
parameters.Add("url", supplementaryURL)
|
|
parameters.Add("url_title", tmpl(n.conf.URLTitle))
|
|
|
|
parameters.Add("priority", tmpl(n.conf.Priority))
|
|
parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
|
|
parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
|
|
parameters.Add("sound", tmpl(n.conf.Sound))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
apiURL := "https://api.pushover.net/1/messages.json"
|
|
u, err := url.Parse(apiURL)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
u.RawQuery = parameters.Encode()
|
|
level.Debug(n.logger).Log("msg", "Sending Pushover message", "incident", key, "url", u.String())
|
|
|
|
c, err := commoncfg.NewClientFromConfig(*n.conf.HTTPConfig, "pushover")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
resp, err := post(ctx, c, u.String(), "text/plain", nil)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return n.retry(resp.StatusCode)
|
|
}
|
|
|
|
func (n *Pushover) retry(statusCode int) (bool, error) {
|
|
// Only documented behaviour is that 2xx response codes are successful and
|
|
// 4xx are unsuccessful, therefore assuming only 5xx are recoverable.
|
|
// https://pushover.net/api#response
|
|
if statusCode/100 == 5 {
|
|
return true, fmt.Errorf("unexpected status code %v", statusCode)
|
|
} else if statusCode/100 != 2 {
|
|
return false, fmt.Errorf("unexpected status code %v", statusCode)
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// tmplText is using monadic error handling in order to make string templating
|
|
// less verbose. Use with care as the final error checking is easily missed.
|
|
func tmplText(tmpl *template.Template, data *template.Data, err *error) func(string) string {
|
|
return func(name string) (s string) {
|
|
if *err != nil {
|
|
return
|
|
}
|
|
s, *err = tmpl.ExecuteTextString(name, data)
|
|
return s
|
|
}
|
|
}
|
|
|
|
// tmplHTML is using monadic error handling in order to make string templating
|
|
// less verbose. Use with care as the final error checking is easily missed.
|
|
func tmplHTML(tmpl *template.Template, data *template.Data, err *error) func(string) string {
|
|
return func(name string) (s string) {
|
|
if *err != nil {
|
|
return
|
|
}
|
|
s, *err = tmpl.ExecuteHTMLString(name, data)
|
|
return s
|
|
}
|
|
}
|
|
|
|
type loginAuth struct {
|
|
username, password string
|
|
}
|
|
|
|
func LoginAuth(username, password string) smtp.Auth {
|
|
return &loginAuth{username, password}
|
|
}
|
|
|
|
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
|
return "LOGIN", []byte{}, nil
|
|
}
|
|
|
|
// Used for AUTH LOGIN. (Maybe password should be encrypted)
|
|
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
|
if more {
|
|
switch strings.ToLower(string(fromServer)) {
|
|
case "username:":
|
|
return []byte(a.username), nil
|
|
case "password:":
|
|
return []byte(a.password), nil
|
|
default:
|
|
return nil, errors.New("unexpected server challenge")
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// hashKey returns the sha256 for a group key as integrations may have
|
|
// maximum length requirements on deduplication keys.
|
|
func hashKey(s string) string {
|
|
h := sha256.New()
|
|
h.Write([]byte(s))
|
|
return fmt.Sprintf("%x", h.Sum(nil))
|
|
}
|
|
|
|
func post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
|
|
req, err := http.NewRequest("POST", url, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", bodyType)
|
|
return client.Do(req.WithContext(ctx))
|
|
}
|
|
|
|
func truncate(s string, n int) (string, bool) {
|
|
r := []rune(s)
|
|
if len(r) <= n {
|
|
return s, false
|
|
}
|
|
if n <= 3 {
|
|
return string(r[:n]), true
|
|
}
|
|
return string(r[:n-3]) + "...", true
|
|
}
|