RepoMirrors/alertmanager
1
0
Fork 0
mirror of https://github.com/prometheus/alertmanager synced 2025-03-02 17:57:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add runbook and alertmanager URLs to PD+email notifications.

Browse Source 
I don't have a way to test all the other notification mechanisms, which
is something we should fix in general. For now, only PagerDuty and email
have the new runbook and alertmanager URL information.

Not very happy with the overall cleanliness of this, and the codebase
overall, of course, but since we need this urgently tomorrow, I hope
this is fine for now.
This commit is contained in:
Julius Volz 2015-06-25 17:59:00 +02:00
parent 8d455091fc
commit 35b3741756
5 changed files with 63 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
main.go
View File

@ -15,6 +15,8 @@ package main
import (
"flag"
"fmt"
"net"
"os"
"strings"
"time"
@ -31,9 +33,27 @@ var (
configFile = flag.String("config.file", "alertmanager.conf", "Alert Manager configuration file name.")
silencesFile = flag.String("silences.file", "silences.json", "Silence storage file name.")
minRefreshPeriod = flag.Duration("alerts.min-refresh-period", 5*time.Minute, "Minimum required alert refresh period before an alert is purged.")
listenAddress = flag.String("web.listen-address", ":9093", "Address to listen on for the web interface and API.")
pathPrefix = flag.String("web.path-prefix", "/", "Prefix for all web paths.")
hostname = flag.String("web.hostname", "", "Hostname on which the Alertmanager is available to the outside world.")
)
func alertmanagerURL(hostname, pathPrefix, addr string) (string, error) {
var err error
if hostname == "" {
hostname, err = os.Hostname()
if err != nil {
return "", err
}
}
_, port, err := net.SplitHostPort(addr)
if err != nil {
return "", err
}
return fmt.Sprintf("http://%s:%s%s", hostname, port, pathPrefix), nil
}
func main() {
flag.Parse()
@ -65,7 +85,11 @@ func main() {
}()
defer saveSilencesTicker.Stop()
notifier := manager.NewNotifier(conf.NotificationConfig)
amURL, err := alertmanagerURL(*hostname, *pathPrefix, *listenAddress)
if err != nil {
log.Fatalln("Error building Alertmanager URL:", err)
}
notifier := manager.NewNotifier(conf.NotificationConfig, amURL)
defer notifier.Close()
inhibitor := new(manager.Inhibitor)
@ -113,7 +137,7 @@ func main() {
},
StatusHandler: statusHandler,
}
go webService.ServeForever(*pathPrefix)
go webService.ServeForever(*listenAddress, *pathPrefix)
// React to configuration changes.
watcher := config.NewFileWatcher(*configFile)

2
manager/alert.go
View File

@ -36,6 +36,8 @@ type Alert struct {
Summary string `json:"summary"`
// Long description of alert.
Description string `json:"description"`
// Runbook link or reference for the alert.
Runbook string `json:"runbook"`
// Label value pairs for purpose of aggregation, matching, and disposition
// dispatching. This must minimally include an "alertname" label.
Labels AlertLabelSet `json:"labels"`

42
manager/notifier.go
View File

@ -51,6 +51,9 @@ Subject: [{{ .Status }}] {{.Alert.Labels.alertname}}: {{.Alert.Summary}}
{{.Alert.Description}}
Alertmanager: {{.AlertmanagerURL}}
{{if .Alert.Runbook}}Runbook entry: {{.Alert.Runbook}}{{end}}
Grouping labels:
{{range $label, $value := .Alert.Labels}}
{{$label}} = "{{$value}}"{{end}}
@ -95,6 +98,8 @@ type notificationReq struct {
type notifier struct {
// Notifications that are queued to be sent.
pendingNotifications chan *notificationReq
// URL that points back to this Alertmanager instance.
alertmanagerURL string
// Mutex to protect the fields below.
mu sync.Mutex
@ -103,9 +108,10 @@ type notifier struct {
}
// NewNotifier construct a new notifier.
func NewNotifier(configs []*pb.NotificationConfig) *notifier {
func NewNotifier(configs []*pb.NotificationConfig, amURL string) *notifier {
notifier := &notifier{
pendingNotifications: make(chan *notificationReq, *notificationBufferSize),
alertmanagerURL: amURL,
}
notifier.SetNotificationConfigs(configs)
return notifier
@ -156,9 +162,13 @@ func (n *notifier) sendPagerDutyNotification(serviceKey string, op notificationO
"event_type": eventType,
"description": a.Description,
"incident_key": incidentKey,
"client": "Prometheus Alertmanager",
"client_url": n.alertmanagerURL,
"details": map[string]interface{}{
"grouping_labels": a.Labels,
"extra_labels": a.Payload,
"runbook": a.Runbook,
"summary": a.Summary,
},
})
if err != nil {
@ -425,23 +435,25 @@ func postJSON(jsonMessage []byte, url string) (*http.Response, error) {
return client.Post(url, contentTypeJSON, bytes.NewBuffer(jsonMessage))
}
func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error {
return writeEmailBodyWithTime(w, from, to, status, a, time.Now())
func writeEmailBody(w io.Writer, from, to, status string, a *Alert, amURL string) error {
return writeEmailBodyWithTime(w, from, to, status, a, time.Now(), amURL)
}
func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time) error {
func writeEmailBodyWithTime(w io.Writer, from, to, status string, a *Alert, moment time.Time, amURL string) error {
err := bodyTmpl.Execute(w, struct {
From string
To string
Date string
Alert *Alert
Status string
From string
To string
Date string
Alert *Alert
Status string
AlertmanagerURL string
}{
From: from,
To: to,
Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
Alert: a,
Status: status,
From: from,
To: to,
Date: moment.Format("Mon, 2 Jan 2006 15:04:05 -0700"),
Alert: a,
Status: status,
AlertmanagerURL: amURL,
})
if err != nil {
return err
@ -529,7 +541,7 @@ func (n *notifier) sendEmailNotification(to string, op notificationOp, a *Alert)
}
defer wc.Close()
return writeEmailBody(wc, *smtpSender, to, status, a)
return writeEmailBody(wc, *smtpSender, to, status, a, n.alertmanagerURL)
}
func (n *notifier) sendPushoverNotification(token string, op notificationOp, userKey string, a *Alert) error {

6
manager/notifier_test.go
View File

@ -33,6 +33,7 @@ func TestWriteEmailBody(t *testing.T) {
event := &Alert{
Summary: "Testsummary",
Description: "Test alert description, something went wrong here.",
Runbook: "http://runbook/",
Labels: AlertLabelSet{
"alertname": "TestAlert",
"grouping_label1": "grouping_value1",
@ -46,7 +47,7 @@ func TestWriteEmailBody(t *testing.T) {
buf := &bytes.Buffer{}
location, _ := time.LoadLocation("Europe/Amsterdam")
moment := time.Date(1918, 11, 11, 11, 0, 3, 0, location)
writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", "ALERT", event, moment)
writeEmailBodyWithTime(buf, "from@prometheus.io", "to@prometheus.io", "ALERT", event, moment, "http://alertmanager/")
expected := `From: Prometheus Alertmanager <from@prometheus.io>
To: to@prometheus.io
@ -55,6 +56,9 @@ Subject: [ALERT] TestAlert: Testsummary
Test alert description, something went wrong here.
Alertmanager: http://alertmanager/
Runbook entry: http://runbook/
Grouping labels:
alertname = "TestAlert"

8
web/web.go
View File

@ -30,7 +30,6 @@ import (
// Commandline flags.
var (
listenAddress = flag.String("web.listen-address", ":9093", "Address to listen on for the web interface and API.")
useLocalAssets = flag.Bool("web.use-local-assets", false, "Serve assets and templates from local files instead of from the binary.")
)
@ -41,8 +40,7 @@ type WebService struct {
StatusHandler *StatusHandler
}
func (w WebService) ServeForever(pathPrefix string) error {
func (w WebService) ServeForever(addr string, pathPrefix string) error {
http.Handle(pathPrefix+"favicon.ico", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "", 404)
}))
@ -75,9 +73,9 @@ func (w WebService) ServeForever(pathPrefix string) error {
}
http.Handle(pathPrefix+"api/", w.AlertManagerService.Handler())
log.Info("listening on ", *listenAddress)
log.Info("listening on ", addr)
return http.ListenAndServe(*listenAddress, nil)
return http.ListenAndServe(addr, nil)
}
func getLocalTemplate(name string, pathPrefix string) (*template.Template, error) {