diff --git a/dispatch.go b/dispatch.go index 0cb5d374..53f67a4f 100644 --- a/dispatch.go +++ b/dispatch.go @@ -150,9 +150,10 @@ func (d *Dispatcher) processAlert(alert *types.Alert, opts *RouteOpts) { // common set of routing options applies. // It emits notifications in the specified intervals. type aggrGroup struct { - labels model.LabelSet - opts *RouteOpts - log log.Logger + labels model.LabelSet + opts *RouteOpts + routeFP model.Fingerprint + log log.Logger ctx context.Context cancel func() @@ -210,6 +211,7 @@ func (ag *aggrGroup) run(nf notifyFunc) { ctx = notify.WithNow(ctx, now) // Populate context with information needed along the pipeline. + ctx = notify.WithGroupKey(ctx, ag.labels.Fingerprint()^ag.routeFP) ctx = notify.WithGroupLabels(ctx, ag.labels) ctx = notify.WithDestination(ctx, ag.opts.SendTo) ctx = notify.WithRepeatInterval(ctx, ag.opts.RepeatInterval) diff --git a/notify/impl.go b/notify/impl.go index 9dc7c434..f6bb29d1 100644 --- a/notify/impl.go +++ b/notify/impl.go @@ -231,13 +231,13 @@ type pagerDutyMessage struct { ServiceKey string `json:"service_key"` EventType string `json:"event_type"` Description string `json:"description"` - IncidentKey uint64 `json:"incident_key"` + IncidentKey model.Fingerprint `json:"incident_key"` Client string `json:"client,omitempty"` ClientURL string `json:"client_url,omitempty"` Details map[string]string `json:"details"` } -func (pd *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error { +func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error { // http://developer.pagerduty.com/documentation/integration/events/trigger alerts := types.Alerts(as...) @@ -246,24 +246,56 @@ func (pd *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error { eventType = pagerDutyEventResolve } + key, ok := GroupKey(ctx) + if !ok { + return fmt.Errorf("group key missing") + } + + log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty") + + groupLabels, ok := GroupLabels(ctx) + if !ok { + log.Error("missing group labels") + } + + data := struct { + Alerts model.Alerts + GroupLabels model.LabelSet + }{ + Alerts: alerts, + GroupLabels: groupLabels, + } + + var err error + tmpl := func(name string) (s string) { + if err != nil { + return + } + s, err = n.tmpl.ExecuteTextString(name, &data) + return s + } + msg := &pagerDutyMessage{ - ServiceKey: pd.conf.ServiceKey, + ServiceKey: n.conf.ServiceKey, EventType: eventType, - IncidentKey: 123, - Description: "", + IncidentKey: key, + Description: tmpl(n.conf.Templates.Description), Details: nil, } if eventType == pagerDutyEventTrigger { msg.Client = "Prometheus Alertmanager" msg.ClientURL = "" } + if err != nil { + return err + } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return err } - resp, err := ctxhttp.Post(ctx, http.DefaultClient, pd.conf.URL, contentTypeJSON, &buf) + resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf) if err != nil { return err } diff --git a/notify/notify.go b/notify/notify.go index 814b0c26..805d603c 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -38,6 +38,7 @@ const ( keyRepeatInterval keySendResolved keyGroupLabels + keyGroupKey keyNow ) @@ -53,6 +54,10 @@ func WithSendResolved(ctx context.Context, b bool) context.Context { return context.WithValue(ctx, keySendResolved, b) } +func WithGroupKey(ctx context.Context, fp model.Fingerprint) context.Context { + return context.WithValue(ctx, keyGroupKey, fp) +} + func WithGroupLabels(ctx context.Context, lset model.LabelSet) context.Context { return context.WithValue(ctx, keyGroupLabels, lset) } @@ -76,6 +81,11 @@ func SendResolved(ctx context.Context) (bool, bool) { return v, ok } +func GroupKey(ctx context.Context) (model.Fingerprint, bool) { + v, ok := ctx.Value(keyGroupKey).(model.Fingerprint) + return v, ok +} + func GroupLabels(ctx context.Context) (model.LabelSet, bool) { v, ok := ctx.Value(keyGroupLabels).(model.LabelSet) return v, ok diff --git a/route.go b/route.go index d53ccd92..f60fe07c 100644 --- a/route.go +++ b/route.go @@ -35,6 +35,8 @@ var DefaultRouteOpts = RouteOpts{ // A Route is a node that contains definitions of how to handle alerts. type Route struct { + parent *Route + // The configuration parameters for matches of this route. RouteOpts RouteOpts @@ -49,13 +51,12 @@ type Route struct { Routes []*Route } -func NewRoute(cr *config.Route, parent *RouteOpts) *Route { - if parent == nil { - parent = &DefaultRouteOpts - } - +func NewRoute(cr *config.Route, parent *Route) *Route { // Create default and overwrite with configured settings. - opts := *parent + opts := DefaultRouteOpts + if parent != nil { + opts = parent.RouteOpts + } if cr.SendTo != "" { opts.SendTo = cr.SendTo @@ -95,16 +96,18 @@ func NewRoute(cr *config.Route, parent *RouteOpts) *Route { } route := &Route{ + parent: parent, RouteOpts: opts, Matchers: matchers, Continue: cr.Continue, - Routes: NewRoutes(cr.Routes, &opts), } + route.Routes = NewRoutes(cr.Routes, route) + return route } -func NewRoutes(croutes []*config.Route, parent *RouteOpts) []*Route { +func NewRoutes(croutes []*config.Route, parent *Route) []*Route { res := []*Route{} for _, cr := range croutes { res = append(res, NewRoute(cr, parent)) @@ -131,6 +134,8 @@ func (r *Route) Match(lset model.LabelSet) []*RouteOpts { } } + // If no child nodes were matches, the current node itself is + // a match. if len(all) == 0 { all = append(all, &r.RouteOpts) } @@ -138,6 +143,30 @@ func (r *Route) Match(lset model.LabelSet) []*RouteOpts { return all } +func (r *Route) SquashMatchers() types.Matchers { + var res types.Matchers + res = append(res, r.Matchers...) + + if r.parent == nil { + return res + } + + pm := r.parent.SquashMatchers() + res = append(pm, res...) + + return res +} + +func (r *Route) Fingerprint() model.Fingerprint { + lset := make(model.LabelSet, len(r.RouteOpts.GroupBy)) + + for ln := range r.RouteOpts.GroupBy { + lset[ln] = "" + } + + return r.SquashMatchers().Fingerprint() ^ lset.Fingerprint() +} + type RouteOpts struct { // The identifier of the associated notification configuration SendTo string diff --git a/route_test.go b/route_test.go index e61cfad4..d3fad08a 100644 --- a/route_test.go +++ b/route_test.go @@ -74,7 +74,7 @@ routes: } var ( def = DefaultRouteOpts - tree = NewRoute(&ctree, &def) + tree = NewRoute(&ctree, nil) ) lset := func(labels ...string) map[model.LabelName]struct{} { s := map[model.LabelName]struct{}{} diff --git a/template/default.tmpl b/template/default.tmpl index 5293a43b..f4e24988 100644 --- a/template/default.tmpl +++ b/template/default.tmpl @@ -20,6 +20,8 @@ my text my text {{ end }} +{{ define "pagerduty.default.description" }}{{ .GroupLabels }} [{{ len .Alerts }} instance{{ if gt (len .Alerts) 1 }}s{{ end }}]{{ end }} + {{ define "email.default.header" }}From: "Prometheus Alertmanager" <{{ .From }}> To: {{ .To }} Date: {{ .Date }} @@ -47,7 +49,7 @@ table.outer { max-width: 700px; background: #fff; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; - color: #202020; + color: #202020 !important; } td { padding: 0; @@ -91,7 +93,7 @@ ul { } .labels.column { max-width: 295px; - color: #fff; + color: #fff !important; background: #4f4f4f; } .annotations.column { diff --git a/types/match.go b/types/match.go index 0c48ae38..c3867459 100644 --- a/types/match.go +++ b/types/match.go @@ -91,3 +91,13 @@ func (ms Matchers) Match(lset model.LabelSet) bool { } return true } + +func (ms Matchers) Fingerprint() model.Fingerprint { + lset := make(model.LabelSet, 3*len(ms)) + + for _, m := range ms { + lset[model.LabelName(fmt.Sprintf("%s-%s-%s", m.Name, m.Value, m.isRegex))] = "" + } + + return lset.Fingerprint() +}