Implement deterministic incident keys, complete PD integration

dispatch.go

@@ -150,9 +150,10 @@ func (d *Dispatcher) processAlert(alert *types.Alert, opts *RouteOpts) {
 // common set of routing options applies.
 // It emits notifications in the specified intervals.
 type aggrGroup struct {
-	labels model.LabelSet
-	opts   *RouteOpts
-	log    log.Logger
+	labels  model.LabelSet
+	opts    *RouteOpts
+	routeFP model.Fingerprint
+	log     log.Logger
 
 	ctx    context.Context
 	cancel func()
@@ -210,6 +211,7 @@ func (ag *aggrGroup) run(nf notifyFunc) {
 			ctx = notify.WithNow(ctx, now)
 
 			// Populate context with information needed along the pipeline.
+			ctx = notify.WithGroupKey(ctx, ag.labels.Fingerprint()^ag.routeFP)
 			ctx = notify.WithGroupLabels(ctx, ag.labels)
 			ctx = notify.WithDestination(ctx, ag.opts.SendTo)
 			ctx = notify.WithRepeatInterval(ctx, ag.opts.RepeatInterval)

notify/impl.go

@@ -231,13 +231,13 @@ type pagerDutyMessage struct {
 	ServiceKey  string            `json:"service_key"`
 	EventType   string            `json:"event_type"`
 	Description string            `json:"description"`
-	IncidentKey uint64            `json:"incident_key"`
+	IncidentKey model.Fingerprint `json:"incident_key"`
 	Client      string            `json:"client,omitempty"`
 	ClientURL   string            `json:"client_url,omitempty"`
 	Details     map[string]string `json:"details"`
 }
 
-func (pd *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
+func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
 	// http://developer.pagerduty.com/documentation/integration/events/trigger
 	alerts := types.Alerts(as...)
 
@@ -246,24 +246,56 @@ func (pd *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
 		eventType = pagerDutyEventResolve
 	}
 
+	key, ok := GroupKey(ctx)
+	if !ok {
+		return fmt.Errorf("group key missing")
+	}
+
+	log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty")
+
+	groupLabels, ok := GroupLabels(ctx)
+	if !ok {
+		log.Error("missing group labels")
+	}
+
+	data := struct {
+		Alerts      model.Alerts
+		GroupLabels model.LabelSet
+	}{
+		Alerts:      alerts,
+		GroupLabels: groupLabels,
+	}
+
+	var err error
+	tmpl := func(name string) (s string) {
+		if err != nil {
+			return
+		}
+		s, err = n.tmpl.ExecuteTextString(name, &data)
+		return s
+	}
+
 	msg := &pagerDutyMessage{
-		ServiceKey:  pd.conf.ServiceKey,
+		ServiceKey:  n.conf.ServiceKey,
 		EventType:   eventType,
-		IncidentKey: 123,
-		Description: "",
+		IncidentKey: key,
+		Description: tmpl(n.conf.Templates.Description),
 		Details:     nil,
 	}
 	if eventType == pagerDutyEventTrigger {
 		msg.Client = "Prometheus Alertmanager"
 		msg.ClientURL = ""
 	}
+	if err != nil {
+		return err
+	}
 
 	var buf bytes.Buffer
 	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
 		return err
 	}
 
-	resp, err := ctxhttp.Post(ctx, http.DefaultClient, pd.conf.URL, contentTypeJSON, &buf)
+	resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf)
 	if err != nil {
 		return err
 	}

notify/notify.go

@@ -38,6 +38,7 @@ const (
 	keyRepeatInterval
 	keySendResolved
 	keyGroupLabels
+	keyGroupKey
 	keyNow
 )
 
@@ -53,6 +54,10 @@ func WithSendResolved(ctx context.Context, b bool) context.Context {
 	return context.WithValue(ctx, keySendResolved, b)
 }
 
+func WithGroupKey(ctx context.Context, fp model.Fingerprint) context.Context {
+	return context.WithValue(ctx, keyGroupKey, fp)
+}
+
 func WithGroupLabels(ctx context.Context, lset model.LabelSet) context.Context {
 	return context.WithValue(ctx, keyGroupLabels, lset)
 }
@@ -76,6 +81,11 @@ func SendResolved(ctx context.Context) (bool, bool) {
 	return v, ok
 }
 
+func GroupKey(ctx context.Context) (model.Fingerprint, bool) {
+	v, ok := ctx.Value(keyGroupKey).(model.Fingerprint)
+	return v, ok
+}
+
 func GroupLabels(ctx context.Context) (model.LabelSet, bool) {
 	v, ok := ctx.Value(keyGroupLabels).(model.LabelSet)
 	return v, ok

route.go

@@ -35,6 +35,8 @@ var DefaultRouteOpts = RouteOpts{
 
 // A Route is a node that contains definitions of how to handle alerts.
 type Route struct {
+	parent *Route
+
 	// The configuration parameters for matches of this route.
 	RouteOpts RouteOpts
 
@@ -49,13 +51,12 @@ type Route struct {
 	Routes []*Route
 }
 
-func NewRoute(cr *config.Route, parent *RouteOpts) *Route {
-	if parent == nil {
-		parent = &DefaultRouteOpts
-	}
-
+func NewRoute(cr *config.Route, parent *Route) *Route {
 	// Create default and overwrite with configured settings.
-	opts := *parent
+	opts := DefaultRouteOpts
+	if parent != nil {
+		opts = parent.RouteOpts
+	}
 
 	if cr.SendTo != "" {
 		opts.SendTo = cr.SendTo
@@ -95,16 +96,18 @@ func NewRoute(cr *config.Route, parent *RouteOpts) *Route {
 	}
 
 	route := &Route{
+		parent:    parent,
 		RouteOpts: opts,
 		Matchers:  matchers,
 		Continue:  cr.Continue,
-		Routes:    NewRoutes(cr.Routes, &opts),
 	}
 
+	route.Routes = NewRoutes(cr.Routes, route)
+
 	return route
 }
 
-func NewRoutes(croutes []*config.Route, parent *RouteOpts) []*Route {
+func NewRoutes(croutes []*config.Route, parent *Route) []*Route {
 	res := []*Route{}
 	for _, cr := range croutes {
 		res = append(res, NewRoute(cr, parent))
@@ -131,6 +134,8 @@ func (r *Route) Match(lset model.LabelSet) []*RouteOpts {
 		}
 	}
 
+	// If no child nodes were matches, the current node itself is
+	// a match.
 	if len(all) == 0 {
 		all = append(all, &r.RouteOpts)
 	}
@@ -138,6 +143,30 @@ func (r *Route) Match(lset model.LabelSet) []*RouteOpts {
 	return all
 }
 
+func (r *Route) SquashMatchers() types.Matchers {
+	var res types.Matchers
+	res = append(res, r.Matchers...)
+
+	if r.parent == nil {
+		return res
+	}
+
+	pm := r.parent.SquashMatchers()
+	res = append(pm, res...)
+
+	return res
+}
+
+func (r *Route) Fingerprint() model.Fingerprint {
+	lset := make(model.LabelSet, len(r.RouteOpts.GroupBy))
+
+	for ln := range r.RouteOpts.GroupBy {
+		lset[ln] = ""
+	}
+
+	return r.SquashMatchers().Fingerprint() ^ lset.Fingerprint()
+}
+
 type RouteOpts struct {
 	// The identifier of the associated notification configuration
 	SendTo       string

route_test.go

@@ -74,7 +74,7 @@ routes:
 	}
 	var (
 		def  = DefaultRouteOpts
-		tree = NewRoute(&ctree, &def)
+		tree = NewRoute(&ctree, nil)
 	)
 	lset := func(labels ...string) map[model.LabelName]struct{} {
 		s := map[model.LabelName]struct{}{}

template/default.tmpl

@@ -20,6 +20,8 @@ my text
 my text
 {{ end }}
 
+{{ define "pagerduty.default.description" }}{{ .GroupLabels }} [{{ len .Alerts }} instance{{ if gt (len .Alerts) 1 }}s{{ end }}]{{ end }}
+
 {{ define "email.default.header" }}From: "Prometheus Alertmanager" <{{ .From }}>
 To: {{ .To }}
 Date: {{ .Date }}
@@ -47,7 +49,7 @@ table.outer {
   max-width: 700px;
   background: #fff;
   font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
-  color: #202020;
+  color: #202020 !important;
 }
 td {
   padding: 0;
@@ -91,7 +93,7 @@ ul {
 }
 .labels.column {
   max-width: 295px;
-  color: #fff;
+  color: #fff !important;
   background: #4f4f4f;
 }
 .annotations.column {

types/match.go

@@ -91,3 +91,13 @@ func (ms Matchers) Match(lset model.LabelSet) bool {
 	}
 	return true
 }
+
+func (ms Matchers) Fingerprint() model.Fingerprint {
+	lset := make(model.LabelSet, 3*len(ms))
+
+	for _, m := range ms {
+		lset[model.LabelName(fmt.Sprintf("%s-%s-%s", m.Name, m.Value, m.isRegex))] = ""
+	}
+
+	return lset.Fingerprint()
+}