Merge pull request #843 from prometheus/fabxc/runbook

promql: add runbook to alert statement.

notification/notification.go

@@ -48,6 +48,8 @@ type NotificationReq struct {
 	Summary string
 	// Longer alert description. May contain text/template-style interpolations.
 	Description string
+	// A reference to the runbook for the alert.
+	Runbook string
 	// Labels associated with this alert notification, including alert name.
 	Labels clientmodel.LabelSet
 	// Current value of alert
@@ -147,6 +149,7 @@ func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error {
 		alerts = append(alerts, map[string]interface{}{
 			"summary":     req.Summary,
 			"description": req.Description,
+			"runbook":     req.Runbook,
 			"labels":      req.Labels,
 			"payload": map[string]interface{}{
 				"value":        req.Value,

notification/notification_test.go

@@ -43,6 +43,7 @@ type testNotificationScenario struct {
 	description string
 	summary     string
 	message     string
+	runbook     string
 }
 
 func (s *testNotificationScenario) test(i int, t *testing.T) {
@@ -63,6 +64,7 @@ func (s *testNotificationScenario) test(i int, t *testing.T) {
 		{
 			Summary:     s.summary,
 			Description: s.description,
+			Runbook:     s.runbook,
 			Labels: clientmodel.LabelSet{
 				clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"),
 			},
@@ -85,7 +87,8 @@ func TestNotificationHandler(t *testing.T) {
 			// Correct message.
 			summary:     "Summary",
 			description: "Description",
-			message:     `[{"description":"Description","labels":{"instance":"testinstance"},"payload":{"activeSince":"0001-01-01T00:00:00Z","alertingRule":"Test rule string","generatorURL":"prometheus_url","value":"0.3333333333333333"},"summary":"Summary"}]`,
+			runbook:     "Runbook",
+			message:     `[{"description":"Description","labels":{"instance":"testinstance"},"payload":{"activeSince":"0001-01-01T00:00:00Z","alertingRule":"Test rule string","generatorURL":"prometheus_url","value":"0.3333333333333333"},"runbook":"Runbook","summary":"Summary"}]`,
 		},
 	}
 

promql/ast.go

@@ -62,6 +62,7 @@ type AlertStmt struct {
 	Labels      clientmodel.LabelSet
 	Summary     string
 	Description string
+	Runbook     string
 }
 
 // EvalStmt holds an expression and information on the range it should

promql/lex.go

@@ -144,6 +144,7 @@ const (
 	itemFor
 	itemWith
 	itemSummary
+	itemRunbook
 	itemDescription
 	itemKeepCommon
 	itemOffset
@@ -174,6 +175,7 @@ var key = map[string]itemType{
 	"for":           itemFor,
 	"with":          itemWith,
 	"summary":       itemSummary,
+	"runbook":       itemRunbook,
 	"description":   itemDescription,
 	"offset":        itemOffset,
 	"by":            itemBy,

promql/lex_test.go

@@ -241,6 +241,9 @@ var tests = []struct {
 	}, {
 		input:    "summary",
 		expected: []item{{itemSummary, 0, "summary"}},
+	}, {
+		input:    "runbook",
+		expected: []item{{itemRunbook, 0, "runbook"}},
 	}, {
 		input:    "offset",
 		expected: []item{{itemOffset, 0, "offset"}},

promql/parse.go

@@ -379,11 +379,45 @@ func (p *parser) alertStmt() *AlertStmt {
 		lset = p.labelSet()
 	}
 
-	p.expect(itemSummary, ctx)
+	var (
-	sum := trimOne(p.expect(itemString, ctx).val)
+		hasSum, hasDesc, hasRunbook bool
+		sum, desc, runbook          string
+	)
+Loop:
+	for {
+		switch p.next().typ {
+		case itemSummary:
+			if hasSum {
+				p.errorf("summary must not be defined twice")
+			}
+			hasSum = true
+			sum = trimOne(p.expect(itemString, ctx).val)
 
-	p.expect(itemDescription, ctx)
+		case itemDescription:
-	desc := trimOne(p.expect(itemString, ctx).val)
+			if hasDesc {
+				p.errorf("description must not be defined twice")
+			}
+			hasDesc = true
+			desc = trimOne(p.expect(itemString, ctx).val)
+
+		case itemRunbook:
+			if hasRunbook {
+				p.errorf("runbook must not be defined twice")
+			}
+			hasRunbook = true
+			runbook = trimOne(p.expect(itemString, ctx).val)
+
+		default:
+			p.backup()
+			break Loop
+		}
+	}
+	if sum == "" {
+		p.errorf("alert summary missing")
+	}
+	if desc == "" {
+		p.errorf("alert description missing")
+	}
 
 	return &AlertStmt{
 		Name:        name.val,
@@ -392,6 +426,7 @@ func (p *parser) alertStmt() *AlertStmt {
 		Labels:      lset,
 		Summary:     sum,
 		Description: desc,
+		Runbook:     runbook,
 	}
 }
 

promql/parse_test.go

@@ -1032,9 +1032,10 @@ var testStatement = []struct {
 
 			foo = bar{label1="value1"}
 
-			ALERT BazAlert IF foo > 10 WITH {}
+			ALERT BazAlert IF foo > 10
-			  SUMMARY "Baz"
 			  DESCRIPTION "BazAlert"
+			  RUNBOOK     "http://my.url"
+			  SUMMARY     "Baz"
 		`,
 		expected: Statements{
 			&RecordStmt{
@@ -1100,6 +1101,7 @@ var testStatement = []struct {
 				Labels:      clientmodel.LabelSet{},
 				Summary:     "Baz",
 				Description: "BazAlert",
+				Runbook:     "http://my.url",
 			},
 		},
 	}, {

rules/alerting.go

@@ -112,6 +112,8 @@ type AlertingRule struct {
 	summary string
 	// More detailed alert description.
 	description string
+	// A reference to a runbook for the alert.
+	runbook string
 
 	// Protects the below.
 	mutex sync.Mutex
@@ -128,6 +130,7 @@ func NewAlertingRule(
 	labels clientmodel.LabelSet,
 	summary string,
 	description string,
+	runbook string,
 ) *AlertingRule {
 	return &AlertingRule{
 		name:         name,
@@ -136,6 +139,7 @@ func NewAlertingRule(
 		labels:       labels,
 		summary:      summary,
 		description:  description,
+		runbook:      runbook,
 
 		activeAlerts: map[clientmodel.Fingerprint]*Alert{},
 	}
@@ -219,6 +223,7 @@ func (rule *AlertingRule) String() string {
 	}
 	s += fmt.Sprintf("\n\tSUMMARY %q", rule.summary)
 	s += fmt.Sprintf("\n\tDESCRIPTION %q", rule.description)
+	s += fmt.Sprintf("\n\tRUNBOOK %q", rule.runbook)
 	return s
 }
 
@@ -240,6 +245,7 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML {
 	}
 	s += fmt.Sprintf("\n  SUMMARY %q", rule.summary)
 	s += fmt.Sprintf("\n  DESCRIPTION %q", rule.description)
+	s += fmt.Sprintf("\n  RUNBOOK %q", rule.runbook)
 	return template.HTML(s)
 }
 

rules/manager.go

@@ -207,6 +207,7 @@ func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmo
 		notifications = append(notifications, &notification.NotificationReq{
 			Summary:     expand(rule.summary),
 			Description: expand(rule.description),
+			Runbook:     rule.runbook,
 			Labels: aa.Labels.Merge(clientmodel.LabelSet{
 				alertNameLabel: clientmodel.LabelValue(rule.Name()),
 			}),
@@ -316,7 +317,7 @@ func (m *Manager) loadRuleFiles(filenames ...string) error {
 		for _, stmt := range stmts {
 			switch r := stmt.(type) {
 			case *promql.AlertStmt:
-				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description)
+				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description, r.Runbook)
 				m.rules = append(m.rules, rule)
 			case *promql.RecordStmt:
 				rule := NewRecordingRule(r.Name, r.Expr, r.Labels)

rules/rules_test.go

@@ -181,7 +181,7 @@ func TestAlertingRule(t *testing.T) {
 	alertLabels := clientmodel.LabelSet{
 		"severity": "critical",
 	}
-	rule := NewAlertingRule("HttpRequestRateLow", expr, time.Minute, alertLabels, "summary", "description")
+	rule := NewAlertingRule("HttpRequestRateLow", expr, time.Minute, alertLabels, "summary", "description", "runbook")
 
 	for i, expectedLines := range evalOutputs {
 		evalTime := testStartTime.Add(testSampleInterval * time.Duration(i))