diff --git a/notification/notification.go b/notification/notification.go index 21c513d88..38574e332 100644 --- a/notification/notification.go +++ b/notification/notification.go @@ -48,6 +48,8 @@ type NotificationReq struct { Summary string // Longer alert description. May contain text/template-style interpolations. Description string + // A reference to the runbook for the alert. + Runbook string // Labels associated with this alert notification, including alert name. Labels clientmodel.LabelSet // Current value of alert @@ -147,6 +149,7 @@ func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error { alerts = append(alerts, map[string]interface{}{ "summary": req.Summary, "description": req.Description, + "runbook": req.Runbook, "labels": req.Labels, "payload": map[string]interface{}{ "value": req.Value, diff --git a/notification/notification_test.go b/notification/notification_test.go index f188c806d..59cc4a00b 100644 --- a/notification/notification_test.go +++ b/notification/notification_test.go @@ -43,6 +43,7 @@ type testNotificationScenario struct { description string summary string message string + runbook string } func (s *testNotificationScenario) test(i int, t *testing.T) { @@ -63,6 +64,7 @@ func (s *testNotificationScenario) test(i int, t *testing.T) { { Summary: s.summary, Description: s.description, + Runbook: s.runbook, Labels: clientmodel.LabelSet{ clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"), }, @@ -85,7 +87,8 @@ func TestNotificationHandler(t *testing.T) { // Correct message. summary: "Summary", description: "Description", - message: `[{"description":"Description","labels":{"instance":"testinstance"},"payload":{"activeSince":"0001-01-01T00:00:00Z","alertingRule":"Test rule string","generatorURL":"prometheus_url","value":"0.3333333333333333"},"summary":"Summary"}]`, + runbook: "Runbook", + message: `[{"description":"Description","labels":{"instance":"testinstance"},"payload":{"activeSince":"0001-01-01T00:00:00Z","alertingRule":"Test rule string","generatorURL":"prometheus_url","value":"0.3333333333333333"},"runbook":"Runbook","summary":"Summary"}]`, }, } diff --git a/promql/ast.go b/promql/ast.go index 5ec7f2356..c8bdc68c2 100644 --- a/promql/ast.go +++ b/promql/ast.go @@ -62,6 +62,7 @@ type AlertStmt struct { Labels clientmodel.LabelSet Summary string Description string + Runbook string } // EvalStmt holds an expression and information on the range it should diff --git a/promql/lex.go b/promql/lex.go index 646116cac..d8db61239 100644 --- a/promql/lex.go +++ b/promql/lex.go @@ -144,6 +144,7 @@ const ( itemFor itemWith itemSummary + itemRunbook itemDescription itemKeepCommon itemOffset @@ -174,6 +175,7 @@ var key = map[string]itemType{ "for": itemFor, "with": itemWith, "summary": itemSummary, + "runbook": itemRunbook, "description": itemDescription, "offset": itemOffset, "by": itemBy, diff --git a/promql/lex_test.go b/promql/lex_test.go index da8b93a47..92855fd7b 100644 --- a/promql/lex_test.go +++ b/promql/lex_test.go @@ -241,6 +241,9 @@ var tests = []struct { }, { input: "summary", expected: []item{{itemSummary, 0, "summary"}}, + }, { + input: "runbook", + expected: []item{{itemRunbook, 0, "runbook"}}, }, { input: "offset", expected: []item{{itemOffset, 0, "offset"}}, diff --git a/promql/parse.go b/promql/parse.go index 19a4cc193..237670323 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -379,11 +379,45 @@ func (p *parser) alertStmt() *AlertStmt { lset = p.labelSet() } - p.expect(itemSummary, ctx) - sum := trimOne(p.expect(itemString, ctx).val) + var ( + hasSum, hasDesc, hasRunbook bool + sum, desc, runbook string + ) +Loop: + for { + switch p.next().typ { + case itemSummary: + if hasSum { + p.errorf("summary must not be defined twice") + } + hasSum = true + sum = trimOne(p.expect(itemString, ctx).val) - p.expect(itemDescription, ctx) - desc := trimOne(p.expect(itemString, ctx).val) + case itemDescription: + if hasDesc { + p.errorf("description must not be defined twice") + } + hasDesc = true + desc = trimOne(p.expect(itemString, ctx).val) + + case itemRunbook: + if hasRunbook { + p.errorf("runbook must not be defined twice") + } + hasRunbook = true + runbook = trimOne(p.expect(itemString, ctx).val) + + default: + p.backup() + break Loop + } + } + if sum == "" { + p.errorf("alert summary missing") + } + if desc == "" { + p.errorf("alert description missing") + } return &AlertStmt{ Name: name.val, @@ -392,6 +426,7 @@ func (p *parser) alertStmt() *AlertStmt { Labels: lset, Summary: sum, Description: desc, + Runbook: runbook, } } diff --git a/promql/parse_test.go b/promql/parse_test.go index 8721d0858..b46dc4a1c 100644 --- a/promql/parse_test.go +++ b/promql/parse_test.go @@ -1032,9 +1032,10 @@ var testStatement = []struct { foo = bar{label1="value1"} - ALERT BazAlert IF foo > 10 WITH {} - SUMMARY "Baz" + ALERT BazAlert IF foo > 10 DESCRIPTION "BazAlert" + RUNBOOK "http://my.url" + SUMMARY "Baz" `, expected: Statements{ &RecordStmt{ @@ -1100,6 +1101,7 @@ var testStatement = []struct { Labels: clientmodel.LabelSet{}, Summary: "Baz", Description: "BazAlert", + Runbook: "http://my.url", }, }, }, { diff --git a/rules/alerting.go b/rules/alerting.go index 49655740d..ff3ac6492 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -112,6 +112,8 @@ type AlertingRule struct { summary string // More detailed alert description. description string + // A reference to a runbook for the alert. + runbook string // Protects the below. mutex sync.Mutex @@ -128,6 +130,7 @@ func NewAlertingRule( labels clientmodel.LabelSet, summary string, description string, + runbook string, ) *AlertingRule { return &AlertingRule{ name: name, @@ -136,6 +139,7 @@ func NewAlertingRule( labels: labels, summary: summary, description: description, + runbook: runbook, activeAlerts: map[clientmodel.Fingerprint]*Alert{}, } @@ -219,6 +223,7 @@ func (rule *AlertingRule) String() string { } s += fmt.Sprintf("\n\tSUMMARY %q", rule.summary) s += fmt.Sprintf("\n\tDESCRIPTION %q", rule.description) + s += fmt.Sprintf("\n\tRUNBOOK %q", rule.runbook) return s } @@ -240,6 +245,7 @@ func (rule *AlertingRule) HTMLSnippet(pathPrefix string) template.HTML { } s += fmt.Sprintf("\n SUMMARY %q", rule.summary) s += fmt.Sprintf("\n DESCRIPTION %q", rule.description) + s += fmt.Sprintf("\n RUNBOOK %q", rule.runbook) return template.HTML(s) } diff --git a/rules/manager.go b/rules/manager.go index 521fa16b0..befcaa2f7 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -207,6 +207,7 @@ func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmo notifications = append(notifications, ¬ification.NotificationReq{ Summary: expand(rule.summary), Description: expand(rule.description), + Runbook: rule.runbook, Labels: aa.Labels.Merge(clientmodel.LabelSet{ alertNameLabel: clientmodel.LabelValue(rule.Name()), }), @@ -316,7 +317,7 @@ func (m *Manager) loadRuleFiles(filenames ...string) error { for _, stmt := range stmts { switch r := stmt.(type) { case *promql.AlertStmt: - rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description) + rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description, r.Runbook) m.rules = append(m.rules, rule) case *promql.RecordStmt: rule := NewRecordingRule(r.Name, r.Expr, r.Labels) diff --git a/rules/rules_test.go b/rules/rules_test.go index c2e5c2f63..d29cc6088 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -181,7 +181,7 @@ func TestAlertingRule(t *testing.T) { alertLabels := clientmodel.LabelSet{ "severity": "critical", } - rule := NewAlertingRule("HttpRequestRateLow", expr, time.Minute, alertLabels, "summary", "description") + rule := NewAlertingRule("HttpRequestRateLow", expr, time.Minute, alertLabels, "summary", "description", "runbook") for i, expectedLines := range evalOutputs { evalTime := testStartTime.Add(testSampleInterval * time.Duration(i))