diff --git a/config/notifiers.go b/config/notifiers.go index 98eeddb4..a00a9247 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -455,16 +455,17 @@ type OpsGenieConfig struct { HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` - APIKey Secret `yaml:"api_key,omitempty" json:"api_key,omitempty"` - APIURL *URL `yaml:"api_url,omitempty" json:"api_url,omitempty"` - Message string `yaml:"message,omitempty" json:"message,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Details map[string]string `yaml:"details,omitempty" json:"details,omitempty"` - Responders []OpsGenieConfigResponder `yaml:"responders,omitempty" json:"responders,omitempty"` - Tags string `yaml:"tags,omitempty" json:"tags,omitempty"` - Note string `yaml:"note,omitempty" json:"note,omitempty"` - Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` + APIKey Secret `yaml:"api_key,omitempty" json:"api_key,omitempty"` + APIURL *URL `yaml:"api_url,omitempty" json:"api_url,omitempty"` + Message string `yaml:"message,omitempty" json:"message,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Details map[string]string `yaml:"details,omitempty" json:"details,omitempty"` + Responders []OpsGenieConfigResponder `yaml:"responders,omitempty" json:"responders,omitempty"` + Tags string `yaml:"tags,omitempty" json:"tags,omitempty"` + Note string `yaml:"note,omitempty" json:"note,omitempty"` + Priority string `yaml:"priority,omitempty" json:"priority,omitempty"` + UpdateAlerts bool `yaml:"update_alerts,omitempty" json:"update_alerts,omitempty"` } const opsgenieValidTypesRe = `^(team|user|escalation|schedule)$` diff --git a/docs/configuration.md b/docs/configuration.md index ec372308..f8677c7d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -843,6 +843,10 @@ responders: # Priority level of alert. Possible values are P1, P2, P3, P4, and P5. [ priority: ] +# Whether or not to update message and description of the alert in OpsGenie if it already exists +# By default, the alert is never updated in OpsGenie, the new message only appears in activity log. +[ update_alerts: | default = false ] + # The HTTP client's configuration. [ http_config: | default = global.http_config ] ``` diff --git a/notify/opsgenie/opsgenie.go b/notify/opsgenie/opsgenie.go index 190ce275..4a64ed65 100644 --- a/notify/opsgenie/opsgenie.go +++ b/notify/opsgenie/opsgenie.go @@ -80,20 +80,33 @@ type opsGenieCloseMessage struct { Source string `json:"source"` } +type opsGenieUpdateMessageMessage struct { + Message string `json:"message,omitempty"` +} + +type opsGenieUpdateDescriptionMessage struct { + Description string `json:"description,omitempty"` +} + // Notify implements the Notifier interface. func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { - req, retry, err := n.createRequest(ctx, as...) + requests, retry, err := n.createRequests(ctx, as...) if err != nil { return retry, err } - resp, err := n.client.Do(req) - if err != nil { - return true, err + for _, req := range requests { + resp, err := n.client.Do(req) + if err != nil { + return true, err + } + shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body) + notify.Drain(resp) + if err != nil { + return shouldRetry, err + } } - defer notify.Drain(resp) - - return n.retrier.Check(resp.StatusCode, resp.Body) + return true, nil } // Like Split but filter out empty strings. @@ -109,7 +122,7 @@ func safeSplit(s string, sep string) []string { } // Create requests for a list of alerts. -func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http.Request, bool, error) { +func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) { key, err := notify.ExtractGroupKey(ctx) if err != nil { return nil, false, err @@ -130,26 +143,37 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http details[k] = tmpl(v) } + requests := []*http.Request{} + var ( - msg interface{} - apiURL = n.conf.APIURL.Copy() alias = key.Hash() alerts = types.Alerts(as...) ) switch alerts.Status() { case model.AlertResolved: - apiURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias) - q := apiURL.Query() + resolvedEndpointURL := n.conf.APIURL.Copy() + resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias) + q := resolvedEndpointURL.Query() q.Set("identifierType", "alias") - apiURL.RawQuery = q.Encode() - msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)} + resolvedEndpointURL.RawQuery = q.Encode() + var msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)} + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(msg); err != nil { + return nil, false, err + } + req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf) + if err != nil { + return nil, true, err + } + requests = append(requests, req.WithContext(ctx)) default: message, truncated := notify.Truncate(tmpl(n.conf.Message), 130) if truncated { level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", message, "alert", key) } - apiURL.Path += "v2/alerts" + createEndpointURL := n.conf.APIURL.Copy() + createEndpointURL.Path += "v2/alerts" var responders []opsGenieCreateMessageResponder for _, r := range n.conf.Responders { @@ -169,7 +193,7 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http responders = append(responders, responder) } - msg = &opsGenieCreateMessage{ + var msg = &opsGenieCreateMessage{ Alias: alias, Message: message, Description: tmpl(n.conf.Description), @@ -180,6 +204,54 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http Note: tmpl(n.conf.Note), Priority: tmpl(n.conf.Priority), } + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(msg); err != nil { + return nil, false, err + } + req, err := http.NewRequest("POST", createEndpointURL.String(), &buf) + if err != nil { + return nil, true, err + } + requests = append(requests, req.WithContext(ctx)) + + if n.conf.UpdateAlerts { + updateMessageEndpointUrl := n.conf.APIURL.Copy() + updateMessageEndpointUrl.Path += fmt.Sprintf("v2/alerts/%s/message", alias) + q := updateMessageEndpointUrl.Query() + q.Set("identifierType", "alias") + updateMessageEndpointUrl.RawQuery = q.Encode() + updateMsgMsg := &opsGenieUpdateMessageMessage{ + Message: msg.Message, + } + var updateMessageBuf bytes.Buffer + if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil { + return nil, false, err + } + req, err := http.NewRequest("PUT", updateMessageEndpointUrl.String(), &updateMessageBuf) + if err != nil { + return nil, true, err + } + requests = append(requests, req) + + updateDescriptionEndpointURL := n.conf.APIURL.Copy() + updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias) + q = updateDescriptionEndpointURL.Query() + q.Set("identifierType", "alias") + updateDescriptionEndpointURL.RawQuery = q.Encode() + updateDescMsg := &opsGenieUpdateDescriptionMessage{ + Description: msg.Description, + } + + var updateDescriptionBuf bytes.Buffer + if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil { + return nil, false, err + } + req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf) + if err != nil { + return nil, true, err + } + requests = append(requests, req.WithContext(ctx)) + } } apiKey := tmpl(string(n.conf.APIKey)) @@ -188,16 +260,10 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http return nil, false, errors.Wrap(err, "templating error") } - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(msg); err != nil { - return nil, false, err + for _, req := range requests { + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey)) } - req, err := http.NewRequest("POST", apiURL.String(), &buf) - if err != nil { - return nil, true, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey)) - return req.WithContext(ctx), true, nil + return requests, true, nil } diff --git a/notify/opsgenie/opsgenie_test.go b/notify/opsgenie/opsgenie_test.go index b2a96b12..543fc357 100644 --- a/notify/opsgenie/opsgenie_test.go +++ b/notify/opsgenie/opsgenie_test.go @@ -167,12 +167,13 @@ func TestOpsGenie(t *testing.T) { }, } - req, retry, err := notifier.createRequest(ctx, alert1) + req, retry, err := notifier.createRequests(ctx, alert1) require.NoError(t, err) + require.Len(t, req, 1) require.Equal(t, true, retry) - require.Equal(t, expectedURL, req.URL) - require.Equal(t, "GenieKey http://am", req.Header.Get("Authorization")) - require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req)) + require.Equal(t, expectedURL, req[0].URL) + require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization")) + require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0])) // Fully defined alert. alert2 := &types.Alert{ @@ -193,20 +194,69 @@ func TestOpsGenie(t *testing.T) { EndsAt: time.Now().Add(time.Hour), }, } - req, retry, err = notifier.createRequest(ctx, alert2) + req, retry, err = notifier.createRequests(ctx, alert2) require.NoError(t, err) require.Equal(t, true, retry) - require.Equal(t, tc.expectedBody, readBody(t, req)) + require.Len(t, req, 1) + require.Equal(t, tc.expectedBody, readBody(t, req[0])) // Broken API Key Template. tc.cfg.APIKey = "{{ kaput " - _, _, err = notifier.createRequest(ctx, alert2) + _, _, err = notifier.createRequests(ctx, alert2) require.Error(t, err) require.Equal(t, err.Error(), "templating error: template: :1: function \"kaput\" not defined") }) } } +func TestOpsGenieWithUpdate(t *testing.T) { + u, err := url.Parse("https://test-opsgenie-url") + require.NoError(t, err) + tmpl := test.CreateTmpl(t) + ctx := context.Background() + ctx = notify.WithGroupKey(ctx, "1") + opsGenieConfigWithUpdate := config.OpsGenieConfig{ + Message: `{{ .CommonLabels.Message }}`, + Description: `{{ .CommonLabels.Description }}`, + UpdateAlerts: true, + APIKey: "test-api-key", + APIURL: &config.URL{URL: u}, + HTTPConfig: &commoncfg.HTTPClientConfig{}, + } + notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, log.NewNopLogger()) + alert := &types.Alert{ + Alert: model.Alert{ + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + Labels: model.LabelSet{ + "Message": "new message", + "Description": "new description", + }, + }, + } + require.NoError(t, err) + requests, retry, err := notifierWithUpdate.createRequests(ctx, alert) + require.NoError(t, err) + require.True(t, retry) + require.Len(t, requests, 3) + + body0 := readBody(t, requests[0]) + body1 := readBody(t, requests[1]) + body2 := readBody(t, requests[2]) + key, _ := notify.ExtractGroupKey(ctx) + alias := key.Hash() + + require.Equal(t, requests[0].URL.String(), "https://test-opsgenie-url/v2/alerts") + require.NotEmpty(t, body0) + + require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias)) + require.Equal(t, body1, `{"message":"new message"} +`) + require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias)) + require.Equal(t, body2, `{"description":"new description"} +`) +} + func readBody(t *testing.T, r *http.Request) string { t.Helper() body, err := ioutil.ReadAll(r.Body)