RepoMirrors/alertmanager
1
0
Fork 0
mirror of https://github.com/prometheus/alertmanager synced 2025-04-11 03:31:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle OpsGenie's response to closing an already closed alert

Browse Source 
OpsGenie returns HTTP 400 to alert close requests if the alert has
already been closed. There is no need to try again if this happens.

When an error is returned from a notifiation service, an error is
logged, and the logged error is more meaningful if it includes a hint
about which notification service that caused a problem.

Defer the resp.body.Close() call in the OpsGenie Notify()
implementation.
This commit is contained in:
Anders Daljord Morken 2016-03-11 15:14:55 +01:00
parent 5d2277a6a3
commit 74c49588cf
1 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
notify/impl.go
View File

@ -198,7 +198,7 @@ func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error {
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
return fmt.Errorf("unexpected status code %v from %s", resp.StatusCode, w.URL)
}
return nil
@ -602,7 +602,7 @@ type OpsGenie struct {
tmpl *template.Template
}
// NewOpsGenieDuty returns a new OpsGenie notifier.
// NewOpsGenie returns a new OpsGenie notifier.
func NewOpsGenie(c *config.OpsGenieConfig, t *template.Template) *OpsGenie {
return &OpsGenie{conf: c, tmpl: t}
}
@ -628,6 +628,11 @@ type opsGenieCloseMessage struct {
*opsGenieMessage `json:",inline"`
}
type opsGenieErrorResponse struct {
Code int `json:"code"`
Error string `json:"error"`
}
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
@ -684,11 +689,25 @@ func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
if err != nil {
return err
}
resp.Body.Close()
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
if resp.StatusCode == 400 && alerts.Status() == model.AlertResolved {
body, _ := ioutil.ReadAll(resp.Body)
log.With("incident", key).Debugf("unexpected OpsGenie response %s: %s", resp.Status, body)
var responseMessage opsGenieErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return fmt.Errorf("could not parse error response %q", body)
}
const alreadyClosedError = 5
if responseMessage.Code == alreadyClosedError {
return nil
}
return fmt.Errorf("error when closing alert: code %d, error %q",
responseMessage.Code, responseMessage.Error)
} else if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
log.With("incident", key).Debugf("unexpected OpsGenie response from %s (POSTed %s), %s: %s",
apiURL, msg, resp.Status, body)
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil