RepoMirrors
/
alertmanager
mirror of https://github.com/prometheus/alertmanager
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add tests to retries (#1084) 

* Add tests to retry logic

* Extract retry logic from OpsGenie,VictorOps,PushOver
This commit is contained in:
Jose Donizetti 2017-11-11 12:45:11 -02:00 committed by stuart nelson
parent 20598bfd71
commit 7fbe63b94f
2 changed files with 200 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
notify/impl.go
View File

@ -20,7 +20,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"mime"
"mime/multipart"
"net"
@ -869,24 +868,18 @@ func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *OpsGenie) retry(statusCode int) (bool, error) {
// https://docs.opsgenie.com/docs/response#section-response-codes
// Response codes 429 (rate limiting) and 5xx are potentially recoverable
if resp.StatusCode/100 == 5 || resp.StatusCode == 429 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
} else if resp.StatusCode/100 == 4 {
return false, fmt.Errorf("unexpected status code %v", resp.StatusCode)
} else if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
level.Debug(n.logger).Log(
"msg", "Unexpected OpsGenie response",
"incident", key,
"url", apiURL,
"posted_message", msg,
"status", resp.Status,
"response_body", body,
)
return false, fmt.Errorf("unexpected status code %v", resp.StatusCode)
if statusCode/100 == 5 || statusCode == 429 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
}
@ -984,31 +977,16 @@ func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *VictorOps) retry(statusCode int) (bool, error) {
// Missing documentation therefore assuming only 5xx response codes are
// recoverable.
if resp.StatusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
var responseMessage victorOpsErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return false, fmt.Errorf("could not parse error response %q", body)
}
level.Debug(n.logger).Log(
"msg", "Unexpected VictorOps response",
"incident", key,
"url", apiURL,
"posted_message", msg,
"status", resp.Status,
"response_body", body,
)
return false, fmt.Errorf("error when posting alert: result %q, message %q",
responseMessage.Result, responseMessage.Message)
if statusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil
@ -1090,19 +1068,17 @@ func (n *Pushover) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
}
defer resp.Body.Close()
return n.retry(resp.StatusCode)
}
func (n *Pushover) retry(statusCode int) (bool, error) {
// Only documented behaviour is that 2xx response codes are successful and
// 4xx are unsuccessful, therefore assuming only 5xx are recoverable.
// https://pushover.net/api#response
if resp.StatusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
if resp.StatusCode/100 != 2 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, fmt.Errorf("unexpected status code %v (body: %s)", resp.StatusCode, string(body))
if statusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", statusCode)
} else if statusCode/100 != 2 {
return false, fmt.Errorf("unexpected status code %v", statusCode)
}
return false, nil

175
notify/impl_test.go Normal file
View File

@ -0,0 +1,175 @@
package notify
import (
"fmt"
"net/http"
"testing"
"github.com/stretchr/testify/require"
)
func TestWebhookRetry(t *testing.T) {
notifier := new(Webhook)
for statusCode, expected := range retryTests(defaultRetryCodes()) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func TestPagerDutyRetryV1(t *testing.T) {
notifier := new(PagerDuty)
retryCodes := append(defaultRetryCodes(), http.StatusForbidden)
for statusCode, expected := range retryTests(retryCodes) {
actual, _ := notifier.retryV1(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("retryv1 - error on status %d", statusCode))
}
}
func TestPagerDutyRetryV2(t *testing.T) {
notifier := new(PagerDuty)
retryCodes := append(defaultRetryCodes(), http.StatusTooManyRequests)
for statusCode, expected := range retryTests(retryCodes) {
actual, _ := notifier.retryV2(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("retryv2 - error on status %d", statusCode))
}
}
func TestSlackRetry(t *testing.T) {
notifier := new(Slack)
for statusCode, expected := range retryTests(defaultRetryCodes()) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func TestHipchatRetry(t *testing.T) {
notifier := new(Hipchat)
retryCodes := append(defaultRetryCodes(), http.StatusTooManyRequests)
for statusCode, expected := range retryTests(retryCodes) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func TestOpsGenieRetry(t *testing.T) {
notifier := new(OpsGenie)
retryCodes := append(defaultRetryCodes(), http.StatusTooManyRequests)
for statusCode, expected := range retryTests(retryCodes) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func TestVictorOpsRetry(t *testing.T) {
notifier := new(VictorOps)
for statusCode, expected := range retryTests(defaultRetryCodes()) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func TestPushoverRetry(t *testing.T) {
notifier := new(Pushover)
for statusCode, expected := range retryTests(defaultRetryCodes()) {
actual, _ := notifier.retry(statusCode)
require.Equal(t, expected, actual, fmt.Sprintf("error on status %d", statusCode))
}
}
func retryTests(retryCodes []int) map[int]bool {
tests := map[int]bool{
// 1xx
http.StatusContinue: false,
http.StatusSwitchingProtocols: false,
http.StatusProcessing: false,
// 2xx
http.StatusOK: false,
http.StatusCreated: false,
http.StatusAccepted: false,
http.StatusNonAuthoritativeInfo: false,
http.StatusNoContent: false,
http.StatusResetContent: false,
http.StatusPartialContent: false,
http.StatusMultiStatus: false,
http.StatusAlreadyReported: false,
http.StatusIMUsed: false,
// 3xx
http.StatusMultipleChoices: false,
http.StatusMovedPermanently: false,
http.StatusFound: false,
http.StatusSeeOther: false,
http.StatusNotModified: false,
http.StatusUseProxy: false,
http.StatusTemporaryRedirect: false,
http.StatusPermanentRedirect: false,
// 4xx
http.StatusBadRequest: false,
http.StatusUnauthorized: false,
http.StatusPaymentRequired: false,
http.StatusForbidden: false,
http.StatusNotFound: false,
http.StatusMethodNotAllowed: false,
http.StatusNotAcceptable: false,
http.StatusProxyAuthRequired: false,
http.StatusRequestTimeout: false,
http.StatusConflict: false,
http.StatusGone: false,
http.StatusLengthRequired: false,
http.StatusPreconditionFailed: false,
http.StatusRequestEntityTooLarge: false,
http.StatusRequestURITooLong: false,
http.StatusUnsupportedMediaType: false,
http.StatusRequestedRangeNotSatisfiable: false,
http.StatusExpectationFailed: false,
http.StatusTeapot: false,
http.StatusUnprocessableEntity: false,
http.StatusLocked: false,
http.StatusFailedDependency: false,
http.StatusUpgradeRequired: false,
http.StatusPreconditionRequired: false,
http.StatusTooManyRequests: false,
http.StatusRequestHeaderFieldsTooLarge: false,
http.StatusUnavailableForLegalReasons: false,
// 5xx
http.StatusInternalServerError: false,
http.StatusNotImplemented: false,
http.StatusBadGateway: false,
http.StatusServiceUnavailable: false,
http.StatusGatewayTimeout: false,
http.StatusHTTPVersionNotSupported: false,
http.StatusVariantAlsoNegotiates: false,
http.StatusInsufficientStorage: false,
http.StatusLoopDetected: false,
http.StatusNotExtended: false,
http.StatusNetworkAuthenticationRequired: false,
}
for _, statusCode := range retryCodes {
tests[statusCode] = true
}
return tests
}
func defaultRetryCodes() []int {
return []int{
http.StatusInternalServerError,
http.StatusNotImplemented,
http.StatusBadGateway,
http.StatusServiceUnavailable,
http.StatusGatewayTimeout,
http.StatusHTTPVersionNotSupported,
http.StatusVariantAlsoNegotiates,
http.StatusInsufficientStorage,
http.StatusLoopDetected,
http.StatusNotExtended,
http.StatusNetworkAuthenticationRequired,
}
}