Implement retrying notifier

Retrying notifier is added to the end of the pipeline where it retries
sending out the final notifications until the context times out.
Exponential backoff is used.

main.go

@@ -65,10 +65,14 @@ func main() {
 	routedNotifier := notify.NewRoutedNotifier(func(confs []*config.NotificationConfig) map[string]notify.Notifier {
 		res := notify.Build(confs)
 		for name, n := range res {
-			res[name] = &notify.LogNotifier{
+			n = &notify.RetryNotifier{
+				Notifier: n,
+			}
+			n = &notify.LogNotifier{
 				Log:      log.With("notifier", fmt.Sprintf("%T", n)),
 				Notifier: n,
 			}
+			res[name] = n
 		}
 		return res
 	})

notify/notify.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cenkalti/backoff"
 	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/model"
 	"golang.org/x/net/context"
@@ -100,6 +101,36 @@ func (ns Notifiers) Notify(ctx context.Context, alerts ...*types.Alert) error {
 	return nil
 }
 
+type RetryNotifier struct {
+	Notifier Notifier
+}
+
+func (n *RetryNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error {
+	var (
+		i    = 0
+		b    = backoff.NewExponentialBackOff()
+		tick = backoff.NewTicker(b)
+	)
+	defer tick.Stop()
+
+	for {
+		i++
+
+		select {
+		case <-tick.C:
+			if err := n.Notifier.Notify(ctx, alerts...); err != nil {
+				log.Warnf("notify attempt %d failed: %s", i, err)
+			} else {
+				return nil
+			}
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+
+	return nil
+}
+
 type DedupingNotifier struct {
 	notifies provider.Notifies
 	notifier Notifier

notify/notify_test.go

@@ -27,7 +27,7 @@ func (n *recordNotifier) Notify(ctx context.Context, as ...*types.Alert) error {
 type failNotifier struct{}
 
 func (n *failNotifier) Notify(ctx context.Context, as ...*types.Alert) error {
-	return fmt.Errorf("")
+	return fmt.Errorf("some error")
 }
 
 func TestDedupingNotifier(t *testing.T) {