Use a fixed now time through notify pipeline

dispatch.go

@@ -199,11 +199,17 @@ func (ag *aggrGroup) run(nf notifyFunc) {
 
 	for {
 		select {
-		case <-ag.next.C:
+		case now := <-ag.next.C:
 			// Give the notifcations time until the next flush to
 			// finish before terminating them.
 			ctx, _ := context.WithTimeout(ag.ctx, ag.opts.GroupInterval)
 
+			// The now time we retrieve from the ticker is the only reliable
+			// point of time reference for the subsequent notification pipeline.
+			// Calculating the current time directly is prone to avoid flaky behavior,
+			// which usually only becomes apparent in tests.
+			ctx = context.WithValue(ctx, notify.NotifyTime, now)
+
 			// Populate context with the destination name and group identifier.
 			ctx = context.WithValue(ctx, notify.NotifyName, ag.opts.SendTo)
 			ctx = context.WithValue(ctx, notify.NotifyGroup, ag.String())

notify/notify.go

@@ -21,6 +21,7 @@ const (
 	NotifyRepeatInterval
 	NotifySendResolved
 	NotifyGroup
+	NotifyTime
 )
 
 type Notifier interface {
@@ -78,6 +79,11 @@ func (n *DedupingNotifier) Notify(ctx context.Context, alerts ...*types.Alert) e
 		return fmt.Errorf("send resolved missing")
 	}
 
+	now, ok := ctx.Value(NotifyTime).(time.Time)
+	if !ok {
+		return fmt.Errorf("now time missing")
+	}
+
 	var fps []model.Fingerprint
 	for _, a := range alerts {
 		fps = append(fps, a.Fingerprint())
@@ -88,8 +94,6 @@ func (n *DedupingNotifier) Notify(ctx context.Context, alerts ...*types.Alert) e
 		return err
 	}
 
-	now := time.Now()
-
 	var (
 		doResend    bool
 		resendQueue []*types.Alert

notify/notify_test.go

@@ -37,11 +37,12 @@ func TestDedupingNotifier(t *testing.T) {
 		deduper  = NewDedupingNotifier(notifies, record)
 		ctx      = context.Background()
 	)
+	now := time.Now()
+
 	ctx = context.WithValue(ctx, NotifyName, "name")
 	ctx = context.WithValue(ctx, NotifyRepeatInterval, time.Duration(100*time.Minute))
 	ctx = context.WithValue(ctx, NotifySendResolved, true)
-
-	now := time.Now()
+	ctx = context.WithValue(ctx, NotifyTime, now)
 
 	alerts := []*types.Alert{
 		{