Fix race condition in dispatcher (#2208)

* fix dispatcher race condition

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* add test to check for race condition in dispatcher

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* return when dispatcher Stop has nil receiver

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* remove unneeded chec

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

dispatch/dispatch.go

@@ -111,9 +111,8 @@ func (d *Dispatcher) Run() {
 	d.mtx.Lock()
 	d.aggrGroups = map[*Route]map[model.Fingerprint]*aggrGroup{}
 	d.metrics.aggrGroups.Set(0)
-	d.mtx.Unlock()
-
 	d.ctx, d.cancel = context.WithCancel(context.Background())
+	d.mtx.Unlock()
 
 	d.run(d.alerts.Subscribe())
 	close(d.done)
@@ -255,11 +254,16 @@ func (d *Dispatcher) Groups(routeFilter func(*Route) bool, alertFilter func(*typ
 
 // Stop the dispatcher.
 func (d *Dispatcher) Stop() {
-	if d == nil || d.cancel == nil {
+	if d == nil {
+		return
+	}
+	d.mtx.Lock()
+	if d.cancel == nil {
 		return
 	}
 	d.cancel()
 	d.cancel = nil
+	d.mtx.Unlock()
 
 	<-d.done
 }

dispatch/dispatch_test.go

@@ -522,3 +522,18 @@ func newAlert(labels model.LabelSet) *types.Alert {
 		Timeout:   false,
 	}
 }
+
+func TestDispatcherRace(t *testing.T) {
+	logger := log.NewNopLogger()
+	marker := types.NewMarker(prometheus.NewRegistry())
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer alerts.Close()
+
+	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
+	dispatcher := NewDispatcher(alerts, nil, nil, marker, timeout, logger, NewDispatcherMetrics(prometheus.NewRegistry()))
+	go dispatcher.Run()
+	dispatcher.Stop()
+}