Test restartability with persistence

dispatch.go

@@ -15,7 +15,7 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
-const ResolveTimeout = 30 * time.Second
+const ResolveTimeout = 5 * time.Minute
 
 // Dispatcher sorts incoming alerts into aggregation groups and
 // assigns the correct notifiers to each.
@@ -180,6 +180,10 @@ func newAggrGroup(ctx context.Context, labels model.LabelSet, opts *RouteOpts) *
 	}
 	ag.ctx, ag.cancel = context.WithCancel(ctx)
 
+	// Set an initial one-time wait before flushing
+	// the first batch of notifications.
+	ag.next = time.NewTimer(ag.opts.GroupWait)
+
 	return ag
 }
 
@@ -190,10 +194,6 @@ func (ag *aggrGroup) String() string {
 func (ag *aggrGroup) run(nf notifyFunc) {
 	ag.done = make(chan struct{})
 
-	// Set an initial one-time wait before flushing
-	// the first batch of notifications.
-	ag.next = time.NewTimer(ag.opts.GroupWait)
-
 	defer close(ag.done)
 	defer ag.next.Stop()
 

test/acceptance.go

@@ -99,6 +99,8 @@ func (t *AcceptanceTest) Alertmanager(conf string) *Alertmanager {
 	if err != nil {
 		t.Fatal(err)
 	}
+	am.dir = dir
+
 	cf, err := os.Create(filepath.Join(dir, "config.yml"))
 	if err != nil {
 		t.Fatal(err)
@@ -118,17 +120,6 @@ func (t *AcceptanceTest) Alertmanager(conf string) *Alertmanager {
 	}
 	am.client = client
 
-	am.cmd = exec.Command("../../alertmanager",
-		"-config.file", cf.Name(),
-		"-log.level", "debug",
-		"-web.listen-address", am.addr,
-		"-data.dir", dir,
-	)
-
-	var outb, errb bytes.Buffer
-	am.cmd.Stdout = &outb
-	am.cmd.Stderr = &errb
-
 	t.ams = append(t.ams, am)
 
 	return am
@@ -212,20 +203,42 @@ type Alertmanager struct {
 	client   alertmanager.Client
 	cmd      *exec.Cmd
 	confFile *os.File
+	dir      string
 }
 
 // Start the alertmanager and wait until it is ready to receive.
 func (am *Alertmanager) Start() {
+	cmd := exec.Command("../../alertmanager",
+		"-config.file", am.confFile.Name(),
+		"-log.level", "debug",
+		"-web.listen-address", am.addr,
+		"-data.dir", am.dir,
+	)
+
+	if am.cmd == nil {
+		var outb, errb bytes.Buffer
+		cmd.Stdout = &outb
+		cmd.Stderr = &errb
+	} else {
+		cmd.Stdout = am.cmd.Stdout
+		cmd.Stderr = am.cmd.Stderr
+	}
+	am.cmd = cmd
+
 	if err := am.cmd.Start(); err != nil {
 		am.t.Fatalf("Starting alertmanager failed: %s", err)
 	}
 
-	time.Sleep(100 * time.Millisecond)
+	time.Sleep(50 * time.Millisecond)
 }
 
 // kill the underlying Alertmanager process and remove intermediate data.
 func (am *Alertmanager) Terminate() {
 	syscall.Kill(am.cmd.Process.Pid, syscall.SIGTERM)
+
+	if err := am.cmd.Wait(); err != nil {
+		am.t.Fatal(err)
+	}
 }
 
 // Reload sends the reloading signal to the Alertmanager process.

test/acceptance/simple_test.go

@@ -45,6 +45,11 @@ notification_configs:
 	// Declare pushes to be made to the Alertmanager at the given time.
 	// Times are provided in fractions of seconds.
 	am.Push(At(1), Alert("alertname", "test").Active(1))
+
+	at.Do(At(1.2), func() {
+		am.Terminate()
+		am.Start()
+	})
 	am.Push(At(3.5), Alert("alertname", "test").Active(1, 3))
 
 	// Declare which alerts are expected to arrive at the collector within