diff --git a/test/acceptance.go b/test/acceptance.go
index 4c06d9cf..d6f60ec7 100644
--- a/test/acceptance.go
+++ b/test/acceptance.go
@@ -32,10 +32,8 @@ type AcceptanceTest struct {
 
 // AcceptanceOpts defines configuration paramters for an acceptance test.
 type AcceptanceOpts struct {
-	baseTime  time.Time
 	Tolerance time.Duration
-
-	Config string
+	baseTime  time.Time
 }
 
 // expandTime returns the absolute time for the relative time
@@ -83,7 +81,7 @@ func (t *AcceptanceTest) Do(at float64, f func()) {
 
 // Alertmanager returns a new structure that allows starting an instance
 // of Alertmanager on a random port.
-func (t *AcceptanceTest) Alertmanager() *Alertmanager {
+func (t *AcceptanceTest) Alertmanager(conf string) *Alertmanager {
 	am := &Alertmanager{
 		t:    t,
 		opts: t.opts,
@@ -95,7 +93,7 @@ func (t *AcceptanceTest) Alertmanager() *Alertmanager {
 	}
 	am.confFile = cf
 
-	if _, err := cf.WriteString(t.opts.Config); err != nil {
+	if _, err := cf.WriteString(conf); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/test/acceptance/simple_test.go b/test/acceptance/simple_test.go
index f70a5384..41a5b7bf 100644
--- a/test/acceptance/simple_test.go
+++ b/test/acceptance/simple_test.go
@@ -1,13 +1,17 @@
 package test
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
 	. "github.com/prometheus/alertmanager/test"
 )
 
-var somethingConfig = `
+func TestSomething(t *testing.T) {
+	t.Parallel()
+
+	conf := `
 routes:
 - send_to: "default"
   group_wait:     1s
@@ -18,29 +22,25 @@ notification_configs:
   send_resolved: true
 
   webhook_configs:
-  - url: 'http://localhost:8088'
+  - url: 'http://%s'
 `
 
-func TestSomething(t *testing.T) {
-	t.Parallel()
-
 	// Create a new acceptance test that instantiates new Alertmanagers
 	// with the given configuration and verifies times with the given
 	// tollerance.
 	at := NewAcceptanceTest(t, &AcceptanceOpts{
 		Tolerance: 150 * time.Millisecond,
-		Config:    somethingConfig,
 	})
 
-	// Create a new Alertmanager process listening to a random port
-	am := at.Alertmanager()
 	// Create a collector to which alerts can be written and verified
 	// against a set of expected alert notifications.
 	co := at.Collector("webhook")
-
 	// Run something that satisfies the webhook interface to which the
 	// Alertmanager pushes as defined by its configuration.
-	go NewWebhook(":8088", co).Run()
+	wh := NewWebhook(co)
+
+	// Create a new Alertmanager process listening to a random port
+	am := at.Alertmanager(fmt.Sprintf(conf, wh.Address()))
 
 	// Declare pushes to be made to the Alertmanager at the given time.
 	// Times are provided in fractions of seconds.
@@ -57,7 +57,10 @@ func TestSomething(t *testing.T) {
 	at.Run()
 }
 
-var silenceConfig = `
+func TestSilencing(t *testing.T) {
+	t.Parallel()
+
+	conf := `
 routes:
 - send_to: "default"
   group_wait:     1s
@@ -68,21 +71,17 @@ notification_configs:
   send_resolved: true
 
   webhook_configs:
-  - url: 'http://localhost:8090'
+  - url: 'http://%s'
 `
 
-func TestSilencing(t *testing.T) {
-	t.Parallel()
-
 	at := NewAcceptanceTest(t, &AcceptanceOpts{
 		Tolerance: 150 * time.Millisecond,
-		Config:    silenceConfig,
 	})
 
-	am := at.Alertmanager()
 	co := at.Collector("webhook")
+	wh := NewWebhook(co)
 
-	go NewWebhook(":8090", co).Run()
+	am := at.Alertmanager(fmt.Sprintf(conf, wh.Address()))
 
 	// No repeat interval is configured. Thus, we receive an alert
 	// notification every second.
@@ -111,7 +110,10 @@ func TestSilencing(t *testing.T) {
 	at.Run()
 }
 
-var batchConfig = `
+func TestBatching(t *testing.T) {
+	t.Parallel()
+
+	conf := `
 routes:
 - send_to: "default"
   group_wait:     1s
@@ -123,21 +125,17 @@ notification_configs:
   repeat_interval: 5s
 
   webhook_configs:
-  - url: 'http://localhost:8089'
+  - url: 'http://%s'
 `
 
-func TestBatching(t *testing.T) {
-	t.Parallel()
-
 	at := NewAcceptanceTest(t, &AcceptanceOpts{
 		Tolerance: 150 * time.Millisecond,
-		Config:    batchConfig,
 	})
 
-	am := at.Alertmanager()
 	co := at.Collector("webhook")
+	wh := NewWebhook(co)
 
-	go NewWebhook(":8089", co).Run()
+	am := at.Alertmanager(fmt.Sprintf(conf, wh.Address()))
 
 	am.Push(At(1.1), Alert("alertname", "test1").Active(1))
 	am.Push(At(1.9), Alert("alertname", "test5").Active(1))
diff --git a/test/mock.go b/test/mock.go
index 738e50cd..1907f81d 100644
--- a/test/mock.go
+++ b/test/mock.go
@@ -3,6 +3,7 @@ package test
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"reflect"
 	"time"
@@ -203,18 +204,24 @@ func equalTime(a, b time.Time, opts *AcceptanceOpts) bool {
 
 type MockWebhook struct {
 	collector *Collector
-	addr      string
+	listener  net.Listener
 }
 
-func NewWebhook(addr string, c *Collector) *MockWebhook {
-	return &MockWebhook{
-		addr:      addr,
+func NewWebhook(c *Collector) *MockWebhook {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		// TODO(fabxc): if shutdown of mock destinations ever becomes a concern
+		// we want to shut them down after test completion. Then we might want to
+		// log the error properly, too.
+		panic(err)
+	}
+	wh := &MockWebhook{
+		listener:  l,
 		collector: c,
 	}
-}
+	go http.Serve(l, wh)
 
-func (ws *MockWebhook) Run() {
-	http.ListenAndServe(ws.addr, ws)
+	return wh
 }
 
 func (ws *MockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -228,3 +235,7 @@ func (ws *MockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 	ws.collector.add(v.Alerts...)
 }
+
+func (ws *MockWebhook) Address() string {
+	return ws.listener.Addr().String()
+}