Split acceptance testing into multiple files

This commit is contained in:
Fabian Reinartz 2015-09-30 16:13:00 +02:00
parent b45dd027bc
commit 4401bb1b82
5 changed files with 323 additions and 311 deletions

View File

@ -8,60 +8,50 @@ import (
"net/http"
"os"
"os/exec"
"reflect"
"sync"
"testing"
"time"
"github.com/prometheus/common/model"
"github.com/prometheus/alertmanager/types"
)
type E2ETest struct {
type AcceptanceTest struct {
*testing.T
opts *E2ETestOpts
opts *AcceptanceOpts
ams []*Alertmanager
collectors []*collector
input map[float64][]*types.Alert
expected map[interval][]*types.Alert
collectors []*Collector
}
type E2ETestOpts struct {
baseTime time.Time
timeFactor float64
tolerance float64
type AcceptanceOpts struct {
baseTime time.Time
Tolerance time.Duration
conf string
Config string
}
func (opts *E2ETestOpts) expandTime(rel float64) time.Time {
func (opts *AcceptanceOpts) expandTime(rel float64) time.Time {
return opts.baseTime.Add(time.Duration(rel * float64(time.Second)))
}
func (opts *E2ETestOpts) relativeTime(act time.Time) float64 {
func (opts *AcceptanceOpts) relativeTime(act time.Time) float64 {
return float64(act.Sub(opts.baseTime)) / float64(time.Second)
}
func NewE2ETest(t *testing.T, opts *E2ETestOpts) *E2ETest {
test := &E2ETest{
func NewAcceptanceTest(t *testing.T, opts *AcceptanceOpts) *AcceptanceTest {
test := &AcceptanceTest{
T: t,
opts: opts,
input: map[float64][]*types.Alert{},
expected: map[interval][]*types.Alert{},
}
opts.baseTime = time.Now()
return test
}
// alertmanager returns a new structure that allows starting an instance
// Alertmanager returns a new structure that allows starting an instance
// of Alertmanager on a random port.
func (t *E2ETest) alertmanager() *Alertmanager {
func (t *AcceptanceTest) Alertmanager() *Alertmanager {
am := &Alertmanager{
t: t.T,
opts: t.opts,
@ -74,12 +64,12 @@ func (t *E2ETest) alertmanager() *Alertmanager {
}
am.confFile = cf
if _, err := cf.WriteString(t.opts.conf); err != nil {
if _, err := cf.WriteString(t.opts.Config); err != nil {
t.Fatal(err)
}
am.url = fmt.Sprintf("http://localhost:%d", 9091)
am.cmd = exec.Command("../alertmanager", "-config.file", cf.Name(), "-log.level=debug")
am.cmd = exec.Command("../../alertmanager", "-config.file", cf.Name(), "-log.level=debug")
var outb, errb bytes.Buffer
am.cmd.Stdout = &outb
@ -90,13 +80,13 @@ func (t *E2ETest) alertmanager() *Alertmanager {
return am
}
func (t *E2ETest) collector(name string) *collector {
co := &collector{
func (t *AcceptanceTest) Collector(name string) *Collector {
co := &Collector{
t: t.T,
name: name,
opts: t.opts,
collected: map[float64][]*types.Alert{},
exepected: map[interval][]*types.Alert{},
exepected: map[Interval][]*types.Alert{},
}
t.collectors = append(t.collectors, co)
@ -105,7 +95,7 @@ func (t *E2ETest) collector(name string) *collector {
// Run starts all Alertmanagers and runs queries against them. It then checks
// whether all expected notifications have arrived at the expected destination.
func (t *E2ETest) Run() {
func (t *AcceptanceTest) Run() {
for _, am := range t.ams {
am.start()
defer am.kill()
@ -142,7 +132,7 @@ type Alertmanager struct {
t *testing.T
url string
cmd *exec.Cmd
opts *E2ETestOpts
opts *AcceptanceOpts
confFile *os.File
@ -151,7 +141,7 @@ type Alertmanager struct {
// push declares alerts that are to be pushed to the Alertmanager
// server at a relative point in time.
func (am *Alertmanager) push(at float64, alerts ...*testAlert) {
func (am *Alertmanager) Push(at float64, alerts ...*TestAlert) {
var nas []*types.Alert
for _, a := range alerts {
nas = append(nas, a.nativeAlert(am.opts))
@ -204,237 +194,3 @@ func (am *Alertmanager) kill() {
am.cmd.Process.Kill()
os.RemoveAll(am.confFile.Name())
}
// collector gathers alerts received by a notification destination
// and verifies whether all arrived and within the correct time boundaries.
type collector struct {
t *testing.T
name string
opts *E2ETestOpts
collected map[float64][]*types.Alert
exepected map[interval][]*types.Alert
}
func (c *collector) String() string {
return c.name
}
// latest returns the latest relative point in time where a notification is
// expected.
func (c *collector) latest() float64 {
var latest float64
for iv := range c.exepected {
if iv.end > latest {
latest = iv.end
}
}
return latest
}
// want declares that the collector expects to receive the given alerts
// within the given time boundaries.
func (c *collector) want(iv interval, alerts ...*testAlert) {
var nas []*types.Alert
for _, a := range alerts {
nas = append(nas, a.nativeAlert(c.opts))
}
c.exepected[iv] = append(c.exepected[iv], nas...)
}
// add the given alerts to the collected alerts.
func (c *collector) add(alerts ...*types.Alert) {
arrival := c.opts.relativeTime(time.Now())
c.collected[arrival] = append(c.collected[arrival], alerts...)
}
func (c *collector) check() string {
report := fmt.Sprintf("\ncollector %q:\n\n", c)
for iv, expected := range c.exepected {
report += fmt.Sprintf("interval %v\n", iv)
for _, exp := range expected {
var found *types.Alert
report += fmt.Sprintf("- %v ", exp)
for at, got := range c.collected {
if !iv.contains(at) {
continue
}
for _, a := range got {
if equalAlerts(exp, a, c.opts) {
found = a
break
}
}
if found != nil {
break
}
}
if found != nil {
report += fmt.Sprintf("✓\n")
} else {
c.t.Fail()
report += fmt.Sprintf("✗\n")
}
}
}
// Detect unexpected notifications.
var totalExp, totalAct int
for _, exp := range c.exepected {
totalExp += len(exp)
}
for _, act := range c.collected {
totalAct += len(act)
}
if totalExp != totalAct {
c.t.Fail()
report += fmt.Sprintf("\nExpected total of %d alerts, got %d", totalExp, totalAct)
}
if c.t.Failed() {
report += "\nreceived:\n"
for at, col := range c.collected {
for _, a := range col {
report += fmt.Sprintf("- %v @ %v\n", a.String(), at)
}
}
}
return report
}
func equalAlerts(a, b *types.Alert, opts *E2ETestOpts) bool {
if !reflect.DeepEqual(a.Labels, b.Labels) {
return false
}
if !reflect.DeepEqual(a.Annotations, b.Annotations) {
return false
}
if !equalTime(a.StartsAt, b.StartsAt, opts) {
return false
}
if !equalTime(a.EndsAt, b.EndsAt, opts) {
return false
}
return true
}
func equalTime(a, b time.Time, opts *E2ETestOpts) bool {
if a.IsZero() != b.IsZero() {
return false
}
tol := time.Duration(float64(time.Second) * opts.tolerance)
diff := a.Sub(b)
if diff < 0 {
diff = -diff
}
return diff <= tol
}
type testAlert struct {
labels model.LabelSet
annotations types.Annotations
startsAt, endsAt float64
}
// at is a convenience method to allow for declarative syntax of e2e
// test definitions.
func at(ts float64) float64 {
return ts
}
type interval struct {
start, end float64
}
func (iv interval) String() string {
return fmt.Sprintf("[%v,%v]", iv.start, iv.end)
}
func (iv interval) contains(f float64) bool {
return f >= iv.start && f <= iv.end
}
// between is a convenience constructor for an interval for declarative syntax
// of e2e test definitions.
func between(start, end float64) interval {
return interval{start: start, end: end}
}
// alert creates a new alert declaration with the given key/value pairs
// as identifying labels.
func alert(keyval ...interface{}) *testAlert {
if len(keyval)%2 == 1 {
panic("bad key/values")
}
a := &testAlert{
labels: model.LabelSet{},
annotations: types.Annotations{},
}
for i := 0; i < len(keyval); i += 2 {
ln := model.LabelName(keyval[i].(string))
lv := model.LabelValue(keyval[i+1].(string))
a.labels[ln] = lv
}
return a
}
// nativeAlert converts the declared test alert into a full alert based
// on the given paramters.
func (a *testAlert) nativeAlert(opts *E2ETestOpts) *types.Alert {
na := &types.Alert{
Labels: a.labels,
Annotations: a.annotations,
}
if a.startsAt > 0 {
na.StartsAt = opts.expandTime(a.startsAt)
}
if a.endsAt > 0 {
na.EndsAt = opts.expandTime(a.endsAt)
}
return na
}
// annotate the alert with the given key/value pairs.
func (a *testAlert) annotate(keyval ...interface{}) *testAlert {
if len(keyval)%2 == 1 {
panic("bad key/values")
}
for i := 0; i < len(keyval); i += 2 {
ln := model.LabelName(keyval[i].(string))
lv := keyval[i+1].(string)
a.annotations[ln] = lv
}
return a
}
// active declares the relative activity time for this alert. It
// must be a single starting value or two values where the second value
// declares the resolved time.
func (a *testAlert) active(tss ...float64) *testAlert {
if len(tss) > 2 || len(tss) == 0 {
panic("only one or two timestamps allowed")
}
if len(tss) == 2 {
a.endsAt = tss[1]
}
a.startsAt = tss[0]
return a
}

View File

@ -0,0 +1,43 @@
package test
import (
"testing"
"time"
. "github.com/prometheus/alertmanager/test"
)
var somethingConfig = `
routes:
- send_to: "default"
group_wait: 1s
group_interval: 1s
notification_configs:
- name: "default"
send_resolved: true
webhook_configs:
- url: 'http://localhost:8088'
`
func TestSomething(t *testing.T) {
at := NewAcceptanceTest(t, &AcceptanceOpts{
Tolerance: 150 * time.Millisecond,
Config: somethingConfig,
})
am := at.Alertmanager()
co := at.Collector("webhook")
go NewWebhook(":8088", co).Run()
am.Push(At(1), Alert("alertname", "test").Active(1))
am.Push(At(3.5), Alert("alertname", "test").Active(1, 3))
co.Want(Between(2, 2.5), Alert("alertname", "test").Active(1))
co.Want(Between(3, 3.5), Alert("alertname", "test").Active(1))
co.Want(Between(3.5, 4.5), Alert("alertname", "test").Active(1, 3))
at.Run()
}

114
test/collector.go Normal file
View File

@ -0,0 +1,114 @@
package test
import (
"fmt"
"testing"
"time"
"github.com/prometheus/alertmanager/types"
)
// Collector gathers alerts received by a notification destination
// and verifies whether all arrived and within the correct time boundaries.
type Collector struct {
t *testing.T
name string
opts *AcceptanceOpts
collected map[float64][]*types.Alert
exepected map[Interval][]*types.Alert
}
func (c *Collector) String() string {
return c.name
}
// latest returns the latest relative point in time where a notification is
// expected.
func (c *Collector) latest() float64 {
var latest float64
for iv := range c.exepected {
if iv.end > latest {
latest = iv.end
}
}
return latest
}
// want declares that the Collector expects to receive the given alerts
// within the given time boundaries.
func (c *Collector) Want(iv Interval, alerts ...*TestAlert) {
var nas []*types.Alert
for _, a := range alerts {
nas = append(nas, a.nativeAlert(c.opts))
}
c.exepected[iv] = append(c.exepected[iv], nas...)
}
// add the given alerts to the collected alerts.
func (c *Collector) add(alerts ...*types.Alert) {
arrival := c.opts.relativeTime(time.Now())
c.collected[arrival] = append(c.collected[arrival], alerts...)
}
func (c *Collector) check() string {
report := fmt.Sprintf("\nCollector %q:\n\n", c)
for iv, expected := range c.exepected {
report += fmt.Sprintf("interval %v\n", iv)
for _, exp := range expected {
var found *types.Alert
report += fmt.Sprintf("- %v ", exp)
for at, got := range c.collected {
if !iv.contains(at) {
continue
}
for _, a := range got {
if equalAlerts(exp, a, c.opts) {
found = a
break
}
}
if found != nil {
break
}
}
if found != nil {
report += fmt.Sprintf("✓\n")
} else {
c.t.Fail()
report += fmt.Sprintf("✗\n")
}
}
}
// Detect unexpected notifications.
var totalExp, totalAct int
for _, exp := range c.exepected {
totalExp += len(exp)
}
for _, act := range c.collected {
totalAct += len(act)
}
if totalExp != totalAct {
c.t.Fail()
report += fmt.Sprintf("\nExpected total of %d alerts, got %d", totalExp, totalAct)
}
if c.t.Failed() {
report += "\nreceived:\n"
for at, col := range c.collected {
for _, a := range col {
report += fmt.Sprintf("- %v @ %v\n", a.String(), at)
}
}
}
return report
}

View File

@ -1,40 +0,0 @@
package test
import (
"testing"
)
var somethingConfig = `
routes:
- send_to: "default"
group_wait: 1s
group_interval: 1s
notification_configs:
- name: "default"
send_resolved: true
webhook_configs:
- url: 'http://localhost:8088'
`
func TestSomething(T *testing.T) {
t := NewE2ETest(T, &E2ETestOpts{
tolerance: 0.2,
conf: somethingConfig,
})
am := t.alertmanager()
co := t.collector("webhook")
go runMockWebhook(":8088", co)
am.push(at(1), alert("alertname", "test").active(1))
am.push(at(3.5), alert("alertname", "test").active(1, 3))
co.want(between(2, 2.5), alert("alertname", "test").active(1))
co.want(between(3, 3.5), alert("alertname", "test").active(1))
co.want(between(3.5, 4.5), alert("alertname", "test").active(1, 3))
t.Run()
}

View File

@ -2,22 +2,161 @@ package test
import (
"encoding/json"
"fmt"
"net/http"
"reflect"
"time"
"github.com/prometheus/common/model"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/types"
)
type mockWebhook struct {
collector *collector
type TestAlert struct {
labels model.LabelSet
annotations types.Annotations
startsAt, endsAt float64
}
func runMockWebhook(addr string, c *collector) {
http.ListenAndServe(addr, &mockWebhook{
// At is a convenience method to allow for declarative syntax of Acceptance
// test definitions.
func At(ts float64) float64 {
return ts
}
type Interval struct {
start, end float64
}
func (iv Interval) String() string {
return fmt.Sprintf("[%v,%v]", iv.start, iv.end)
}
func (iv Interval) contains(f float64) bool {
return f >= iv.start && f <= iv.end
}
// Between is a convenience constructor for an interval for declarative syntax
// of Acceptance test definitions.
func Between(start, end float64) Interval {
return Interval{start: start, end: end}
}
// alert creates a new alert declaration with the given key/value pairs
// as identifying labels.
func Alert(keyval ...interface{}) *TestAlert {
if len(keyval)%2 == 1 {
panic("bad key/values")
}
a := &TestAlert{
labels: model.LabelSet{},
annotations: types.Annotations{},
}
for i := 0; i < len(keyval); i += 2 {
ln := model.LabelName(keyval[i].(string))
lv := model.LabelValue(keyval[i+1].(string))
a.labels[ln] = lv
}
return a
}
// nativeAlert converts the declared test alert into a full alert based
// on the given paramters.
func (a *TestAlert) nativeAlert(opts *AcceptanceOpts) *types.Alert {
na := &types.Alert{
Labels: a.labels,
Annotations: a.annotations,
}
if a.startsAt > 0 {
na.StartsAt = opts.expandTime(a.startsAt)
}
if a.endsAt > 0 {
na.EndsAt = opts.expandTime(a.endsAt)
}
return na
}
// Annotate the alert with the given key/value pairs.
func (a *TestAlert) Annotate(keyval ...interface{}) *TestAlert {
if len(keyval)%2 == 1 {
panic("bad key/values")
}
for i := 0; i < len(keyval); i += 2 {
ln := model.LabelName(keyval[i].(string))
lv := keyval[i+1].(string)
a.annotations[ln] = lv
}
return a
}
// Active declares the relative activity time for this alert. It
// must be a single starting value or two values where the second value
// declares the resolved time.
func (a *TestAlert) Active(tss ...float64) *TestAlert {
if len(tss) > 2 || len(tss) == 0 {
panic("only one or two timestamps allowed")
}
if len(tss) == 2 {
a.endsAt = tss[1]
}
a.startsAt = tss[0]
return a
}
func equalAlerts(a, b *types.Alert, opts *AcceptanceOpts) bool {
if !reflect.DeepEqual(a.Labels, b.Labels) {
return false
}
if !reflect.DeepEqual(a.Annotations, b.Annotations) {
return false
}
if !equalTime(a.StartsAt, b.StartsAt, opts) {
return false
}
if !equalTime(a.EndsAt, b.EndsAt, opts) {
return false
}
return true
}
func equalTime(a, b time.Time, opts *AcceptanceOpts) bool {
if a.IsZero() != b.IsZero() {
return false
}
diff := a.Sub(b)
if diff < 0 {
diff = -diff
}
return diff <= opts.Tolerance
}
type MockWebhook struct {
collector *Collector
addr string
}
func NewWebhook(addr string, c *Collector) *MockWebhook {
return &MockWebhook{
addr: addr,
collector: c,
})
}
}
func (ws *mockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) {
func (ws *MockWebhook) Run() {
http.ListenAndServe(ws.addr, ws)
}
func (ws *MockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) {
dec := json.NewDecoder(req.Body)
defer req.Body.Close()