From 4401bb1b821ef012444fb0f21943679d24f8bb86 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Wed, 30 Sep 2015 16:13:00 +0200 Subject: [PATCH] Split acceptance testing into multiple files --- test/acceptance.go | 286 +++------------------------------ test/acceptance/simple_test.go | 43 +++++ test/collector.go | 114 +++++++++++++ test/misc_test.go | 40 ----- test/mock.go | 151 ++++++++++++++++- 5 files changed, 323 insertions(+), 311 deletions(-) create mode 100644 test/acceptance/simple_test.go create mode 100644 test/collector.go delete mode 100644 test/misc_test.go diff --git a/test/acceptance.go b/test/acceptance.go index 1f741896..9f1cd29c 100644 --- a/test/acceptance.go +++ b/test/acceptance.go @@ -8,60 +8,50 @@ import ( "net/http" "os" "os/exec" - "reflect" "sync" "testing" "time" - "github.com/prometheus/common/model" - "github.com/prometheus/alertmanager/types" ) -type E2ETest struct { +type AcceptanceTest struct { *testing.T - opts *E2ETestOpts + opts *AcceptanceOpts ams []*Alertmanager - collectors []*collector - - input map[float64][]*types.Alert - expected map[interval][]*types.Alert + collectors []*Collector } -type E2ETestOpts struct { - baseTime time.Time - timeFactor float64 - tolerance float64 +type AcceptanceOpts struct { + baseTime time.Time + Tolerance time.Duration - conf string + Config string } -func (opts *E2ETestOpts) expandTime(rel float64) time.Time { +func (opts *AcceptanceOpts) expandTime(rel float64) time.Time { return opts.baseTime.Add(time.Duration(rel * float64(time.Second))) } -func (opts *E2ETestOpts) relativeTime(act time.Time) float64 { +func (opts *AcceptanceOpts) relativeTime(act time.Time) float64 { return float64(act.Sub(opts.baseTime)) / float64(time.Second) } -func NewE2ETest(t *testing.T, opts *E2ETestOpts) *E2ETest { - test := &E2ETest{ +func NewAcceptanceTest(t *testing.T, opts *AcceptanceOpts) *AcceptanceTest { + test := &AcceptanceTest{ T: t, opts: opts, - - input: map[float64][]*types.Alert{}, - expected: map[interval][]*types.Alert{}, } opts.baseTime = time.Now() return test } -// alertmanager returns a new structure that allows starting an instance +// Alertmanager returns a new structure that allows starting an instance // of Alertmanager on a random port. -func (t *E2ETest) alertmanager() *Alertmanager { +func (t *AcceptanceTest) Alertmanager() *Alertmanager { am := &Alertmanager{ t: t.T, opts: t.opts, @@ -74,12 +64,12 @@ func (t *E2ETest) alertmanager() *Alertmanager { } am.confFile = cf - if _, err := cf.WriteString(t.opts.conf); err != nil { + if _, err := cf.WriteString(t.opts.Config); err != nil { t.Fatal(err) } am.url = fmt.Sprintf("http://localhost:%d", 9091) - am.cmd = exec.Command("../alertmanager", "-config.file", cf.Name(), "-log.level=debug") + am.cmd = exec.Command("../../alertmanager", "-config.file", cf.Name(), "-log.level=debug") var outb, errb bytes.Buffer am.cmd.Stdout = &outb @@ -90,13 +80,13 @@ func (t *E2ETest) alertmanager() *Alertmanager { return am } -func (t *E2ETest) collector(name string) *collector { - co := &collector{ +func (t *AcceptanceTest) Collector(name string) *Collector { + co := &Collector{ t: t.T, name: name, opts: t.opts, collected: map[float64][]*types.Alert{}, - exepected: map[interval][]*types.Alert{}, + exepected: map[Interval][]*types.Alert{}, } t.collectors = append(t.collectors, co) @@ -105,7 +95,7 @@ func (t *E2ETest) collector(name string) *collector { // Run starts all Alertmanagers and runs queries against them. It then checks // whether all expected notifications have arrived at the expected destination. -func (t *E2ETest) Run() { +func (t *AcceptanceTest) Run() { for _, am := range t.ams { am.start() defer am.kill() @@ -142,7 +132,7 @@ type Alertmanager struct { t *testing.T url string cmd *exec.Cmd - opts *E2ETestOpts + opts *AcceptanceOpts confFile *os.File @@ -151,7 +141,7 @@ type Alertmanager struct { // push declares alerts that are to be pushed to the Alertmanager // server at a relative point in time. -func (am *Alertmanager) push(at float64, alerts ...*testAlert) { +func (am *Alertmanager) Push(at float64, alerts ...*TestAlert) { var nas []*types.Alert for _, a := range alerts { nas = append(nas, a.nativeAlert(am.opts)) @@ -204,237 +194,3 @@ func (am *Alertmanager) kill() { am.cmd.Process.Kill() os.RemoveAll(am.confFile.Name()) } - -// collector gathers alerts received by a notification destination -// and verifies whether all arrived and within the correct time boundaries. -type collector struct { - t *testing.T - name string - opts *E2ETestOpts - - collected map[float64][]*types.Alert - exepected map[interval][]*types.Alert -} - -func (c *collector) String() string { - return c.name -} - -// latest returns the latest relative point in time where a notification is -// expected. -func (c *collector) latest() float64 { - var latest float64 - for iv := range c.exepected { - if iv.end > latest { - latest = iv.end - } - } - return latest -} - -// want declares that the collector expects to receive the given alerts -// within the given time boundaries. -func (c *collector) want(iv interval, alerts ...*testAlert) { - var nas []*types.Alert - for _, a := range alerts { - nas = append(nas, a.nativeAlert(c.opts)) - } - - c.exepected[iv] = append(c.exepected[iv], nas...) -} - -// add the given alerts to the collected alerts. -func (c *collector) add(alerts ...*types.Alert) { - arrival := c.opts.relativeTime(time.Now()) - - c.collected[arrival] = append(c.collected[arrival], alerts...) -} - -func (c *collector) check() string { - report := fmt.Sprintf("\ncollector %q:\n\n", c) - - for iv, expected := range c.exepected { - report += fmt.Sprintf("interval %v\n", iv) - - for _, exp := range expected { - var found *types.Alert - report += fmt.Sprintf("- %v ", exp) - - for at, got := range c.collected { - if !iv.contains(at) { - continue - } - for _, a := range got { - if equalAlerts(exp, a, c.opts) { - found = a - break - } - } - if found != nil { - break - } - } - - if found != nil { - report += fmt.Sprintf("✓\n") - } else { - c.t.Fail() - report += fmt.Sprintf("✗\n") - } - } - } - - // Detect unexpected notifications. - var totalExp, totalAct int - for _, exp := range c.exepected { - totalExp += len(exp) - } - for _, act := range c.collected { - totalAct += len(act) - } - if totalExp != totalAct { - c.t.Fail() - report += fmt.Sprintf("\nExpected total of %d alerts, got %d", totalExp, totalAct) - } - - if c.t.Failed() { - report += "\nreceived:\n" - - for at, col := range c.collected { - for _, a := range col { - report += fmt.Sprintf("- %v @ %v\n", a.String(), at) - } - } - } - - return report -} - -func equalAlerts(a, b *types.Alert, opts *E2ETestOpts) bool { - if !reflect.DeepEqual(a.Labels, b.Labels) { - return false - } - if !reflect.DeepEqual(a.Annotations, b.Annotations) { - return false - } - - if !equalTime(a.StartsAt, b.StartsAt, opts) { - return false - } - if !equalTime(a.EndsAt, b.EndsAt, opts) { - return false - } - return true -} - -func equalTime(a, b time.Time, opts *E2ETestOpts) bool { - if a.IsZero() != b.IsZero() { - return false - } - - tol := time.Duration(float64(time.Second) * opts.tolerance) - diff := a.Sub(b) - - if diff < 0 { - diff = -diff - } - return diff <= tol -} - -type testAlert struct { - labels model.LabelSet - annotations types.Annotations - startsAt, endsAt float64 -} - -// at is a convenience method to allow for declarative syntax of e2e -// test definitions. -func at(ts float64) float64 { - return ts -} - -type interval struct { - start, end float64 -} - -func (iv interval) String() string { - return fmt.Sprintf("[%v,%v]", iv.start, iv.end) -} - -func (iv interval) contains(f float64) bool { - return f >= iv.start && f <= iv.end -} - -// between is a convenience constructor for an interval for declarative syntax -// of e2e test definitions. -func between(start, end float64) interval { - return interval{start: start, end: end} -} - -// alert creates a new alert declaration with the given key/value pairs -// as identifying labels. -func alert(keyval ...interface{}) *testAlert { - if len(keyval)%2 == 1 { - panic("bad key/values") - } - a := &testAlert{ - labels: model.LabelSet{}, - annotations: types.Annotations{}, - } - - for i := 0; i < len(keyval); i += 2 { - ln := model.LabelName(keyval[i].(string)) - lv := model.LabelValue(keyval[i+1].(string)) - - a.labels[ln] = lv - } - - return a -} - -// nativeAlert converts the declared test alert into a full alert based -// on the given paramters. -func (a *testAlert) nativeAlert(opts *E2ETestOpts) *types.Alert { - na := &types.Alert{ - Labels: a.labels, - Annotations: a.annotations, - } - if a.startsAt > 0 { - na.StartsAt = opts.expandTime(a.startsAt) - } - if a.endsAt > 0 { - na.EndsAt = opts.expandTime(a.endsAt) - } - return na -} - -// annotate the alert with the given key/value pairs. -func (a *testAlert) annotate(keyval ...interface{}) *testAlert { - if len(keyval)%2 == 1 { - panic("bad key/values") - } - - for i := 0; i < len(keyval); i += 2 { - ln := model.LabelName(keyval[i].(string)) - lv := keyval[i+1].(string) - - a.annotations[ln] = lv - } - - return a -} - -// active declares the relative activity time for this alert. It -// must be a single starting value or two values where the second value -// declares the resolved time. -func (a *testAlert) active(tss ...float64) *testAlert { - if len(tss) > 2 || len(tss) == 0 { - panic("only one or two timestamps allowed") - } - if len(tss) == 2 { - a.endsAt = tss[1] - } - a.startsAt = tss[0] - - return a -} diff --git a/test/acceptance/simple_test.go b/test/acceptance/simple_test.go new file mode 100644 index 00000000..c3dbb48b --- /dev/null +++ b/test/acceptance/simple_test.go @@ -0,0 +1,43 @@ +package test + +import ( + "testing" + "time" + + . "github.com/prometheus/alertmanager/test" +) + +var somethingConfig = ` +routes: +- send_to: "default" + group_wait: 1s + group_interval: 1s + +notification_configs: +- name: "default" + send_resolved: true + + webhook_configs: + - url: 'http://localhost:8088' +` + +func TestSomething(t *testing.T) { + at := NewAcceptanceTest(t, &AcceptanceOpts{ + Tolerance: 150 * time.Millisecond, + Config: somethingConfig, + }) + + am := at.Alertmanager() + co := at.Collector("webhook") + + go NewWebhook(":8088", co).Run() + + am.Push(At(1), Alert("alertname", "test").Active(1)) + am.Push(At(3.5), Alert("alertname", "test").Active(1, 3)) + + co.Want(Between(2, 2.5), Alert("alertname", "test").Active(1)) + co.Want(Between(3, 3.5), Alert("alertname", "test").Active(1)) + co.Want(Between(3.5, 4.5), Alert("alertname", "test").Active(1, 3)) + + at.Run() +} diff --git a/test/collector.go b/test/collector.go new file mode 100644 index 00000000..b4a8a763 --- /dev/null +++ b/test/collector.go @@ -0,0 +1,114 @@ +package test + +import ( + "fmt" + "testing" + "time" + + "github.com/prometheus/alertmanager/types" +) + +// Collector gathers alerts received by a notification destination +// and verifies whether all arrived and within the correct time boundaries. +type Collector struct { + t *testing.T + name string + opts *AcceptanceOpts + + collected map[float64][]*types.Alert + exepected map[Interval][]*types.Alert +} + +func (c *Collector) String() string { + return c.name +} + +// latest returns the latest relative point in time where a notification is +// expected. +func (c *Collector) latest() float64 { + var latest float64 + for iv := range c.exepected { + if iv.end > latest { + latest = iv.end + } + } + return latest +} + +// want declares that the Collector expects to receive the given alerts +// within the given time boundaries. +func (c *Collector) Want(iv Interval, alerts ...*TestAlert) { + var nas []*types.Alert + for _, a := range alerts { + nas = append(nas, a.nativeAlert(c.opts)) + } + + c.exepected[iv] = append(c.exepected[iv], nas...) +} + +// add the given alerts to the collected alerts. +func (c *Collector) add(alerts ...*types.Alert) { + arrival := c.opts.relativeTime(time.Now()) + + c.collected[arrival] = append(c.collected[arrival], alerts...) +} + +func (c *Collector) check() string { + report := fmt.Sprintf("\nCollector %q:\n\n", c) + + for iv, expected := range c.exepected { + report += fmt.Sprintf("interval %v\n", iv) + + for _, exp := range expected { + var found *types.Alert + report += fmt.Sprintf("- %v ", exp) + + for at, got := range c.collected { + if !iv.contains(at) { + continue + } + for _, a := range got { + if equalAlerts(exp, a, c.opts) { + found = a + break + } + } + if found != nil { + break + } + } + + if found != nil { + report += fmt.Sprintf("✓\n") + } else { + c.t.Fail() + report += fmt.Sprintf("✗\n") + } + } + } + + // Detect unexpected notifications. + var totalExp, totalAct int + for _, exp := range c.exepected { + totalExp += len(exp) + } + for _, act := range c.collected { + totalAct += len(act) + } + if totalExp != totalAct { + c.t.Fail() + report += fmt.Sprintf("\nExpected total of %d alerts, got %d", totalExp, totalAct) + } + + if c.t.Failed() { + report += "\nreceived:\n" + + for at, col := range c.collected { + for _, a := range col { + report += fmt.Sprintf("- %v @ %v\n", a.String(), at) + } + } + } + + return report +} diff --git a/test/misc_test.go b/test/misc_test.go deleted file mode 100644 index 82dc9d84..00000000 --- a/test/misc_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package test - -import ( - "testing" -) - -var somethingConfig = ` -routes: -- send_to: "default" - group_wait: 1s - group_interval: 1s - -notification_configs: -- name: "default" - send_resolved: true - - webhook_configs: - - url: 'http://localhost:8088' -` - -func TestSomething(T *testing.T) { - t := NewE2ETest(T, &E2ETestOpts{ - tolerance: 0.2, - conf: somethingConfig, - }) - - am := t.alertmanager() - co := t.collector("webhook") - - go runMockWebhook(":8088", co) - - am.push(at(1), alert("alertname", "test").active(1)) - am.push(at(3.5), alert("alertname", "test").active(1, 3)) - - co.want(between(2, 2.5), alert("alertname", "test").active(1)) - co.want(between(3, 3.5), alert("alertname", "test").active(1)) - co.want(between(3.5, 4.5), alert("alertname", "test").active(1, 3)) - - t.Run() -} diff --git a/test/mock.go b/test/mock.go index bc3ac63c..d0fd1fd4 100644 --- a/test/mock.go +++ b/test/mock.go @@ -2,22 +2,161 @@ package test import ( "encoding/json" + "fmt" "net/http" + "reflect" + "time" + + "github.com/prometheus/common/model" "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/types" ) -type mockWebhook struct { - collector *collector +type TestAlert struct { + labels model.LabelSet + annotations types.Annotations + startsAt, endsAt float64 } -func runMockWebhook(addr string, c *collector) { - http.ListenAndServe(addr, &mockWebhook{ +// At is a convenience method to allow for declarative syntax of Acceptance +// test definitions. +func At(ts float64) float64 { + return ts +} + +type Interval struct { + start, end float64 +} + +func (iv Interval) String() string { + return fmt.Sprintf("[%v,%v]", iv.start, iv.end) +} + +func (iv Interval) contains(f float64) bool { + return f >= iv.start && f <= iv.end +} + +// Between is a convenience constructor for an interval for declarative syntax +// of Acceptance test definitions. +func Between(start, end float64) Interval { + return Interval{start: start, end: end} +} + +// alert creates a new alert declaration with the given key/value pairs +// as identifying labels. +func Alert(keyval ...interface{}) *TestAlert { + if len(keyval)%2 == 1 { + panic("bad key/values") + } + a := &TestAlert{ + labels: model.LabelSet{}, + annotations: types.Annotations{}, + } + + for i := 0; i < len(keyval); i += 2 { + ln := model.LabelName(keyval[i].(string)) + lv := model.LabelValue(keyval[i+1].(string)) + + a.labels[ln] = lv + } + + return a +} + +// nativeAlert converts the declared test alert into a full alert based +// on the given paramters. +func (a *TestAlert) nativeAlert(opts *AcceptanceOpts) *types.Alert { + na := &types.Alert{ + Labels: a.labels, + Annotations: a.annotations, + } + if a.startsAt > 0 { + na.StartsAt = opts.expandTime(a.startsAt) + } + if a.endsAt > 0 { + na.EndsAt = opts.expandTime(a.endsAt) + } + return na +} + +// Annotate the alert with the given key/value pairs. +func (a *TestAlert) Annotate(keyval ...interface{}) *TestAlert { + if len(keyval)%2 == 1 { + panic("bad key/values") + } + + for i := 0; i < len(keyval); i += 2 { + ln := model.LabelName(keyval[i].(string)) + lv := keyval[i+1].(string) + + a.annotations[ln] = lv + } + + return a +} + +// Active declares the relative activity time for this alert. It +// must be a single starting value or two values where the second value +// declares the resolved time. +func (a *TestAlert) Active(tss ...float64) *TestAlert { + if len(tss) > 2 || len(tss) == 0 { + panic("only one or two timestamps allowed") + } + if len(tss) == 2 { + a.endsAt = tss[1] + } + a.startsAt = tss[0] + + return a +} + +func equalAlerts(a, b *types.Alert, opts *AcceptanceOpts) bool { + if !reflect.DeepEqual(a.Labels, b.Labels) { + return false + } + if !reflect.DeepEqual(a.Annotations, b.Annotations) { + return false + } + + if !equalTime(a.StartsAt, b.StartsAt, opts) { + return false + } + if !equalTime(a.EndsAt, b.EndsAt, opts) { + return false + } + return true +} + +func equalTime(a, b time.Time, opts *AcceptanceOpts) bool { + if a.IsZero() != b.IsZero() { + return false + } + + diff := a.Sub(b) + if diff < 0 { + diff = -diff + } + return diff <= opts.Tolerance +} + +type MockWebhook struct { + collector *Collector + addr string +} + +func NewWebhook(addr string, c *Collector) *MockWebhook { + return &MockWebhook{ + addr: addr, collector: c, - }) + } } -func (ws *mockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (ws *MockWebhook) Run() { + http.ListenAndServe(ws.addr, ws) +} + +func (ws *MockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) { dec := json.NewDecoder(req.Body) defer req.Body.Close()