From 5393ec22cb8dccae49232dca2dce4531cfc3d633 Mon Sep 17 00:00:00 2001 From: David Leadbeater Date: Fri, 9 Oct 2020 12:53:20 +0100 Subject: [PATCH] promtool: Don't end alert tests early, in some failure situations If an alert test had a failing test, then any other alert test interval specified after that point would result in the test exiting early. This made debugging some tests more difficult than needed. Now only exit early for evaluation failures. Signed-off-by: David Leadbeater --- cmd/promtool/unittest.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index cbdb8931e..ed2aad260 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -234,6 +234,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou var errs []error for ts := mint; ts.Before(maxt); ts = ts.Add(evalInterval) { // Collects the alerts asked for unit testing. + var evalErrs []error suite.WithSamplesTill(ts, func(err error) { if err != nil { errs = append(errs, err) @@ -243,13 +244,16 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou g.Eval(suite.Context(), ts) for _, r := range g.Rules() { if r.LastError() != nil { - errs = append(errs, errors.Errorf(" rule: %s, time: %s, err: %v", + evalErrs = append(evalErrs, errors.Errorf(" rule: %s, time: %s, err: %v", r.Name(), ts.Sub(time.Unix(0, 0).UTC()), r.LastError())) } } } }) - if len(errs) > 0 { + errs = append(errs, evalErrs...) + // Only end testing at this point if errors occurred evaluating above, + // rather than any test failures already collected in errs. + if len(evalErrs) > 0 { return errs }