RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-25 16:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enhanced visibility for promtool test rules with JSON colored formatting (#13342)

Browse Source 
* Added diff flag for unit test to improvise readability & debugging

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Removed blank spaces

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Fixed linting error

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Added cli flags to documentation

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Revert unrrelated linting fixes

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Fixed review suggestions

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Cleanup

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Updated flag description

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

* Updated flag description

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>

---------

Signed-off-by: Rewanth Tammana <22347290+rewanthtammana@users.noreply.github.com>
This commit is contained in:
Rewanth Tammana 2024-01-18 18:49:16 +04:00 committed by GitHub
parent 4ad9b6df2e
commit 102fd8cc88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 56 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/promtool/main.go
View File

@ -210,6 +210,7 @@ func main() {
"test-rule-file",
"The unit test file.",
).Required().ExistingFiles()
testRulesDiff := testRulesCmd.Flag("diff", "[Experimental] Print colored differential output between expected & received output.").Default("false").Bool()
defaultDBPath := "data/"
tsdbCmd := app.Command("tsdb", "Run tsdb commands.")
@ -375,6 +376,7 @@ func main() {
EnableNegativeOffset: true,
},
*testRulesRun,
*testRulesDiff,
*testRulesFiles...),
)

52
cmd/promtool/unittest.go
View File

@ -15,6 +15,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
@ -27,6 +28,7 @@ import (
"github.com/go-kit/log"
"github.com/grafana/regexp"
"github.com/nsf/jsondiff"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v2"
@ -40,7 +42,7 @@ import (
// RulesUnitTest does unit testing of rules based on the unit testing files provided.
// More info about the file format can be found in the docs.
func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files ...string) int {
func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
failed := false
var run *regexp.Regexp
@ -49,7 +51,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
}
for _, f := range files {
if errs := ruleUnitTest(f, queryOpts, run); errs != nil {
if errs := ruleUnitTest(f, queryOpts, run, diffFlag); errs != nil {
fmt.Fprintln(os.Stderr, " FAILED:")
for _, e := range errs {
fmt.Fprintln(os.Stderr, e.Error())
@ -67,7 +69,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, files .
return successExitCode
}
func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp) []error {
func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
fmt.Println("Unit Testing: ", filename)
b, err := os.ReadFile(filename)
@ -109,7 +111,7 @@ func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.
if t.Interval == 0 {
t.Interval = unitTestInp.EvaluationInterval
}
ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...)
ers := t.test(evalInterval, groupOrderMap, queryOpts, diffFlag, unitTestInp.RuleFiles...)
if ers != nil {
errs = append(errs, ers...)
}
@ -173,7 +175,7 @@ type testGroup struct {
}
// test performs the unit tests.
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error {
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) []error {
// Setup testing suite.
suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts)
if err != nil {
@ -345,8 +347,44 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
}
expString := indentLines(expAlerts.String(), " ")
gotString := indentLines(gotAlerts.String(), " ")
errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n exp:%v, \n got:%v",
testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
if diffFlag {
// If empty, populates an empty value
if gotAlerts.Len() == 0 {
gotAlerts = append(gotAlerts, labelAndAnnotation{
Labels: labels.Labels{},
Annotations: labels.Labels{},
})
}
// If empty, populates an empty value
if expAlerts.Len() == 0 {
expAlerts = append(expAlerts, labelAndAnnotation{
Labels: labels.Labels{},
Annotations: labels.Labels{},
})
}
diffOpts := jsondiff.DefaultConsoleOptions()
expAlertsJSON, err := json.Marshal(expAlerts)
if err != nil {
errs = append(errs, fmt.Errorf("error marshaling expected %s alert: [%s]", tg.TestGroupName, err.Error()))
continue
}
gotAlertsJSON, err := json.Marshal(gotAlerts)
if err != nil {
errs = append(errs, fmt.Errorf("error marshaling received %s alert: [%s]", tg.TestGroupName, err.Error()))
continue
}
res, diff := jsondiff.Compare(expAlertsJSON, gotAlertsJSON, &diffOpts)
if res != jsondiff.FullMatch {
errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n diff: %v",
testName, testcase.Alertname, testcase.EvalTime.String(), indentLines(diff, " ")))
}
} else {
errs = append(errs, fmt.Errorf("%s alertname: %s, time: %s, \n exp:%v, \n got:%v",
testName, testcase.Alertname, testcase.EvalTime.String(), expString, gotString))
}
}
}

4
cmd/promtool/unittest_test.go
View File

@ -125,7 +125,7 @@ func TestRulesUnitTest(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, nil, tt.args.files...); got != tt.want {
if got := RulesUnitTest(tt.queryOpts, nil, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
@ -178,7 +178,7 @@ func TestRulesUnitTestRun(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.queryOpts, tt.args.run, tt.args.files...); got != tt.want {
if got := RulesUnitTest(tt.queryOpts, tt.args.run, false, tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})

7
docs/command-line/promtool.md
View File

@ -450,9 +450,10 @@ Unit tests for rules.
###### Flags
| Flag | Description |
| --- | --- |
| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. |
| Flag | Description | Default |
| --- | --- | --- |
| <code class="text-nowrap">--run</code> | If set, will only run test groups whose names match the regular expression. Can be specified multiple times. | |
| <code class="text-nowrap">--diff</code> | [Experimental] Print colored differential output between expected & received output. | `false` |

1
go.mod
View File

@ -43,6 +43,7 @@ require (
github.com/miekg/dns v1.1.57
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
github.com/oklog/run v1.1.0
github.com/oklog/ulid v1.3.1
github.com/ovh/go-ovh v1.4.3

2
go.sum
View File

@ -579,6 +579,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM=
github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=