diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index 61dcabdc6..5a7f21d3b 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -236,7 +236,7 @@ func main() { dumpPath := tsdbDumpCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String() dumpMinTime := tsdbDumpCmd.Flag("min-time", "Minimum timestamp to dump.").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() dumpMaxTime := tsdbDumpCmd.Flag("max-time", "Maximum timestamp to dump.").Default(strconv.FormatInt(math.MaxInt64, 10)).Int64() - dumpMatch := tsdbDumpCmd.Flag("match", "Series selector.").Default("{__name__=~'(?s:.*)'}").String() + dumpMatch := tsdbDumpCmd.Flag("match", "Series selector. Can be specified multiple times.").Default("{__name__=~'(?s:.*)'}").Strings() importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.") importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool() diff --git a/cmd/promtool/testdata/dump-test-1.prom b/cmd/promtool/testdata/dump-test-1.prom new file mode 100644 index 000000000..878cdecab --- /dev/null +++ b/cmd/promtool/testdata/dump-test-1.prom @@ -0,0 +1,15 @@ +{__name__="heavy_metric", foo="bar"} 5 0 +{__name__="heavy_metric", foo="bar"} 4 60000 +{__name__="heavy_metric", foo="bar"} 3 120000 +{__name__="heavy_metric", foo="bar"} 2 180000 +{__name__="heavy_metric", foo="bar"} 1 240000 +{__name__="heavy_metric", foo="foo"} 5 0 +{__name__="heavy_metric", foo="foo"} 4 60000 +{__name__="heavy_metric", foo="foo"} 3 120000 +{__name__="heavy_metric", foo="foo"} 2 180000 +{__name__="heavy_metric", foo="foo"} 1 240000 +{__name__="metric", baz="abc", foo="bar"} 1 0 +{__name__="metric", baz="abc", foo="bar"} 2 60000 +{__name__="metric", baz="abc", foo="bar"} 3 120000 +{__name__="metric", baz="abc", foo="bar"} 4 180000 +{__name__="metric", baz="abc", foo="bar"} 5 240000 diff --git a/cmd/promtool/testdata/dump-test-2.prom b/cmd/promtool/testdata/dump-test-2.prom new file mode 100644 index 000000000..4ac2ffa5a --- /dev/null +++ b/cmd/promtool/testdata/dump-test-2.prom @@ -0,0 +1,10 @@ +{__name__="heavy_metric", foo="foo"} 5 0 +{__name__="heavy_metric", foo="foo"} 4 60000 +{__name__="heavy_metric", foo="foo"} 3 120000 +{__name__="heavy_metric", foo="foo"} 2 180000 +{__name__="heavy_metric", foo="foo"} 1 240000 +{__name__="metric", baz="abc", foo="bar"} 1 0 +{__name__="metric", baz="abc", foo="bar"} 2 60000 +{__name__="metric", baz="abc", foo="bar"} 3 120000 +{__name__="metric", baz="abc", foo="bar"} 4 180000 +{__name__="metric", baz="abc", foo="bar"} 5 240000 diff --git a/cmd/promtool/testdata/dump-test-3.prom b/cmd/promtool/testdata/dump-test-3.prom new file mode 100644 index 000000000..faa278101 --- /dev/null +++ b/cmd/promtool/testdata/dump-test-3.prom @@ -0,0 +1,2 @@ +{__name__="metric", baz="abc", foo="bar"} 2 60000 +{__name__="metric", baz="abc", foo="bar"} 3 120000 diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index e6df9b78c..a9239d937 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -706,7 +706,7 @@ func analyzeCompaction(ctx context.Context, block tsdb.BlockReader, indexr tsdb. return nil } -func dumpSamples(ctx context.Context, path string, mint, maxt int64, match string) (err error) { +func dumpSamples(ctx context.Context, path string, mint, maxt int64, match []string) (err error) { db, err := tsdb.OpenDBReadOnly(path, nil) if err != nil { return err @@ -720,11 +720,21 @@ func dumpSamples(ctx context.Context, path string, mint, maxt int64, match strin } defer q.Close() - matchers, err := parser.ParseMetricSelector(match) + matcherSets, err := parser.ParseMetricSelectors(match) if err != nil { return err } - ss := q.Select(ctx, false, nil, matchers...) + + var ss storage.SeriesSet + if len(matcherSets) > 1 { + var sets []storage.SeriesSet + for _, mset := range matcherSets { + sets = append(sets, q.Select(ctx, true, nil, mset...)) + } + ss = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) + } else { + ss = q.Select(ctx, false, nil, matcherSets[0]...) + } for ss.Next() { series := ss.At() diff --git a/cmd/promtool/tsdb_test.go b/cmd/promtool/tsdb_test.go index 0f0040cd3..aeb51a07e 100644 --- a/cmd/promtool/tsdb_test.go +++ b/cmd/promtool/tsdb_test.go @@ -14,9 +14,18 @@ package main import ( + "bytes" + "context" + "io" + "math" + "os" + "runtime" + "strings" "testing" "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/promql" ) func TestGenerateBucket(t *testing.T) { @@ -41,3 +50,101 @@ func TestGenerateBucket(t *testing.T) { require.Equal(t, tc.step, step) } } + +// getDumpedSamples dumps samples and returns them. +func getDumpedSamples(t *testing.T, path string, mint, maxt int64, match []string) string { + t.Helper() + + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + err := dumpSamples( + context.Background(), + path, + mint, + maxt, + match, + ) + require.NoError(t, err) + + w.Close() + os.Stdout = oldStdout + + var buf bytes.Buffer + io.Copy(&buf, r) + return buf.String() +} + +func TestTSDBDump(t *testing.T) { + storage := promql.LoadedStorage(t, ` + load 1m + metric{foo="bar", baz="abc"} 1 2 3 4 5 + heavy_metric{foo="bar"} 5 4 3 2 1 + heavy_metric{foo="foo"} 5 4 3 2 1 + `) + + tests := []struct { + name string + mint int64 + maxt int64 + match []string + expectedDump string + }{ + { + name: "default match", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__=~'(?s:.*)'}"}, + expectedDump: "testdata/dump-test-1.prom", + }, + { + name: "same matcher twice", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{foo=~'.+'}", "{foo=~'.+'}"}, + expectedDump: "testdata/dump-test-1.prom", + }, + { + name: "no duplication", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__=~'(?s:.*)'}", "{baz='abc'}"}, + expectedDump: "testdata/dump-test-1.prom", + }, + { + name: "well merged", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__='heavy_metric'}", "{baz='abc'}"}, + expectedDump: "testdata/dump-test-1.prom", + }, + { + name: "multi matchers", + mint: math.MinInt64, + maxt: math.MaxInt64, + match: []string{"{__name__='heavy_metric',foo='foo'}", "{__name__='metric'}"}, + expectedDump: "testdata/dump-test-2.prom", + }, + { + name: "with reduced mint and maxt", + mint: int64(60000), + maxt: int64(120000), + match: []string{"{__name__='metric'}"}, + expectedDump: "testdata/dump-test-3.prom", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dumpedMetrics := getDumpedSamples(t, storage.Dir(), tt.mint, tt.maxt, tt.match) + expectedMetrics, err := os.ReadFile(tt.expectedDump) + require.NoError(t, err) + if strings.Contains(runtime.GOOS, "windows") { + // We use "/n" while dumping on windows as well. + expectedMetrics = bytes.ReplaceAll(expectedMetrics, []byte("\r\n"), []byte("\n")) + } + // even though in case of one matcher samples are not sorted, the order in the cases above should stay the same. + require.Equal(t, string(expectedMetrics), dumpedMetrics) + }) + } +} diff --git a/docs/command-line/promtool.md b/docs/command-line/promtool.md index e9ee7597e..ba948685b 100644 --- a/docs/command-line/promtool.md +++ b/docs/command-line/promtool.md @@ -567,7 +567,7 @@ Dump samples from a TSDB. | --- | --- | --- | | --min-time | Minimum timestamp to dump. | `-9223372036854775808` | | --max-time | Maximum timestamp to dump. | `9223372036854775807` | -| --match | Series selector. | `{__name__=~'(?s:.*)'}` | +| --match | Series selector. Can be specified multiple times. | `{__name__=~'(?s:.*)'}` | diff --git a/promql/parser/parse.go b/promql/parser/parse.go index 122286c55..c2a42ed15 100644 --- a/promql/parser/parse.go +++ b/promql/parser/parse.go @@ -208,6 +208,20 @@ func ParseMetricSelector(input string) (m []*labels.Matcher, err error) { return m, err } +// ParseMetricSelectors parses a list of provided textual metric selectors into lists of +// label matchers. +func ParseMetricSelectors(matchers []string) (m [][]*labels.Matcher, err error) { + var matcherSets [][]*labels.Matcher + for _, s := range matchers { + matchers, err := ParseMetricSelector(s) + if err != nil { + return nil, err + } + matcherSets = append(matcherSets, matchers) + } + return matcherSets, nil +} + // SequenceValue is an omittable value in a sequence of time series values. type SequenceValue struct { Value float64 diff --git a/web/api/v1/api.go b/web/api/v1/api.go index dd35d1fe9..d9d4cfd1d 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -1848,13 +1848,9 @@ func parseDuration(s string) (time.Duration, error) { } func parseMatchersParam(matchers []string) ([][]*labels.Matcher, error) { - var matcherSets [][]*labels.Matcher - for _, s := range matchers { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, err - } - matcherSets = append(matcherSets, matchers) + matcherSets, err := parser.ParseMetricSelectors(matchers) + if err != nil { + return nil, err } OUTER: diff --git a/web/federate.go b/web/federate.go index 2e7bac21d..22384a696 100644 --- a/web/federate.go +++ b/web/federate.go @@ -65,14 +65,10 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { return } - var matcherSets [][]*labels.Matcher - for _, s := range req.Form["match[]"] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - matcherSets = append(matcherSets, matchers) + matcherSets, err := parser.ParseMetricSelectors(req.Form["match[]"]) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } var (