diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 584037c57..41d022e91 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -266,7 +266,10 @@ func TestBlockSize(t *testing.T) { // createBlock creates a block with given set of series and returns its dir. func createBlock(tb testing.TB, dir string, series []Series) string { - head := createHead(tb, series) + return createBlockFromHead(tb, dir, createHead(tb, series)) +} + +func createBlockFromHead(tb testing.TB, dir string, head *Head) string { compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) testutil.Ok(tb, err) diff --git a/tsdb/head_bench_test.go b/tsdb/head_bench_test.go index 6fee2122f..0f807814e 100644 --- a/tsdb/head_bench_test.go +++ b/tsdb/head_bench_test.go @@ -48,73 +48,3 @@ func BenchmarkHeadStripeSeriesCreateParallel(b *testing.B) { } }) } - -func BenchmarkHeadPostingForMatchers(b *testing.B) { - h, err := NewHead(nil, nil, nil, 1000) - testutil.Ok(b, err) - defer func() { - testutil.Ok(b, h.Close()) - }() - - var ref uint64 - - addSeries := func(l labels.Labels) { - ref++ - h.getOrCreateWithID(ref, l.Hash(), l) - } - - for n := 0; n < 10; n++ { - for i := 0; i < 100000; i++ { - addSeries(labels.FromStrings("i", strconv.Itoa(i), "n", strconv.Itoa(n), "j", "foo")) - // Have some series that won't be matched, to properly test inverted matches. - addSeries(labels.FromStrings("i", strconv.Itoa(i), "n", strconv.Itoa(n), "j", "bar")) - addSeries(labels.FromStrings("i", strconv.Itoa(i), "n", "0_"+strconv.Itoa(n), "j", "bar")) - addSeries(labels.FromStrings("i", strconv.Itoa(i), "n", "1_"+strconv.Itoa(n), "j", "bar")) - addSeries(labels.FromStrings("i", strconv.Itoa(i), "n", "2_"+strconv.Itoa(n), "j", "foo")) - } - } - - n1 := labels.MustNewMatcher(labels.MatchEqual, "n", "1") - - jFoo := labels.MustNewMatcher(labels.MatchEqual, "j", "foo") - jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo") - - iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$") - iPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$") - i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^1.+$") - iEmptyRe := labels.MustNewMatcher(labels.MatchRegexp, "i", "^$") - iNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "i", "") - iNot2 := labels.MustNewMatcher(labels.MatchNotEqual, "n", "2") - iNot2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^2.*$") - - cases := []struct { - name string - matchers []*labels.Matcher - }{ - {`n="1"`, []*labels.Matcher{n1}}, - {`n="1",j="foo"`, []*labels.Matcher{n1, jFoo}}, - {`j="foo",n="1"`, []*labels.Matcher{jFoo, n1}}, - {`n="1",j!="foo"`, []*labels.Matcher{n1, jNotFoo}}, - {`i=~".*"`, []*labels.Matcher{iStar}}, - {`i=~".+"`, []*labels.Matcher{iPlus}}, - {`i=~""`, []*labels.Matcher{iEmptyRe}}, - {`i!=""`, []*labels.Matcher{iNotEmpty}}, - {`n="1",i=~".*",j="foo"`, []*labels.Matcher{n1, iStar, jFoo}}, - {`n="1",i=~".*",i!="2",j="foo"`, []*labels.Matcher{n1, iStar, iNot2, jFoo}}, - {`n="1",i!=""`, []*labels.Matcher{n1, iNotEmpty}}, - {`n="1",i!="",j="foo"`, []*labels.Matcher{n1, iNotEmpty, jFoo}}, - {`n="1",i=~".+",j="foo"`, []*labels.Matcher{n1, iPlus, jFoo}}, - {`n="1",i=~"1.+",j="foo"`, []*labels.Matcher{n1, i1Plus, jFoo}}, - {`n="1",i=~".+",i!="2",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2, jFoo}}, - {`n="1",i=~".+",i!~"2.*",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2Star, jFoo}}, - } - - for _, c := range cases { - b.Run(c.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := PostingsForMatchers(h.indexRange(0, 1000), c.matchers...) - testutil.Ok(b, err) - } - }) - } -} diff --git a/tsdb/querier_bench_test.go b/tsdb/querier_bench_test.go new file mode 100644 index 000000000..48e73fe85 --- /dev/null +++ b/tsdb/querier_bench_test.go @@ -0,0 +1,125 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdb + +import ( + "io/ioutil" + "os" + "strconv" + "testing" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/util/testutil" +) + +// Make entries ~50B in size, to emulate real-world high cardinality. +const ( + postingsBenchSuffix = "aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd" +) + +func BenchmarkPostingsForMatchers(b *testing.B) { + h, err := NewHead(nil, nil, nil, 1000) + testutil.Ok(b, err) + defer func() { + testutil.Ok(b, h.Close()) + }() + + app := h.Appender() + addSeries := func(l labels.Labels) { + app.Add(l, 0, 0) + } + + for n := 0; n < 10; n++ { + for i := 0; i < 100000; i++ { + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", strconv.Itoa(n)+postingsBenchSuffix, "j", "foo")) + // Have some series that won't be matched, to properly test inverted matches. + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "0_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "1_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "bar")) + addSeries(labels.FromStrings("i", strconv.Itoa(i)+postingsBenchSuffix, "n", "2_"+strconv.Itoa(n)+postingsBenchSuffix, "j", "foo")) + } + } + testutil.Ok(b, app.Commit()) + + ir, err := h.Index() + testutil.Ok(b, err) + b.Run("Head", func(b *testing.B) { + benchmarkPostingsForMatchers(b, ir) + }) + + tmpdir, err := ioutil.TempDir("", "test_benchpostingsformatchers") + testutil.Ok(b, err) + defer func() { + testutil.Ok(b, os.RemoveAll(tmpdir)) + }() + + blockdir := createBlockFromHead(b, tmpdir, h) + block, err := OpenBlock(nil, blockdir, nil) + testutil.Ok(b, err) + defer func() { + testutil.Ok(b, block.Close()) + }() + ir, err = block.Index() + testutil.Ok(b, err) + defer ir.Close() + b.Run("Block", func(b *testing.B) { + benchmarkPostingsForMatchers(b, ir) + }) +} + +func benchmarkPostingsForMatchers(b *testing.B, ir IndexReader) { + n1 := labels.MustNewMatcher(labels.MatchEqual, "n", "1"+postingsBenchSuffix) + + jFoo := labels.MustNewMatcher(labels.MatchEqual, "j", "foo") + jNotFoo := labels.MustNewMatcher(labels.MatchNotEqual, "j", "foo") + + iStar := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.*$") + iPlus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^.+$") + i1Plus := labels.MustNewMatcher(labels.MatchRegexp, "i", "^1.+$") + iEmptyRe := labels.MustNewMatcher(labels.MatchRegexp, "i", "^$") + iNotEmpty := labels.MustNewMatcher(labels.MatchNotEqual, "i", "") + iNot2 := labels.MustNewMatcher(labels.MatchNotEqual, "n", "2"+postingsBenchSuffix) + iNot2Star := labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^2.*$") + + cases := []struct { + name string + matchers []*labels.Matcher + }{ + {`n="1"`, []*labels.Matcher{n1}}, + {`n="1",j="foo"`, []*labels.Matcher{n1, jFoo}}, + {`j="foo",n="1"`, []*labels.Matcher{jFoo, n1}}, + {`n="1",j!="foo"`, []*labels.Matcher{n1, jNotFoo}}, + {`i=~".*"`, []*labels.Matcher{iStar}}, + {`i=~".+"`, []*labels.Matcher{iPlus}}, + {`i=~""`, []*labels.Matcher{iEmptyRe}}, + {`i!=""`, []*labels.Matcher{iNotEmpty}}, + {`n="1",i=~".*",j="foo"`, []*labels.Matcher{n1, iStar, jFoo}}, + {`n="1",i=~".*",i!="2",j="foo"`, []*labels.Matcher{n1, iStar, iNot2, jFoo}}, + {`n="1",i!=""`, []*labels.Matcher{n1, iNotEmpty}}, + {`n="1",i!="",j="foo"`, []*labels.Matcher{n1, iNotEmpty, jFoo}}, + {`n="1",i=~".+",j="foo"`, []*labels.Matcher{n1, iPlus, jFoo}}, + {`n="1",i=~"1.+",j="foo"`, []*labels.Matcher{n1, i1Plus, jFoo}}, + {`n="1",i=~".+",i!="2",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2, jFoo}}, + {`n="1",i=~".+",i!~"2.*",j="foo"`, []*labels.Matcher{n1, iPlus, iNot2Star, jFoo}}, + } + + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := PostingsForMatchers(ir, c.matchers...) + testutil.Ok(b, err) + } + }) + } +}