prometheus/postings_test.go

366 lines
7.4 KiB
Go
Raw Normal View History

2017-04-10 18:59:45 +00:00
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2016-12-27 10:32:10 +00:00
package tsdb
import (
2017-03-27 10:16:55 +00:00
"encoding/binary"
"math/rand"
2016-12-27 10:32:10 +00:00
"testing"
2017-03-27 10:16:55 +00:00
"github.com/stretchr/testify/require"
2016-12-27 10:32:10 +00:00
)
func TestMemPostings_addFor(t *testing.T) {
p := newMemPostings()
p.m[allPostingsKey] = []uint64{1, 2, 3, 4, 6, 7, 8}
p.addFor(5, allPostingsKey)
require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8}, p.m[allPostingsKey])
}
2016-12-27 10:32:10 +00:00
type mockPostings struct {
next func() bool
2017-09-04 14:08:38 +00:00
seek func(uint64) bool
value func() uint64
2016-12-27 10:32:10 +00:00
err func() error
}
func (m *mockPostings) Next() bool { return m.next() }
2017-09-04 14:08:38 +00:00
func (m *mockPostings) Seek(v uint64) bool { return m.seek(v) }
func (m *mockPostings) Value() uint64 { return m.value() }
2016-12-27 10:32:10 +00:00
func (m *mockPostings) Err() error { return m.err() }
2016-12-28 10:02:19 +00:00
func TestIntersect(t *testing.T) {
2016-12-27 10:32:10 +00:00
var cases = []struct {
2017-09-04 14:08:38 +00:00
a, b []uint64
res []uint64
2016-12-27 10:32:10 +00:00
}{
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{6, 7, 8, 9, 10},
2016-12-27 10:32:10 +00:00
res: nil,
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{4, 5, 6, 7, 8},
res: []uint64{4, 5},
2016-12-27 10:32:10 +00:00
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 9, 10},
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
res: []uint64{1, 4, 10},
2016-12-27 10:32:10 +00:00
}, {
2017-09-04 14:08:38 +00:00
a: []uint64{1},
b: []uint64{0, 1},
res: []uint64{1},
2016-12-27 10:32:10 +00:00
},
}
for _, c := range cases {
2016-12-27 10:32:10 +00:00
a := newListPostings(c.a)
b := newListPostings(c.b)
res, err := expandPostings(Intersect(a, b))
require.NoError(t, err)
require.Equal(t, c.res, res)
2016-12-27 10:32:10 +00:00
}
}
func TestMultiIntersect(t *testing.T) {
var cases = []struct {
2017-09-04 14:08:38 +00:00
p [][]uint64
res []uint64
2016-12-27 10:32:10 +00:00
}{
{
2017-09-04 14:08:38 +00:00
p: [][]uint64{
{1, 2, 3, 4, 5, 6, 1000, 1001},
{2, 4, 5, 6, 7, 8, 999, 1001},
{1, 2, 5, 6, 7, 8, 1001, 1200},
},
2017-09-04 14:08:38 +00:00
res: []uint64{2, 5, 6, 1001},
2016-12-27 10:32:10 +00:00
},
// One of the reproduceable cases for:
// https://github.com/prometheus/prometheus/issues/2616
// The initialisation of intersectPostings was moving the iterator forward
// prematurely making us miss some postings.
{
2017-09-04 14:08:38 +00:00
p: [][]uint64{
{1, 2},
{1, 2},
{1, 2},
{2},
},
2017-09-04 14:08:38 +00:00
res: []uint64{2},
},
2016-12-27 10:32:10 +00:00
}
for _, c := range cases {
ps := make([]Postings, 0, len(c.p))
for _, postings := range c.p {
ps = append(ps, newListPostings(postings))
}
2016-12-27 10:32:10 +00:00
res, err := expandPostings(Intersect(ps...))
require.NoError(t, err)
require.Equal(t, c.res, res)
2016-12-27 10:32:10 +00:00
}
}
func BenchmarkIntersect(t *testing.B) {
2017-09-04 14:08:38 +00:00
var a, b, c, d []uint64
2016-12-27 10:32:10 +00:00
for i := 0; i < 10000000; i += 2 {
2017-09-04 14:08:38 +00:00
a = append(a, uint64(i))
2016-12-27 10:32:10 +00:00
}
for i := 5000000; i < 5000100; i += 4 {
2017-09-04 14:08:38 +00:00
b = append(b, uint64(i))
2016-12-27 10:32:10 +00:00
}
for i := 5090000; i < 5090600; i += 4 {
2017-09-04 14:08:38 +00:00
b = append(b, uint64(i))
2016-12-27 10:32:10 +00:00
}
for i := 4990000; i < 5100000; i++ {
2017-09-04 14:08:38 +00:00
c = append(c, uint64(i))
2016-12-27 10:32:10 +00:00
}
for i := 4000000; i < 6000000; i++ {
2017-09-04 14:08:38 +00:00
d = append(d, uint64(i))
2016-12-27 10:32:10 +00:00
}
i1 := newListPostings(a)
i2 := newListPostings(b)
i3 := newListPostings(c)
i4 := newListPostings(d)
t.ResetTimer()
for i := 0; i < t.N; i++ {
if _, err := expandPostings(Intersect(i1, i2, i3, i4)); err != nil {
t.Fatal(err)
}
}
}
2016-12-28 10:02:19 +00:00
func TestMultiMerge(t *testing.T) {
var cases = []struct {
2017-09-04 14:08:38 +00:00
a, b, c []uint64
res []uint64
2016-12-28 10:02:19 +00:00
}{
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5, 6, 1000, 1001},
b: []uint64{2, 4, 5, 6, 7, 8, 999, 1001},
c: []uint64{1, 2, 5, 6, 7, 8, 1001, 1200},
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 999, 1000, 1001, 1200},
2016-12-28 10:02:19 +00:00
},
}
for _, c := range cases {
i1 := newListPostings(c.a)
i2 := newListPostings(c.b)
i3 := newListPostings(c.c)
res, err := expandPostings(Merge(i1, i2, i3))
require.NoError(t, err)
require.Equal(t, c.res, res)
2016-12-28 10:02:19 +00:00
}
}
func TestMergedPostings(t *testing.T) {
2016-12-28 10:02:19 +00:00
var cases = []struct {
2017-09-04 14:08:38 +00:00
a, b []uint64
res []uint64
2016-12-28 10:02:19 +00:00
}{
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{6, 7, 8, 9, 10},
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
2016-12-28 10:02:19 +00:00
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{4, 5, 6, 7, 8},
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8},
2016-12-28 10:02:19 +00:00
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 9, 10},
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
2016-12-28 10:02:19 +00:00
},
}
for _, c := range cases {
a := newListPostings(c.a)
b := newListPostings(c.b)
res, err := expandPostings(newMergedPostings(a, b))
require.NoError(t, err)
require.Equal(t, c.res, res)
}
}
func TestMergedPostingsSeek(t *testing.T) {
var cases = []struct {
2017-09-04 14:08:38 +00:00
a, b []uint64
2017-09-04 14:08:38 +00:00
seek uint64
success bool
2017-09-04 14:08:38 +00:00
res []uint64
}{
{
2017-09-04 14:08:38 +00:00
a: []uint64{2, 3, 4, 5},
b: []uint64{6, 7, 8, 9, 10},
seek: 1,
success: true,
2017-09-04 14:08:38 +00:00
res: []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10},
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{6, 7, 8, 9, 10},
seek: 2,
success: true,
2017-09-04 14:08:38 +00:00
res: []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10},
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 5},
b: []uint64{4, 5, 6, 7, 8},
seek: 9,
success: false,
res: nil,
},
{
2017-09-04 14:08:38 +00:00
a: []uint64{1, 2, 3, 4, 9, 10},
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
seek: 10,
success: true,
2017-09-04 14:08:38 +00:00
res: []uint64{10, 11},
},
}
for _, c := range cases {
a := newListPostings(c.a)
b := newListPostings(c.b)
p := newMergedPostings(a, b)
require.Equal(t, c.success, p.Seek(c.seek))
// After Seek(), At() should be called.
if c.success {
start := p.At()
lst, err := expandPostings(p)
require.NoError(t, err)
2017-09-04 14:08:38 +00:00
lst = append([]uint64{start}, lst...)
require.Equal(t, c.res, lst)
}
2016-12-28 10:02:19 +00:00
}
return
2016-12-28 10:02:19 +00:00
}
2017-03-27 10:16:55 +00:00
func TestBigEndian(t *testing.T) {
num := 1000
// mock a list as postings
ls := make([]uint32, num)
ls[0] = 2
for i := 1; i < num; i++ {
ls[i] = ls[i-1] + uint32(rand.Int31n(25)) + 2
}
beLst := make([]byte, num*4)
for i := 0; i < num; i++ {
b := beLst[i*4 : i*4+4]
binary.BigEndian.PutUint32(b, ls[i])
}
t.Run("Iteration", func(t *testing.T) {
bep := newBigEndianPostings(beLst)
for i := 0; i < num; i++ {
require.True(t, bep.Next())
2017-09-04 14:08:38 +00:00
require.Equal(t, uint64(ls[i]), bep.At())
2017-03-27 10:16:55 +00:00
}
require.False(t, bep.Next())
require.Nil(t, bep.Err())
})
t.Run("Seek", func(t *testing.T) {
table := []struct {
seek uint32
val uint32
found bool
}{
{
ls[0] - 1, ls[0], true,
},
{
ls[4], ls[4], true,
},
{
ls[500] - 1, ls[500], true,
},
{
ls[600] + 1, ls[601], true,
},
{
ls[600] + 1, ls[601], true,
2017-03-27 10:16:55 +00:00
},
{
ls[600] + 1, ls[601], true,
2017-03-27 10:16:55 +00:00
},
{
ls[0], ls[601], true,
2017-03-27 10:16:55 +00:00
},
{
ls[600], ls[601], true,
2017-03-27 10:16:55 +00:00
},
{
ls[999], ls[999], true,
},
{
ls[999] + 10, ls[999], false,
},
}
bep := newBigEndianPostings(beLst)
for _, v := range table {
2017-09-04 14:08:38 +00:00
require.Equal(t, v.found, bep.Seek(uint64(v.seek)))
require.Equal(t, uint64(v.val), bep.At())
require.Nil(t, bep.Err())
2017-03-27 10:16:55 +00:00
}
})
}
func TestIntersectWithMerge(t *testing.T) {
// One of the reproduceable cases for:
// https://github.com/prometheus/prometheus/issues/2616
2017-09-04 14:08:38 +00:00
a := newListPostings([]uint64{21, 22, 23, 24, 25, 30})
b := newMergedPostings(
2017-09-04 14:08:38 +00:00
newListPostings([]uint64{10, 20, 30}),
newListPostings([]uint64{15, 26, 30}),
)
p := Intersect(a, b)
res, err := expandPostings(p)
require.NoError(t, err)
2017-09-04 14:08:38 +00:00
require.Equal(t, []uint64{30}, res)
}