535 lines
10 KiB
Go
535 lines
10 KiB
Go
// Copyright 2017 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package index
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"math/rand"
|
|
"sort"
|
|
"testing"
|
|
|
|
"github.com/prometheus/tsdb/labels"
|
|
"github.com/prometheus/tsdb/testutil"
|
|
)
|
|
|
|
func TestMemPostings_addFor(t *testing.T) {
|
|
p := NewMemPostings()
|
|
p.m[allPostingsKey] = []uint64{1, 2, 3, 4, 6, 7, 8}
|
|
|
|
p.addFor(5, allPostingsKey)
|
|
|
|
testutil.Equals(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8}, p.m[allPostingsKey])
|
|
}
|
|
|
|
func TestMemPostings_ensureOrder(t *testing.T) {
|
|
p := NewUnorderedMemPostings()
|
|
|
|
for i := 0; i < 100; i++ {
|
|
l := make([]uint64, 100)
|
|
for j := range l {
|
|
l[j] = rand.Uint64()
|
|
}
|
|
v := fmt.Sprintf("%d", i)
|
|
|
|
p.m[labels.Label{"a", v}] = l
|
|
}
|
|
|
|
p.EnsureOrder()
|
|
|
|
for _, l := range p.m {
|
|
ok := sort.SliceIsSorted(l, func(i, j int) bool {
|
|
return l[i] < l[j]
|
|
})
|
|
if !ok {
|
|
t.Fatalf("postings list %v is not sorted", l)
|
|
}
|
|
}
|
|
}
|
|
|
|
type mockPostings struct {
|
|
next func() bool
|
|
seek func(uint64) bool
|
|
value func() uint64
|
|
err func() error
|
|
}
|
|
|
|
func (m *mockPostings) Next() bool { return m.next() }
|
|
func (m *mockPostings) Seek(v uint64) bool { return m.seek(v) }
|
|
func (m *mockPostings) Value() uint64 { return m.value() }
|
|
func (m *mockPostings) Err() error { return m.err() }
|
|
|
|
func TestIntersect(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b []uint64
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
res: nil,
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{4, 5, 6, 7, 8},
|
|
res: []uint64{4, 5},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
|
|
res: []uint64{1, 4, 10},
|
|
}, {
|
|
a: []uint64{1},
|
|
b: []uint64{0, 1},
|
|
res: []uint64{1},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
a := newListPostings(c.a)
|
|
b := newListPostings(c.b)
|
|
|
|
res, err := ExpandPostings(Intersect(a, b))
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, c.res, res)
|
|
}
|
|
}
|
|
|
|
func TestMultiIntersect(t *testing.T) {
|
|
var cases = []struct {
|
|
p [][]uint64
|
|
res []uint64
|
|
}{
|
|
{
|
|
p: [][]uint64{
|
|
{1, 2, 3, 4, 5, 6, 1000, 1001},
|
|
{2, 4, 5, 6, 7, 8, 999, 1001},
|
|
{1, 2, 5, 6, 7, 8, 1001, 1200},
|
|
},
|
|
res: []uint64{2, 5, 6, 1001},
|
|
},
|
|
// One of the reproduceable cases for:
|
|
// https://github.com/prometheus/prometheus/issues/2616
|
|
// The initialisation of intersectPostings was moving the iterator forward
|
|
// prematurely making us miss some postings.
|
|
{
|
|
p: [][]uint64{
|
|
{1, 2},
|
|
{1, 2},
|
|
{1, 2},
|
|
{2},
|
|
},
|
|
res: []uint64{2},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
ps := make([]Postings, 0, len(c.p))
|
|
for _, postings := range c.p {
|
|
ps = append(ps, newListPostings(postings))
|
|
}
|
|
|
|
res, err := ExpandPostings(Intersect(ps...))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, c.res, res)
|
|
}
|
|
}
|
|
|
|
func BenchmarkIntersect(t *testing.B) {
|
|
var a, b, c, d []uint64
|
|
|
|
for i := 0; i < 10000000; i += 2 {
|
|
a = append(a, uint64(i))
|
|
}
|
|
for i := 5000000; i < 5000100; i += 4 {
|
|
b = append(b, uint64(i))
|
|
}
|
|
for i := 5090000; i < 5090600; i += 4 {
|
|
b = append(b, uint64(i))
|
|
}
|
|
for i := 4990000; i < 5100000; i++ {
|
|
c = append(c, uint64(i))
|
|
}
|
|
for i := 4000000; i < 6000000; i++ {
|
|
d = append(d, uint64(i))
|
|
}
|
|
|
|
i1 := newListPostings(a)
|
|
i2 := newListPostings(b)
|
|
i3 := newListPostings(c)
|
|
i4 := newListPostings(d)
|
|
|
|
t.ResetTimer()
|
|
|
|
for i := 0; i < t.N; i++ {
|
|
if _, err := ExpandPostings(Intersect(i1, i2, i3, i4)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMultiMerge(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b, c []uint64
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5, 6, 1000, 1001},
|
|
b: []uint64{2, 4, 5, 6, 7, 8, 999, 1001},
|
|
c: []uint64{1, 2, 5, 6, 7, 8, 1001, 1200},
|
|
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 999, 1000, 1001, 1200},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
i1 := newListPostings(c.a)
|
|
i2 := newListPostings(c.b)
|
|
i3 := newListPostings(c.c)
|
|
|
|
res, err := ExpandPostings(Merge(i1, i2, i3))
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, c.res, res)
|
|
}
|
|
}
|
|
|
|
func TestMergedPostings(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b []uint64
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{4, 5, 6, 7, 8},
|
|
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
|
|
res: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
a := newListPostings(c.a)
|
|
b := newListPostings(c.b)
|
|
|
|
res, err := ExpandPostings(newMergedPostings(a, b))
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, c.res, res)
|
|
}
|
|
|
|
}
|
|
|
|
func TestMergedPostingsSeek(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b []uint64
|
|
|
|
seek uint64
|
|
success bool
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: []uint64{2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
|
|
seek: 1,
|
|
success: true,
|
|
res: []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
|
|
seek: 2,
|
|
success: true,
|
|
res: []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{4, 5, 6, 7, 8},
|
|
|
|
seek: 9,
|
|
success: false,
|
|
res: nil,
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
|
|
|
|
seek: 10,
|
|
success: true,
|
|
res: []uint64{10, 11},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
a := newListPostings(c.a)
|
|
b := newListPostings(c.b)
|
|
|
|
p := newMergedPostings(a, b)
|
|
|
|
testutil.Equals(t, c.success, p.Seek(c.seek))
|
|
|
|
// After Seek(), At() should be called.
|
|
if c.success {
|
|
start := p.At()
|
|
lst, err := ExpandPostings(p)
|
|
testutil.Ok(t, err)
|
|
|
|
lst = append([]uint64{start}, lst...)
|
|
testutil.Equals(t, c.res, lst)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func TestRemovedPostings(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b []uint64
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: nil,
|
|
b: nil,
|
|
res: []uint64(nil),
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4},
|
|
b: nil,
|
|
res: []uint64{1, 2, 3, 4},
|
|
},
|
|
{
|
|
a: nil,
|
|
b: []uint64{1, 2, 3, 4},
|
|
res: []uint64(nil),
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
res: []uint64{1, 2, 3, 4, 5},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{4, 5, 6, 7, 8},
|
|
res: []uint64{1, 2, 3},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
|
|
res: []uint64{2, 3, 9},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
|
|
res: []uint64(nil),
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
a := newListPostings(c.a)
|
|
b := newListPostings(c.b)
|
|
|
|
res, err := ExpandPostings(newRemovedPostings(a, b))
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, c.res, res)
|
|
}
|
|
|
|
}
|
|
|
|
func TestRemovedPostingsSeek(t *testing.T) {
|
|
var cases = []struct {
|
|
a, b []uint64
|
|
|
|
seek uint64
|
|
success bool
|
|
res []uint64
|
|
}{
|
|
{
|
|
a: []uint64{2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
|
|
seek: 1,
|
|
success: true,
|
|
res: []uint64{2, 3, 4, 5},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{6, 7, 8, 9, 10},
|
|
|
|
seek: 2,
|
|
success: true,
|
|
res: []uint64{2, 3, 4, 5},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 5},
|
|
b: []uint64{4, 5, 6, 7, 8},
|
|
|
|
seek: 9,
|
|
success: false,
|
|
res: nil,
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 10, 11},
|
|
|
|
seek: 10,
|
|
success: false,
|
|
res: nil,
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 11},
|
|
|
|
seek: 4,
|
|
success: true,
|
|
res: []uint64{9, 10},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 11},
|
|
|
|
seek: 5,
|
|
success: true,
|
|
res: []uint64{9, 10},
|
|
},
|
|
{
|
|
a: []uint64{1, 2, 3, 4, 9, 10},
|
|
b: []uint64{1, 4, 5, 6, 7, 8, 11},
|
|
|
|
seek: 10,
|
|
success: true,
|
|
res: []uint64{10},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
a := newListPostings(c.a)
|
|
b := newListPostings(c.b)
|
|
|
|
p := newRemovedPostings(a, b)
|
|
|
|
testutil.Equals(t, c.success, p.Seek(c.seek))
|
|
|
|
// After Seek(), At() should be called.
|
|
if c.success {
|
|
start := p.At()
|
|
lst, err := ExpandPostings(p)
|
|
testutil.Ok(t, err)
|
|
|
|
lst = append([]uint64{start}, lst...)
|
|
testutil.Equals(t, c.res, lst)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func TestBigEndian(t *testing.T) {
|
|
num := 1000
|
|
// mock a list as postings
|
|
ls := make([]uint32, num)
|
|
ls[0] = 2
|
|
for i := 1; i < num; i++ {
|
|
ls[i] = ls[i-1] + uint32(rand.Int31n(25)) + 2
|
|
}
|
|
|
|
beLst := make([]byte, num*4)
|
|
for i := 0; i < num; i++ {
|
|
b := beLst[i*4 : i*4+4]
|
|
binary.BigEndian.PutUint32(b, ls[i])
|
|
}
|
|
|
|
t.Run("Iteration", func(t *testing.T) {
|
|
bep := newBigEndianPostings(beLst)
|
|
for i := 0; i < num; i++ {
|
|
testutil.Assert(t, bep.Next() == true, "")
|
|
testutil.Equals(t, uint64(ls[i]), bep.At())
|
|
}
|
|
|
|
testutil.Assert(t, bep.Next() == false, "")
|
|
testutil.Assert(t, bep.Err() == nil, "")
|
|
})
|
|
|
|
t.Run("Seek", func(t *testing.T) {
|
|
table := []struct {
|
|
seek uint32
|
|
val uint32
|
|
found bool
|
|
}{
|
|
{
|
|
ls[0] - 1, ls[0], true,
|
|
},
|
|
{
|
|
ls[4], ls[4], true,
|
|
},
|
|
{
|
|
ls[500] - 1, ls[500], true,
|
|
},
|
|
{
|
|
ls[600] + 1, ls[601], true,
|
|
},
|
|
{
|
|
ls[600] + 1, ls[601], true,
|
|
},
|
|
{
|
|
ls[600] + 1, ls[601], true,
|
|
},
|
|
{
|
|
ls[0], ls[601], true,
|
|
},
|
|
{
|
|
ls[600], ls[601], true,
|
|
},
|
|
{
|
|
ls[999], ls[999], true,
|
|
},
|
|
{
|
|
ls[999] + 10, ls[999], false,
|
|
},
|
|
}
|
|
|
|
bep := newBigEndianPostings(beLst)
|
|
|
|
for _, v := range table {
|
|
testutil.Equals(t, v.found, bep.Seek(uint64(v.seek)))
|
|
testutil.Equals(t, uint64(v.val), bep.At())
|
|
testutil.Assert(t, bep.Err() == nil, "")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestIntersectWithMerge(t *testing.T) {
|
|
// One of the reproduceable cases for:
|
|
// https://github.com/prometheus/prometheus/issues/2616
|
|
a := newListPostings([]uint64{21, 22, 23, 24, 25, 30})
|
|
|
|
b := newMergedPostings(
|
|
newListPostings([]uint64{10, 20, 30}),
|
|
newListPostings([]uint64{15, 26, 30}),
|
|
)
|
|
|
|
p := Intersect(a, b)
|
|
res, err := ExpandPostings(p)
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, []uint64{30}, res)
|
|
}
|