RepoMirrors
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve WAL/mmap chunks test for histograms (#11208) 

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
This commit is contained in:
Ganesh Vernekar 2022-08-29 16:21:32 +05:30 committed by GitHub
parent d209a29a5b
commit 6383994f3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 158 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
tsdb/head_test.go
View File

@ -2855,29 +2855,71 @@ func TestAppendHistogram(t *testing.T) {
}
}
func TestHistogramInWAL(t *testing.T) {
l := labels.Labels{{Name: "a", Value: "b"}}
numHistograms := 10
func TestHistogramInWALAndMmapChunk(t *testing.T) {
head, _ := newTestHead(t, 1000, false)
t.Cleanup(func() {
require.NoError(t, head.Close())
})
require.NoError(t, head.Init(0))
app := head.Appender(context.Background())
type timedHistogram struct {
t int64
h *histogram.Histogram
}
expHistograms := make([]timedHistogram, 0, numHistograms)
// Series with only histograms.
s1 := labels.Labels{{Name: "a", Value: "b1"}}
k1 := s1.String()
numHistograms := 450
exp := map[string][]tsdbutil.Sample{}
app := head.Appender(context.Background())
for i, h := range GenerateTestHistograms(numHistograms) {
h.Count = h.Count * 2
h.NegativeSpans = h.PositiveSpans
h.NegativeBuckets = h.PositiveBuckets
_, err := app.AppendHistogram(0, l, int64(i), h)
_, err := app.AppendHistogram(0, s1, int64(i), h)
require.NoError(t, err)
expHistograms = append(expHistograms, timedHistogram{int64(i), h})
exp[k1] = append(exp[k1], sample{t: int64(i), h: h.Copy()})
if i%5 == 0 {
require.NoError(t, app.Commit())
app = head.Appender(context.Background())
}
}
require.NoError(t, app.Commit())
// There should be 3 mmap chunks in s1.
ms := head.series.getByHash(s1.Hash(), s1)
require.Len(t, ms.mmappedChunks, 3)
expMmapChunks := make([]*mmappedChunk, 0, 3)
for _, mmap := range ms.mmappedChunks {
require.Greater(t, mmap.numSamples, uint16(0))
cpy := *mmap
expMmapChunks = append(expMmapChunks, &cpy)
}
expHeadChunkSamples := ms.headChunk.chunk.NumSamples()
require.Greater(t, expHeadChunkSamples, 0)
// Series with mix of histograms and float.
s2 := labels.Labels{{Name: "a", Value: "b2"}}
k2 := s2.String()
app = head.Appender(context.Background())
ts := 0
for _, h := range GenerateTestHistograms(200) {
ts++
h.Count = h.Count * 2
h.NegativeSpans = h.PositiveSpans
h.NegativeBuckets = h.PositiveBuckets
_, err := app.AppendHistogram(0, s2, int64(ts), h)
require.NoError(t, err)
exp[k2] = append(exp[k2], sample{t: int64(ts), h: h.Copy()})
if ts%20 == 0 {
require.NoError(t, app.Commit())
app = head.Appender(context.Background())
// Add some float.
for i := 0; i < 10; i++ {
ts++
_, err := app.Append(0, s2, int64(ts), float64(ts))
require.NoError(t, err)
exp[k2] = append(exp[k2], sample{t: int64(ts), v: float64(ts)})
}
require.NoError(t, app.Commit())
app = head.Appender(context.Background())
}
}
require.NoError(t, app.Commit())
@ -2889,26 +2931,18 @@ func TestHistogramInWAL(t *testing.T) {
require.NoError(t, err)
require.NoError(t, head.Init(0))
// Checking contents of s1.
ms = head.series.getByHash(s1.Hash(), s1)
require.Equal(t, expMmapChunks, ms.mmappedChunks)
for _, mmap := range ms.mmappedChunks {
require.Greater(t, mmap.numSamples, uint16(0))
}
require.Equal(t, expHeadChunkSamples, ms.headChunk.chunk.NumSamples())
q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime())
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, q.Close())
})
ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
require.True(t, ss.Next())
s := ss.At()
require.False(t, ss.Next())
it := s.Iterator()
actHistograms := make([]timedHistogram, 0, len(expHistograms))
for it.Next() == chunkenc.ValHistogram {
t, h := it.AtHistogram()
actHistograms = append(actHistograms, timedHistogram{t, h})
}
require.Equal(t, expHistograms, actHistograms)
act := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "a", "b.*"))
require.Equal(t, exp, act)
}
func TestChunkSnapshot(t *testing.T) {

4
tsdb/record/record.go
View File

@ -44,10 +44,10 @@ const (
Tombstones Type = 3
// Exemplars is used to match WAL records of type Exemplars.
Exemplars Type = 4
// Histograms is used to match WAL records of type Histograms.
Histograms Type = 5
// Metadata is used to match WAL records of type Metadata.
Metadata Type = 6
// Histograms is used to match WAL records of type Histograms.
Histograms Type = 7
)
func (rt Type) String() string {

92
tsdb/record/record_test.go
View File

@ -15,11 +15,13 @@
package record
import (
"math/rand"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb/encoding"
"github.com/prometheus/prometheus/tsdb/tombstones"
@ -107,6 +109,50 @@ func TestRecord_EncodeDecode(t *testing.T) {
decExemplars, err := dec.Exemplars(enc.Exemplars(exemplars, nil), nil)
require.NoError(t, err)
require.Equal(t, exemplars, decExemplars)
histograms := []RefHistogram{
{
Ref: 56,
T: 1234,
H: &histogram.Histogram{
Count: 5,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 18.4 * rand.Float64(),
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
},
},
{
Ref: 42,
T: 5678,
H: &histogram.Histogram{
Count: 11,
ZeroCount: 4,
ZeroThreshold: 0.001,
Sum: 35.5,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
NegativeSpans: []histogram.Span{
{Offset: 0, Length: 1},
{Offset: 1, Length: 2},
},
NegativeBuckets: []int64{1, 2, -1},
},
},
}
decHistograms, err := dec.Histograms(enc.Histograms(histograms, nil), nil)
require.NoError(t, err)
require.Equal(t, histograms, decHistograms)
}
// TestRecord_Corrupted ensures that corrupted records return the correct error.
@ -170,6 +216,31 @@ func TestRecord_Corrupted(t *testing.T) {
_, err := dec.Metadata(corrupted, nil)
require.Equal(t, errors.Cause(err), encoding.ErrInvalidSize)
})
t.Run("Test corrupted histogram record", func(t *testing.T) {
histograms := []RefHistogram{
{
Ref: 56,
T: 1234,
H: &histogram.Histogram{
Count: 5,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 18.4 * rand.Float64(),
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
},
},
}
corrupted := enc.Histograms(histograms, nil)[:8]
_, err := dec.Histograms(corrupted, nil)
require.Equal(t, errors.Cause(err), encoding.ErrInvalidSize)
})
}
func TestRecord_Type(t *testing.T) {
@ -192,6 +263,27 @@ func TestRecord_Type(t *testing.T) {
recordType = dec.Type(enc.Metadata(metadata, nil))
require.Equal(t, Metadata, recordType)
histograms := []RefHistogram{
{
Ref: 56,
T: 1234,
H: &histogram.Histogram{
Count: 5,
ZeroCount: 2,
ZeroThreshold: 0.001,
Sum: 18.4 * rand.Float64(),
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
},
},
}
recordType = dec.Type(enc.Histograms(histograms, nil))
require.Equal(t, Histograms, recordType)
recordType = dec.Type(nil)
require.Equal(t, Unknown, recordType)