RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-28 09:42:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10075 from prometheus/beorn7/histogram

Browse Source 
model: Implement FloatHistogram.Compact
This commit is contained in:
Björn Rabenstein 2022-01-05 16:09:39 +01:00 committed by GitHub
commit ec80745884
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 355 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
model/histogram/float_histogram.go
View File

@ -305,14 +305,164 @@ func addBucket(
// Compact eliminates empty buckets at the beginning and end of each span, then
// merges spans that are consecutive or at most maxEmptyBuckets apart, and
// finally splits spans that contain more than maxEmptyBuckets. The compaction
// happens "in place" in the receiving histogram, but a pointer to it is
// returned for convenience.
// finally splits spans that contain more consecutive empty buckets than
// maxEmptyBuckets. (The actual implementation might do something more efficient
// but with the same result.) The compaction happens "in place" in the
// receiving histogram, but a pointer to it is returned for convenience.
func (h *FloatHistogram) Compact(maxEmptyBuckets int) *FloatHistogram {
// TODO(beorn7): Implement.
h.PositiveBuckets, h.PositiveSpans = compactBuckets(
h.PositiveBuckets, h.PositiveSpans, maxEmptyBuckets,
)
h.NegativeBuckets, h.NegativeSpans = compactBuckets(
h.NegativeBuckets, h.NegativeSpans, maxEmptyBuckets,
)
return h
}
func compactBuckets(buckets []float64, spans []Span, maxEmptyBuckets int) ([]float64, []Span) {
if len(buckets) == 0 {
return buckets, spans
}
var iBucket, iSpan int
var posInSpan uint32
// Helper function.
emptyBucketsHere := func() int {
i := 0
for i+iBucket < len(buckets) &&
uint32(i)+posInSpan < spans[iSpan].Length &&
buckets[i+iBucket] == 0 {
i++
}
return i
}
// Merge spans with zero-offset to avoid special cases later.
if len(spans) > 1 {
for i, span := range spans[1:] {
if span.Offset == 0 {
spans[iSpan].Length += span.Length
continue
}
iSpan++
if i+1 != iSpan {
spans[iSpan] = span
}
}
spans = spans[:iSpan+1]
iSpan = 0
}
// Merge spans with zero-length to avoid special cases later.
for i, span := range spans {
if span.Length == 0 {
if i+1 < len(spans) {
spans[i+1].Offset += span.Offset
}
continue
}
if i != iSpan {
spans[iSpan] = span
}
iSpan++
}
spans = spans[:iSpan]
iSpan = 0
// Cut out empty buckets from start and end of spans, no matter
// what. Also cut out empty buckets from the middle of a span but only
// if there are more than maxEmptyBuckets consecutive empty buckets.
for iBucket < len(buckets) {
if nEmpty := emptyBucketsHere(); nEmpty > 0 {
if posInSpan > 0 &&
nEmpty < int(spans[iSpan].Length-posInSpan) &&
nEmpty <= maxEmptyBuckets {
// The empty buckets are in the middle of a
// span, and there are few enough to not bother.
// Just fast-forward.
iBucket += nEmpty
posInSpan += uint32(nEmpty)
continue
}
// In all other cases, we cut out the empty buckets.
buckets = append(buckets[:iBucket], buckets[iBucket+nEmpty:]...)
if posInSpan == 0 {
// Start of span.
if nEmpty == int(spans[iSpan].Length) {
// The whole span is empty.
spans = append(spans[:iSpan], spans[iSpan+1:]...)
continue
}
spans[iSpan].Length -= uint32(nEmpty)
spans[iSpan].Offset += int32(nEmpty)
continue
}
// It's in the middle or in the end of the span.
// Split the current span.
newSpan := Span{
Offset: int32(nEmpty),
Length: spans[iSpan].Length - posInSpan - uint32(nEmpty),
}
spans[iSpan].Length = posInSpan
// In any case, we have to split to the next span.
iSpan++
posInSpan = 0
if newSpan.Length == 0 {
// The span is empty, so we were already at the end of a span.
// We don't have to insert the new span, just adjust the next
// span's offset, if there is one.
if iSpan < len(spans) {
spans[iSpan].Offset += int32(nEmpty)
}
continue
}
// Insert the new span.
spans = append(spans, Span{})
if iSpan+1 < len(spans) {
copy(spans[iSpan+1:], spans[iSpan:])
}
spans[iSpan] = newSpan
continue
}
iBucket++
posInSpan++
if posInSpan >= spans[iSpan].Length {
posInSpan = 0
iSpan++
}
}
if maxEmptyBuckets == 0 {
return buckets, spans
}
// Finally, check if any offsets between spans are small enough to merge
// the spans.
iBucket = int(spans[0].Length)
iSpan = 1
for iSpan < len(spans) {
if int(spans[iSpan].Offset) > maxEmptyBuckets {
iBucket += int(spans[iSpan].Length)
iSpan++
continue
}
// Merge span with previous one and insert empty buckets.
offset := int(spans[iSpan].Offset)
spans[iSpan-1].Length += uint32(offset) + spans[iSpan].Length
spans = append(spans[:iSpan], spans[iSpan+1:]...)
newBuckets := make([]float64, len(buckets)+offset)
copy(newBuckets, buckets[:iBucket])
copy(newBuckets[iBucket+offset:], buckets[iBucket:])
iBucket += offset
buckets = newBuckets
// Note that with many merges, it would be more efficient to
// first record all the chunks of empty buckets to insert and
// then do it in one go through all the buckets.
}
return buckets, spans
}
// DetectReset returns true if the receiving histogram is missing any buckets
// that have a non-zero population in the provided previous histogram. It also
// returns true if any count (in any bucket, in the zero count, or in the count

195
model/histogram/float_histogram_test.go
View File

@ -495,7 +495,200 @@ func TestFloatHistogramCompact(t *testing.T) {
maxEmptyBuckets int
expected *FloatHistogram
}{
// TODO(beorn7): Add test cases.
{
"empty histogram",
&FloatHistogram{},
0,
&FloatHistogram{},
},
{
"nothing should happen",
&FloatHistogram{
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
NegativeSpans: []Span{{3, 2}, {3, 2}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 1}, {2, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
NegativeSpans: []Span{{3, 2}, {3, 2}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000},
},
},
{
"eliminate zero offsets",
&FloatHistogram{
PositiveSpans: []Span{{-2, 1}, {0, 3}, {0, 1}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {0, 2}, {2, 1}, {0, 1}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 5}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 4}, {2, 2}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"eliminate zero length",
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {2, 0}, {3, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {0, 0}, {2, 0}, {1, 4}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"eliminate multiple zero length spans",
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {9, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
},
},
{
"cut empty buckets at start or end",
&FloatHistogram{
PositiveSpans: []Span{{-4, 4}, {5, 3}},
PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4, 0},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"cut empty buckets at start and end",
&FloatHistogram{
PositiveSpans: []Span{{-4, 4}, {5, 6}},
PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3, 0, 0, 0},
NegativeSpans: []Span{{-2, 4}, {3, 5}},
NegativeBuckets: []float64{0, 0, 3.1, 3, 1.234e5, 1000, 3, 4, 0},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"cut empty buckets at start or end of chunks, even in the middle",
&FloatHistogram{
PositiveSpans: []Span{{-4, 6}, {3, 6}},
PositiveBuckets: []float64{0, 0, 1, 3.3, 0, 0, 4.2, 0.1, 3.3, 0, 0, 0},
NegativeSpans: []Span{{0, 2}, {2, 6}},
NegativeBuckets: []float64{3.1, 3, 0, 1.234e5, 1000, 3, 4, 0},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 2}, {5, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 4}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"cut empty buckets at start or end but merge spans due to maxEmptyBuckets",
&FloatHistogram{
PositiveSpans: []Span{{-4, 4}, {5, 3}},
PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4, 0},
},
10,
&FloatHistogram{
PositiveSpans: []Span{{-2, 10}},
PositiveBuckets: []float64{1, 3.3, 0, 0, 0, 0, 0, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 9}},
NegativeBuckets: []float64{3.1, 3, 0, 0, 0, 1.234e5, 1000, 3, 4},
},
},
{
"cut empty buckets from the middle of a chunk",
&FloatHistogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
0,
&FloatHistogram{
PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 2}, {1, 2}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
},
},
{
"cut empty buckets from the middle of a chunk, avoiding some due to maxEmptyBuckets",
&FloatHistogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
1,
&FloatHistogram{
PositiveSpans: []Span{{-2, 1}, {2, 1}, {3, 3}},
PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
},
{
"avoiding all cutting of empty buckets from the middle of a chunk due to maxEmptyBuckets",
&FloatHistogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
2,
&FloatHistogram{
PositiveSpans: []Span{{-2, 4}, {3, 3}},
PositiveBuckets: []float64{1, 0, 0, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
},
{
"everything merged into one span due to maxEmptyBuckets",
&FloatHistogram{
PositiveSpans: []Span{{-4, 6}, {3, 3}},
PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 2}, {3, 5}},
NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
},
3,
&FloatHistogram{
PositiveSpans: []Span{{-2, 10}},
PositiveBuckets: []float64{1, 0, 0, 3.3, 0, 0, 0, 4.2, 0.1, 3.3},
NegativeSpans: []Span{{0, 10}},
NegativeBuckets: []float64{3.1, 3, 0, 0, 0, 1.234e5, 1000, 0, 3, 4},
},
},
}
for _, c := range cases {

17
promql/engine_test.go
View File

@ -2650,16 +2650,13 @@ func TestSparseHistogramRate(t *testing.T) {
require.Len(t, vector, 1)
actualHistogram := vector[0].H
expectedHistogram := &histogram.FloatHistogram{
Schema: 1,
ZeroThreshold: 0.001,
ZeroCount: 1. / 15.,
Count: 4. / 15.,
Sum: 1.226666666666667,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 1, Length: 2},
},
PositiveBuckets: []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
Schema: 1,
ZeroThreshold: 0.001,
ZeroCount: 1. / 15.,
Count: 4. / 15.,
Sum: 1.226666666666667,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 5}},
PositiveBuckets: []float64{1. / 15., 1. / 15., 0, 1. / 15., 1. / 15.},
}
require.Equal(t, expectedHistogram, actualHistogram)
}