RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-26 00:23:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

histograms: parse float histograms from proto definition (#11149)

Browse Source 
* histograms: parse float histograms from proto definition

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Improve comment

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Ignore float buckets

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Refactor Histogram() function

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Fix test_float_histogram

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Update model/textparse/protobufparse.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update protobufparse.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape/scrape.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
This commit is contained in:
Marc Tudurí 2022-08-25 17:07:41 +02:00 committed by GitHub
parent 0f4e5196c4
commit f7df3b86ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 163 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
model/textparse/interface.go
View File

@ -30,7 +30,9 @@ type Parser interface {
// Histogram returns the bytes of a series with a sparse histogram as a
// value, the timestamp if set, and the histogram in the current sample.
Histogram() ([]byte, *int64, *histogram.Histogram)
// Depending on the parsed input, the function returns an (integer) Histogram
// or a FloatHistogram, with the respective other return value being nil.
Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram)
// Help returns the metric name and help text in the current entry.
// Must only be called after Next returned a help entry.

6
model/textparse/openmetricsparse.go
View File

@ -113,10 +113,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
return p.series, nil, p.val
}
// Histogram always returns (nil, nil, nil) because OpenMetrics does not support
// Histogram always returns (nil, nil, nil, nil) because OpenMetrics does not support
// sparse histograms.
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
return nil, nil, nil
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
return nil, nil, nil, nil
}
// Help returns the metric name and help text in the current entry.

6
model/textparse/promparse.go
View File

@ -168,10 +168,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) {
return p.series, nil, p.val
}
// Histogram always returns (nil, nil, nil) because the Prometheus text format
// Histogram always returns (nil, nil, nil, nil) because the Prometheus text format
// does not support sparse histograms.
func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
return nil, nil, nil
func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
return nil, nil, nil, nil
}
// Help returns the metric name and help text in the current entry.

40
model/textparse/protobufparse.go
View File

@ -135,12 +135,42 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
// Histogram returns the bytes of a series with a native histogram as a
// value, the timestamp if set, and the native histogram in the current
// sample.
func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
var (
m = p.mf.GetMetric()[p.metricPos]
ts = m.GetTimestampMs()
h = m.GetHistogram()
)
if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
// It is a float histogram.
fh := histogram.FloatHistogram{
Count: h.GetSampleCountFloat(),
Sum: h.GetSampleSum(),
ZeroThreshold: h.GetZeroThreshold(),
ZeroCount: h.GetZeroCountFloat(),
Schema: h.GetSchema(),
PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())),
PositiveBuckets: h.GetPositiveCount(),
NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())),
NegativeBuckets: h.GetNegativeCount(),
}
for i, span := range h.GetPositiveSpan() {
fh.PositiveSpans[i].Offset = span.GetOffset()
fh.PositiveSpans[i].Length = span.GetLength()
}
for i, span := range h.GetNegativeSpan() {
fh.NegativeSpans[i].Offset = span.GetOffset()
fh.NegativeSpans[i].Length = span.GetLength()
}
if ts != 0 {
return p.metricBytes.Bytes(), &ts, nil, &fh
}
// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
// general, but proto3 has no distinction between unset and
// default. Need to avoid in the final format.
return p.metricBytes.Bytes(), nil, nil, &fh
}
sh := histogram.Histogram{
Count: h.GetSampleCount(),
Sum: h.GetSampleSum(),
@ -160,13 +190,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
sh.NegativeSpans[i].Offset = span.GetOffset()
sh.NegativeSpans[i].Length = span.GetLength()
}
if ts != 0 {
return p.metricBytes.Bytes(), &ts, &sh
return p.metricBytes.Bytes(), &ts, &sh, nil
}
// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
// general, but proto3 has no distinction between unset and
// default. Need to avoid in the final format.
return p.metricBytes.Bytes(), nil, &sh
return p.metricBytes.Bytes(), nil, &sh, nil
}
// Help returns the metric name and help text in the current entry.

118
model/textparse/protobufparse_test.go
View File

@ -154,6 +154,78 @@ metric: <
timestamp_ms: 1234568
>
`,
`name: "test_float_histogram"
help: "Test float histogram with many buckets removed to keep it manageable in size."
type: HISTOGRAM
metric: <
histogram: <
sample_count: 175
sample_count_float: 175.0
sample_sum: 0.0008280461746287094
bucket: <
cumulative_count_float: 2.0
upper_bound: -0.0004899999999999998
>
bucket: <
cumulative_count_float: 4.0
upper_bound: -0.0003899999999999998
exemplar: <
label: <
name: "dummyID"
value: "59727"
>
value: -0.00039
timestamp: <
seconds: 1625851155
nanos: 146848499
>
>
>
bucket: <
cumulative_count_float: 16
upper_bound: -0.0002899999999999998
exemplar: <
label: <
name: "dummyID"
value: "5617"
>
value: -0.00029
>
>
schema: 3
zero_threshold: 2.938735877055719e-39
zero_count_float: 2.0
negative_span: <
offset: -162
length: 1
>
negative_span: <
offset: 23
length: 4
>
negative_count: 1.0
negative_count: 3.0
negative_count: -2.0
negative_count: -1.0
negative_count: 1.0
positive_span: <
offset: -161
length: 1
>
positive_span: <
offset: 8
length: 3
>
positive_count: 1.0
positive_count: 2.0
positive_count: -1.0
positive_count: -1.0
>
timestamp_ms: 1234568
>
`,
`name: "test_histogram2"
help: "Similar histogram as before but now without sparse buckets."
@ -263,6 +335,7 @@ metric: <
unit string
comment string
shs *histogram.Histogram
fhs *histogram.FloatHistogram
e []exemplar.Exemplar
}{
{
@ -353,6 +426,42 @@ metric: <
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{
m: "test_float_histogram",
help: "Test float histogram with many buckets removed to keep it manageable in size.",
},
{
m: "test_float_histogram",
typ: MetricTypeHistogram,
},
{
m: "test_float_histogram",
t: 1234568,
fhs: &histogram.FloatHistogram{
Count: 175.0,
ZeroCount: 2.0,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
},
lset: labels.FromStrings(
"__name__", "test_float_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{
m: "test_histogram2",
help: "Similar histogram as before but now without sparse buckets.",
@ -524,8 +633,7 @@ metric: <
res = res[:0]
case EntryHistogram:
m, ts, shs := p.Histogram()
m, ts, shs, fhs := p.Histogram()
p.Metric(&res)
require.Equal(t, exp[i].m, string(m))
if ts != nil {
@ -536,7 +644,11 @@ metric: <
require.Equal(t, exp[i].lset, res)
res = res[:0]
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].shs, shs)
if shs != nil {
require.Equal(t, exp[i].shs, shs)
} else {
require.Equal(t, exp[i].fhs, fhs)
}
j := 0
for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
require.Equal(t, exp[i].e[j], e)

7
scrape/scrape.go
View File

@ -1512,7 +1512,8 @@ loop:
t := defTime
if isHistogram {
met, parsedTimestamp, h = p.Histogram()
met, parsedTimestamp, h, _ = p.Histogram()
// TODO: ingest float histograms in tsdb.
} else {
met, parsedTimestamp, val = p.Series()
}
@ -1564,7 +1565,9 @@ loop:
}
if isHistogram {
ref, err = app.AppendHistogram(ref, lset, t, h)
if h != nil {
ref, err = app.AppendHistogram(ref, lset, t, h)
}
} else {
ref, err = app.Append(ref, lset, t, val)
}