From 37bbc07118f2873d893621d6f06de3d7ba96f5b9 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 18 Apr 2022 01:46:26 +0200 Subject: [PATCH] Histogram: Add jsoniter marshaling This now even enables jsoniter marshaling of Points in an instant query (which previously used the traditional JSON marshaling). Signed-off-by: beorn7 --- promql/value.go | 5 ++ web/api/v1/api.go | 187 ++++++++++++++++++++++++++++++++++++++++- web/api/v1/api_test.go | 1 + 3 files changed, 190 insertions(+), 3 deletions(-) diff --git a/promql/value.go b/promql/value.go index 8627ed5a4c..cd36c9a7e9 100644 --- a/promql/value.go +++ b/promql/value.go @@ -77,6 +77,9 @@ func (s Series) String() string { return fmt.Sprintf("%s =>\n%s", s.Metric, strings.Join(vals, "\n")) } +// MarshalJSON is mirrored in web/api/v1/api.go for efficiency reasons. +// This implementation is still provided for debug purposes and usage +// without jsoniter. func (s Series) MarshalJSON() ([]byte, error) { // Note that this is rather inefficient because it re-creates the whole // series, just separated by Histogram Points and Value Points. For API @@ -177,6 +180,8 @@ func (s Sample) String() string { return fmt.Sprintf("%s => %s", s.Metric, s.Point) } +// MarshalJSON is mirrored in web/api/v1/api.go with jsoniter because Point +// wouldn't be marshaled with jsoniter in all cases otherwise. func (s Sample) MarshalJSON() ([]byte, error) { if s.Point.H == nil { v := struct { diff --git a/web/api/v1/api.go b/web/api/v1/api.go index c0846cde58..92dc254c34 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -203,7 +203,8 @@ type API struct { } func init() { - // TODO(beorn7): Need this for promql.Series and promql.Sample, too. + jsoniter.RegisterTypeEncoderFunc("promql.Series", marshalSeriesJSON, marshalSeriesJSONIsEmpty) + jsoniter.RegisterTypeEncoderFunc("promql.Sample", marshalSampleJSON, marshalSampleJSONIsEmpty) jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty) jsoniter.RegisterTypeEncoderFunc("exemplar.Exemplar", marshalExemplarJSON, marshalExemplarJSONEmpty) } @@ -1815,6 +1816,123 @@ OUTER: return matcherSets, nil } +// marshalSeriesJSON writes something like the following: +// +// { +// "metric" : { +// "__name__" : "up", +// "job" : "prometheus", +// "instance" : "localhost:9090" +// }, +// "values": [ +// [ 1435781451.781, "1" ], +// < more values> +// ], +// "histograms": [ +// [ 1435781451.781, { < histogram, see below > } ], +// < more histograms > +// ], +// }, +func marshalSeriesJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + s := *((*promql.Series)(ptr)) + stream.WriteObjectStart() + stream.WriteObjectField(`metric`) + m, err := s.Metric.MarshalJSON() + if err != nil { + stream.Error = err + return + } + stream.SetBuffer(append(stream.Buffer(), m...)) + + // We make two passes through the series here: In the first marshaling + // all value points, in the second marshaling all histogram + // points. That's probably cheaper than just one pass in which we copy + // out histogram Points into a newly allocated slice for separate + // marshaling. (Could be benchmarked, though.) + var foundValue, foundHistogram bool + for _, p := range s.Points { + if p.H == nil { + stream.WriteMore() + if !foundValue { + stream.WriteObjectField(`values`) + stream.WriteArrayStart() + } + foundValue = true + marshalPointJSON(unsafe.Pointer(&p), stream) + } else { + foundHistogram = true + } + } + if foundValue { + stream.WriteArrayEnd() + } + if foundHistogram { + firstHistogram := true + for _, p := range s.Points { + if p.H != nil { + stream.WriteMore() + if firstHistogram { + stream.WriteObjectField(`histograms`) + stream.WriteArrayStart() + } + firstHistogram = false + marshalPointJSON(unsafe.Pointer(&p), stream) + } + } + stream.WriteArrayEnd() + } + stream.WriteObjectEnd() +} + +func marshalSeriesJSONIsEmpty(ptr unsafe.Pointer) bool { + return false +} + +// marshalSampleJSON writes something like the following for normal value samples: +// +// { +// "metric" : { +// "__name__" : "up", +// "job" : "prometheus", +// "instance" : "localhost:9090" +// }, +// "value": [ 1435781451.781, "1" ] +// }, +// +// For histogram samples, it writes something like this: +// +// { +// "metric" : { +// "__name__" : "up", +// "job" : "prometheus", +// "instance" : "localhost:9090" +// }, +// "histogram": [ 1435781451.781, { < histogram, see below > } ] +// }, +func marshalSampleJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + s := *((*promql.Sample)(ptr)) + stream.WriteObjectStart() + stream.WriteObjectField(`metric`) + m, err := s.Metric.MarshalJSON() + if err != nil { + stream.Error = err + return + } + stream.SetBuffer(append(stream.Buffer(), m...)) + stream.WriteMore() + if s.Point.H == nil { + stream.WriteObjectField(`value`) + } else { + stream.WriteObjectField(`histogram`) + } + marshalPointJSON(unsafe.Pointer(&s.Point), stream) + stream.WriteObjectEnd() +} + +func marshalSampleJSONIsEmpty(ptr unsafe.Pointer) bool { + return false +} + // marshalPointJSON writes `[ts, "val"]`. func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { p := *((*promql.Point)(ptr)) @@ -1833,9 +1951,72 @@ func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool { return false } +// marshalHistogramJSON writes something like: +// +// { +// "count": "42", +// "sum": "34593.34", +// "buckets": [ +// [ 3, "-0.25", "0.25", "3"], +// [ 0, "0.25", "0.5", "12"], +// [ 0, "0.5", "1", "21"], +// [ 0, "2", "4", "6"] +// ] +// } +// +// The 1st element in each bucket array determines if the boundaries are +// inclusive (AKA closed) or exclusive (AKA open): +// 0: lower exclusive, upper inclusive +// 1: lower inclusive, upper exclusive +// 2: both exclusive +// 3: both inclusive +// +// The 2nd and 3rd elements are the lower and upper boundary. The 4th element is +// the bucket count. func marshalHistogram(h *histogram.FloatHistogram, stream *jsoniter.Stream) { - // TODO(beorn7): Implement. - stream.WriteString("TODO render histogram") + stream.WriteObjectStart() + stream.WriteObjectField(`count`) + marshalValue(h.Count, stream) + stream.WriteMore() + stream.WriteObjectField(`sum`) + marshalValue(h.Sum, stream) + + bucketFound := false + it := h.AllBucketIterator() + for it.Next() { + stream.WriteMore() + if !bucketFound { + stream.WriteObjectField(`buckets`) + stream.WriteArrayStart() + } + bucketFound = true + bucket := it.At() + boundaries := 2 // Exclusive on both sides AKA open interval. + if bucket.LowerInclusive { + if bucket.UpperInclusive { + boundaries = 3 // Inclusive on both sides AKA closed interval. + } else { + boundaries = 1 // Inclusive only on lower end AKA right open. + } + } else { + if bucket.UpperInclusive { + boundaries = 0 // Inclusive only on upper end AKA left open. + } + } + stream.WriteArrayStart() + stream.WriteInt(boundaries) + stream.WriteMore() + marshalValue(bucket.Lower, stream) + stream.WriteMore() + marshalValue(bucket.Upper, stream) + stream.WriteMore() + marshalValue(bucket.Count, stream) + stream.WriteArrayEnd() + } + if bucketFound { + stream.WriteArrayEnd() + } + stream.WriteObjectEnd() } // marshalExemplarJSON writes. diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 55155f8bcf..919274bee8 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -2784,6 +2784,7 @@ func TestRespond(t *testing.T) { Result: promql.Matrix{ promql.Series{ Points: []promql.Point{{V: 1, T: 1000}}, + // TODO(beorn7): Add histogram points. Metric: labels.FromStrings("__name__", "foo"), }, },