# Two histograms with 4 buckets each (x_sum and x_count not included, # only buckets). Lowest bucket for one histogram < 0, for the other > # 0. They have the same name, just separated by label. Not useful in # practice, but can happen (if clients change bucketing), and the # server has to cope with it. # Test histogram. load 5m testhistogram_bucket{le="0.1", start="positive"} 0+5x10 testhistogram_bucket{le=".2", start="positive"} 0+7x10 testhistogram_bucket{le="1e0", start="positive"} 0+11x10 testhistogram_bucket{le="+Inf", start="positive"} 0+12x10 testhistogram_bucket{le="-.2", start="negative"} 0+1x10 testhistogram_bucket{le="-0.1", start="negative"} 0+2x10 testhistogram_bucket{le="0.3", start="negative"} 0+2x10 testhistogram_bucket{le="+Inf", start="negative"} 0+3x10 # Now a more realistic histogram per job and instance to test aggregation. load 5m request_duration_seconds_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 request_duration_seconds_bucket{job="job1", instance="ins1", le="0.2"} 0+3x10 request_duration_seconds_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="0.1"} 0+2x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="0.2"} 0+5x10 request_duration_seconds_bucket{job="job1", instance="ins2", le="+Inf"} 0+6x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="0.1"} 0+3x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="0.2"} 0+4x10 request_duration_seconds_bucket{job="job2", instance="ins1", le="+Inf"} 0+6x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="0.1"} 0+4x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="0.2"} 0+7x10 request_duration_seconds_bucket{job="job2", instance="ins2", le="+Inf"} 0+9x10 # Different le representations in one histogram. load 5m mixed_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="0.2"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="2e-1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="2.0e-1"} 0+1x10 mixed_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 mixed_bucket{job="job1", instance="ins2", le="+inf"} 0+0x10 mixed_bucket{job="job1", instance="ins2", le="+Inf"} 0+0x10 # Quantile too low. eval instant at 50m histogram_quantile(-0.1, testhistogram_bucket) {start="positive"} -Inf {start="negative"} -Inf # Quantile too high. eval instant at 50m histogram_quantile(1.01, testhistogram_bucket) {start="positive"} +Inf {start="negative"} +Inf # Quantile value in lowest bucket, which is positive. eval instant at 50m histogram_quantile(0, testhistogram_bucket{start="positive"}) {start="positive"} 0 # Quantile value in lowest bucket, which is negative. eval instant at 50m histogram_quantile(0, testhistogram_bucket{start="negative"}) {start="negative"} -0.2 # Quantile value in highest bucket. eval instant at 50m histogram_quantile(1, testhistogram_bucket) {start="positive"} 1 {start="negative"} 0.3 # Finally some useful quantiles. eval instant at 50m histogram_quantile(0.2, testhistogram_bucket) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.5, testhistogram_bucket) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.8, testhistogram_bucket) {start="positive"} 0.72 {start="negative"} 0.3 # More realistic with rates. eval instant at 50m histogram_quantile(0.2, rate(testhistogram_bucket[5m])) {start="positive"} 0.048 {start="negative"} -0.2 eval instant at 50m histogram_quantile(0.5, rate(testhistogram_bucket[5m])) {start="positive"} 0.15 {start="negative"} -0.15 eval instant at 50m histogram_quantile(0.8, rate(testhistogram_bucket[5m])) {start="positive"} 0.72 {start="negative"} 0.3 # Aggregated histogram: Everything in one. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le)) {} 0.075 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le)) {} 0.1277777777777778 # Aggregated histogram: Everything in one. Now with avg, which does not change anything. eval instant at 50m histogram_quantile(0.3, avg(rate(request_duration_seconds_bucket[5m])) by (le)) {} 0.075 eval instant at 50m histogram_quantile(0.5, avg(rate(request_duration_seconds_bucket[5m])) by (le)) {} 0.12777777777777778 # Aggregated histogram: By job. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, instance)) {instance="ins1"} 0.075 {instance="ins2"} 0.075 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, instance)) {instance="ins1"} 0.1333333333 {instance="ins2"} 0.125 # Aggregated histogram: By instance. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, job)) {job="job1"} 0.1 {job="job2"} 0.0642857142857143 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, job)) {job="job1"} 0.14 {job="job2"} 0.1125 # Aggregated histogram: By job and instance. eval instant at 50m histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[5m])) by (le, job, instance)) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[5m])) by (le, job, instance)) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.1333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.1166666666666667 # The unaggregated histogram for comparison. Same result as the previous one. eval instant at 50m histogram_quantile(0.3, rate(request_duration_seconds_bucket[5m])) {instance="ins1", job="job1"} 0.11 {instance="ins2", job="job1"} 0.09 {instance="ins1", job="job2"} 0.06 {instance="ins2", job="job2"} 0.0675 eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds_bucket[5m])) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} 0.13333333333333333 {instance="ins1", job="job2"} 0.1 {instance="ins2", job="job2"} 0.11666666666666667 # A histogram with nonmonotonic bucket counts. This may happen when recording # rule evaluation or federation races scrape ingestion, causing some buckets # counts to be derived from fewer samples. The wrong answer we want to avoid # is for histogram_quantile(0.99, nonmonotonic_bucket) to return ~1000 instead # of 1. load 5m nonmonotonic_bucket{le="0.1"} 0+1x10 nonmonotonic_bucket{le="1"} 0+9x10 nonmonotonic_bucket{le="10"} 0+8x10 nonmonotonic_bucket{le="100"} 0+8x10 nonmonotonic_bucket{le="1000"} 0+9x10 nonmonotonic_bucket{le="+Inf"} 0+9x10 # Nonmonotonic buckets eval instant at 50m histogram_quantile(0.99, nonmonotonic_bucket) {} 0.989875 # Buckets with different representations of the same upper bound. eval instant at 50m histogram_quantile(0.5, rate(mixed_bucket[5m])) {instance="ins1", job="job1"} 0.15 {instance="ins2", job="job1"} NaN eval instant at 50m histogram_quantile(0.75, rate(mixed_bucket[5m])) {instance="ins1", job="job1"} 0.2 {instance="ins2", job="job1"} NaN eval instant at 50m histogram_quantile(1, rate(mixed_bucket[5m])) {instance="ins1", job="job1"} 0.2 {instance="ins2", job="job1"} NaN load 5m empty_bucket{le="0.1", job="job1", instance="ins1"} 0x10 empty_bucket{le="0.2", job="job1", instance="ins1"} 0x10 empty_bucket{le="+Inf", job="job1", instance="ins1"} 0x10 eval instant at 50m histogram_quantile(0.2, rate(empty_bucket[5m])) {instance="ins1", job="job1"} NaN