From 4d4d822c36b7fdda555aec124eb7b031ed0daa6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Fri, 1 Mar 2024 14:04:54 +0100 Subject: [PATCH] Add native histograms to latency/duration metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dogfood native histograms. Allow dependent projects to migrate to native histograms. I took the defaults from client_golang. Signed-off-by: György Krajcsovits --- .../remote_storage/remote_storage_adapter/main.go | 9 ++++++--- storage/remote/client.go | 13 ++++++++----- storage/remote/queue_manager.go | 15 +++++++++------ tsdb/compact.go | 9 ++++++--- tsdb/db.go | 7 +++++-- tsdb/head.go | 3 +++ web/web.go | 9 ++++++--- 7 files changed, 43 insertions(+), 22 deletions(-) diff --git a/documentation/examples/remote_storage/remote_storage_adapter/main.go b/documentation/examples/remote_storage/remote_storage_adapter/main.go index 85ef1839a..bb348aba7 100644 --- a/documentation/examples/remote_storage/remote_storage_adapter/main.go +++ b/documentation/examples/remote_storage/remote_storage_adapter/main.go @@ -83,9 +83,12 @@ var ( ) sentBatchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "sent_batch_duration_seconds", - Help: "Duration of sample batch send calls to the remote storage.", - Buckets: prometheus.DefBuckets, + Name: "sent_batch_duration_seconds", + Help: "Duration of sample batch send calls to the remote storage.", + Buckets: prometheus.DefBuckets, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"remote"}, ) diff --git a/storage/remote/client.go b/storage/remote/client.go index e765b47c3..5ba0f7117 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -64,11 +64,14 @@ var ( ) remoteReadQueryDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "read_request_duration_seconds", - Help: "Histogram of the latency for remote read requests.", - Buckets: append(prometheus.DefBuckets, 25, 60), + Namespace: namespace, + Subsystem: subsystem, + Name: "read_request_duration_seconds", + Help: "Histogram of the latency for remote read requests.", + Buckets: append(prometheus.DefBuckets, 25, 60), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{remoteName, endpoint}, ) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index 98e4aa54c..01d2db06a 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -214,12 +214,15 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager ConstLabels: constLabels, }) m.sentBatchDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "sent_batch_duration_seconds", - Help: "Duration of send calls to the remote storage.", - Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), - ConstLabels: constLabels, + Namespace: namespace, + Subsystem: subsystem, + Name: "sent_batch_duration_seconds", + Help: "Duration of send calls to the remote storage.", + Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), + ConstLabels: constLabels, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }) m.highestSentTimestamp = &maxTimestamp{ Gauge: prometheus.NewGauge(prometheus.GaugeOpts{ diff --git a/tsdb/compact.go b/tsdb/compact.go index b21ac4d31..3d8d9130c 100644 --- a/tsdb/compact.go +++ b/tsdb/compact.go @@ -112,9 +112,12 @@ func newCompactorMetrics(r prometheus.Registerer) *CompactorMetrics { Help: "Total number of compactions done on overlapping blocks.", }) m.Duration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "prometheus_tsdb_compaction_duration_seconds", - Help: "Duration of compaction runs", - Buckets: prometheus.ExponentialBuckets(1, 2, 14), + Name: "prometheus_tsdb_compaction_duration_seconds", + Help: "Duration of compaction runs", + Buckets: prometheus.ExponentialBuckets(1, 2, 14), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }) m.ChunkSize = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "prometheus_tsdb_compaction_chunk_size_bytes", diff --git a/tsdb/db.go b/tsdb/db.go index f29315517..4998da6aa 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -319,8 +319,11 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { return float64(db.blocks[0].meta.MinTime) }) m.tombCleanTimer = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "prometheus_tsdb_tombstone_cleanup_seconds", - Help: "The time taken to recompact blocks to remove tombstones.", + Name: "prometheus_tsdb_tombstone_cleanup_seconds", + Help: "The time taken to recompact blocks to remove tombstones.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }) m.blocksBytes = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "prometheus_tsdb_storage_blocks_bytes", diff --git a/tsdb/head.go b/tsdb/head.go index dd2142e52..7e6fda9c7 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -480,6 +480,9 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { 60 * 60 * 6, // 6h 60 * 60 * 12, // 12h }, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }), mmapChunksTotal: prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_mmap_chunks_total", diff --git a/web/web.go b/web/web.go index 43b79c235..a87759fb2 100644 --- a/web/web.go +++ b/web/web.go @@ -123,9 +123,12 @@ func newMetrics(r prometheus.Registerer) *metrics { ), requestDuration: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "prometheus_http_request_duration_seconds", - Help: "Histogram of latencies for HTTP requests.", - Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120}, + Name: "prometheus_http_request_duration_seconds", + Help: "Histogram of latencies for HTTP requests.", + Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"handler"}, ),