From c40a83f386d2d95faac91c237e180ff8beb60ce1 Mon Sep 17 00:00:00 2001
From: Callum Styan <callumstyan@gmail.com>
Date: Fri, 19 Jul 2019 14:53:26 -0700
Subject: [PATCH] Add metrics for max shards, min shards, and desired shards.

Signed-off-by: Callum Styan <callumstyan@gmail.com>
---
 storage/remote/queue_manager.go | 40 +++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go
index e36a6c987..0388512cb 100644
--- a/storage/remote/queue_manager.go
+++ b/storage/remote/queue_manager.go
@@ -143,6 +143,33 @@ var (
 		},
 		[]string{queue},
 	)
+	maxNumShards = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "shards_max",
+			Help:      "The maximum number of shards that the queue is allowed to run.",
+		},
+		[]string{queue},
+	)
+	minNumShards = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "shards_min",
+			Help:      "The minimum number of shards that the queue is allowed to run.",
+		},
+		[]string{queue},
+	)
+	desiredNumShards = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "shards_desired",
+			Help:      "The number of shards that the queues shard calculation wants to run based on the rate of samples in vs. samples out.",
+		},
+		[]string{queue},
+	)
 )
 
 // StorageClient defines an interface for sending a batch of samples to an
@@ -189,6 +216,9 @@ type QueueManager struct {
 	succeededSamplesTotal      prometheus.Counter
 	retriedSamplesTotal        prometheus.Counter
 	shardCapacity              prometheus.Gauge
+	maxNumShards               prometheus.Gauge
+	minNumShards               prometheus.Gauge
+	desiredNumShards           prometheus.Gauge
 }
 
 // NewQueueManager builds a new QueueManager.
@@ -293,10 +323,16 @@ func (t *QueueManager) Start() {
 	t.succeededSamplesTotal = succeededSamplesTotal.WithLabelValues(name)
 	t.retriedSamplesTotal = retriedSamplesTotal.WithLabelValues(name)
 	t.shardCapacity = shardCapacity.WithLabelValues(name)
+	t.maxNumShards = maxNumShards.WithLabelValues(name)
+	t.minNumShards = minNumShards.WithLabelValues(name)
+	t.desiredNumShards = desiredNumShards.WithLabelValues(name)
 
 	// Initialise some metrics.
 	t.shardCapacity.Set(float64(t.cfg.Capacity))
 	t.pendingSamplesMetric.Set(0)
+	t.maxNumShards.Set(float64(t.cfg.MaxShards))
+	t.minNumShards.Set(float64(t.cfg.MinShards))
+	t.desiredNumShards.Set(float64(t.cfg.MinShards))
 
 	t.shards.start(t.numShards)
 	t.watcher.Start()
@@ -336,6 +372,9 @@ func (t *QueueManager) Stop() {
 	succeededSamplesTotal.DeleteLabelValues(name)
 	retriedSamplesTotal.DeleteLabelValues(name)
 	shardCapacity.DeleteLabelValues(name)
+	maxNumShards.DeleteLabelValues(name)
+	minNumShards.DeleteLabelValues(name)
+	desiredNumShards.DeleteLabelValues(name)
 }
 
 // StoreSeries keeps track of which series we know about for lookups when sending samples to remote.
@@ -483,6 +522,7 @@ func (t *QueueManager) calculateDesiredShards() {
 	}
 
 	numShards := int(math.Ceil(desiredShards))
+	t.desiredNumShards.Set(float64(numShards))
 	if numShards > t.cfg.MaxShards {
 		numShards = t.cfg.MaxShards
 	} else if numShards < t.cfg.MinShards {