From 0e684ca20505fc16f286dd5e040b59d6b717d0a3 Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Tue, 26 Nov 2019 06:22:56 -0700 Subject: [PATCH 1/2] Fix unknown type in sharding up log Signed-off-by: Chris Marchbanks --- storage/remote/queue_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index c47039b4b..6b50b76d1 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -500,7 +500,7 @@ func (t *QueueManager) updateShardsLoop() { // to stay close to shardUpdateDuration. select { case t.reshardChan <- desiredShards: - level.Info(t.logger).Log("msg", "Remote storage resharding", "from", t.numShards, "to", numShards) + level.Info(t.logger).Log("msg", "Remote storage resharding", "from", t.numShards, "to", desiredShards) t.numShards = desiredShards default: level.Info(t.logger).Log("msg", "Currently resharding, skipping.") From 6f34e35b3e0dbb1e9b932bc96cbb283a76e0355b Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Tue, 26 Nov 2019 06:26:03 -0700 Subject: [PATCH 2/2] Record the exact value of desired shards in metric It is possible that desired shards is always a bit higher than the number of shards (less than 30%) and by exporting desired shards as the raw number it will be easy to tell if a Prometheus is in that situation. Signed-off-by: Chris Marchbanks --- storage/remote/queue_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index 6b50b76d1..923cb14e9 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -565,6 +565,7 @@ func (t *QueueManager) calculateDesiredShards() int { timePerSample = samplesOutDuration / samplesOutRate desiredShards = timePerSample * (samplesInRate + t.integralAccumulator) ) + t.desiredNumShards.Set(desiredShards) level.Debug(t.logger).Log("msg", "QueueManager.calculateDesiredShards", "samplesInRate", samplesInRate, "samplesOutRate", samplesOutRate, @@ -591,7 +592,6 @@ func (t *QueueManager) calculateDesiredShards() int { } numShards := int(math.Ceil(desiredShards)) - t.desiredNumShards.Set(float64(numShards)) if numShards > t.cfg.MaxShards { numShards = t.cfg.MaxShards } else if numShards < t.cfg.MinShards {