From 7f3aca62c4178649fe271cf31fc4652a3ef7e4b4 Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Thu, 2 Jan 2020 10:30:56 -0700 Subject: [PATCH] Only reduce the number of shards when caught up. Signed-off-by: Chris Marchbanks --- storage/remote/queue_manager.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index c3d97430d..799fdc9b4 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -526,7 +526,8 @@ func (t *QueueManager) calculateDesiredShards() int { samplesPendingRate = samplesInRate*samplesKeptRatio - samplesOutRate highestSent = t.highestSentTimestampMetric.Get() highestRecv = highestTimestamp.Get() - samplesPending = (highestRecv - highestSent) * samplesInRate * samplesKeptRatio + delay = highestRecv - highestSent + samplesPending = delay * samplesInRate * samplesKeptRatio ) if samplesOutRate <= 0 { @@ -577,6 +578,12 @@ func (t *QueueManager) calculateDesiredShards() int { } numShards := int(math.Ceil(desiredShards)) + // Do not downshard if we are more than ten seconds back. + if numShards < t.numShards && delay > 10.0 { + level.Debug(t.logger).Log("msg", "Not downsharding due to being too far behind") + return t.numShards + } + if numShards > t.cfg.MaxShards { numShards = t.cfg.MaxShards } else if numShards < t.cfg.MinShards {