From 2c8fdcbc233cf2bf9f09e94ca195aaa0197877d3 Mon Sep 17 00:00:00 2001
From: Bjoern Rabenstein <beorn@soundcloud.com>
Date: Thu, 22 Jan 2015 14:42:15 +0100
Subject: [PATCH] Remove a deadlock during shutdown.

If queries are still running when the shutdown is initiated, they will
finish _during_ the shutdown. In that case, they might request chunk
eviction upon unpinning their pinned chunks. That might completely
fill the evict request queue _after_ draining it during storage
shutdown. If that ever happens (which is the case if there are _many_
queries still running during shutdown), the affected queries will be
stuck while keeping a fingerprint locked. The checkpointing can then
not process that fingerprint (or one that shares the same lock). And
then we are deadlocked.
---
 storage/local/storage.go | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/storage/local/storage.go b/storage/local/storage.go
index 9b8c385d6..e778bf915 100644
--- a/storage/local/storage.go
+++ b/storage/local/storage.go
@@ -467,18 +467,17 @@ func (s *memorySeriesStorage) handleEvictList() {
 				s.maybeEvict()
 			}
 		case <-s.evictStopping:
-			// Drain evictRequests to not let requesters hang.
-			for {
-				select {
-				case <-s.evictRequests:
-					// Do nothing.
-				default:
-					ticker.Stop()
-					glog.Info("Chunk eviction stopped.")
-					close(s.evictStopped)
-					return
+			// Drain evictRequests forever in a goroutine to not let
+			// requesters hang.
+			go func() {
+				for {
+					<-s.evictRequests
 				}
-			}
+			}()
+			ticker.Stop()
+			glog.Info("Chunk eviction stopped.")
+			close(s.evictStopped)
+			return
 		}
 	}
 }