From a52980e0a879eccff6eca7612b5bb8454b52ba00 Mon Sep 17 00:00:00 2001
From: Fabian Reinartz <fab.reinartz@gmail.com>
Date: Mon, 27 Mar 2017 19:05:34 +0200
Subject: [PATCH] Add workaround for deadlocks

This adds a workaround to avoid deadlocks for inconsistent write lock
order across headBlocks.
Things keep working if transactions only append data for the same
timestamp, which is generally the case for Prometheus.

Full behavior should be restored in a subsequent change.
---
 db.go   | 43 ++++++++++++++++++++++++++++++-------------
 head.go |  4 ++++
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/db.go b/db.go
index 9e50d5b20..93e2dfd1a 100644
--- a/db.go
+++ b/db.go
@@ -462,18 +462,22 @@ func (db *DB) Appender() Appender {
 	db.mtx.RLock()
 	a := &dbAppender{db: db}
 
-	// Only instantiate appender after returning the headmtx to avoid
-	// questionable locking order.
-	db.headmtx.RLock()
-	app := db.appendable()
-	db.headmtx.RUnlock()
+	// XXX(fabxc): turn off creating initial appender as it will happen on-demand
+	// anyway. For now this, with combination of only having a single timestamp per batch,
+	// prevents opening more than one appender and hitting an unresolved deadlock (#11).
+	//
+	// // Only instantiate appender after returning the headmtx to avoid
+	// // questionable locking order.
+	// db.headmtx.RLock()
+	// app := db.appendable()
+	// db.headmtx.RUnlock()
 
-	for _, b := range app {
-		a.heads = append(a.heads, &metaAppender{
-			meta: b.Meta(),
-			app:  b.Appender().(*headAppender),
-		})
-	}
+	// for _, b := range app {
+	// 	a.heads = append(a.heads, &metaAppender{
+	// 		meta: b.Meta(),
+	// 		app:  b.Appender().(*headAppender),
+	// 	})
+	// }
 
 	return a
 }
@@ -551,14 +555,27 @@ func (a *dbAppender) appenderFor(t int64) (*metaAppender, error) {
 
 		a.db.headmtx.Unlock()
 
-		// Instantiate appenders after returning headmtx to avoid questionable
-		// locking order.
+		// XXX(fabxc): temporary workaround. See comment on instantiating DB.Appender.
 		for _, b := range newHeads {
+			// Only get appender for the block with the specific timestamp.
+			if t >= b.Meta().MaxTime {
+				continue
+			}
 			a.heads = append(a.heads, &metaAppender{
 				app:  b.Appender(),
 				meta: b.Meta(),
 			})
+			break
 		}
+
+		// // Instantiate appenders after returning headmtx to avoid questionable
+		// // locking order.
+		// for _, b := range newHeads {
+		// 	a.heads = append(a.heads, &metaAppender{
+		// 		app:  b.Appender(),
+		// 		meta: b.Meta(),
+		// 	})
+		// }
 	}
 	for i := len(a.heads) - 1; i >= 0; i-- {
 		if h := a.heads[i]; t >= h.meta.MinTime {
diff --git a/head.go b/head.go
index d1b9a9200..328a8e564 100644
--- a/head.go
+++ b/head.go
@@ -34,6 +34,10 @@ var (
 	ErrOutOfBounds = errors.New("out of bounds")
 )
 
+// func init() {
+// 	deadlock.Opts.OnPotentialDeadlock = func() { fmt.Println("found deadlock") }
+// }
+
 // headBlock handles reads and writes of time series data within a time window.
 type headBlock struct {
 	mtx sync.RWMutex