Merge pull request #82 from prometheus/julius-fix-targets

Fix target pool deadlock and health checking bugs

retrieval/target.go

@@ -138,7 +138,8 @@ func (t *target) Scrape(earliest time.Time, results chan format.Result) (err err
 			done <- true
 		}()
 
-		resp, err := http.Get(t.Address())
+		var resp *http.Response // Don't shadow "err" from the enclosing function.
+		resp, err = http.Get(t.Address())
 		if err != nil {
 			return
 		}

retrieval/targetpool.go

@@ -90,7 +90,11 @@ func (p *TargetPool) runIteration(results chan format.Result, interval time.Dura
 
 		if target.scheduledFor().After(now) {
 			heap.Push(p, target)
-
+			// None of the remaining targets are ready to be scheduled. Signal that
+			// we're done processing them in this scrape iteration.
+			for j := i; j < targetCount; j++ {
+				finished <- true
+			}
 			break
 		}
 
@@ -107,6 +111,6 @@ func (p *TargetPool) runIteration(results chan format.Result, interval time.Dura
 
 	close(finished)
 
-	duration := float64(time.Now().Sub(begin) / time.Millisecond)
+	duration := float64(time.Since(begin) / time.Millisecond)
 	retrievalDurations.Add(map[string]string{intervalKey: interval.String()}, duration)
 }

retrieval/targetpool_test.go

@@ -15,6 +15,7 @@ package retrieval
 
 import (
 	"container/heap"
+	"github.com/prometheus/prometheus/retrieval/format"
 	"github.com/prometheus/prometheus/utility/test"
 	"testing"
 	"time"
@@ -45,8 +46,8 @@ func testTargetPool(t test.Tester) {
 
 	var scenarios = []struct {
 		name    string
-		outputs []output
 		inputs  []input
+		outputs []output
 	}{
 		{
 			name:    "empty",
@@ -160,6 +161,28 @@ func TestTargetPool(t *testing.T) {
 	testTargetPool(t)
 }
 
+func TestTargetPoolIterationWithUnhealthyTargetsFinishes(t *testing.T) {
+	pool := TargetPool{}
+	target := &target{
+		address:   "http://example.com/metrics.json",
+		scheduler: literalScheduler(time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC)),
+	}
+	pool.Push(target)
+
+	done := make(chan bool)
+	go func() {
+		pool.runIteration(make(chan format.Result), time.Duration(0))
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		break
+	case <-time.After(time.Duration(1) * time.Second):
+		t.Fatalf("Targetpool iteration is stuck")
+	}
+}
+
 func BenchmarkTargetPool(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		testTargetPool(b)