Fix race during shutdown.

Change-Id: I2f8bf48d92a14f1e5ecde27c1b138734d7653394

main.go

@@ -104,20 +104,10 @@ func (p *prometheus) close() {
 	glog.Info("Rule Executor: Done")
 
 	close(p.unwrittenSamples)
-
+	// Note: Before closing the remaining subsystems (storage, ...), we have
-	if err := p.storage.Close(); err != nil {
+	// to wait until p.unwrittenSamples is actually drained. Therefore,
-		glog.Error("Error closing local storage: ", err)
+	// things are closed in main(), after the loop consuming
-	}
+	// p.unwrittenSamples has finished.
-	glog.Info("Local Storage: Done")
-
-	if p.remoteTSDBQueue != nil {
-		p.remoteTSDBQueue.Close()
-		glog.Info("Remote Storage: Done")
-	}
-
-	close(p.notifications)
-	glog.Info("Sundry Queues: Done")
-	glog.Info("See you next time!")
 }
 
 func main() {
@@ -263,4 +253,21 @@ func main() {
 			}
 		}
 	}
+
+	// Note: It might appear tempting to move the code below into
+	// prometheus.Close(), but we have to wait for the unwrittenSamples loop
+	// above to exit before we can do the below.
+	if err := prometheus.storage.Close(); err != nil {
+		glog.Error("Error closing local storage: ", err)
+	}
+	glog.Info("Local Storage: Done")
+
+	if prometheus.remoteTSDBQueue != nil {
+		prometheus.remoteTSDBQueue.Close()
+		glog.Info("Remote Storage: Done")
+	}
+
+	close(prometheus.notifications)
+	glog.Info("Sundry Queues: Done")
+	glog.Info("See you next time!")
 }

storage/metric/sample.go

@@ -30,9 +30,9 @@ type SamplePair struct {
 	Value     clientmodel.SampleValue
 }
 
-// TODO: can this method be deleted, or is it used in tests?
 // Equal returns true if this SamplePair and o have equal Values and equal
 // Timestamps.
+// TODO: can this method be deleted, or is it used in tests?
 func (s *SamplePair) Equal(o *SamplePair) bool {
 	if s == o {
 		return true