Use boolean function instead of postings to drop WAL series

There is not guarantee or requirement for WAL writers to only add
series entries in increasing order of IDs. A postings list cannot look
back and thus unordered WAL entries would skip over IDs to not truncate
from the WAL.
We replace it with a simple boolean check function that does not require
order.

head.go

@@ -190,6 +190,10 @@ func (h *Head) ReadWAL() error {
 	r := h.wal.Reader()
 	mint := h.MinTime()
 
+	// Track number of samples that referenced a series we don't know about
+	// for error reporting.
+	var unknownRefs int
+
 	seriesFunc := func(series []RefSeries) error {
 		for _, s := range series {
 			h.getOrCreateWithID(s.Ref, s.Labels.Hash(), s.Labels)
@@ -207,7 +211,7 @@ func (h *Head) ReadWAL() error {
 			}
 			ms := h.series.getByID(s.Ref)
 			if ms == nil {
-				h.logger.Log("msg", "unknown series reference in WAL", "ref", s.Ref, "ts", s.T, "mint", mint)
+				unknownRefs++
 				continue
 			}
 			_, chunkCreated := ms.append(s.T, s.V)
@@ -230,6 +234,8 @@ func (h *Head) ReadWAL() error {
 		return nil
 	}
 
+	h.logger.Log("msg", "unknown series references in WAL samples", "count", unknownRefs)
+
 	if err := r.Read(seriesFunc, samplesFunc, deletesFunc); err != nil {
 		return errors.Wrap(err, "consume WAL")
 	}
@@ -267,12 +273,10 @@ func (h *Head) Truncate(mint int64) error {
 
 	start = time.Now()
 
-	p, err := h.indexRange(mint, math.MaxInt64).Postings(allPostingsKey.Name, allPostingsKey.Value)
+	keep := func(id uint64) bool {
-	if err != nil {
+		return h.series.getByID(id) != nil
-		return err
 	}
-
+	if err := h.wal.Truncate(mint, keep); err == nil {
-	if err := h.wal.Truncate(mint, p); err == nil {
 		h.logger.Log("msg", "WAL truncation completed", "duration", time.Since(start))
 	} else {
 		h.logger.Log("msg", "WAL truncation failed", "err", err, "duration", time.Since(start))

wal.go

@@ -71,7 +71,7 @@ type WAL interface {
 	LogSeries([]RefSeries) error
 	LogSamples([]RefSample) error
 	LogDeletes([]Stone) error
-	Truncate(int64, Postings) error
+	Truncate(mint int64, keep func(uint64) bool) error
 	Close() error
 }
 
@@ -87,7 +87,7 @@ func (w nopWAL) Reader() WALReader                       { return w }
 func (nopWAL) LogSeries([]RefSeries) error               { return nil }
 func (nopWAL) LogSamples([]RefSample) error              { return nil }
 func (nopWAL) LogDeletes([]Stone) error                  { return nil }
-func (nopWAL) Truncate(int64, Postings) error            { return nil }
+func (nopWAL) Truncate(int64, func(uint64) bool) error   { return nil }
 func (nopWAL) Close() error                              { return nil }
 
 // WALReader reads entries from a WAL.
@@ -272,8 +272,9 @@ func (w *SegmentWAL) putBuffer(b *encbuf) {
 	w.buffers.Put(b)
 }
 
-// Truncate deletes the values prior to mint and the series entries not in p.
+// Truncate deletes the values prior to mint and the series which the keep function
-func (w *SegmentWAL) Truncate(mint int64, p Postings) error {
+// does not indiciate to preserve.
+func (w *SegmentWAL) Truncate(mint int64, keep func(uint64) bool) error {
 	// The last segment is always active.
 	if len(w.files) < 2 {
 		return nil
@@ -314,7 +315,6 @@ func (w *SegmentWAL) Truncate(mint int64, p Postings) error {
 		activeSeries = []RefSeries{}
 	)
 
-Loop:
 	for r.next() {
 		rt, flag, byt := r.at()
 
@@ -328,10 +328,7 @@ Loop:
 		activeSeries = activeSeries[:0]
 
 		for _, s := range series {
-			if !p.Seek(s.Ref) {
+			if keep(s.Ref) {
-				break Loop
-			}
-			if p.At() == s.Ref {
 				activeSeries = append(activeSeries, s)
 			}
 		}

wal_test.go

@@ -154,12 +154,16 @@ func TestSegmentWAL_Truncate(t *testing.T) {
 	boundarySeries := w.files[len(w.files)/2].minSeries
 
 	// We truncate while keeping every 2nd series.
-	keep := []uint64{}
+	keep := map[uint64]struct{}{}
 	for i := 1; i <= numMetrics; i += 2 {
-		keep = append(keep, uint64(i))
+		keep[uint64(i)] = struct{}{}
+	}
+	keepf := func(id uint64) bool {
+		_, ok := keep[id]
+		return ok
 	}
 
-	err = w.Truncate(1000, newListPostings(keep))
+	err = w.Truncate(1000, keepf)
 	require.NoError(t, err)
 
 	var expected []RefSeries
@@ -172,7 +176,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
 
 	// Call Truncate once again to see whether we can read the written file without
 	// creating a new WAL.
-	err = w.Truncate(1000, newListPostings(keep))
+	err = w.Truncate(1000, keepf)
 	require.NoError(t, err)
 	require.NoError(t, w.Close())