diff --git a/CHANGELOG.md b/CHANGELOG.md index d65c6bd99..00919b057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## 2.30.3 / 2021-10-05 + +* [BUGFIX] TSDB: Fix panic on failed snapshot replay. #9438 +* [BUGFIX] TSDB: Don't fail snapshot replay with exemplar storage disabled when the snapshot contains exemplars. #9438 + +## 2.30.2 / 2021-10-01 + +* [BUGFIX] TSDB: Don't error on overlapping m-mapped chunks during WAL replay. #9381 + ## 2.30.1 / 2021-09-28 * [ENHANCEMENT] Remote Write: Redact remote write URL when used for metric label. #9383 diff --git a/VERSION b/VERSION index bcec02eeb..e88ba89ba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.30.1 +2.30.3 diff --git a/tsdb/head.go b/tsdb/head.go index 9cf68d6ce..cd0f9bd16 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -176,6 +176,10 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOpti stats = NewHeadStats() } + if !opts.EnableExemplarStorage { + opts.MaxExemplars.Store(0) + } + h := &Head{ wal: wal, logger: l, @@ -211,7 +215,16 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOpti func (h *Head) resetInMemoryState() error { var err error - em := NewExemplarMetrics(h.reg) + var em *ExemplarMetrics + if h.exemplars != nil { + ce, ok := h.exemplars.(*CircularExemplarStorage) + if ok { + em = ce.metrics + } + } + if em == nil { + em = NewExemplarMetrics(h.reg) + } es, err := NewCircularExemplarStorage(h.opts.MaxExemplars.Load(), em) if err != nil { return err diff --git a/tsdb/head_test.go b/tsdb/head_test.go index 7d4b63843..e404e94f6 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -2753,7 +2754,16 @@ func TestChunkSnapshot(t *testing.T) { // Test the replay of snapshot. head.opts.EnableMemorySnapshotOnShutdown = true // Enabled to read from snapshot. + + // Disabling exemplars to check that it does not hard fail replay + // https://github.com/prometheus/prometheus/issues/9437#issuecomment-933285870. + head.opts.EnableExemplarStorage = false + head.opts.MaxExemplars.Store(0) + expExemplars = expExemplars[:0] + openHeadAndCheckReplay() + + require.Equal(t, 0.0, prom_testutil.ToFloat64(head.metrics.snapshotReplayErrorTotal)) } } @@ -2805,7 +2815,8 @@ func TestSnapshotError(t *testing.T) { // Create new Head which should replay this snapshot. w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false) require.NoError(t, err) - head, err = NewHead(nil, nil, w, head.opts, nil) + // Testing https://github.com/prometheus/prometheus/issues/9437 with the registry. + head, err = NewHead(prometheus.NewRegistry(), nil, w, head.opts, nil) require.NoError(t, err) require.NoError(t, head.Init(math.MinInt64)) diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index ba98c2e46..478829824 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -948,6 +948,11 @@ Outer: } } + if !h.opts.EnableExemplarStorage || h.opts.MaxExemplars.Load() <= 0 { + // Exemplar storage is disabled. + continue Outer + } + decbuf := encoding.Decbuf{B: rec[1:]} exemplarBuf = exemplarBuf[:0] @@ -969,7 +974,7 @@ Outer: Value: e.V, Ts: e.T, }); err != nil { - loopErr = errors.Wrap(err, "append exemplar") + loopErr = errors.Wrap(err, "add exemplar") break Outer } }