From 3d8d8928befefd8452e966d7584e20196367d8a4 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 18 Mar 2015 19:58:47 +0100 Subject: [PATCH] Increase resilience of the storage against data corruption - step 1. Step 1: Admit the problem by turning the various "panic"s into logged errors, followed by marking the persistence as dirty. --- storage/local/storage.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/storage/local/storage.go b/storage/local/storage.go index 897310dc3..689edbbe9 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -809,7 +809,8 @@ func (s *memorySeriesStorage) writeMemorySeries( series.dropChunks(beforeTime) if len(series.chunkDescs) == 0 { // All chunks dropped from memory series. if !allDroppedFromPersistence { - panic("all chunks dropped from memory but chunks left in persistence") + glog.Errorf("All chunks dropped from memory but chunks left in persistence for fingerprint %v, series %v.", fp, series) + s.persistence.setDirty(true) } s.fpToSeries.del(fp) s.numSeries.Dec() @@ -823,7 +824,9 @@ func (s *memorySeriesStorage) writeMemorySeries( } else { series.chunkDescsOffset -= numDroppedFromPersistence if series.chunkDescsOffset < 0 { - panic("dropped more chunks from persistence than from memory") + glog.Errorf("Dropped more chunks from persistence than from memory for fingerprint %v, series %v.", fp, series) + s.persistence.setDirty(true) + series.chunkDescsOffset = -1 // Makes sure it will be looked at during crash recovery. } } return false