From cd73b3d33e064bbd846fc7a26dc8c313d46af382 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Tue, 7 Apr 2020 11:25:57 +0100 Subject: [PATCH] Reduce how much old WAL we keep around. (#7098) Previously we were keeping up to around 6 hours of WAL around by removing 1/3 every hours. This was excessive, so switch to removing 2/3 which will up to around 3 hours of WAL around. This will roughly halve the size of the WAL and halve startup time for those who are I/O bound. This may increase the checkpoint size for those with certain churn patterns, but by much less than we're saving from the segments. Signed-off-by: Brian Brazil --- tsdb/head.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tsdb/head.go b/tsdb/head.go index b8494f618..c0abe150e 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -681,9 +681,11 @@ func (h *Head) Truncate(mint int64) (err error) { if last < 0 { return nil // no segments yet. } - // The lower third of segments should contain mostly obsolete samples. - // If we have less than three segments, it's not worth checkpointing yet. - last = first + (last-first)/3 + // The lower two thirds of segments should contain mostly obsolete samples. + // If we have less than two segments, it's not worth checkpointing yet. + // With the default 2h blocks, this will keeping up to around 3h worth + // of WAL segments. + last = first + (last-first)*2/3 if last <= first { return nil }