From 6544f95403301979fa5b2668e4750ac29c6aaf03 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 25 Sep 2020 15:44:47 +0200 Subject: [PATCH] Introduce timestamp tolerance in scrapes Signed-off-by: Julien Pivotto --- scrape/scrape.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scrape/scrape.go b/scrape/scrape.go index eb0294c85..f8de16bed 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -47,6 +47,11 @@ import ( "github.com/prometheus/prometheus/storage" ) +// Temporary tolerance for scrape appends timestamps alignment, to enable better +// compression at the TSDB level. +// See https://github.com/prometheus/prometheus/issues/7846 +const scrapeTimestampTolerance = 2 * time.Millisecond + var errNameLabelMandatory = fmt.Errorf("missing metric name (%s label)", labels.MetricName) var ( @@ -1033,6 +1038,16 @@ func (sl *scrapeLoop) scrapeAndReport(interval, timeout time.Duration, last time ) } + // Temporary workaround for a jitter in go timers that causes disk space + // increase in TSDB. + // See https://github.com/prometheus/prometheus/issues/7846 + if last.IsZero() && interval > 100*scrapeTimestampTolerance { + exactStart := last.Add(interval) + if t := start.Sub(exactStart); t >= -scrapeTimestampTolerance && t <= scrapeTimestampTolerance { + start = exactStart + } + } + b := sl.buffers.Get(sl.lastScrapeSize).([]byte) defer sl.buffers.Put(b) buf := bytes.NewBuffer(b)