From ec3d02019e84d9d793d2e137891dd7ea6d19ea60 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Thu, 14 Apr 2022 10:03:07 -0400 Subject: [PATCH] Pass the correct context to staleness Appender (#10588) OTel Collector prints the following error when a target disappears: ``` 2022-04-13T14:20:24.932-0400 warn scrape/scrape.go:1408 Stale append failed {"kind": "receiver", "name": "prometheus", "scrape_pool": "beep-boop", "target": "http://localhost:9090/metrics", "error": "transaction aborted"} ``` This `transaction aborted` error is returned by the custom appender that is used by the collector when the context of the appender is cancelled: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b7bf11174ec4d16727118b69b71c832b8d17d9b2/receiver/prometheusreceiver/internal/otlp_transaction.go#L81-L82 We call `endOfRunStaleness` after `sl.stop()` which cancels `sl.ctx`. The other `.Appender()` calls use `parentCtx` for the same reason. This hasn't come up so far because Prometheus' Appender implementation just ignores the context passed. Signed-off-by: Goutham Veeramachaneni --- scrape/scrape.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrape/scrape.go b/scrape/scrape.go index e8d5147df..e34cc16e1 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1379,7 +1379,8 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int // Call sl.append again with an empty scrape to trigger stale markers. // If the target has since been recreated and scraped, the // stale markers will be out of order and ignored. - app := sl.appender(sl.ctx) + // sl.context would have been cancelled, hence using sl.parentCtx. + app := sl.appender(sl.parentCtx) var err error defer func() { if err != nil { @@ -1393,7 +1394,7 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int }() if _, _, _, err = sl.append(app, []byte{}, "", staleTime); err != nil { app.Rollback() - app = sl.appender(sl.ctx) + app = sl.appender(sl.parentCtx) level.Warn(sl.l).Log("msg", "Stale append failed", "err", err) } if err = sl.reportStale(app, staleTime); err != nil {