Pass the correct context to staleness Appender (#10588)
OTel Collector prints the following error when a target disappears:
```
2022-04-13T14:20:24.932-0400 warn scrape/scrape.go:1408 Stale append failed {"kind": "receiver", "name": "prometheus", "scrape_pool": "beep-boop", "target": "http://localhost:9090/metrics", "error": "transaction aborted"}
```
This `transaction aborted` error is returned by the custom appender that is
used by the collector when the context of the appender is cancelled:
b7bf11174e/receiver/prometheusreceiver/internal/otlp_transaction.go (L81-L82)
We call `endOfRunStaleness` after `sl.stop()` which cancels `sl.ctx`.
The other `.Appender()` calls use `parentCtx` for the same reason.
This hasn't come up so far because Prometheus' Appender implementation just
ignores the context passed.
Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>
This commit is contained in:
parent
580e852f10
commit
ec3d02019e
|
@ -1379,7 +1379,8 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int
|
|||
// Call sl.append again with an empty scrape to trigger stale markers.
|
||||
// If the target has since been recreated and scraped, the
|
||||
// stale markers will be out of order and ignored.
|
||||
app := sl.appender(sl.ctx)
|
||||
// sl.context would have been cancelled, hence using sl.parentCtx.
|
||||
app := sl.appender(sl.parentCtx)
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
|
@ -1393,7 +1394,7 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int
|
|||
}()
|
||||
if _, _, _, err = sl.append(app, []byte{}, "", staleTime); err != nil {
|
||||
app.Rollback()
|
||||
app = sl.appender(sl.ctx)
|
||||
app = sl.appender(sl.parentCtx)
|
||||
level.Warn(sl.l).Log("msg", "Stale append failed", "err", err)
|
||||
}
|
||||
if err = sl.reportStale(app, staleTime); err != nil {
|
||||
|
|
Loading…
Reference in New Issue