RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-24 23:42:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10102 from prometheus/update-metrics-on-rw-fails

Browse Source 
Update sent timestamp when write irrecoverably fails
This commit is contained in:
Chris Marchbanks 2022-01-05 10:46:09 -07:00 committed by GitHub
commit ba03f7fc23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
storage/remote/queue_manager.go
View File

@ -15,6 +15,7 @@ package remote
import (
"context"
"errors"
"math"
"strconv"
"sync"
@ -1311,12 +1312,16 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
}
err = sendWriteRequestWithBackoff(ctx, s.qm.cfg, s.qm.logger, attemptStore, onRetry)
if err != nil {
if errors.Is(err, context.Canceled) {
// When there is resharding, we cancel the context for this queue, which means the data is not sent.
// So we exit early to not update the metrics.
return err
}
s.qm.metrics.sentBytesTotal.Add(float64(reqSize))
s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000))
return nil
return err
}
func sendWriteRequestWithBackoff(ctx context.Context, cfg config.QueueConfig, l log.Logger, attempt func(int) error, onRetry func()) error {