RepoMirrors
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10102 from prometheus/update-metrics-on-rw-fails 

Update sent timestamp when write irrecoverably fails
This commit is contained in:
Chris Marchbanks 2022-01-05 10:46:09 -07:00 committed by GitHub
parent d26fd5c97b 6696b7a5f0
commit ba03f7fc23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
storage/remote/queue_manager.go
View File

@ -15,6 +15,7 @@ package remote
import ( import (
"context" "context"
"errors"
"math" "math"
"strconv" "strconv"
"sync" "sync"
@ -1311,12 +1312,16 @@ func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.Ti
} }
err = sendWriteRequestWithBackoff(ctx, s.qm.cfg, s.qm.logger, attemptStore, onRetry) err = sendWriteRequestWithBackoff(ctx, s.qm.cfg, s.qm.logger, attemptStore, onRetry)
if err != nil { if errors.Is(err, context.Canceled) {
// When there is resharding, we cancel the context for this queue, which means the data is not sent.
// So we exit early to not update the metrics.
return err return err
} }
s.qm.metrics.sentBytesTotal.Add(float64(reqSize)) s.qm.metrics.sentBytesTotal.Add(float64(reqSize))
s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000)) s.qm.metrics.highestSentTimestamp.Set(float64(highest / 1000))
return nil
return err
} }
func sendWriteRequestWithBackoff(ctx context.Context, cfg config.QueueConfig, l log.Logger, attempt func(int) error, onRetry func()) error { func sendWriteRequestWithBackoff(ctx context.Context, cfg config.QueueConfig, l log.Logger, attempt func(int) error, onRetry func()) error {