mirror of
https://github.com/prometheus/prometheus
synced 2024-12-25 07:52:28 +00:00
Add missing logging of out-of-order samples
So far, out-of-order samples during rule evaluation were not logged, and neither scrape health samples. The latter are unlikely to cause any errors. That's why I'm logging them always now. (It's alway highly irregular should it happen.) For rules, I have used the same plumbing as for samples, just with a different wording in the message to mark them as a result of rule evaluation.
This commit is contained in:
parent
d9394eb359
commit
45e5775f9b
@ -496,6 +496,10 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, err error)
|
||||
Value: model.SampleValue(float64(duration) / float64(time.Second)),
|
||||
}
|
||||
|
||||
sl.reportAppender.Append(healthSample)
|
||||
sl.reportAppender.Append(durationSample)
|
||||
if err := sl.reportAppender.Append(healthSample); err != nil {
|
||||
log.With("sample", healthSample).With("error", err).Warn("Scrape health sample discarded")
|
||||
}
|
||||
if err := sl.reportAppender.Append(durationSample); err != nil {
|
||||
log.With("sample", durationSample).With("error", err).Warn("Scrape duration sample discarded")
|
||||
}
|
||||
}
|
||||
|
@ -31,6 +31,7 @@ import (
|
||||
"github.com/prometheus/prometheus/notifier"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/storage/local"
|
||||
"github.com/prometheus/prometheus/template"
|
||||
"github.com/prometheus/prometheus/util/strutil"
|
||||
)
|
||||
@ -270,8 +271,29 @@ func (g *Group) eval() {
|
||||
if ar, ok := rule.(*AlertingRule); ok {
|
||||
g.sendAlerts(ar, now)
|
||||
}
|
||||
var (
|
||||
numOutOfOrder = 0
|
||||
numDuplicates = 0
|
||||
)
|
||||
for _, s := range vector {
|
||||
g.opts.SampleAppender.Append(s)
|
||||
if err := g.opts.SampleAppender.Append(s); err != nil {
|
||||
switch err {
|
||||
case local.ErrOutOfOrderSample:
|
||||
numOutOfOrder++
|
||||
log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
|
||||
case local.ErrDuplicateSampleForTimestamp:
|
||||
numDuplicates++
|
||||
log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
|
||||
default:
|
||||
log.With("sample", s).With("error", err).Warn("Rule evaluation result discarded")
|
||||
}
|
||||
}
|
||||
}
|
||||
if numOutOfOrder > 0 {
|
||||
log.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order result from rule evaluation")
|
||||
}
|
||||
if numDuplicates > 0 {
|
||||
log.With("numDropped", numDuplicates).Warn("Error on ingesting results from rule evaluation with different value but same timestamp")
|
||||
}
|
||||
}(rule)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user