mirror of
https://github.com/prometheus/prometheus
synced 2024-12-24 07:23:02 +00:00
Handle scrapes with OutOfBounds metrics better
fixes #2894 Signed-off-by: Goutham Veeramachaneni <goutham@boomerangcommerce.com>
This commit is contained in:
parent
426125298e
commit
3069bd3996
@ -722,11 +722,12 @@ func (s samples) Less(i, j int) bool {
|
||||
|
||||
func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) {
|
||||
var (
|
||||
app = sl.appender()
|
||||
p = textparse.New(b)
|
||||
defTime = timestamp.FromTime(ts)
|
||||
numOutOfOrder = 0
|
||||
numDuplicates = 0
|
||||
app = sl.appender()
|
||||
p = textparse.New(b)
|
||||
defTime = timestamp.FromTime(ts)
|
||||
numOutOfOrder = 0
|
||||
numDuplicates = 0
|
||||
numOutOfBounds = 0
|
||||
)
|
||||
var sampleLimitErr error
|
||||
|
||||
@ -761,6 +762,10 @@ loop:
|
||||
numDuplicates++
|
||||
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
|
||||
continue
|
||||
case storage.ErrOutOfBounds:
|
||||
numOutOfBounds++
|
||||
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
|
||||
continue
|
||||
case errSampleLimit:
|
||||
// Keep on parsing output if we hit the limit, so we report the correct
|
||||
// total number of samples scraped.
|
||||
@ -804,6 +809,10 @@ loop:
|
||||
numDuplicates++
|
||||
sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
|
||||
continue
|
||||
case storage.ErrOutOfBounds:
|
||||
numOutOfBounds++
|
||||
sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
|
||||
continue
|
||||
case errSampleLimit:
|
||||
sampleLimitErr = err
|
||||
added++
|
||||
@ -832,6 +841,9 @@ loop:
|
||||
if numDuplicates > 0 {
|
||||
sl.l.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp")
|
||||
}
|
||||
if numOutOfBounds > 0 {
|
||||
sl.l.With("numOutOfBounds", numOutOfBounds).Warn("Error on ingesting samples that are too old")
|
||||
}
|
||||
if err == nil {
|
||||
sl.cache.forEachStale(func(lset labels.Labels) bool {
|
||||
// Series no longer exposed, mark it stale.
|
||||
|
@ -19,10 +19,12 @@ import (
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
)
|
||||
|
||||
// The errors exposed.
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrOutOfOrderSample = errors.New("out of order sample")
|
||||
ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp")
|
||||
ErrOutOfBounds = errors.New("out of bounds")
|
||||
)
|
||||
|
||||
// Storage ingests and manages samples, along with various indexes. All methods
|
||||
|
Loading…
Reference in New Issue
Block a user