RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-24 07:23:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

tsdb: Do a full rollback upon commit error

Browse Source 
I think the previous behavior is problematic as it will leave
`memSeries` around that still have `pendingCommit` set to `true`.

The only case where this can happen in this code path is a failure to
write to the WAL, in which case we are probably in trouble anyway. I
believe, however, we should still try to do the right thing and do the
full rollback. This will implicitly try to write to the WAL again, but
this time without samples, which may even succeed. (But we propagate
the previous error in any case.)

This also adds `a.head.putSeriesBuffer(a.sampleSeries)` to Rollback,
which was previously missing.

Signed-off-by: beorn7 <beorn@grafana.com>
This commit is contained in:
beorn7 2020-03-09 18:24:18 +01:00
parent a17ddcf0f9
commit f6f4fd6556
2 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
storage/interface.go
View File

@ -116,7 +116,10 @@ type Appender interface {
// faster than adding a sample by providing its full label set.
AddFast(ref uint64, t int64, v float64) error
// Commit submits the collected samples and purges the batch.
// Commit submits the collected samples and purges the batch. If Commit
// returns a non-nil error, it also rolls back all modifications made in
// the appender so far, as Rollback would do. In any case, an Appender
// must not be used anymore after Commit has been called.
Commit() error
// Rollback rolls back all modifications made in the appender so far.

11
tsdb/head.go
View File

@ -1042,15 +1042,17 @@ func (a *headAppender) log() error {
}
func (a *headAppender) Commit() error {
if err := a.log(); err != nil {
//nolint: errcheck
a.Rollback() // Most likely the same error will happen again.
return errors.Wrap(err, "write to WAL")
}
defer a.head.metrics.activeAppenders.Dec()
defer a.head.putAppendBuffer(a.samples)
defer a.head.putSeriesBuffer(a.sampleSeries)
defer a.head.iso.closeAppend(a.appendID)
if err := a.log(); err != nil {
return errors.Wrap(err, "write to WAL")
}
total := len(a.samples)
var series *memSeries
for i, s := range a.samples {
@ -1088,6 +1090,7 @@ func (a *headAppender) Rollback() error {
}
a.head.putAppendBuffer(a.samples)
a.samples = nil
a.head.putSeriesBuffer(a.sampleSeries)
a.head.iso.closeAppend(a.appendID)
// Series are created in the head memory regardless of rollback. Thus we have