RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-25 16:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compress records before checking segment size (#8501)

Browse Source 
Right now a new segment might be created unnecessarily if the
uncompressed record would not fit, but after compression (typically
reducing record size in half) it would.

Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
This commit is contained in:
Chris Marchbanks 2021-02-17 06:15:49 -07:00 committed by GitHub
parent 6a3d55db0a
commit bedcd88343
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tsdb/wal/wal.go
View File

@ -613,18 +613,8 @@ func (w *WAL) log(rec []byte, final bool) error {
return err return err
} }
} }
// If the record is too big to fit within the active page in the current
// segment, terminate the active segment and advance to the next one.
// This ensures that records do not cross segment boundaries.
left := w.page.remaining() - recordHeaderSize // Free space in the active page.
left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment.
if len(rec) > left {
if err := w.nextSegment(); err != nil {
return err
}
}
// Compress the record before calculating if a new segment is needed.
compressed := false compressed := false
if w.compress && len(rec) > 0 { if w.compress && len(rec) > 0 {
// The snappy library uses `len` to calculate if we need a new buffer. // The snappy library uses `len` to calculate if we need a new buffer.
@ -638,6 +628,18 @@ func (w *WAL) log(rec []byte, final bool) error {
} }
} }
// If the record is too big to fit within the active page in the current
// segment, terminate the active segment and advance to the next one.
// This ensures that records do not cross segment boundaries.
left := w.page.remaining() - recordHeaderSize // Free space in the active page.
left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment.
if len(rec) > left {
if err := w.nextSegment(); err != nil {
return err
}
}
// Populate as many pages as necessary to fit the record. // Populate as many pages as necessary to fit the record.
// Be careful to always do one pass to ensure we write zero-length records. // Be careful to always do one pass to ensure we write zero-length records.
for i := 0; i == 0 || len(rec) > 0; i++ { for i := 0; i == 0 || len(rec) > 0; i++ {