RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-26 08:33:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Export refdSample

Browse Source 
The type was part of a exported method signatures and should therefore
be exported as well.
This commit is contained in:
Fabian Reinartz 2017-05-12 17:06:26 +02:00
parent 5534e6c53c
commit 535532ca02
3 changed files with 71 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
head.go
View File

@ -131,13 +131,13 @@ Outer:
h.meta.Stats.NumSeries++
}
for _, s := range samples {
if int(s.ref) >= len(h.series) {
l.Log("msg", "unknown series reference, abort WAL restore", "got", s.ref, "max", len(h.series)-1)
if int(s.Ref) >= len(h.series) {
l.Log("msg", "unknown series reference, abort WAL restore", "got", s.Ref, "max", len(h.series)-1)
break Outer
}
h.series[s.ref].append(s.t, s.v)
h.series[s.Ref].append(s.T, s.V)
if !h.inBounds(s.t) {
if !h.inBounds(s.T) {
return nil, errors.Wrap(ErrOutOfBounds, "consume WAL")
}
h.meta.Stats.NumSamples++
@ -262,15 +262,15 @@ func (h *HeadBlock) Busy() bool {
var headPool = sync.Pool{}
func getHeadAppendBuffer() []refdSample {
func getHeadAppendBuffer() []RefSample {
b := headPool.Get()
if b == nil {
return make([]refdSample, 0, 512)
return make([]RefSample, 0, 512)
}
return b.([]refdSample)
return b.([]RefSample)
}
func putHeadAppendBuffer(b []refdSample) {
func putHeadAppendBuffer(b []RefSample) {
headPool.Put(b[:0])
}
@ -282,7 +282,7 @@ type headAppender struct {
refmap map[uint64]uint64
newLabels []labels.Labels
samples []refdSample
samples []RefSample
}
type hashedLabels struct {
@ -290,12 +290,6 @@ type hashedLabels struct {
labels labels.Labels
}
type refdSample struct {
ref uint64
t int64
v float64
}
func (a *headAppender) Add(lset labels.Labels, t int64, v float64) (uint64, error) {
if !a.inBounds(t) {
return 0, ErrOutOfBounds
@ -370,10 +364,10 @@ func (a *headAppender) AddFast(ref uint64, t int64, v float64) error {
}
}
a.samples = append(a.samples, refdSample{
ref: ref,
t: t,
v: v,
a.samples = append(a.samples, RefSample{
Ref: ref,
T: t,
V: v,
})
return nil
}
@ -419,8 +413,8 @@ func (a *headAppender) Commit() error {
for i := range a.samples {
s := &a.samples[i]
if s.ref&(1<<32) > 0 {
s.ref = a.refmap[s.ref]
if s.Ref&(1<<32) > 0 {
s.Ref = a.refmap[s.Ref]
}
}
@ -434,7 +428,7 @@ func (a *headAppender) Commit() error {
total := uint64(len(a.samples))
for _, s := range a.samples {
if !a.series[s.ref].append(s.t, s.v) {
if !a.series[s.Ref].append(s.T, s.V) {
total--
}
}

71
wal.go
View File

@ -50,7 +50,7 @@ const (
)
// WAL is a write ahead log for series data. It can only be written to.
// Use WALReader to read back from a write ahead log.
// Use walReader to read back from a write ahead log.
type WAL struct {
mtx sync.Mutex
@ -69,6 +69,13 @@ type WAL struct {
donec chan struct{}
}
// RefSample is a timestamp/value pair associated with a reference to a series.
type RefSample struct {
Ref uint64
T int64
V float64
}
const (
walDirName = "wal"
walSegmentSizeBytes = 256 * 1024 * 1024 // 256 MB
@ -119,12 +126,12 @@ func OpenWAL(dir string, logger log.Logger, flushInterval time.Duration) (*WAL,
// Reader returns a new reader over the the write ahead log data.
// It must be completely consumed before writing to the WAL.
func (w *WAL) Reader() *WALReader {
return NewWALReader(w.logger, w)
func (w *WAL) Reader() WALReader {
return newWALReader(w, w.logger)
}
// Log writes a batch of new series labels and samples to the log.
func (w *WAL) Log(series []labels.Labels, samples []refdSample) error {
func (w *WAL) Log(series []labels.Labels, samples []RefSample) error {
if err := w.encodeSeries(series); err != nil {
return err
}
@ -395,7 +402,7 @@ func (w *WAL) encodeSeries(series []labels.Labels) error {
return w.entry(WALEntrySeries, walSeriesSimple, buf)
}
func (w *WAL) encodeSamples(samples []refdSample) error {
func (w *WAL) encodeSamples(samples []RefSample) error {
if len(samples) == 0 {
return nil
}
@ -409,27 +416,27 @@ func (w *WAL) encodeSamples(samples []refdSample) error {
// TODO(fabxc): optimize for all samples having the same timestamp.
first := samples[0]
binary.BigEndian.PutUint64(b, first.ref)
binary.BigEndian.PutUint64(b, first.Ref)
buf = append(buf, b[:8]...)
binary.BigEndian.PutUint64(b, uint64(first.t))
binary.BigEndian.PutUint64(b, uint64(first.T))
buf = append(buf, b[:8]...)
for _, s := range samples {
n := binary.PutVarint(b, int64(s.ref)-int64(first.ref))
n := binary.PutVarint(b, int64(s.Ref)-int64(first.Ref))
buf = append(buf, b[:n]...)
n = binary.PutVarint(b, s.t-first.t)
n = binary.PutVarint(b, s.T-first.T)
buf = append(buf, b[:n]...)
binary.BigEndian.PutUint64(b, math.Float64bits(s.v))
binary.BigEndian.PutUint64(b, math.Float64bits(s.V))
buf = append(buf, b[:8]...)
}
return w.entry(WALEntrySamples, walSamplesSimple, buf)
}
// WALReader decodes and emits write ahead log entries.
type WALReader struct {
// walReader decodes and emits write ahead log entries.
type walReader struct {
logger log.Logger
wal *WAL
@ -439,37 +446,35 @@ type WALReader struct {
err error
labels []labels.Labels
samples []refdSample
samples []RefSample
}
// NewWALReader returns a new WALReader over the sequence of the given ReadClosers.
func NewWALReader(logger log.Logger, w *WAL) *WALReader {
if logger == nil {
logger = log.NewNopLogger()
func newWALReader(w *WAL, l log.Logger) *walReader {
if l == nil {
l = log.NewNopLogger()
}
r := &WALReader{
logger: logger,
return &walReader{
logger: l,
wal: w,
buf: make([]byte, 0, 128*4096),
crc32: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
}
return r
}
// At returns the last decoded entry of labels or samples.
// The returned slices are only valid until the next call to Next(). Their elements
// have to be copied to preserve them.
func (r *WALReader) At() ([]labels.Labels, []refdSample) {
func (r *walReader) At() ([]labels.Labels, []RefSample) {
return r.labels, r.samples
}
// Err returns the last error the reader encountered.
func (r *WALReader) Err() error {
func (r *walReader) Err() error {
return r.err
}
// nextEntry retrieves the next entry. It is also used as a testing hook.
func (r *WALReader) nextEntry() (WALEntryType, byte, []byte, error) {
func (r *walReader) nextEntry() (WALEntryType, byte, []byte, error) {
if r.cur >= len(r.wal.files) {
return 0, 0, nil, io.EOF
}
@ -492,7 +497,7 @@ func (r *WALReader) nextEntry() (WALEntryType, byte, []byte, error) {
// Next returns decodes the next entry pair and returns true
// if it was succesful.
func (r *WALReader) Next() bool {
func (r *walReader) Next() bool {
r.labels = r.labels[:0]
r.samples = r.samples[:0]
@ -549,12 +554,12 @@ func (r *WALReader) Next() bool {
return r.err == nil
}
func (r *WALReader) current() *os.File {
func (r *walReader) current() *os.File {
return r.wal.files[r.cur]
}
// truncate the WAL after the last valid entry.
func (r *WALReader) truncate(lastOffset int64) error {
func (r *walReader) truncate(lastOffset int64) error {
r.logger.Log("msg", "WAL corruption detected; truncating",
"err", r.err, "file", r.current().Name(), "pos", lastOffset)
@ -582,7 +587,7 @@ func walCorruptionErrf(s string, args ...interface{}) error {
return walCorruptionErr(errors.Errorf(s, args...))
}
func (r *WALReader) entry(cr io.Reader) (WALEntryType, byte, []byte, error) {
func (r *walReader) entry(cr io.Reader) (WALEntryType, byte, []byte, error) {
r.crc32.Reset()
tr := io.TeeReader(cr, r.crc32)
@ -629,7 +634,7 @@ func (r *WALReader) entry(cr io.Reader) (WALEntryType, byte, []byte, error) {
return etype, flag, buf, nil
}
func (r *WALReader) decodeSeries(flag byte, b []byte) error {
func (r *walReader) decodeSeries(flag byte, b []byte) error {
for len(b) > 0 {
l, n := binary.Uvarint(b)
if n < 1 {
@ -659,7 +664,7 @@ func (r *WALReader) decodeSeries(flag byte, b []byte) error {
return nil
}
func (r *WALReader) decodeSamples(flag byte, b []byte) error {
func (r *walReader) decodeSamples(flag byte, b []byte) error {
if len(b) < 16 {
return errors.Wrap(errInvalidSize, "header length")
}
@ -670,7 +675,7 @@ func (r *WALReader) decodeSamples(flag byte, b []byte) error {
b = b[16:]
for len(b) > 0 {
var smpl refdSample
var smpl RefSample
dref, n := binary.Varint(b)
if n < 1 {
@ -678,19 +683,19 @@ func (r *WALReader) decodeSamples(flag byte, b []byte) error {
}
b = b[n:]
smpl.ref = uint64(int64(baseRef) + dref)
smpl.Ref = uint64(int64(baseRef) + dref)
dtime, n := binary.Varint(b)
if n < 1 {
return errors.Wrap(errInvalidSize, "sample timestamp delta")
}
b = b[n:]
smpl.t = baseTime + dtime
smpl.T = baseTime + dtime
if len(b) < 8 {
return errors.Wrapf(errInvalidSize, "sample value bits %d", len(b))
}
smpl.v = float64(math.Float64frombits(binary.BigEndian.Uint64(b)))
smpl.V = float64(math.Float64frombits(binary.BigEndian.Uint64(b)))
b = b[8:]
r.samples = append(r.samples, smpl)

34
wal_test.go
View File

@ -122,7 +122,7 @@ func TestWAL_cut(t *testing.T) {
// We cannot actually check for correct pre-allocation as it is
// optional per filesystem and handled transparently.
et, flag, b, err := NewWALReader(nil, nil).entry(f)
et, flag, b, err := newWALReader(nil, nil).entry(f)
require.NoError(t, err)
require.Equal(t, WALEntrySeries, et)
require.Equal(t, flag, byte(walSeriesSimple))
@ -148,7 +148,7 @@ func TestWAL_Log_Restore(t *testing.T) {
var (
recordedSeries [][]labels.Labels
recordedSamples [][]refdSample
recordedSamples [][]RefSample
)
var totalSamples int
@ -165,7 +165,7 @@ func TestWAL_Log_Restore(t *testing.T) {
var (
resultSeries [][]labels.Labels
resultSamples [][]refdSample
resultSamples [][]RefSample
)
for r.Next() {
@ -177,7 +177,7 @@ func TestWAL_Log_Restore(t *testing.T) {
resultSeries = append(resultSeries, clsets)
}
if len(smpls) > 0 {
csmpls := make([]refdSample, len(smpls))
csmpls := make([]RefSample, len(smpls))
copy(csmpls, smpls)
resultSamples = append(resultSamples, csmpls)
}
@ -191,13 +191,13 @@ func TestWAL_Log_Restore(t *testing.T) {
// Insert in batches and generate different amounts of samples for each.
for i := 0; i < len(series); i += stepSize {
var samples []refdSample
var samples []RefSample
for j := 0; j < i*10; j++ {
samples = append(samples, refdSample{
ref: uint64(j % 10000),
t: int64(j * 2),
v: rand.Float64(),
samples = append(samples, RefSample{
Ref: uint64(j % 10000),
T: int64(j * 2),
V: rand.Float64(),
})
}
@ -292,13 +292,13 @@ func TestWALRestoreCorrupted(t *testing.T) {
w, err := OpenWAL(dir, nil, 0)
require.NoError(t, err)
require.NoError(t, w.Log(nil, []refdSample{{t: 1, v: 2}}))
require.NoError(t, w.Log(nil, []refdSample{{t: 2, v: 3}}))
require.NoError(t, w.Log(nil, []RefSample{{T: 1, V: 2}}))
require.NoError(t, w.Log(nil, []RefSample{{T: 2, V: 3}}))
require.NoError(t, w.cut())
require.NoError(t, w.Log(nil, []refdSample{{t: 3, v: 4}}))
require.NoError(t, w.Log(nil, []refdSample{{t: 5, v: 6}}))
require.NoError(t, w.Log(nil, []RefSample{{T: 3, V: 4}}))
require.NoError(t, w.Log(nil, []RefSample{{T: 5, V: 6}}))
require.NoError(t, w.Close())
@ -318,13 +318,13 @@ func TestWALRestoreCorrupted(t *testing.T) {
require.True(t, r.Next())
l, s := r.At()
require.Equal(t, 0, len(l))
require.Equal(t, []refdSample{{t: 1, v: 2}}, s)
require.Equal(t, []RefSample{{T: 1, V: 2}}, s)
// Truncation should happen transparently and now cause an error.
require.False(t, r.Next())
require.Nil(t, r.Err())
require.NoError(t, w2.Log(nil, []refdSample{{t: 99, v: 100}}))
require.NoError(t, w2.Log(nil, []RefSample{{T: 99, V: 100}}))
require.NoError(t, w2.Close())
files, err := fileutil.ReadDir(dir)
@ -341,12 +341,12 @@ func TestWALRestoreCorrupted(t *testing.T) {
require.True(t, r.Next())
l, s = r.At()
require.Equal(t, 0, len(l))
require.Equal(t, []refdSample{{t: 1, v: 2}}, s)
require.Equal(t, []RefSample{{T: 1, V: 2}}, s)
require.True(t, r.Next())
l, s = r.At()
require.Equal(t, 0, len(l))
require.Equal(t, []refdSample{{t: 99, v: 100}}, s)
require.Equal(t, []RefSample{{T: 99, V: 100}}, s)
require.False(t, r.Next())
require.Nil(t, r.Err())