prometheus/wal_test.go

169 lines
4.2 KiB
Go
Raw Normal View History

2016-12-22 14:18:33 +00:00
package tsdb
import (
2017-02-14 23:54:52 +00:00
"encoding/binary"
"fmt"
2016-12-22 14:18:33 +00:00
"io/ioutil"
2017-02-15 05:54:59 +00:00
"math/rand"
2016-12-22 14:18:33 +00:00
"os"
"testing"
2017-02-14 23:54:52 +00:00
"github.com/coreos/etcd/pkg/fileutil"
2016-12-22 14:18:33 +00:00
"github.com/stretchr/testify/require"
)
2017-02-14 23:54:52 +00:00
func TestWAL_initSegments(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "test_wal_open")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
df, err := fileutil.OpenDir(tmpdir)
require.NoError(t, err)
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
w := &WAL{dirFile: df}
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// Create segment files with an appropriate header.
for i := 1; i <= 5; i++ {
metab := make([]byte, 8)
binary.BigEndian.PutUint32(metab[:4], WALMagic)
metab[4] = WALFormatDefault
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
f, err := os.Create(fmt.Sprintf("%s/000%d", tmpdir, i))
require.NoError(t, err)
_, err = f.Write(metab)
require.NoError(t, err)
require.NoError(t, f.Close())
2016-12-22 14:18:33 +00:00
}
2017-02-14 23:54:52 +00:00
// Initialize 5 correct segment files.
require.NoError(t, w.initSegments())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.Equal(t, 5, len(w.files), "unexpected number of segments loaded")
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// Validate that files are locked properly.
for _, of := range w.files {
f, err := os.Open(of.Name())
require.NoError(t, err, "open locked segment %s", f.Name())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
_, err = f.Read([]byte{0})
require.NoError(t, err, "read locked segment %s", f.Name())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
_, err = f.Write([]byte{0})
require.Error(t, err, "write to tail segment file %s", f.Name())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.NoError(t, f.Close())
}
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
for _, f := range w.files {
require.NoError(t, f.Close())
}
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// Make initialization fail by corrupting the header of one file.
f, err := os.OpenFile(w.files[3].Name(), os.O_WRONLY, 0666)
require.NoError(t, err)
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
_, err = f.WriteAt([]byte{0}, 4)
require.NoError(t, err)
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
w = &WAL{dirFile: df}
require.Error(t, w.initSegments(), "init corrupted segments")
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
for _, f := range w.files {
require.NoError(t, f.Close())
}
}
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
func TestWAL_cut(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "test_wal_cut")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// This calls cut() implicitly the first time without a previous tail.
w, err := OpenWAL(tmpdir, nil, 0)
require.NoError(t, err)
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.NoError(t, w.entry(WALEntrySeries, 1, []byte("Hello World!!")))
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.NoError(t, w.cut(), "cut failed")
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// Cutting creates a new file and close the previous tail file.
require.Equal(t, 2, len(w.files))
require.Equal(t, os.ErrInvalid.Error(), w.files[0].Close().Error())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.NoError(t, w.entry(WALEntrySeries, 1, []byte("Hello World!!")))
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
require.NoError(t, w.Close())
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
for _, of := range w.files {
f, err := os.Open(of.Name())
require.NoError(t, err)
// Verify header data.
metab := make([]byte, 8)
_, err = f.Read(metab)
require.NoError(t, err, "read meta data %s", f.Name())
require.Equal(t, WALMagic, binary.BigEndian.Uint32(metab[:4]), "verify magic")
require.Equal(t, WALFormatDefault, metab[4], "verify format flag")
2016-12-22 14:18:33 +00:00
2017-02-14 23:54:52 +00:00
// We cannot actually check for correct pre-allocation as it is
// optional per filesystem and handled transparently.
2017-02-15 05:54:59 +00:00
et, flag, b, err := NewWALReader(f).nextEntry()
2017-02-14 23:54:52 +00:00
require.NoError(t, err)
require.Equal(t, WALEntrySeries, et)
require.Equal(t, flag, byte(walSeriesSimple))
require.Equal(t, []byte("Hello World!!"), b)
}
2016-12-22 14:18:33 +00:00
}
2017-02-15 05:54:59 +00:00
// Symmetrical test of reading and writing to the WAL via its main interface.
func TestWAL_Log_Restore(t *testing.T) {
// Generate testing data. It does not make semantical sense but
// for the purpose of this test.
series, err := readPrometheusLabels("testdata/20k.series", 10000)
require.NoError(t, err)
var samples []refdSample
for i := 0; i < 200000; i++ {
samples = append(samples, refdSample{
ref: uint64(i % 10000),
t: int64(i * 2),
v: rand.Float64(),
})
}
dir, err := ioutil.TempDir("", "test_wal_log_restore")
require.NoError(t, err)
defer os.RemoveAll(dir)
w, err := OpenWAL(dir, nil, 0)
require.NoError(t, err)
// Set smaller segment size so we can actually write several files.
w.segmentSize = 300 * 1000
for i := 0; i < len(series); i += 100 {
require.NoError(t, w.Log(series[i:i+100], samples[i*10:(i+100)*10]))
}
require.NoError(t, w.Close())
w, err = OpenWAL(dir, nil, 0)
r := w.Reader()
var i, j int
for r.Next() {
lsets, smpls := r.At()
if l := len(lsets); l > 0 {
require.Equal(t, series[i:i+l], lsets)
i += l
}
if l := len(smpls); l > 0 {
require.Equal(t, samples[j:j+l], smpls)
j += l
}
}
require.NoError(t, r.Err())
}