2016-12-09 19:45:46 +00:00
|
|
|
package tsdb
|
|
|
|
|
|
|
|
import (
|
2016-12-09 20:23:34 +00:00
|
|
|
"encoding/binary"
|
2016-12-09 20:40:38 +00:00
|
|
|
"fmt"
|
2016-12-09 19:45:46 +00:00
|
|
|
"hash/crc32"
|
|
|
|
"io"
|
|
|
|
"os"
|
2016-12-09 20:23:34 +00:00
|
|
|
"sort"
|
2016-12-09 19:45:46 +00:00
|
|
|
"unsafe"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// MagicSeries 4 bytes at the head of series file.
|
|
|
|
MagicSeries = 0x85BD40DD
|
|
|
|
|
|
|
|
// MagicIndex 4 bytes at the head of an index file.
|
|
|
|
MagicIndex = 0xBAAAD700
|
|
|
|
)
|
|
|
|
|
|
|
|
// SeriesWriter serializes a time block of chunked series data.
|
|
|
|
type SeriesWriter interface {
|
|
|
|
// WriteSeries writes the time series data chunks for a single series.
|
2016-12-10 08:44:00 +00:00
|
|
|
// The reference is used to resolve the correct series in the written index.
|
|
|
|
// It only has to be valid for the duration of the write.
|
|
|
|
WriteSeries(ref uint32, l Labels, cds []*chunkDesc) error
|
2016-12-09 19:45:46 +00:00
|
|
|
|
|
|
|
// Size returns the size of the data written so far.
|
|
|
|
Size() int64
|
|
|
|
|
|
|
|
// Close writes any required finalization and closes the resources
|
|
|
|
// associated with the underlying writer.
|
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
|
|
|
// seriesWriter implements the SeriesWriter interface for the standard
|
|
|
|
// serialization format.
|
|
|
|
type seriesWriter struct {
|
|
|
|
w io.Writer
|
|
|
|
n int64
|
|
|
|
c int
|
|
|
|
|
|
|
|
baseTimestamp int64
|
|
|
|
index IndexWriter
|
|
|
|
}
|
|
|
|
|
2016-12-09 20:23:34 +00:00
|
|
|
func newSeriesWriter(w io.Writer, index IndexWriter, base int64) *seriesWriter {
|
2016-12-09 19:45:46 +00:00
|
|
|
return &seriesWriter{
|
|
|
|
w: w,
|
|
|
|
n: 0,
|
2016-12-09 20:23:34 +00:00
|
|
|
index: index,
|
2016-12-09 19:45:46 +00:00
|
|
|
baseTimestamp: base,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *seriesWriter) write(wr io.Writer, b []byte) error {
|
|
|
|
n, err := wr.Write(b)
|
|
|
|
w.n += int64(n)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *seriesWriter) writeMeta() error {
|
|
|
|
meta := &meta{magic: MagicSeries, flag: flagStd}
|
|
|
|
metab := ((*[metaSize]byte)(unsafe.Pointer(meta)))[:]
|
|
|
|
|
|
|
|
return w.write(w.w, metab)
|
|
|
|
}
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
func (w *seriesWriter) WriteSeries(ref uint32, lset Labels, chks []*chunkDesc) error {
|
2016-12-09 19:45:46 +00:00
|
|
|
// Initialize with meta data.
|
|
|
|
if w.n == 0 {
|
|
|
|
if err := w.writeMeta(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO(fabxc): is crc32 enough for chunks of one series?
|
|
|
|
h := crc32.NewIEEE()
|
|
|
|
wr := io.MultiWriter(h, w.w)
|
|
|
|
|
|
|
|
l := uint32(0)
|
|
|
|
for _, cd := range chks {
|
|
|
|
l += uint32(len(cd.chunk.Bytes()))
|
|
|
|
}
|
|
|
|
// For normal reads we don't need the length of the chunk section but
|
|
|
|
// it allows us to verify checksums without reading the index file.
|
2016-12-09 21:12:16 +00:00
|
|
|
if err := w.write(wr, ((*[4]byte)(unsafe.Pointer(&l)))[:]); err != nil {
|
2016-12-09 19:45:46 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
offsets := make([]ChunkOffset, 0, len(chks))
|
|
|
|
lastTimestamp := w.baseTimestamp
|
|
|
|
|
|
|
|
for _, cd := range chks {
|
|
|
|
offsets = append(offsets, ChunkOffset{
|
|
|
|
Value: lastTimestamp,
|
|
|
|
Offset: uint32(w.n),
|
|
|
|
})
|
|
|
|
|
|
|
|
if err := w.write(wr, []byte{byte(cd.chunk.Encoding())}); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := w.write(wr, cd.chunk.Bytes()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
lastTimestamp = cd.lastTimestamp
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := w.write(w.w, h.Sum(nil)); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if w.index != nil {
|
2016-12-10 08:44:00 +00:00
|
|
|
w.index.AddSeries(ref, lset, offsets...)
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *seriesWriter) Size() int64 {
|
|
|
|
return w.n
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *seriesWriter) Close() error {
|
|
|
|
if f, ok := w.w.(*os.File); ok {
|
|
|
|
if err := f.Sync(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if c, ok := w.w.(io.Closer); ok {
|
|
|
|
return c.Close()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type ChunkOffset struct {
|
|
|
|
Value int64
|
|
|
|
Offset uint32
|
|
|
|
}
|
|
|
|
|
|
|
|
type BlockStats struct {
|
|
|
|
}
|
|
|
|
|
|
|
|
// IndexWriter serialized the index for a block of series data.
|
|
|
|
// The methods must generally be called in order they are specified.
|
|
|
|
type IndexWriter interface {
|
2016-12-10 08:44:00 +00:00
|
|
|
// AddSeries populates the index writer witha series and its offsets
|
|
|
|
// of chunks that the index can reference.
|
|
|
|
// The reference number is used to resolve a series against the postings
|
|
|
|
// list iterator. It only has to be available during the write processing.
|
|
|
|
AddSeries(ref uint32, l Labels, o ...ChunkOffset)
|
2016-12-09 19:45:46 +00:00
|
|
|
|
|
|
|
// WriteStats writes final stats for the indexed block.
|
|
|
|
WriteStats(*BlockStats) error
|
|
|
|
|
|
|
|
// WriteLabelIndex serializes an index from label names to values.
|
|
|
|
// The passed in values chained tuples of strings of the length of names.
|
|
|
|
WriteLabelIndex(names []string, values []string) error
|
|
|
|
|
|
|
|
// WritePostings writes a postings list for a single label pair.
|
|
|
|
WritePostings(name, value string, it Iterator) error
|
|
|
|
|
|
|
|
// Size returns the size of the data written so far.
|
|
|
|
Size() int64
|
|
|
|
|
2016-12-09 20:23:34 +00:00
|
|
|
// Close writes any finalization and closes theresources associated with
|
2016-12-09 19:45:46 +00:00
|
|
|
// the underlying writer.
|
|
|
|
Close() error
|
|
|
|
}
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
type indexWriterSeries struct {
|
|
|
|
labels Labels
|
|
|
|
chunks []ChunkOffset // series file offset of chunks
|
|
|
|
offset uint32 // index file offset of series reference
|
|
|
|
}
|
|
|
|
|
2016-12-09 19:45:46 +00:00
|
|
|
// indexWriter implements the IndexWriter interface for the standard
|
|
|
|
// serialization format.
|
|
|
|
type indexWriter struct {
|
|
|
|
w io.Writer
|
|
|
|
n int64
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
series map[uint32]*indexWriterSeries
|
2016-12-09 20:23:34 +00:00
|
|
|
|
2016-12-09 20:40:38 +00:00
|
|
|
symbols map[string]uint32 // symbol offsets
|
2016-12-09 21:12:16 +00:00
|
|
|
labelIndexes []hashEntry // label index offsets
|
2016-12-10 08:44:00 +00:00
|
|
|
postings []hashEntry // postings lists offsets
|
2016-12-09 20:23:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func newIndexWriter(w io.Writer) *indexWriter {
|
|
|
|
return &indexWriter{
|
2016-12-09 20:40:38 +00:00
|
|
|
w: w,
|
|
|
|
n: 0,
|
2016-12-09 21:12:16 +00:00
|
|
|
symbols: make(map[string]uint32, 4096),
|
2016-12-10 08:44:00 +00:00
|
|
|
series: make(map[uint32]*indexWriterSeries, 4096),
|
2016-12-09 21:12:16 +00:00
|
|
|
labelIndexes: make([]hashEntry, 10),
|
2016-12-09 20:23:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) write(wr io.Writer, b []byte) error {
|
|
|
|
n, err := wr.Write(b)
|
|
|
|
w.n += int64(n)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
// section writes a CRC32 checksummed section of length l and guarded by flag.
|
|
|
|
func (w *indexWriter) section(l uint32, flag byte, f func(w io.Writer) error) error {
|
|
|
|
h := crc32.NewIEEE()
|
|
|
|
wr := io.MultiWriter(h, w.w)
|
|
|
|
|
|
|
|
if err := w.write(wr, ((*[4]byte)(unsafe.Pointer(&l)))[:]); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := w.write(wr, []byte{flagStd}); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := f(wr); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return w.write(w.w, h.Sum(nil))
|
|
|
|
}
|
|
|
|
|
2016-12-09 20:23:34 +00:00
|
|
|
func (w *indexWriter) writeMeta() error {
|
|
|
|
meta := &meta{magic: MagicSeries, flag: flagStd}
|
|
|
|
metab := ((*[metaSize]byte)(unsafe.Pointer(meta)))[:]
|
|
|
|
|
|
|
|
return w.write(w.w, metab)
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
func (w *indexWriter) AddSeries(ref uint32, lset Labels, offsets ...ChunkOffset) {
|
2016-12-09 20:23:34 +00:00
|
|
|
// Populate the symbol table from all label sets we have to reference.
|
|
|
|
for _, l := range lset {
|
|
|
|
w.symbols[l.Name] = 0
|
|
|
|
w.symbols[l.Value] = 0
|
|
|
|
}
|
2016-12-10 08:44:00 +00:00
|
|
|
|
|
|
|
w.series[ref] = &indexWriterSeries{
|
|
|
|
labels: lset,
|
|
|
|
chunks: offsets,
|
|
|
|
}
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) WriteStats(*BlockStats) error {
|
2016-12-09 20:23:34 +00:00
|
|
|
if w.n == 0 {
|
|
|
|
if err := w.writeMeta(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := w.writeSymbols(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-12-09 21:27:43 +00:00
|
|
|
if err := w.writeSeries(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-12-09 20:23:34 +00:00
|
|
|
}
|
2016-12-09 19:45:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-09 20:23:34 +00:00
|
|
|
func (w *indexWriter) writeSymbols() error {
|
|
|
|
// Generate sorted list of strings we will store as reference table.
|
|
|
|
symbols := make([]string, 0, len(w.symbols))
|
|
|
|
for s := range w.symbols {
|
|
|
|
symbols = append(symbols, s)
|
|
|
|
}
|
|
|
|
sort.Strings(symbols)
|
|
|
|
|
2016-12-09 21:27:43 +00:00
|
|
|
buf := [binary.MaxVarintLen32]byte{}
|
2016-12-09 21:12:16 +00:00
|
|
|
b := append(make([]byte, 4096), flagStd)
|
2016-12-09 20:23:34 +00:00
|
|
|
|
|
|
|
for _, s := range symbols {
|
2016-12-09 21:12:16 +00:00
|
|
|
w.symbols[s] = uint32(w.n) + uint32(len(b))
|
|
|
|
|
2016-12-09 21:27:43 +00:00
|
|
|
n := binary.PutUvarint(buf[:], uint64(len(s)))
|
2016-12-09 21:12:16 +00:00
|
|
|
b = append(b, buf[:n]...)
|
|
|
|
b = append(b, s...)
|
|
|
|
}
|
2016-12-09 20:23:34 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
l := uint32(len(b) + 1)
|
2016-12-09 21:27:43 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
return w.section(l, flagStd, func(wr io.Writer) error {
|
|
|
|
return w.write(wr, b)
|
|
|
|
})
|
2016-12-09 21:27:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) writeSeries() error {
|
|
|
|
b := make([]byte, 0, 4096)
|
|
|
|
buf := [binary.MaxVarintLen32]byte{}
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
for _, s := range w.series {
|
|
|
|
s.offset = uint32(w.n) + uint32(len(b))
|
|
|
|
|
|
|
|
n := binary.PutUvarint(buf[:], uint64(len(s.labels)))
|
|
|
|
b = append(b, buf[:n]...)
|
2016-12-09 21:27:43 +00:00
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
for _, l := range s.labels {
|
2016-12-09 21:27:43 +00:00
|
|
|
n = binary.PutUvarint(buf[:], uint64(w.symbols[l.Name]))
|
|
|
|
b = append(b, buf[:n]...)
|
|
|
|
n = binary.PutUvarint(buf[:], uint64(w.symbols[l.Value]))
|
|
|
|
b = append(b, buf[:n]...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
l := uint32(len(b) + 1)
|
2016-12-09 21:27:43 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
return w.section(l, flagStd, func(wr io.Writer) error {
|
|
|
|
return w.write(wr, b)
|
|
|
|
})
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) WriteLabelIndex(names []string, values []string) error {
|
2016-12-09 20:40:38 +00:00
|
|
|
if len(names) != 1 {
|
|
|
|
return fmt.Errorf("not supported")
|
|
|
|
}
|
|
|
|
sort.Strings(values)
|
|
|
|
|
2016-12-09 21:12:16 +00:00
|
|
|
w.labelIndexes = append(w.labelIndexes, hashEntry{
|
|
|
|
name: names[0],
|
|
|
|
offset: uint32(w.n),
|
|
|
|
})
|
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
l := uint32(1 + len(values)*4)
|
2016-12-09 21:12:16 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
return w.section(l, flagStd, func(wr io.Writer) error {
|
|
|
|
for _, v := range values {
|
|
|
|
o := w.symbols[v]
|
|
|
|
b := ((*[4]byte)(unsafe.Pointer(&o)))[:]
|
2016-12-09 20:40:38 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
if err := w.write(wr, b); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-12-09 20:40:38 +00:00
|
|
|
}
|
2016-12-09 21:36:31 +00:00
|
|
|
return nil
|
|
|
|
})
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) WritePostings(name, value string, it Iterator) error {
|
2016-12-10 08:44:00 +00:00
|
|
|
key := name + string(sep) + value
|
|
|
|
|
|
|
|
w.postings = append(w.postings, hashEntry{
|
|
|
|
name: key,
|
|
|
|
offset: uint32(w.n),
|
|
|
|
})
|
|
|
|
|
|
|
|
b := make([]byte, 0, 4096)
|
|
|
|
|
|
|
|
for it.Next() {
|
|
|
|
v := w.series[it.Value()].offset
|
|
|
|
b = append(b, ((*[4]byte)(unsafe.Pointer(&v)))[:]...)
|
|
|
|
}
|
|
|
|
|
|
|
|
return w.section(uint32(len(b)), flagStd, func(wr io.Writer) error {
|
|
|
|
return w.write(wr, b)
|
|
|
|
})
|
2016-12-09 19:45:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (w *indexWriter) Size() int64 {
|
|
|
|
return w.n
|
|
|
|
}
|
2016-12-09 20:23:34 +00:00
|
|
|
|
2016-12-09 21:12:16 +00:00
|
|
|
type hashEntry struct {
|
|
|
|
name string
|
|
|
|
offset uint32
|
|
|
|
}
|
|
|
|
|
|
|
|
const hashEntrySize = uint32(unsafe.Sizeof(hashEntry{}))
|
|
|
|
|
|
|
|
func (w *indexWriter) finalize() error {
|
|
|
|
l := 1 + uint32(len(w.labelIndexes))*hashEntrySize
|
|
|
|
|
2016-12-10 08:44:00 +00:00
|
|
|
err := w.section(l, flagStd, func(wr io.Writer) error {
|
2016-12-09 21:36:31 +00:00
|
|
|
for _, e := range w.labelIndexes {
|
|
|
|
b := ((*[hashEntrySize]byte)(unsafe.Pointer(&e)))[:]
|
2016-12-09 21:12:16 +00:00
|
|
|
|
2016-12-09 21:36:31 +00:00
|
|
|
if err := w.write(w.w, b); err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
2016-12-09 21:12:16 +00:00
|
|
|
}
|
2016-12-09 21:36:31 +00:00
|
|
|
return nil
|
|
|
|
})
|
2016-12-10 08:44:00 +00:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = w.section(l, flagStd, func(wr io.Writer) error {
|
|
|
|
for _, e := range w.postings {
|
|
|
|
b := ((*[hashEntrySize]byte)(unsafe.Pointer(&e)))[:]
|
|
|
|
|
|
|
|
if err := w.write(w.w, b); err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
// TODO(fabxc): write hashmap offsets.
|
|
|
|
|
|
|
|
return err
|
2016-12-09 21:12:16 +00:00
|
|
|
}
|
|
|
|
|
2016-12-09 19:45:46 +00:00
|
|
|
func (w *indexWriter) Close() error {
|
2016-12-09 21:12:16 +00:00
|
|
|
if err := w.finalize(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-12-09 19:45:46 +00:00
|
|
|
if f, ok := w.w.(*os.File); ok {
|
|
|
|
if err := f.Sync(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if c, ok := w.w.(io.Closer); ok {
|
|
|
|
return c.Close()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|