2017-04-10 18:59:45 +00:00
|
|
|
// Copyright 2017 The Prometheus Authors
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2016-12-04 12:16:11 +00:00
|
|
|
package tsdb
|
|
|
|
|
|
|
|
import (
|
2017-01-16 13:18:32 +00:00
|
|
|
"fmt"
|
2017-01-04 13:06:40 +00:00
|
|
|
"math"
|
2017-01-13 15:14:40 +00:00
|
|
|
"math/rand"
|
2017-01-19 10:22:47 +00:00
|
|
|
"os"
|
2016-12-14 17:38:46 +00:00
|
|
|
"sort"
|
2016-12-04 12:16:11 +00:00
|
|
|
"sync"
|
2017-02-04 10:53:52 +00:00
|
|
|
"sync/atomic"
|
2017-01-06 14:18:06 +00:00
|
|
|
"time"
|
2016-12-04 12:16:11 +00:00
|
|
|
|
2017-01-06 14:18:06 +00:00
|
|
|
"github.com/go-kit/kit/log"
|
2017-02-27 09:46:15 +00:00
|
|
|
"github.com/oklog/ulid"
|
2017-01-19 10:22:47 +00:00
|
|
|
"github.com/pkg/errors"
|
2017-04-04 09:27:26 +00:00
|
|
|
"github.com/prometheus/tsdb/chunks"
|
|
|
|
"github.com/prometheus/tsdb/labels"
|
2016-12-04 12:16:11 +00:00
|
|
|
)
|
|
|
|
|
2017-01-17 15:33:58 +00:00
|
|
|
var (
|
|
|
|
// ErrNotFound is returned if a looked up resource was not found.
|
2017-03-21 09:11:23 +00:00
|
|
|
ErrNotFound = errors.Errorf("not found")
|
2017-01-17 15:33:58 +00:00
|
|
|
|
|
|
|
// ErrOutOfOrderSample is returned if an appended sample has a
|
|
|
|
// timestamp larger than the most recent sample.
|
|
|
|
ErrOutOfOrderSample = errors.New("out of order sample")
|
|
|
|
|
|
|
|
// ErrAmendSample is returned if an appended sample has the same timestamp
|
|
|
|
// as the most recent sample but a different value.
|
|
|
|
ErrAmendSample = errors.New("amending sample")
|
|
|
|
|
|
|
|
// ErrOutOfBounds is returned if an appended sample is out of the
|
|
|
|
// writable time range.
|
|
|
|
ErrOutOfBounds = errors.New("out of bounds")
|
|
|
|
)
|
|
|
|
|
2017-05-12 14:34:41 +00:00
|
|
|
// HeadBlock handles reads and writes of time series data within a time window.
|
|
|
|
type HeadBlock struct {
|
2017-03-20 07:41:56 +00:00
|
|
|
mtx sync.RWMutex
|
|
|
|
dir string
|
2017-05-13 15:09:26 +00:00
|
|
|
wal WAL
|
2017-01-19 10:22:47 +00:00
|
|
|
|
2017-02-04 10:53:52 +00:00
|
|
|
activeWriters uint64
|
2017-03-17 11:12:50 +00:00
|
|
|
closed bool
|
2017-02-04 10:53:52 +00:00
|
|
|
|
2016-12-22 00:12:28 +00:00
|
|
|
// descs holds all chunk descs for the head block. Each chunk implicitly
|
|
|
|
// is assigned the index as its ID.
|
2017-01-11 12:02:38 +00:00
|
|
|
series []*memSeries
|
2016-12-22 00:12:28 +00:00
|
|
|
// hashes contains a collision map of label set hashes of chunks
|
2016-12-31 09:19:02 +00:00
|
|
|
// to their chunk descs.
|
2017-01-11 12:02:38 +00:00
|
|
|
hashes map[uint64][]*memSeries
|
2016-12-22 00:12:28 +00:00
|
|
|
|
|
|
|
values map[string]stringset // label names to possible values
|
|
|
|
postings *memPostings // postings lists for terms
|
2016-12-07 16:10:49 +00:00
|
|
|
|
2017-03-23 17:27:20 +00:00
|
|
|
meta BlockMeta
|
2017-01-19 13:01:38 +00:00
|
|
|
}
|
|
|
|
|
2017-05-13 16:14:18 +00:00
|
|
|
// TouchHeadBlock atomically touches a new head block in dir for
|
|
|
|
// samples in the range [mint,maxt).
|
|
|
|
func TouchHeadBlock(dir string, seq int, mint, maxt int64) error {
|
2017-03-02 08:13:29 +00:00
|
|
|
// Make head block creation appear atomic.
|
|
|
|
tmp := dir + ".tmp"
|
|
|
|
|
|
|
|
if err := os.MkdirAll(tmp, 0777); err != nil {
|
2017-05-13 16:14:18 +00:00
|
|
|
return err
|
2017-02-27 09:46:15 +00:00
|
|
|
}
|
2017-03-20 13:45:27 +00:00
|
|
|
|
|
|
|
entropy := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
|
|
2017-02-27 09:46:15 +00:00
|
|
|
ulid, err := ulid.New(ulid.Now(), entropy)
|
|
|
|
if err != nil {
|
2017-05-13 16:14:18 +00:00
|
|
|
return err
|
2017-01-19 13:01:38 +00:00
|
|
|
}
|
2016-12-22 11:05:24 +00:00
|
|
|
|
2017-03-02 08:13:29 +00:00
|
|
|
if err := writeMetaFile(tmp, &BlockMeta{
|
2017-02-27 09:46:15 +00:00
|
|
|
ULID: ulid,
|
2017-01-29 07:11:47 +00:00
|
|
|
Sequence: seq,
|
2017-02-01 14:29:48 +00:00
|
|
|
MinTime: mint,
|
|
|
|
MaxTime: maxt,
|
2017-01-29 07:11:47 +00:00
|
|
|
}); err != nil {
|
2017-05-13 16:14:18 +00:00
|
|
|
return err
|
2017-03-02 08:13:29 +00:00
|
|
|
}
|
2017-05-13 16:14:18 +00:00
|
|
|
return renameFile(tmp, dir)
|
2016-12-04 12:16:11 +00:00
|
|
|
}
|
|
|
|
|
2017-05-12 14:34:41 +00:00
|
|
|
// OpenHeadBlock opens the head block in dir.
|
2017-05-13 16:14:18 +00:00
|
|
|
func OpenHeadBlock(dir string, l log.Logger, wal WAL) (*HeadBlock, error) {
|
2017-01-19 13:01:38 +00:00
|
|
|
meta, err := readMetaFile(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-12-22 11:05:24 +00:00
|
|
|
|
2017-05-12 14:34:41 +00:00
|
|
|
h := &HeadBlock{
|
2017-01-10 14:28:22 +00:00
|
|
|
dir: dir,
|
2017-01-19 13:01:38 +00:00
|
|
|
wal: wal,
|
Fix missing postings in Merge and Intersect (#77)
* Test for a previous implematation of Intersect
Before we were moving the postings list everytime we create a new
chained `intersectPostings`. That was causing some postings to be
skipped. This test fails on the older version.
Signed-off-by: Goutham Veeramachaneni <cs14btech11014@iith.ac.in>
* Advance on Seek only when valid.
Issue:
Before in mergedPostings and others we advance everytime we `Seek`,
which causes issues with `Intersect`.
Take the case, where we have a mergedPostings = m merging, a: {10, 20, 30} and
b: {15, 25, 35}. Everytime we `Seek`, we do a.Seek and b.Seek.
Now if we Intersect m with {21, 22, 23, 30}, we would do Seek({21,22,23}) which
would advance a and b beyond 30.
Fix:
Now we advance only when the seeking value is greater than the current
value, as the definition specifies.
Also, posting 0 will not be a valid posting and will be used to signal
finished or un-initialized PostingsList.
Signed-off-by: Goutham Veeramachaneni <cs14btech11014@iith.ac.in>
* Add test for Merge+Intersect edgecase.
Signed-off-by: Goutham Veeramachaneni <cs14btech11014@iith.ac.in>
* Add comments to trivial tests.
Signed-off-by: Goutham Veeramachaneni <cs14btech11014@iith.ac.in>
2017-05-12 07:44:41 +00:00
|
|
|
series: []*memSeries{nil}, // 0 is not a valid posting, filled with nil.
|
2017-01-11 12:02:38 +00:00
|
|
|
hashes: map[uint64][]*memSeries{},
|
2016-12-22 00:12:28 +00:00
|
|
|
values: map[string]stringset{},
|
|
|
|
postings: &memPostings{m: make(map[term][]uint32)},
|
2017-01-19 13:01:38 +00:00
|
|
|
meta: *meta,
|
2017-01-07 15:20:32 +00:00
|
|
|
}
|
2017-05-13 16:14:18 +00:00
|
|
|
return h, h.init()
|
|
|
|
}
|
2017-01-06 11:37:28 +00:00
|
|
|
|
2017-05-13 16:14:18 +00:00
|
|
|
func (h *HeadBlock) init() error {
|
|
|
|
r := h.wal.Reader()
|
2017-02-15 05:54:59 +00:00
|
|
|
|
|
|
|
for r.Next() {
|
|
|
|
series, samples := r.At()
|
|
|
|
|
|
|
|
for _, lset := range series {
|
2017-01-19 10:22:47 +00:00
|
|
|
h.create(lset.Hash(), lset)
|
|
|
|
h.meta.Stats.NumSeries++
|
2017-02-15 05:54:59 +00:00
|
|
|
}
|
|
|
|
for _, s := range samples {
|
2017-05-12 15:06:26 +00:00
|
|
|
if int(s.Ref) >= len(h.series) {
|
2017-05-13 16:14:18 +00:00
|
|
|
return errors.Errorf("unknown series reference %d (max %d); abort WAL restore", s.Ref, len(h.series))
|
2017-03-23 17:27:20 +00:00
|
|
|
}
|
2017-05-12 15:06:26 +00:00
|
|
|
h.series[s.Ref].append(s.T, s.V)
|
2017-01-06 08:26:39 +00:00
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
if !h.inBounds(s.T) {
|
2017-05-13 16:14:18 +00:00
|
|
|
return errors.Wrap(ErrOutOfBounds, "consume WAL")
|
2017-01-19 10:22:47 +00:00
|
|
|
}
|
|
|
|
h.meta.Stats.NumSamples++
|
2017-02-15 05:54:59 +00:00
|
|
|
}
|
|
|
|
}
|
2017-05-13 16:14:18 +00:00
|
|
|
return errors.Wrap(r.Err(), "consume WAL")
|
2016-12-22 11:05:24 +00:00
|
|
|
}
|
|
|
|
|
2017-01-19 13:01:38 +00:00
|
|
|
// inBounds returns true if the given timestamp is within the valid
|
|
|
|
// time bounds of the block.
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) inBounds(t int64) bool {
|
2017-02-01 14:29:48 +00:00
|
|
|
return t >= h.meta.MinTime && t <= h.meta.MaxTime
|
2017-01-19 13:01:38 +00:00
|
|
|
}
|
|
|
|
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) String() string {
|
2017-03-20 09:41:43 +00:00
|
|
|
return fmt.Sprintf("(%d, %s)", h.meta.Sequence, h.meta.ULID)
|
|
|
|
}
|
|
|
|
|
2016-12-22 11:05:24 +00:00
|
|
|
// Close syncs all data and closes underlying resources of the head block.
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Close() error {
|
2017-03-04 15:50:48 +00:00
|
|
|
h.mtx.Lock()
|
2017-03-17 11:12:50 +00:00
|
|
|
defer h.mtx.Unlock()
|
2017-03-04 15:50:48 +00:00
|
|
|
|
2017-02-27 09:46:15 +00:00
|
|
|
if err := h.wal.Close(); err != nil {
|
2017-03-08 15:53:07 +00:00
|
|
|
return errors.Wrapf(err, "close WAL for head %s", h.dir)
|
2017-02-02 06:58:54 +00:00
|
|
|
}
|
2017-02-27 09:46:15 +00:00
|
|
|
// Check whether the head block still exists in the underlying dir
|
2017-03-02 13:32:09 +00:00
|
|
|
// or has already been replaced with a compacted version or removed.
|
2017-02-27 09:46:15 +00:00
|
|
|
meta, err := readMetaFile(h.dir)
|
2017-03-02 13:32:09 +00:00
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
}
|
2017-02-27 09:46:15 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if meta.ULID == h.meta.ULID {
|
|
|
|
return writeMetaFile(h.dir, &h.meta)
|
|
|
|
}
|
2017-03-17 11:12:50 +00:00
|
|
|
|
|
|
|
h.closed = true
|
2017-02-27 09:46:15 +00:00
|
|
|
return nil
|
2016-12-09 09:41:51 +00:00
|
|
|
}
|
|
|
|
|
2017-05-14 09:06:26 +00:00
|
|
|
// Meta implements headBlock
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Meta() BlockMeta {
|
2017-03-23 17:27:20 +00:00
|
|
|
m := BlockMeta{
|
|
|
|
ULID: h.meta.ULID,
|
|
|
|
Sequence: h.meta.Sequence,
|
|
|
|
MinTime: h.meta.MinTime,
|
|
|
|
MaxTime: h.meta.MaxTime,
|
|
|
|
Compaction: h.meta.Compaction,
|
|
|
|
}
|
2017-01-19 10:22:47 +00:00
|
|
|
|
2017-03-23 17:27:20 +00:00
|
|
|
m.Stats.NumChunks = atomic.LoadUint64(&h.meta.Stats.NumChunks)
|
|
|
|
m.Stats.NumSeries = atomic.LoadUint64(&h.meta.Stats.NumSeries)
|
|
|
|
m.Stats.NumSamples = atomic.LoadUint64(&h.meta.Stats.NumSamples)
|
|
|
|
|
|
|
|
return m
|
2017-01-19 10:22:47 +00:00
|
|
|
}
|
|
|
|
|
2017-05-14 09:06:26 +00:00
|
|
|
// Dir implements headBlock
|
|
|
|
func (h *HeadBlock) Dir() string { return h.dir }
|
|
|
|
|
|
|
|
// Persisted implements headBlock
|
|
|
|
func (h *HeadBlock) Persisted() bool { return false }
|
|
|
|
|
|
|
|
// Index implements headBlock
|
|
|
|
func (h *HeadBlock) Index() IndexReader { return &headIndexReader{h} }
|
|
|
|
|
|
|
|
// Chunks implements headBlock
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Chunks() ChunkReader { return &headChunkReader{h} }
|
2017-01-10 14:28:22 +00:00
|
|
|
|
2017-05-14 09:06:26 +00:00
|
|
|
// Delete implements headBlock
|
|
|
|
func (h *HeadBlock) Delete(int64, int64, ...labels.Matcher) error { return nil }
|
|
|
|
|
|
|
|
// Querier implements Queryable and headBlock
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Querier(mint, maxt int64) Querier {
|
2017-03-20 09:21:21 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
|
|
|
if h.closed {
|
|
|
|
panic(fmt.Sprintf("block %s already closed", h.dir))
|
|
|
|
}
|
2017-03-21 11:12:33 +00:00
|
|
|
|
|
|
|
// Reference on the original slice to use for postings mapping.
|
|
|
|
series := h.series[:]
|
|
|
|
|
2017-03-20 09:21:21 +00:00
|
|
|
return &blockQuerier{
|
2017-03-21 11:12:33 +00:00
|
|
|
mint: mint,
|
|
|
|
maxt: maxt,
|
|
|
|
index: h.Index(),
|
|
|
|
chunks: h.Chunks(),
|
|
|
|
postingsMapper: func(p Postings) Postings {
|
|
|
|
ep := make([]uint32, 0, 64)
|
|
|
|
|
|
|
|
for p.Next() {
|
2017-03-23 17:27:20 +00:00
|
|
|
// Skip posting entries that include series added after we
|
|
|
|
// instantiated the querier.
|
|
|
|
if int(p.At()) >= len(series) {
|
|
|
|
break
|
|
|
|
}
|
2017-03-21 11:12:33 +00:00
|
|
|
ep = append(ep, p.At())
|
|
|
|
}
|
|
|
|
if err := p.Err(); err != nil {
|
|
|
|
return errPostings{err: errors.Wrap(err, "expand postings")}
|
|
|
|
}
|
|
|
|
|
|
|
|
sort.Slice(ep, func(i, j int) bool {
|
|
|
|
return labels.Compare(series[ep[i]].lset, series[ep[j]].lset) < 0
|
|
|
|
})
|
|
|
|
return newListPostings(ep)
|
|
|
|
},
|
2017-03-20 09:21:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-05-14 09:06:26 +00:00
|
|
|
// Appender implements headBlock
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Appender() Appender {
|
2017-02-04 10:53:52 +00:00
|
|
|
atomic.AddUint64(&h.activeWriters, 1)
|
|
|
|
|
2017-01-12 18:18:51 +00:00
|
|
|
h.mtx.RLock()
|
2017-03-17 11:12:50 +00:00
|
|
|
|
|
|
|
if h.closed {
|
|
|
|
panic(fmt.Sprintf("block %s already closed", h.dir))
|
|
|
|
}
|
2017-05-12 14:34:41 +00:00
|
|
|
return &headAppender{HeadBlock: h, samples: getHeadAppendBuffer()}
|
2017-01-12 19:00:36 +00:00
|
|
|
}
|
|
|
|
|
2017-05-14 09:06:26 +00:00
|
|
|
// Busy implements headBlock
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) Busy() bool {
|
2017-03-20 07:41:56 +00:00
|
|
|
return atomic.LoadUint64(&h.activeWriters) > 0
|
|
|
|
}
|
|
|
|
|
2017-01-12 19:00:36 +00:00
|
|
|
var headPool = sync.Pool{}
|
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
func getHeadAppendBuffer() []RefSample {
|
2017-01-12 19:00:36 +00:00
|
|
|
b := headPool.Get()
|
|
|
|
if b == nil {
|
2017-05-12 15:06:26 +00:00
|
|
|
return make([]RefSample, 0, 512)
|
2017-01-12 19:00:36 +00:00
|
|
|
}
|
2017-05-12 15:06:26 +00:00
|
|
|
return b.([]RefSample)
|
2017-01-12 19:00:36 +00:00
|
|
|
}
|
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
func putHeadAppendBuffer(b []RefSample) {
|
2017-01-12 19:00:36 +00:00
|
|
|
headPool.Put(b[:0])
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type headAppender struct {
|
2017-05-12 14:34:41 +00:00
|
|
|
*HeadBlock
|
2017-01-12 18:18:51 +00:00
|
|
|
|
2017-01-13 15:14:40 +00:00
|
|
|
newSeries map[uint64]hashedLabels
|
|
|
|
newHashes map[uint64]uint64
|
|
|
|
refmap map[uint64]uint64
|
2017-01-12 18:18:51 +00:00
|
|
|
newLabels []labels.Labels
|
2017-01-13 14:25:11 +00:00
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
samples []RefSample
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type hashedLabels struct {
|
|
|
|
hash uint64
|
|
|
|
labels labels.Labels
|
|
|
|
}
|
|
|
|
|
2017-02-01 14:29:48 +00:00
|
|
|
func (a *headAppender) Add(lset labels.Labels, t int64, v float64) (uint64, error) {
|
2017-05-05 14:22:11 +00:00
|
|
|
if !a.inBounds(t) {
|
|
|
|
return 0, ErrOutOfBounds
|
|
|
|
}
|
|
|
|
|
2017-03-06 13:27:33 +00:00
|
|
|
hash := lset.Hash()
|
2017-01-12 18:18:51 +00:00
|
|
|
|
|
|
|
if ms := a.get(hash, lset); ms != nil {
|
2017-02-02 10:09:19 +00:00
|
|
|
return uint64(ms.ref), a.AddFast(uint64(ms.ref), t, v)
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
2017-01-13 14:25:11 +00:00
|
|
|
if ref, ok := a.newHashes[hash]; ok {
|
2017-02-02 10:09:19 +00:00
|
|
|
return uint64(ref), a.AddFast(uint64(ref), t, v)
|
2017-01-13 14:25:11 +00:00
|
|
|
}
|
2017-01-12 18:18:51 +00:00
|
|
|
|
2017-01-13 15:14:40 +00:00
|
|
|
// We only know the actual reference after committing. We generate an
|
|
|
|
// intermediate reference only valid for this batch.
|
|
|
|
// It is indicated by the the LSB of the 4th byte being set to 1.
|
|
|
|
// We use a random ID to avoid collisions when new series are created
|
2017-01-17 15:33:58 +00:00
|
|
|
// in two subsequent batches.
|
|
|
|
// TODO(fabxc): Provide method for client to determine whether a ref
|
|
|
|
// is valid beyond the current transaction.
|
2017-01-13 15:14:40 +00:00
|
|
|
ref := uint64(rand.Int31()) | (1 << 32)
|
|
|
|
|
2017-01-12 18:18:51 +00:00
|
|
|
if a.newSeries == nil {
|
2017-01-13 15:14:40 +00:00
|
|
|
a.newSeries = map[uint64]hashedLabels{}
|
|
|
|
a.newHashes = map[uint64]uint64{}
|
|
|
|
a.refmap = map[uint64]uint64{}
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
2017-01-13 15:14:40 +00:00
|
|
|
a.newSeries[ref] = hashedLabels{hash: hash, labels: lset}
|
|
|
|
a.newHashes[hash] = ref
|
2017-01-12 18:18:51 +00:00
|
|
|
|
2017-02-01 14:29:48 +00:00
|
|
|
return ref, a.AddFast(ref, t, v)
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
2017-02-01 14:29:48 +00:00
|
|
|
func (a *headAppender) AddFast(ref uint64, t int64, v float64) error {
|
2017-01-19 10:22:47 +00:00
|
|
|
// We only own the last 5 bytes of the reference. Anything before is
|
2017-01-13 15:14:40 +00:00
|
|
|
// used by higher-order appenders. We erase it to avoid issues.
|
2017-01-17 07:40:31 +00:00
|
|
|
ref = (ref << 24) >> 24
|
2017-01-12 18:18:51 +00:00
|
|
|
|
|
|
|
// Distinguish between existing series and series created in
|
|
|
|
// this transaction.
|
2017-01-17 07:40:31 +00:00
|
|
|
if ref&(1<<32) != 0 {
|
2017-01-13 15:14:40 +00:00
|
|
|
if _, ok := a.newSeries[ref]; !ok {
|
2017-01-16 13:18:32 +00:00
|
|
|
return ErrNotFound
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
// TODO(fabxc): we also have to validate here that the
|
|
|
|
// sample sequence is valid.
|
|
|
|
// We also have to revalidate it as we switch locks an create
|
|
|
|
// the new series.
|
2017-03-20 07:41:56 +00:00
|
|
|
} else if ref > uint64(len(a.series)) {
|
|
|
|
return ErrNotFound
|
2017-01-17 07:40:31 +00:00
|
|
|
} else {
|
|
|
|
ms := a.series[int(ref)]
|
|
|
|
if ms == nil {
|
|
|
|
return ErrNotFound
|
|
|
|
}
|
|
|
|
// TODO(fabxc): memory series should be locked here already.
|
|
|
|
// Only problem is release of locks in case of a rollback.
|
2017-01-19 14:03:57 +00:00
|
|
|
c := ms.head()
|
|
|
|
|
|
|
|
if !a.inBounds(t) {
|
2017-02-01 14:29:48 +00:00
|
|
|
return ErrOutOfBounds
|
2017-01-19 14:03:57 +00:00
|
|
|
}
|
2017-01-17 07:40:31 +00:00
|
|
|
if t < c.maxTime {
|
|
|
|
return ErrOutOfOrderSample
|
|
|
|
}
|
2017-05-02 21:06:40 +00:00
|
|
|
|
|
|
|
// We are allowing exact duplicates as we can encounter them in valid cases
|
|
|
|
// like federation and erroring out at that time would be extremely noisy.
|
2017-04-12 15:25:32 +00:00
|
|
|
if c.maxTime == t && math.Float64bits(ms.lastValue) != math.Float64bits(v) {
|
2017-01-17 07:40:31 +00:00
|
|
|
return ErrAmendSample
|
|
|
|
}
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
a.samples = append(a.samples, RefSample{
|
|
|
|
Ref: ref,
|
|
|
|
T: t,
|
|
|
|
V: v,
|
2017-01-12 18:18:51 +00:00
|
|
|
})
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *headAppender) createSeries() {
|
|
|
|
if len(a.newSeries) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
a.newLabels = make([]labels.Labels, 0, len(a.newSeries))
|
2017-01-13 15:14:40 +00:00
|
|
|
base0 := len(a.series)
|
2017-01-12 18:18:51 +00:00
|
|
|
|
|
|
|
a.mtx.RUnlock()
|
|
|
|
a.mtx.Lock()
|
|
|
|
|
2017-01-13 15:14:40 +00:00
|
|
|
base1 := len(a.series)
|
|
|
|
|
|
|
|
for ref, l := range a.newSeries {
|
2017-01-12 18:18:51 +00:00
|
|
|
// We switched locks and have to re-validate that the series were not
|
|
|
|
// created by another goroutine in the meantime.
|
2017-01-13 15:14:40 +00:00
|
|
|
if base1 > base0 {
|
|
|
|
if ms := a.get(l.hash, l.labels); ms != nil {
|
|
|
|
a.refmap[ref] = uint64(ms.ref)
|
|
|
|
continue
|
|
|
|
}
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
// Series is still new.
|
|
|
|
a.newLabels = append(a.newLabels, l.labels)
|
2017-01-13 15:14:40 +00:00
|
|
|
a.refmap[ref] = uint64(len(a.series))
|
2017-01-12 18:18:51 +00:00
|
|
|
|
2017-01-13 15:14:40 +00:00
|
|
|
a.create(l.hash, l.labels)
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
a.mtx.Unlock()
|
|
|
|
a.mtx.RLock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *headAppender) Commit() error {
|
2017-02-04 10:53:52 +00:00
|
|
|
defer atomic.AddUint64(&a.activeWriters, ^uint64(0))
|
2017-01-12 19:00:36 +00:00
|
|
|
defer putHeadAppendBuffer(a.samples)
|
2017-01-12 18:18:51 +00:00
|
|
|
|
|
|
|
a.createSeries()
|
2017-01-13 14:25:11 +00:00
|
|
|
|
2017-01-13 15:14:40 +00:00
|
|
|
for i := range a.samples {
|
|
|
|
s := &a.samples[i]
|
|
|
|
|
2017-05-12 15:06:26 +00:00
|
|
|
if s.Ref&(1<<32) > 0 {
|
|
|
|
s.Ref = a.refmap[s.Ref]
|
2017-01-13 15:14:40 +00:00
|
|
|
}
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
2017-01-17 07:40:31 +00:00
|
|
|
|
|
|
|
// Write all new series and samples to the WAL and add it to the
|
|
|
|
// in-mem database on success.
|
|
|
|
if err := a.wal.Log(a.newLabels, a.samples); err != nil {
|
2017-01-19 10:22:47 +00:00
|
|
|
a.mtx.RUnlock()
|
2017-01-17 07:40:31 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-04-20 12:45:07 +00:00
|
|
|
total := uint64(len(a.samples))
|
2017-01-19 13:01:38 +00:00
|
|
|
|
2017-01-17 15:33:58 +00:00
|
|
|
for _, s := range a.samples {
|
2017-05-12 15:06:26 +00:00
|
|
|
if !a.series[s.Ref].append(s.T, s.V) {
|
2017-01-17 15:33:58 +00:00
|
|
|
total--
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
a.mtx.RUnlock()
|
|
|
|
|
2017-03-23 17:27:20 +00:00
|
|
|
atomic.AddUint64(&a.meta.Stats.NumSamples, total)
|
|
|
|
atomic.AddUint64(&a.meta.Stats.NumSeries, uint64(len(a.newSeries)))
|
2017-01-12 18:18:51 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *headAppender) Rollback() error {
|
|
|
|
a.mtx.RUnlock()
|
2017-02-04 10:53:52 +00:00
|
|
|
atomic.AddUint64(&a.activeWriters, ^uint64(0))
|
|
|
|
putHeadAppendBuffer(a.samples)
|
2017-01-12 18:18:51 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-02-23 09:50:22 +00:00
|
|
|
type headChunkReader struct {
|
2017-05-12 14:34:41 +00:00
|
|
|
*HeadBlock
|
2017-01-07 17:02:17 +00:00
|
|
|
}
|
2016-12-14 17:38:46 +00:00
|
|
|
|
2016-12-15 15:14:33 +00:00
|
|
|
// Chunk returns the chunk for the reference number.
|
2017-02-23 09:50:22 +00:00
|
|
|
func (h *headChunkReader) Chunk(ref uint64) (chunks.Chunk, error) {
|
2017-01-06 16:23:12 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
2017-02-18 16:33:20 +00:00
|
|
|
si := ref >> 32
|
|
|
|
ci := (ref << 32) >> 32
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
c := &safeChunk{
|
2017-02-18 16:33:20 +00:00
|
|
|
Chunk: h.series[si].chunks[ci].chunk,
|
|
|
|
s: h.series[si],
|
|
|
|
i: int(ci),
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
2017-01-11 12:02:38 +00:00
|
|
|
return c, nil
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-09 15:51:39 +00:00
|
|
|
type safeChunk struct {
|
2017-01-11 12:02:38 +00:00
|
|
|
chunks.Chunk
|
|
|
|
s *memSeries
|
|
|
|
i int
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *safeChunk) Iterator() chunks.Iterator {
|
2017-01-11 12:02:38 +00:00
|
|
|
c.s.mtx.RLock()
|
|
|
|
defer c.s.mtx.RUnlock()
|
|
|
|
return c.s.iterator(c.i)
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
// func (c *safeChunk) Appender() (chunks.Appender, error) { panic("illegal") }
|
|
|
|
// func (c *safeChunk) Bytes() []byte { panic("illegal") }
|
|
|
|
// func (c *safeChunk) Encoding() chunks.Encoding { panic("illegal") }
|
2017-01-09 15:51:39 +00:00
|
|
|
|
2017-01-10 14:28:22 +00:00
|
|
|
type headIndexReader struct {
|
2017-05-12 14:34:41 +00:00
|
|
|
*HeadBlock
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// LabelValues returns the possible label values
|
2017-01-10 14:28:22 +00:00
|
|
|
func (h *headIndexReader) LabelValues(names ...string) (StringTuples, error) {
|
2017-01-06 16:23:12 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
2016-12-14 17:38:46 +00:00
|
|
|
if len(names) != 1 {
|
|
|
|
return nil, errInvalidSize
|
|
|
|
}
|
|
|
|
var sl []string
|
|
|
|
|
2016-12-22 00:12:28 +00:00
|
|
|
for s := range h.values[names[0]] {
|
2016-12-14 17:38:46 +00:00
|
|
|
sl = append(sl, s)
|
|
|
|
}
|
|
|
|
sort.Strings(sl)
|
|
|
|
|
2017-01-05 14:13:01 +00:00
|
|
|
return &stringTuples{l: len(names), s: sl}, nil
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Postings returns the postings list iterator for the label pair.
|
2017-01-10 14:28:22 +00:00
|
|
|
func (h *headIndexReader) Postings(name, value string) (Postings, error) {
|
2017-01-06 16:23:12 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
2016-12-22 00:12:28 +00:00
|
|
|
return h.postings.get(term{name: name, value: value}), nil
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Series returns the series for the given reference.
|
2017-03-14 14:40:16 +00:00
|
|
|
func (h *headIndexReader) Series(ref uint32) (labels.Labels, []*ChunkMeta, error) {
|
2017-01-06 16:23:12 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
if int(ref) >= len(h.series) {
|
2017-01-16 13:18:32 +00:00
|
|
|
return nil, nil, ErrNotFound
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
2017-01-11 12:02:38 +00:00
|
|
|
s := h.series[ref]
|
2017-03-14 14:40:16 +00:00
|
|
|
metas := make([]*ChunkMeta, 0, len(s.chunks))
|
2017-01-11 12:02:38 +00:00
|
|
|
|
|
|
|
s.mtx.RLock()
|
|
|
|
defer s.mtx.RUnlock()
|
|
|
|
|
|
|
|
for i, c := range s.chunks {
|
2017-03-14 14:40:16 +00:00
|
|
|
metas = append(metas, &ChunkMeta{
|
2017-01-11 12:02:38 +00:00
|
|
|
MinTime: c.minTime,
|
|
|
|
MaxTime: c.maxTime,
|
2017-02-18 16:33:20 +00:00
|
|
|
Ref: (uint64(ref) << 32) | uint64(i),
|
2017-01-11 12:02:38 +00:00
|
|
|
})
|
2017-01-03 14:43:26 +00:00
|
|
|
}
|
2017-01-11 12:02:38 +00:00
|
|
|
|
|
|
|
return s.lset, metas, nil
|
2016-12-31 14:35:08 +00:00
|
|
|
}
|
|
|
|
|
2017-01-10 14:28:22 +00:00
|
|
|
func (h *headIndexReader) LabelIndices() ([][]string, error) {
|
2017-01-06 16:23:12 +00:00
|
|
|
h.mtx.RLock()
|
|
|
|
defer h.mtx.RUnlock()
|
|
|
|
|
2016-12-31 14:35:08 +00:00
|
|
|
res := [][]string{}
|
|
|
|
|
|
|
|
for s := range h.values {
|
|
|
|
res = append(res, []string{s})
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
2016-12-31 14:35:08 +00:00
|
|
|
return res, nil
|
2016-12-14 17:38:46 +00:00
|
|
|
}
|
|
|
|
|
2016-12-04 12:16:11 +00:00
|
|
|
// get retrieves the chunk with the hash and label set and creates
|
|
|
|
// a new one if it doesn't exist yet.
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) get(hash uint64, lset labels.Labels) *memSeries {
|
2017-01-11 12:02:38 +00:00
|
|
|
series := h.hashes[hash]
|
2016-12-22 00:12:28 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
for _, s := range series {
|
|
|
|
if s.lset.Equals(lset) {
|
|
|
|
return s
|
2016-12-04 12:16:11 +00:00
|
|
|
}
|
|
|
|
}
|
2016-12-31 09:19:02 +00:00
|
|
|
return nil
|
2016-12-22 11:05:24 +00:00
|
|
|
}
|
|
|
|
|
2017-05-12 14:34:41 +00:00
|
|
|
func (h *HeadBlock) create(hash uint64, lset labels.Labels) *memSeries {
|
2017-01-12 18:18:51 +00:00
|
|
|
s := &memSeries{
|
|
|
|
lset: lset,
|
2017-01-13 15:14:40 +00:00
|
|
|
ref: uint32(len(h.series)),
|
2017-01-12 18:18:51 +00:00
|
|
|
}
|
2017-04-05 18:29:10 +00:00
|
|
|
// create the initial chunk and appender
|
|
|
|
s.cut()
|
2017-01-04 13:06:40 +00:00
|
|
|
|
2017-01-12 18:18:51 +00:00
|
|
|
// Allocate empty space until we can insert at the given index.
|
2017-01-13 15:14:40 +00:00
|
|
|
h.series = append(h.series, s)
|
2016-12-22 00:12:28 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
h.hashes[hash] = append(h.hashes[hash], s)
|
2016-12-22 00:12:28 +00:00
|
|
|
|
|
|
|
for _, l := range lset {
|
|
|
|
valset, ok := h.values[l.Name]
|
|
|
|
if !ok {
|
|
|
|
valset = stringset{}
|
|
|
|
h.values[l.Name] = valset
|
|
|
|
}
|
|
|
|
valset.set(l.Value)
|
2017-01-03 14:43:26 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
h.postings.add(s.ref, term{name: l.Name, value: l.Value})
|
2016-12-22 00:12:28 +00:00
|
|
|
}
|
2017-01-03 14:43:26 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
h.postings.add(s.ref, term{})
|
2016-12-04 12:16:11 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
return s
|
2016-12-04 12:16:11 +00:00
|
|
|
}
|
|
|
|
|
2017-03-24 09:20:39 +00:00
|
|
|
type sample struct {
|
|
|
|
t int64
|
|
|
|
v float64
|
|
|
|
}
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
type memSeries struct {
|
2017-01-09 15:51:39 +00:00
|
|
|
mtx sync.RWMutex
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
ref uint32
|
|
|
|
lset labels.Labels
|
|
|
|
chunks []*memChunk
|
2017-01-09 15:51:39 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
lastValue float64
|
2017-01-09 15:51:39 +00:00
|
|
|
sampleBuf [4]sample
|
|
|
|
|
2017-03-06 16:34:49 +00:00
|
|
|
app chunks.Appender // Current appender for the chunk.
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
func (s *memSeries) cut() *memChunk {
|
|
|
|
c := &memChunk{
|
|
|
|
chunk: chunks.NewXORChunk(),
|
|
|
|
maxTime: math.MinInt64,
|
|
|
|
}
|
|
|
|
s.chunks = append(s.chunks, c)
|
|
|
|
|
|
|
|
app, err := c.chunk.Appender()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
s.app = app
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *memSeries) append(t int64, v float64) bool {
|
2017-03-20 13:45:27 +00:00
|
|
|
s.mtx.Lock()
|
|
|
|
defer s.mtx.Unlock()
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
var c *memChunk
|
|
|
|
|
2017-05-09 14:22:19 +00:00
|
|
|
if s.head().samples > 130 {
|
2017-01-11 12:02:38 +00:00
|
|
|
c = s.cut()
|
|
|
|
c.minTime = t
|
|
|
|
} else {
|
|
|
|
c = s.head()
|
2017-05-02 21:06:40 +00:00
|
|
|
// Skip duplicate and out of order samples.
|
|
|
|
if c.maxTime >= t {
|
2017-01-11 12:02:38 +00:00
|
|
|
return false
|
|
|
|
}
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
2017-01-11 12:02:38 +00:00
|
|
|
s.app.Append(t, v)
|
|
|
|
|
|
|
|
c.maxTime = t
|
|
|
|
c.samples++
|
|
|
|
|
|
|
|
s.lastValue = v
|
2017-01-09 15:51:39 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
s.sampleBuf[0] = s.sampleBuf[1]
|
|
|
|
s.sampleBuf[1] = s.sampleBuf[2]
|
|
|
|
s.sampleBuf[2] = s.sampleBuf[3]
|
|
|
|
s.sampleBuf[3] = sample{t: t, v: v}
|
2017-01-09 15:51:39 +00:00
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
return true
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
func (s *memSeries) iterator(i int) chunks.Iterator {
|
|
|
|
c := s.chunks[i]
|
|
|
|
|
|
|
|
if i < len(s.chunks)-1 {
|
|
|
|
return c.chunk.Iterator()
|
|
|
|
}
|
|
|
|
|
2017-01-09 15:51:39 +00:00
|
|
|
it := &memSafeIterator{
|
2017-01-11 12:02:38 +00:00
|
|
|
Iterator: c.chunk.Iterator(),
|
2017-01-09 15:51:39 +00:00
|
|
|
i: -1,
|
2017-01-11 12:02:38 +00:00
|
|
|
total: c.samples,
|
|
|
|
buf: s.sampleBuf,
|
2017-01-09 15:51:39 +00:00
|
|
|
}
|
|
|
|
return it
|
|
|
|
}
|
|
|
|
|
2017-01-11 12:02:38 +00:00
|
|
|
func (s *memSeries) head() *memChunk {
|
|
|
|
return s.chunks[len(s.chunks)-1]
|
|
|
|
}
|
|
|
|
|
|
|
|
type memChunk struct {
|
|
|
|
chunk chunks.Chunk
|
|
|
|
minTime, maxTime int64
|
|
|
|
samples int
|
|
|
|
}
|
|
|
|
|
2017-01-09 15:51:39 +00:00
|
|
|
type memSafeIterator struct {
|
|
|
|
chunks.Iterator
|
|
|
|
|
|
|
|
i int
|
|
|
|
total int
|
|
|
|
buf [4]sample
|
|
|
|
}
|
|
|
|
|
|
|
|
func (it *memSafeIterator) Next() bool {
|
|
|
|
if it.i+1 >= it.total {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
it.i++
|
|
|
|
if it.total-it.i > 4 {
|
|
|
|
return it.Iterator.Next()
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func (it *memSafeIterator) At() (int64, float64) {
|
|
|
|
if it.total-it.i > 4 {
|
|
|
|
return it.Iterator.At()
|
|
|
|
}
|
|
|
|
s := it.buf[4-(it.total-it.i)]
|
|
|
|
return s.t, s.v
|
|
|
|
}
|