From 61f1953c50148b3857bad6ab9a5622e1f1277695 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 18 Dec 2019 16:57:51 +0000 Subject: [PATCH 1/7] Cut 2.15.0-rc.0 release. (#6479) * Cut 2.15.0-rc.0 Signed-off-by: Bartek Plotka --- CHANGELOG.md | 19 +++++++++++++++++++ VERSION | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc36461c..1571bd697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## 2.15.0-rc.0 / 2019-12-18 + +* [CHANGE] Discovery: Removed `prometheus_sd_kubernetes_cache_*` metrics. Additionally `prometheus_sd_kubernetes_workqueue_latency_seconds` and `prometheus_sd_kubernetes_workqueue_work_duration_seconds` metrics now show correct values in seconds. #6393 +* [CHANGE] Remote write: Changed `query` label on `prometheus_remote_storage_*` metrics to `remote_name` and `url`. #6043 +* [FEATURE] API: Added new endpoint for exposing per metric metadata `/metadata`. #6420 #6442 +* [ENHANCEMENT] TSDB: Significantly reduced memory footprint of loaded TSDB blocks. #6418 #6461 +* [ENHANCEMENT] TSDB: Significantly optimized what we buffer during compaction which should result in lower memory footprint during compaction. #6422 #6452 #6468 #6475 +* [ENHANCEMENT] TSDB: Improve replay latency. #6230 +* [ENHANCEMENT] TSDB: WAL size is now used for size based retention calculation. #5886 +* [ENHANCEMENT] Remote read: Added query grouping and range hints to the remote read request #6401 +* [ENHANCEMENT] Remote write: Added `prometheus_remote_storage_sent_bytes_total` counter per queue. #6344 +* [ENHANCEMENT] promql: Improved PromQL parser performance. #6356 +* [ENHANCEMENT] React UI: Implemented missing pages like `/targets` #6276, TSDB status page #6267 and many other fixes and performance improvements. +* [ENHANCEMENT] promql: Prometheus now accepts spaces between time range and square bracket. e.g `[ 5m]` #6065 +* [BUGFIX] Config: Fixed alertmanager configuration to not miss targets when configurations are similar. #6455 +* [BUGFIX] Remote write: Value of `prometheus_remote_storage_shards_desired` gauge shows raw value of desired shards and it's updated correctly. #6378 +* [BUGFIX] Rules: Prometheus now fails the evaluation of rules and alerts where metric results collide with labels specified in `labels` field. #6469 +* [BUGFIX] API: Targets Metadata API `/targets/metadata` now accepts empty `match_targets` parameter as in the spec. #6303 + ## 2.14.0 / 2019-11-11 * [SECURITY/BUGFIX] UI: Ensure warnings from the API are escaped. #6279 diff --git a/VERSION b/VERSION index edcfe40d1..90edc54ef 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.0 +2.15.0-rc.0 From ec1868b0267d13cb5967286fd5ec6afff507905b Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 23 Dec 2019 11:57:27 +0000 Subject: [PATCH 2/7] Cut 2.15.0 release. (#6509) Signed-off-by: Bartek Plotka --- CHANGELOG.md | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1571bd697..3e0b15e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 2.15.0-rc.0 / 2019-12-18 +## 2.15.0 / 2019-12-23 * [CHANGE] Discovery: Removed `prometheus_sd_kubernetes_cache_*` metrics. Additionally `prometheus_sd_kubernetes_workqueue_latency_seconds` and `prometheus_sd_kubernetes_workqueue_work_duration_seconds` metrics now show correct values in seconds. #6393 * [CHANGE] Remote write: Changed `query` label on `prometheus_remote_storage_*` metrics to `remote_name` and `url`. #6043 diff --git a/VERSION b/VERSION index 90edc54ef..68e69e405 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.15.0-rc.0 +2.15.0 From c2e083c30f2cf94bfb5bc8bdfcb27dafb37e2349 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 24 Dec 2019 22:55:22 +0100 Subject: [PATCH 3/7] Fixed race in Chunks method. (#6515) Added regression test. Fixes #6512 Before (not deterministic result due to concurrency): ``` === RUN TestChunkReader_ConcurrentRead --- FAIL: TestChunkReader_ConcurrentRead (0.01s) db_test.go:2702: unexpected error: checksum mismatch expected:597ad276, actual:597ad276 db_test.go:2702: unexpected error: checksum mismatch expected:dd0cdbc2, actual:dd0cdbc2 FAIL ``` After succuess on multiple runs. Signed-off-by: Bartlomiej Plotka --- tsdb/chunks/chunks.go | 21 ++++++++------- tsdb/db_test.go | 60 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go index 418541f58..1a11924d1 100644 --- a/tsdb/chunks/chunks.go +++ b/tsdb/chunks/chunks.go @@ -456,16 +456,14 @@ func (b realByteSlice) Sub(start, end int) ByteSlice { type Reader struct { // The underlying bytes holding the encoded series data. // Each slice holds the data for a different segment. - bs []ByteSlice - cs []io.Closer // Closers for resources behind the byte slices. - size int64 // The total size of bytes in the reader. - pool chunkenc.Pool - crc32 hash.Hash - buf [binary.MaxVarintLen32]byte + bs []ByteSlice + cs []io.Closer // Closers for resources behind the byte slices. + size int64 // The total size of bytes in the reader. + pool chunkenc.Pool } func newReader(bs []ByteSlice, cs []io.Closer, pool chunkenc.Pool) (*Reader, error) { - cr := Reader{pool: pool, bs: bs, cs: cs, crc32: newCRC32()} + cr := Reader{pool: pool, bs: bs, cs: cs} var totalSize int64 for i, b := range cr.bs { @@ -541,6 +539,7 @@ func (s *Reader) Chunk(ref uint64) (chunkenc.Chunk, error) { // Get the lower 4 bytes. // These contain the segment offset where the data for this chunk starts. chkStart = int((ref << 32) >> 32) + chkCRC32 = newCRC32() ) if sgmIndex >= len(s.bs) { @@ -569,12 +568,12 @@ func (s *Reader) Chunk(ref uint64) (chunkenc.Chunk, error) { return nil, errors.Errorf("segment doesn't include enough bytes to read the chunk - required:%v, available:%v", chkEnd, sgmBytes.Len()) } - sum := sgmBytes.Range(chkEnd-crc32.Size, chkEnd) - s.crc32.Reset() - if _, err := s.crc32.Write(sgmBytes.Range(chkEncStart, chkDataEnd)); err != nil { + sum := sgmBytes.Range(chkDataEnd, chkEnd) + if _, err := chkCRC32.Write(sgmBytes.Range(chkEncStart, chkDataEnd)); err != nil { return nil, err } - if act := s.crc32.Sum(s.buf[:0]); !bytes.Equal(act, sum) { + + if act := chkCRC32.Sum(nil); !bytes.Equal(act, sum) { return nil, errors.Errorf("checksum mismatch expected:%x, actual:%x", sum, act) } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 1168460ec..70d79118f 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -25,6 +25,7 @@ import ( "path/filepath" "sort" "strconv" + "sync" "testing" "time" @@ -2485,9 +2486,9 @@ func TestDBReadOnly_FlushWAL(t *testing.T) { testutil.Equals(t, 1000.0, sum) } -// TestChunkWriter ensures that chunk segment are cut at the set segment size and +// TestChunkWriter_ReadAfterWrite ensures that chunk segment are cut at the set segment size and // that the resulted segments includes the expected chunks data. -func TestChunkWriter(t *testing.T) { +func TestChunkWriter_ReadAfterWrite(t *testing.T) { chk1 := tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 1}}) chk2 := tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 2}}) chk3 := tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 3}}) @@ -2611,22 +2612,19 @@ func TestChunkWriter(t *testing.T) { for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "test_chunk_witer") + tempDir, err := ioutil.TempDir("", "test_chunk_writer") testutil.Ok(t, err) - defer func() { - testutil.Ok(t, os.RemoveAll(tmpdir)) - }() + defer func() { testutil.Ok(t, os.RemoveAll(tempDir)) }() - chunkw, err := chunks.NewWriterWithSegSize(tmpdir, chunks.SegmentHeaderSize+int64(test.segmentSize)) + chunkw, err := chunks.NewWriterWithSegSize(tempDir, chunks.SegmentHeaderSize+int64(test.segmentSize)) testutil.Ok(t, err) for _, chks := range test.chks { - chunkw.WriteChunks(chks...) + testutil.Ok(t, chunkw.WriteChunks(chks...)) } - testutil.Ok(t, chunkw.Close()) - files, err := ioutil.ReadDir(tmpdir) + files, err := ioutil.ReadDir(tempDir) testutil.Ok(t, err) testutil.Equals(t, test.expSegmentsCount, len(files), "expected segments count mismatch") @@ -2655,7 +2653,7 @@ func TestChunkWriter(t *testing.T) { testutil.Equals(t, sizeExp, sizeAct) // Check the content of the chunks. - r, err := chunks.NewDirReader(tmpdir, nil) + r, err := chunks.NewDirReader(tempDir, nil) testutil.Ok(t, err) for _, chks := range test.chks { @@ -2668,3 +2666,43 @@ func TestChunkWriter(t *testing.T) { }) } } + +// TestChunkReader_ConcurrentReads checks that the chunk result can be read concurrently. +// Regression test for https://github.com/prometheus/prometheus/pull/6514. +func TestChunkReader_ConcurrentReads(t *testing.T) { + chks := []chunks.Meta{ + tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 1}}), + tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 2}}), + tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 3}}), + tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 4}}), + tsdbutil.ChunkFromSamples([]tsdbutil.Sample{sample{1, 5}}), + } + + tempDir, err := ioutil.TempDir("", "test_chunk_writer") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tempDir)) }() + + chunkw, err := chunks.NewWriter(tempDir) + testutil.Ok(t, err) + + testutil.Ok(t, chunkw.WriteChunks(chks...)) + testutil.Ok(t, chunkw.Close()) + + r, err := chunks.NewDirReader(tempDir, nil) + testutil.Ok(t, err) + + var wg sync.WaitGroup + for _, chk := range chks { + for i := 0; i < 100; i++ { + wg.Add(1) + go func(chunk chunks.Meta) { + defer wg.Done() + + chkAct, err := r.Chunk(chunk.Ref) + testutil.Ok(t, err) + testutil.Equals(t, chunk.Chunk.Bytes(), chkAct.Bytes()) + }(chk) + } + wg.Wait() + } +} From e6e48e6dce7b8483cd718209a5562d6bdda1d503 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 24 Dec 2019 22:51:12 +0100 Subject: [PATCH 4/7] Cut release 2.15.1 Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0b15e17..33a611ea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.15.1 / 2019-12-25 + +* [BUGFIX] Fixed race on concurrent queries against same data. #6512 + ## 2.15.0 / 2019-12-23 * [CHANGE] Discovery: Removed `prometheus_sd_kubernetes_cache_*` metrics. Additionally `prometheus_sd_kubernetes_workqueue_latency_seconds` and `prometheus_sd_kubernetes_workqueue_work_duration_seconds` metrics now show correct values in seconds. #6393 diff --git a/VERSION b/VERSION index 68e69e405..3b1fc7950 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.15.0 +2.15.1 From ada0945b8fcbdc7ead552111424b6bc003242e13 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Sat, 4 Jan 2020 14:55:02 +0000 Subject: [PATCH 5/7] Fix tsdb code and tests to work on Windows. (#6547) Add back Windows CI, we lost it when tsdb was merged into the prometheus repo. There's many tests failing outside tsdb, so only test tsdb for now. Fixes #6513 Signed-off-by: Brian Brazil --- .circleci/config.yml | 12 ++++++++++++ tsdb/block_test.go | 2 ++ tsdb/db_test.go | 2 ++ tsdb/index/index.go | 25 ++++++++++++++++++------- tsdb/index/index_test.go | 1 + tsdb/wal/watcher_test.go | 1 + 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55496f2f6..e7992d245 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,7 @@ version: 2.1 orbs: prometheus: prometheus/prometheus@0.3.0 go: circleci/go@0.2.0 + win: circleci/windows@2.3.0 executors: # Whenever the Go version is updated here, .promu.yml @@ -49,6 +50,13 @@ jobs: key: v1-npm-deps-{{ checksum "web/ui/react-app/yarn.lock" }} paths: - web/ui/react-app/node_modules + test_windows: + executor: win/default + working_directory: /go/src/github.com/prometheus/prometheus + steps: + - checkout + # TSDB is where the most risk is Windows wise, so only test there for now. + - run: go test ./tsdb/... fuzzit_regression: executor: fuzzit working_directory: /go/src/github.com/prometheus/prometheus @@ -78,6 +86,10 @@ workflows: filters: tags: only: /.*/ + - test_windows: + filters: + tags: + only: /.*/ - fuzzit_regression: filters: tags: diff --git a/tsdb/block_test.go b/tsdb/block_test.go index f746ccd4b..0f83ac5c0 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -197,9 +197,11 @@ func TestCorruptedChunk(t *testing.T) { testutil.Equals(t, test.openErr.Error(), err.Error()) return } + defer func() { testutil.Ok(t, b.Close()) }() querier, err := NewBlockQuerier(b, 0, 1) testutil.Ok(t, err) + defer func() { testutil.Ok(t, querier.Close()) }() set, err := querier.Select(labels.MustNewMatcher(labels.MatchEqual, "a", "b")) testutil.Ok(t, err) diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 70d79118f..9063b741a 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2655,6 +2655,7 @@ func TestChunkWriter_ReadAfterWrite(t *testing.T) { // Check the content of the chunks. r, err := chunks.NewDirReader(tempDir, nil) testutil.Ok(t, err) + defer func() { testutil.Ok(t, r.Close()) }() for _, chks := range test.chks { for _, chkExp := range chks { @@ -2705,4 +2706,5 @@ func TestChunkReader_ConcurrentReads(t *testing.T) { } wg.Wait() } + testutil.Ok(t, r.Close()) } diff --git a/tsdb/index/index.go b/tsdb/index/index.go index 700c5da7e..18fa0360e 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -511,11 +511,11 @@ func (w *Writer) finishSymbols() error { return err } - var err error - w.symbolFile, err = fileutil.OpenMmapFile(w.f.name) + sf, err := fileutil.OpenMmapFile(w.f.name) if err != nil { return err } + w.symbolFile = sf hash := crc32.Checksum(w.symbolFile.Bytes()[w.toc.Symbols+4:hashPos], castagnoliTable) w.buf1.Reset() w.buf1.PutBE32(hash) @@ -700,7 +700,11 @@ func (w *Writer) writePostingsOffsetTable() error { if err != nil { return err } - defer f.Close() + defer func() { + if f != nil { + f.Close() + } + }() d := encoding.NewDecbufRaw(realByteSlice(f.Bytes()), int(w.fPO.pos)) cnt := w.cntPO for d.Err() == nil && cnt > 0 { @@ -720,6 +724,10 @@ func (w *Writer) writePostingsOffsetTable() error { } // Cleanup temporary file. + if err := f.Close(); err != nil { + return err + } + f = nil if err := w.fPO.close(); err != nil { return err } @@ -962,9 +970,9 @@ type labelIndexHashEntry struct { } func (w *Writer) Close() error { - if err := w.ensureStage(idxStageDone); err != nil { - return err - } + // Even if this fails, we need to close all the files. + ensureErr := w.ensureStage(idxStageDone) + if w.symbolFile != nil { if err := w.symbolFile.Close(); err != nil { return err @@ -980,7 +988,10 @@ func (w *Writer) Close() error { return err } } - return w.f.close() + if err := w.f.close(); err != nil { + return err + } + return ensureErr } // StringTuples provides access to a sorted list of string tuples. diff --git a/tsdb/index/index_test.go b/tsdb/index/index_test.go index a8bf59112..4a0b6b8df 100644 --- a/tsdb/index/index_test.go +++ b/tsdb/index/index_test.go @@ -280,6 +280,7 @@ func TestPostingsMany(t *testing.T) { ir, err := NewFileReader(fn) testutil.Ok(t, err) + defer func() { testutil.Ok(t, ir.Close()) }() cases := []struct { in []string diff --git a/tsdb/wal/watcher_test.go b/tsdb/wal/watcher_test.go index 8378e997d..db8e3e89f 100644 --- a/tsdb/wal/watcher_test.go +++ b/tsdb/wal/watcher_test.go @@ -427,6 +427,7 @@ func TestReadCheckpointMultipleSegments(t *testing.T) { } } } + testutil.Ok(t, w.Close()) // At this point we should have at least 6 segments, lets create a checkpoint dir of the first 5. checkpointDir := dir + "/wal/checkpoint.000004" From 13e88e5a49d5179ea44a58a1f1f78cece657fb8f Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Mon, 6 Jan 2020 14:06:11 +0000 Subject: [PATCH 6/7] Handle V1 indexes, some of which have unsorted posting offset tables. (#6564) Fixes #6535 Signed-off-by: Brian Brazil --- tsdb/block_test.go | 26 ++-- tsdb/index/index.go | 128 ++++++++++++++------ tsdb/testdata/index_format_v1/chunks/000001 | Bin 44 -> 1844 bytes tsdb/testdata/index_format_v1/index | Bin 238 -> 5407 bytes tsdb/testdata/index_format_v1/meta.json | 10 +- 5 files changed, 112 insertions(+), 52 deletions(-) diff --git a/tsdb/block_test.go b/tsdb/block_test.go index 0f83ac5c0..ef19468a3 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -267,16 +267,20 @@ func TestBlockSize(t *testing.T) { } func TestReadIndexFormatV1(t *testing.T) { - /* The block here was produced at commit - 07ef80820ef1250db82f9544f3fcf7f0f63ccee0 with: - db, _ := Open("v1db", nil, nil, nil) - app := db.Appender() - app.Add(labels.FromStrings("foo", "bar"), 1, 2) - app.Add(labels.FromStrings("foo", "baz"), 3, 4) - app.Add(labels.FromStrings("foo", "meh"), 1000*3600*4, 4) // Not in the block. - app.Commit() - db.compact() - db.Close() + /* The block here was produced at the commit + 706602daed1487f7849990678b4ece4599745905 used in 2.0.0 with: + db, _ := Open("v1db", nil, nil, nil) + app := db.Appender() + app.Add(labels.FromStrings("foo", "bar"), 1, 2) + app.Add(labels.FromStrings("foo", "baz"), 3, 4) + app.Add(labels.FromStrings("foo", "meh"), 1000*3600*4, 4) // Not in the block. + // Make sure we've enough values for the lack of sorting of postings offsets to show up. + for i := 0; i < 100; i++ { + app.Add(labels.FromStrings("bar", strconv.FormatInt(int64(i), 10)), 0, 0) + } + app.Commit() + db.compact() + db.Close() */ blockDir := filepath.Join("testdata", "index_format_v1") @@ -290,7 +294,7 @@ func TestReadIndexFormatV1(t *testing.T) { q, err = NewBlockQuerier(block, 0, 1000) testutil.Ok(t, err) - testutil.Equals(t, query(t, q, labels.MustNewMatcher(labels.MatchNotRegexp, "foo", "^.$")), + testutil.Equals(t, query(t, q, labels.MustNewMatcher(labels.MatchNotRegexp, "foo", "^.?$")), map[string][]tsdbutil.Sample{ `{foo="bar"}`: []tsdbutil.Sample{sample{t: 1, v: 2}}, `{foo="baz"}`: []tsdbutil.Sample{sample{t: 3, v: 4}}, diff --git a/tsdb/index/index.go b/tsdb/index/index.go index 18fa0360e..dd1a0dd37 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -1024,6 +1024,8 @@ type Reader struct { // Map of LabelName to a list of some LabelValues's position in the offset table. // The first and last values for each name are always present. postings map[string][]postingOffset + // For the v1 format, labelname -> labelvalue -> offset. + postingsV1 map[string]map[string]uint64 symbols *Symbols nameSymbols map[uint32]string // Cache of the label name symbol lookups, @@ -1113,45 +1115,64 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) { return nil, errors.Wrap(err, "read symbols") } - var lastKey []string - lastOff := 0 - valueCount := 0 - // For the postings offset table we keep every label name but only every nth - // label value (plus the first and last one), to save memory. - if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []string, _ uint64, off int) error { - if len(key) != 2 { - return errors.Errorf("unexpected key length for posting table %d", len(key)) - } - if _, ok := r.postings[key[0]]; !ok { - // Next label name. - r.postings[key[0]] = []postingOffset{} - if lastKey != nil { - // Always include last value for each label name. - r.postings[lastKey[0]] = append(r.postings[lastKey[0]], postingOffset{value: lastKey[1], off: lastOff}) + if r.version == FormatV1 { + // Earlier V1 formats don't have a sorted postings offset table, so + // load the whole offset table into memory. + r.postingsV1 = map[string]map[string]uint64{} + if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []string, off uint64, _ int) error { + if len(key) != 2 { + return errors.Errorf("unexpected key length for posting table %d", len(key)) } - lastKey = nil - valueCount = 0 + if _, ok := r.postingsV1[key[0]]; !ok { + r.postingsV1[key[0]] = map[string]uint64{} + r.postings[key[0]] = nil // Used to get a list of labelnames in places. + } + r.postingsV1[key[0]][key[1]] = off + return nil + }); err != nil { + return nil, errors.Wrap(err, "read postings table") } - if valueCount%32 == 0 { - r.postings[key[0]] = append(r.postings[key[0]], postingOffset{value: key[1], off: off}) - lastKey = nil - } else { - lastKey = key - lastOff = off + } else { + var lastKey []string + lastOff := 0 + valueCount := 0 + // For the postings offset table we keep every label name but only every nth + // label value (plus the first and last one), to save memory. + if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []string, _ uint64, off int) error { + if len(key) != 2 { + return errors.Errorf("unexpected key length for posting table %d", len(key)) + } + if _, ok := r.postings[key[0]]; !ok { + // Next label name. + r.postings[key[0]] = []postingOffset{} + if lastKey != nil { + // Always include last value for each label name. + r.postings[lastKey[0]] = append(r.postings[lastKey[0]], postingOffset{value: lastKey[1], off: lastOff}) + } + lastKey = nil + valueCount = 0 + } + if valueCount%32 == 0 { + r.postings[key[0]] = append(r.postings[key[0]], postingOffset{value: key[1], off: off}) + lastKey = nil + } else { + lastKey = key + lastOff = off + } + valueCount++ + return nil + }); err != nil { + return nil, errors.Wrap(err, "read postings table") + } + if lastKey != nil { + r.postings[lastKey[0]] = append(r.postings[lastKey[0]], postingOffset{value: lastKey[1], off: lastOff}) + } + // Trim any extra space in the slices. + for k, v := range r.postings { + l := make([]postingOffset, len(v)) + copy(l, v) + r.postings[k] = l } - valueCount++ - return nil - }); err != nil { - return nil, errors.Wrap(err, "read postings table") - } - if lastKey != nil { - r.postings[lastKey[0]] = append(r.postings[lastKey[0]], postingOffset{value: lastKey[1], off: lastOff}) - } - // Trim any extra space in the slices. - for k, v := range r.postings { - l := make([]postingOffset, len(v)) - copy(l, v) - r.postings[k] = l } r.nameSymbols = make(map[uint32]string, len(r.postings)) @@ -1408,6 +1429,19 @@ func (r *Reader) LabelValues(names ...string) (StringTuples, error) { if len(names) != 1 { return nil, errors.Errorf("only one label name supported") } + if r.version == FormatV1 { + e, ok := r.postingsV1[names[0]] + if !ok { + return emptyStringTuples{}, nil + } + values := make([]string, 0, len(e)) + for k := range e { + values = append(values, k) + } + sort.Strings(values) + return NewStringTuples(values, 1) + + } e, ok := r.postings[names[0]] if !ok { return emptyStringTuples{}, nil @@ -1467,6 +1501,28 @@ func (r *Reader) Series(id uint64, lbls *labels.Labels, chks *[]chunks.Meta) err } func (r *Reader) Postings(name string, values ...string) (Postings, error) { + if r.version == FormatV1 { + e, ok := r.postingsV1[name] + if !ok { + return EmptyPostings(), nil + } + res := make([]Postings, 0, len(values)) + for _, v := range values { + postingsOff, ok := e[v] + if !ok { + continue + } + // Read from the postings table. + d := encoding.NewDecbufAt(r.b, int(postingsOff), castagnoliTable) + _, p, err := r.dec.Postings(d.Get()) + if err != nil { + return nil, errors.Wrap(err, "decode postings") + } + res = append(res, p) + } + return Merge(res...), nil + } + e, ok := r.postings[name] if !ok { return EmptyPostings(), nil diff --git a/tsdb/testdata/index_format_v1/chunks/000001 b/tsdb/testdata/index_format_v1/chunks/000001 index 1f2250b865ef7234a1bcc92dee7f76e033aadc03..ac544dbbef997224e7b69248bab6e0b945bbc7a4 100644 GIT binary patch literal 1844 zcmZqO>u{Hmfq{XCkpVqT-kXDdqcr+Q)Q+qOoeJp)tPEA)nuwAXiW1lO=6nN zG=*s@)6k+b&0w0zG)vH#?qj;dbeZW2(^aNxOh==^bd%|pU@*hS42c;sGZbd1%+Q#j zGXw1=Gc3VmrjMBtGi7Ef%v71FF;i!z!Az7a!D5z=SrW5kW+}{4nWZsHXO_V%lUbH~ z+}L-(9CzH?Z@Z^WnE3G%Tl$9z*(?%s7+RRWD1@XgBzn+)ZhlWvAJTcj^#LkLBS^2E zC7pMZG=U86I%Ky%5)BzwmitORNlVDAxyrn-Fh~p}NjTQyUwg>f!P0~t8>+^Tb%5;< zyn|PcC+i5y6ZSVgKaQ+3?36IGVEP2IZm?3JAm*GwCc*9qEzj=wflP%ZvQK_YC?hjq zYuNUxCBwyU1T$!5bgdgfJ} z2ql{X+v|CvTiblHd9V|n&^g_s!eI#lw8&$&|L9xNg``(KmpeUmmo$mA#xwWW*wLhm zNtb#z_j{%pX$onEx9skwWYT4%UwM1SH#tnYg7mmonH_eBbT#RDZ$+ave$rIZO7GRg zb8AW0kv{as=L>nH>q%2WUOaPPA!!EbhLCsCn;#_IM7lfV`ID`uk!~eD8q!E_(2{gJ z>A8^lcOTCp%_6M``KE>HBF!eP4)KSyeT8%n>5|axX=8Vg?j=nRmE)hPJ^29Xm!a1` z`beW{F6oib==&vqkRBvG6WV&!uEwNCNH2y?pOUzY^cd;=&{H>jIix2@lf!Z^w#X(u zNxClVcGmJ}(gM<~u*Rple??kIdMIpP?9xw2Pm}%_7MH%U_VUk>mW36}nskz?7fA1h zwf!)!Eom`nQh29758osGg)}X^hE1JKT1vVj{Oj-A#gdkj=7kUU-_IbeAUze%j7L9` zULh?FFV6mc0cjQKo$$H^H*5FCP13|VtJL(`OSnzCrcUSfCu%p+UDC`tuMW*VO|N}G znp@}Ih2|Bc4@nE^WGq`!yOI7P{k6`PtB)R2HGx5I)#?4w(@CTYNf$)y&-!&FX%gw` zh%PUr-X~p5x;3I%QuXtsDWo|Otv`IHc72zT=0`j`u6!I-SCE!OoT@IGK)RarM#Nk7 z8rCOGB~6GNVGTV&x{h>ZqM7oL8AK9b$+HTUV zq{Wd}zCKpl)ppXV$W^y~*hSSW(!V47uDXP-4zO${FxT>^Ya7!I9rOgxfj*!w=nn>hm%$J) z3=9V&z$h>Vj0bUG5||3!2Jv7Pm<{HDc_0BS1W8~qNCC^h3a}caf^}d$$N-zbR;(rvE;tB|fMehUI0*_sAvg`rfeWA*`~pfrIj8_vKoz(NZiBnv0eA@h3QViu z#?cGHKm@1@>Vrn035W(QK@4aII)IL#a~H*!ihk+=S|5f(H=yV{Kkf$)4Ya#^$82ay zf0w&h9)9*R`nFvdeE-nr3mg|0yB>@jcx zzrEnNd+p@oLuv2WLKkRvb$6$6xQ=GGJhrFCAy4 zGX}nJ#&PeY;|q?{y7BWpw<{a@A35%I{c=Ag==xdTyIY&2J9Xw-HTSwd$k$EQt;>!J zv)RdaX>aiUg~_`P|6Rv%mif&M*B6e9`=^I={Mm7DtVis=D#(vTKK(|}^$O>QUg^`o zsf+cSCRML-+>`u!uYTPf_rBlZK*D*)-Qm}Ij9L`rKgHD~&{+9+S<1+h&Bg@z{t*BB zv`=~6t_U`+1O0Bhm?5G(;}0s3*{esE(=+&=+~$Nh)PB{G10B*p+dm&9J+B`^|9 z0doMJ10vpE#BBg$5x)cHKsmrQdGK8G)C1Trp01!Lz}$M?1ehDo2Ot@wf~{a5!1Z{} zf--R1=$y>&q=&;|4Wv0wz41l|YO&)zj)3)l;=zFy3U_jhm$Jib3UdnP&x1+`=P z^W}2%Z#%nDqCUJA4Qpw;c&D;r6UJq` zFslEWvqKygx24F_Fz0rinH~GqVE$|uvR+*E?30drgywU$Ee-On6V5fB73BM--|PLl z<7`IR#jmjzc7LAZ*S(*;=D2swq{pe(92dJKw#3%^=Y03ntT#K-{|bD*ggg6Ajli1N z&N>yfi^Y1{E+m#cxS~7m-JAKrq@#|r7N^Eu!hK+$&uY31d1;#Cp49A_dGU_3aScC? z!u?~P?;bsGVLhz5?cU!Dc5Mq9pZkw7rB|>&|Eb^VnpQs6anIR~D|PG#yDqZk$pt~L z7dEVo^#+~K<_}8u2JH_qt?X#iw$AIlt;YU%7x#hPpKx)ZG-sIO-X0mRg}>{#(C6On zok8!h;uKbsxXG!D=WaaAALO{ZL(C2-8`jEc6zR@c`33upUsQjBz z1>ZgR>j#o<76p6e+n;pg&1Yvy+s`lOpHvq-*{_ z609QLWEqt_NmeyanwH!+(4AqX@np%FJSkQVPnzVn+k8cKQYv{;rE2>;U-A?Da+NeY znVCE(S`JT=;SVH1E#gh8QpuB{RP&^n$xZn1q%@vXBQuZ$J%=|*s^3m#5l@;`$&;y9 z+ih}k)7tI?Gc6GCHBqLWj2xaUpPwgPFS4szWmAR^wYK`wTLcEC7+E~YMjlVPRKSy= zmhhxnRd&^?;YsqPv<&p3tLZ$+S{6@|o)<`hQNWuFxr8UvSH+WT)!1!PN~=J3mYU9! zW@PcCYI!`#z5+X`B|J$=6;GC4V^@`w*8C#Pbe;?=iznI4<4IKuc(SY#o;0NjDR9rc zQ^OlgIVC30p{AzuBxzZJBq({jNmmMZGK~^ml&kC}tA;oEQrhsz^QGI#$l^)$<+XV* mZ|8J80O-dPIMBZx^b=D19LJ}Mf`Etm1IKz52Y-0J(fN*1>0|SEq5Hm9;B^Cj36?0mCzVvtbJa$G=UPdMc4sVkHCLmjok&QvBuYYeSO05M3dth9gFEF`|>8?z1*AfF2) z&j_;DLfOS}3y>!R#LPgW1DHUfAeW`USXCu;zg{i?%7Xwagkk{M1_GuKilGHc2SDj* JGkz-^2LQA1BPsv@ diff --git a/tsdb/testdata/index_format_v1/meta.json b/tsdb/testdata/index_format_v1/meta.json index 62347db26..d99ae6b49 100644 --- a/tsdb/testdata/index_format_v1/meta.json +++ b/tsdb/testdata/index_format_v1/meta.json @@ -1,17 +1,17 @@ { "version": 1, - "ulid": "01DVZX4CHY2EGZ6JQVS80AB9CF", + "ulid": "01DXXFZDYD1MQW6079WK0K6EDQ", "minTime": 0, "maxTime": 7200000, "stats": { - "numSamples": 2, - "numSeries": 2, - "numChunks": 2 + "numSamples": 102, + "numSeries": 102, + "numChunks": 102 }, "compaction": { "level": 1, "sources": [ - "01DVZX4CHY2EGZ6JQVS80AB9CF" + "01DXXFZDYD1MQW6079WK0K6EDQ" ] } } From bfe3dafcf0e4ce63c3efb2f8e9d53fe999592c64 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 6 Jan 2020 14:36:22 +0000 Subject: [PATCH 7/7] Cut release 2.15.2 Cherry-picked 2 commits. Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 7 ++++++- VERSION | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33a611ea3..05320f7cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ +## 2.15.2 / 2020-01-06 + +* [BUGFIX] TSDB: Fixed support for TSDB blocks built with Prometheus before 2.1.0. #6564 +* [BUGFIX] TSDB: Fixed block compaction issues on Windows. #6547 + ## 2.15.1 / 2019-12-25 -* [BUGFIX] Fixed race on concurrent queries against same data. #6512 +* [BUGFIX] TSDB: Fixed race on concurrent queries against same data. #6512 ## 2.15.0 / 2019-12-23 diff --git a/VERSION b/VERSION index 3b1fc7950..07d875c2d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.15.1 +2.15.2