mirror of
https://github.com/prometheus/prometheus
synced 2025-01-12 18:01:36 +00:00
971dafdfbe
When compacting rather than doing a read of all series in the index per label name, do many at once but only when it won't use (much) more ram than writing the special all index does. original in-memory postings: BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 1 1202383447 ns/op 158936496 B/op 1031511 allocs/op BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 1 1141792706 ns/op 154453408 B/op 1093453 allocs/op BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 1 1169288829 ns/op 161072336 B/op 1110021 allocs/op BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 1 1115700103 ns/op 149480472 B/op 1129180 allocs/op BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 1 1283813141 ns/op 162937800 B/op 1202771 allocs/op before: BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 1 1145195941 ns/op 131749984 B/op 834400 allocs/op BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 1 1233526345 ns/op 127889416 B/op 897033 allocs/op BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 1 1821942296 ns/op 131665648 B/op 914836 allocs/op BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 1 8035568665 ns/op 123811832 B/op 934312 allocs/op BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 1 71325926267 ns/op 140722648 B/op 1016824 allocs/op after: BenchmarkCompactionFromHead/labelnames=1,labelvalues=100000-4 1 1101429174 ns/op 129063496 B/op 832571 allocs/op BenchmarkCompactionFromHead/labelnames=10,labelvalues=10000-4 1 1074466374 ns/op 124154888 B/op 894875 allocs/op BenchmarkCompactionFromHead/labelnames=100,labelvalues=1000-4 1 1166510282 ns/op 128790648 B/op 912931 allocs/op BenchmarkCompactionFromHead/labelnames=1000,labelvalues=100-4 1 1075013071 ns/op 120570696 B/op 933511 allocs/op BenchmarkCompactionFromHead/labelnames=10000,labelvalues=10-4 1 1231673790 ns/op 138754288 B/op 1022791 allocs/op Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
80 lines
2.2 KiB
Go
80 lines
2.2 KiB
Go
// Copyright 2017 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package tsdb
|
|
|
|
import (
|
|
"github.com/prometheus/prometheus/pkg/labels"
|
|
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
|
"github.com/prometheus/prometheus/tsdb/chunks"
|
|
"github.com/prometheus/prometheus/tsdb/tombstones"
|
|
)
|
|
|
|
type mockIndexWriter struct {
|
|
series []seriesSamples
|
|
}
|
|
|
|
func (mockIndexWriter) AddSymbols(sym map[string]struct{}) error { return nil }
|
|
func (m *mockIndexWriter) AddSeries(ref uint64, l labels.Labels, chunks ...chunks.Meta) error {
|
|
i := -1
|
|
for j, s := range m.series {
|
|
if !labels.Equal(labels.FromMap(s.lset), l) {
|
|
continue
|
|
}
|
|
i = j
|
|
break
|
|
}
|
|
if i == -1 {
|
|
m.series = append(m.series, seriesSamples{
|
|
lset: l.Map(),
|
|
})
|
|
i = len(m.series) - 1
|
|
}
|
|
|
|
var iter chunkenc.Iterator
|
|
for _, chk := range chunks {
|
|
samples := make([]sample, 0, chk.Chunk.NumSamples())
|
|
|
|
iter = chk.Chunk.Iterator(iter)
|
|
for iter.Next() {
|
|
s := sample{}
|
|
s.t, s.v = iter.At()
|
|
|
|
samples = append(samples, s)
|
|
}
|
|
if err := iter.Err(); err != nil {
|
|
return err
|
|
}
|
|
|
|
m.series[i].chunks = append(m.series[i].chunks, samples)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (mockIndexWriter) WriteLabelIndex(names []string, values []string) error { return nil }
|
|
func (mockIndexWriter) Close() error { return nil }
|
|
|
|
type mockBReader struct {
|
|
ir IndexReader
|
|
cr ChunkReader
|
|
mint int64
|
|
maxt int64
|
|
}
|
|
|
|
func (r *mockBReader) Index() (IndexReader, error) { return r.ir, nil }
|
|
func (r *mockBReader) Chunks() (ChunkReader, error) { return r.cr, nil }
|
|
func (r *mockBReader) Tombstones() (tombstones.Reader, error) {
|
|
return tombstones.NewMemTombstones(), nil
|
|
}
|
|
func (r *mockBReader) Meta() BlockMeta { return BlockMeta{MinTime: r.mint, MaxTime: r.maxt} }
|