prometheus/cmd/promtool/tsdb.go

627 lines
14 KiB
Go
Raw Normal View History

2017-04-10 18:59:45 +00:00
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2016-12-07 16:30:10 +00:00
package main
import (
"bufio"
"context"
2016-12-07 16:30:10 +00:00
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"sort"
"strconv"
"strings"
2016-12-07 16:30:10 +00:00
"sync"
2017-10-19 16:14:37 +00:00
"text/tabwriter"
2016-12-07 16:30:10 +00:00
"time"
"github.com/alecthomas/units"
"github.com/go-kit/log"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunks"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
"github.com/prometheus/prometheus/tsdb/fileutil"
2016-12-07 16:30:10 +00:00
)
const timeDelta = 30000
2016-12-07 16:30:10 +00:00
type writeBenchmark struct {
outPath string
samplesFile string
cleanup bool
numMetrics int
2016-12-07 16:30:10 +00:00
2017-02-19 15:04:37 +00:00
storage *tsdb.DB
2016-12-07 16:30:10 +00:00
cpuprof *os.File
memprof *os.File
blockprof *os.File
2017-05-14 09:51:56 +00:00
mtxprof *os.File
logger log.Logger
2016-12-07 16:30:10 +00:00
}
func benchmarkWrite(outPath, samplesFile string, numMetrics, numScrapes int) error {
b := &writeBenchmark{
outPath: outPath,
samplesFile: samplesFile,
numMetrics: numMetrics,
logger: log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)),
}
2016-12-07 16:30:10 +00:00
if b.outPath == "" {
dir, err := ioutil.TempDir("", "tsdb_bench")
if err != nil {
return err
2016-12-07 16:30:10 +00:00
}
b.outPath = dir
b.cleanup = true
}
if err := os.RemoveAll(b.outPath); err != nil {
return err
2016-12-07 16:30:10 +00:00
}
if err := os.MkdirAll(b.outPath, 0777); err != nil {
return err
2016-12-07 16:30:10 +00:00
}
dir := filepath.Join(b.outPath, "storage")
l := log.With(b.logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller)
st, err := tsdb.Open(dir, l, nil, &tsdb.Options{
RetentionDuration: int64(15 * 24 * time.Hour / time.Millisecond),
MinBlockDuration: int64(2 * time.Hour / time.Millisecond),
React UI: Add Starting Screen (#8662) * Added walreplay API endpoint Signed-off-by: Levi Harrison <git@leviharrison.dev> * Added starting page to react-ui Signed-off-by: Levi Harrison <git@leviharrison.dev> * Documented the new endpoint Signed-off-by: Levi Harrison <git@leviharrison.dev> * Fixed typos Signed-off-by: Levi Harrison <git@leviharrison.dev> Co-authored-by: Julius Volz <julius.volz@gmail.com> * Removed logo Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed isResponding to isUnexpected Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed width of progress bar Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed width of progress bar Signed-off-by: Levi Harrison <git@leviharrison.dev> * Added DB stats object Signed-off-by: Levi Harrison <git@leviharrison.dev> * Updated starting page to work with new fields Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil (pt. 2) Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil (pt. 3) Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil (and also implementing a method this time) (pt. 4) Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil (and also implementing a method this time) (pt. 5) Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed const to let Signed-off-by: Levi Harrison <git@leviharrison.dev> * Passing nil (pt. 6) Signed-off-by: Levi Harrison <git@leviharrison.dev> * Remove SetStats method Signed-off-by: Levi Harrison <git@leviharrison.dev> * Added comma Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed api Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed to triple equals Signed-off-by: Levi Harrison <git@leviharrison.dev> * Fixed data response types Signed-off-by: Levi Harrison <git@leviharrison.dev> * Don't return pointer Signed-off-by: Levi Harrison <git@leviharrison.dev> * Changed version Signed-off-by: Levi Harrison <git@leviharrison.dev> * Fixed interface issue Signed-off-by: Levi Harrison <git@leviharrison.dev> * Fixed pointer Signed-off-by: Levi Harrison <git@leviharrison.dev> * Fixed copying lock value error Signed-off-by: Levi Harrison <git@leviharrison.dev> Co-authored-by: Julius Volz <julius.volz@gmail.com>
2021-06-05 14:29:32 +00:00
}, tsdb.NewDBStats())
if err != nil {
return err
2016-12-07 16:30:10 +00:00
}
st.DisableCompactions()
b.storage = st
var lbs []labels.Labels
2016-12-07 16:30:10 +00:00
if _, err = measureTime("readData", func() error {
f, err := os.Open(b.samplesFile)
2016-12-07 16:30:10 +00:00
if err != nil {
return err
2016-12-07 16:30:10 +00:00
}
defer f.Close()
lbs, err = readPrometheusLabels(f, b.numMetrics)
2016-12-07 16:30:10 +00:00
if err != nil {
return err
2016-12-07 16:30:10 +00:00
}
return nil
}); err != nil {
return err
}
2016-12-07 16:30:10 +00:00
var total uint64
dur, err := measureTime("ingestScrapes", func() error {
if err := b.startProfiling(); err != nil {
return err
}
total, err = b.ingestScrapes(lbs, numScrapes)
if err != nil {
return err
2016-12-07 16:30:10 +00:00
}
return nil
2016-12-07 16:30:10 +00:00
})
if err != nil {
return err
}
fmt.Println(" > total samples:", total)
fmt.Println(" > samples/sec:", float64(total)/dur.Seconds())
if _, err = measureTime("stopStorage", func() error {
if err := b.storage.Close(); err != nil {
return err
2016-12-07 16:30:10 +00:00
}
return b.stopProfiling()
}); err != nil {
return err
}
return nil
2016-12-07 16:30:10 +00:00
}
func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (uint64, error) {
var mu sync.Mutex
var total uint64
2016-12-07 16:30:10 +00:00
for i := 0; i < scrapeCount; i += 100 {
var wg sync.WaitGroup
lbls := lbls
for len(lbls) > 0 {
l := 1000
if len(lbls) < 1000 {
l = len(lbls)
2016-12-07 16:30:10 +00:00
}
batch := lbls[:l]
lbls = lbls[l:]
wg.Add(1)
go func() {
n, err := b.ingestScrapesShard(batch, 100, int64(timeDelta*i))
if err != nil {
// exitWithError(err)
fmt.Println(" err", err)
}
mu.Lock()
total += n
mu.Unlock()
wg.Done()
}()
}
wg.Wait()
2016-12-07 16:30:10 +00:00
}
fmt.Println("ingestion completed")
return total, nil
2016-12-07 16:30:10 +00:00
}
func (b *writeBenchmark) ingestScrapesShard(lbls []labels.Labels, scrapeCount int, baset int64) (uint64, error) {
ts := baset
2016-12-07 16:30:10 +00:00
2016-12-08 09:04:24 +00:00
type sample struct {
2016-12-21 08:39:01 +00:00
labels labels.Labels
2016-12-08 09:04:24 +00:00
value int64
ref *uint64
2016-12-08 09:04:24 +00:00
}
scrape := make([]*sample, 0, len(lbls))
2016-12-08 09:04:24 +00:00
for _, m := range lbls {
scrape = append(scrape, &sample{
labels: m,
2016-12-08 09:04:24 +00:00
value: 123456789,
})
2016-12-08 09:04:24 +00:00
}
total := uint64(0)
2016-12-08 09:04:24 +00:00
2016-12-07 16:30:10 +00:00
for i := 0; i < scrapeCount; i++ {
app := b.storage.Appender(context.TODO())
ts += timeDelta
2016-12-07 16:30:10 +00:00
2016-12-08 09:04:24 +00:00
for _, s := range scrape {
2016-12-09 09:00:14 +00:00
s.value += 1000
var ref uint64
if s.ref != nil {
ref = *s.ref
}
ref, err := app.Append(ref, s.labels, ts, float64(s.value))
if err != nil {
panic(err)
}
if s.ref == nil {
s.ref = &ref
}
total++
2016-12-07 16:30:10 +00:00
}
if err := app.Commit(); err != nil {
return total, err
2016-12-07 16:30:10 +00:00
}
}
return total, nil
2016-12-07 16:30:10 +00:00
}
func (b *writeBenchmark) startProfiling() error {
2016-12-07 16:30:10 +00:00
var err error
// Start CPU profiling.
b.cpuprof, err = os.Create(filepath.Join(b.outPath, "cpu.prof"))
if err != nil {
return fmt.Errorf("bench: could not create cpu profile: %v", err)
2016-12-07 16:30:10 +00:00
}
if err := pprof.StartCPUProfile(b.cpuprof); err != nil {
return fmt.Errorf("bench: could not start CPU profile: %v", err)
}
2016-12-07 16:30:10 +00:00
// Start memory profiling.
b.memprof, err = os.Create(filepath.Join(b.outPath, "mem.prof"))
if err != nil {
return fmt.Errorf("bench: could not create memory profile: %v", err)
2016-12-07 16:30:10 +00:00
}
2017-05-14 09:51:56 +00:00
runtime.MemProfileRate = 64 * 1024
2016-12-07 16:30:10 +00:00
// Start fatal profiling.
b.blockprof, err = os.Create(filepath.Join(b.outPath, "block.prof"))
if err != nil {
return fmt.Errorf("bench: could not create block profile: %v", err)
2016-12-07 16:30:10 +00:00
}
2017-05-14 09:51:56 +00:00
runtime.SetBlockProfileRate(20)
b.mtxprof, err = os.Create(filepath.Join(b.outPath, "mutex.prof"))
if err != nil {
return fmt.Errorf("bench: could not create mutex profile: %v", err)
2017-05-14 09:51:56 +00:00
}
runtime.SetMutexProfileFraction(20)
return nil
2016-12-07 16:30:10 +00:00
}
func (b *writeBenchmark) stopProfiling() error {
2016-12-07 16:30:10 +00:00
if b.cpuprof != nil {
pprof.StopCPUProfile()
b.cpuprof.Close()
b.cpuprof = nil
}
if b.memprof != nil {
if err := pprof.Lookup("heap").WriteTo(b.memprof, 0); err != nil {
return fmt.Errorf("error writing mem profile: %v", err)
}
2016-12-07 16:30:10 +00:00
b.memprof.Close()
b.memprof = nil
}
if b.blockprof != nil {
if err := pprof.Lookup("block").WriteTo(b.blockprof, 0); err != nil {
return fmt.Errorf("error writing block profile: %v", err)
}
2016-12-07 16:30:10 +00:00
b.blockprof.Close()
b.blockprof = nil
runtime.SetBlockProfileRate(0)
}
2017-05-14 09:51:56 +00:00
if b.mtxprof != nil {
if err := pprof.Lookup("mutex").WriteTo(b.mtxprof, 0); err != nil {
return fmt.Errorf("error writing mutex profile: %v", err)
}
2017-05-14 09:51:56 +00:00
b.mtxprof.Close()
b.mtxprof = nil
runtime.SetMutexProfileFraction(0)
}
return nil
2016-12-07 16:30:10 +00:00
}
func measureTime(stage string, f func() error) (time.Duration, error) {
2016-12-07 16:30:10 +00:00
fmt.Printf(">> start stage=%s\n", stage)
start := time.Now()
if err := f(); err != nil {
return 0, err
}
2016-12-07 16:30:10 +00:00
fmt.Printf(">> completed stage=%s duration=%s\n", stage, time.Since(start))
return time.Since(start), nil
2016-12-07 16:30:10 +00:00
}
func readPrometheusLabels(r io.Reader, n int) ([]labels.Labels, error) {
scanner := bufio.NewScanner(r)
2016-12-07 16:30:10 +00:00
2017-01-16 20:29:53 +00:00
var mets []labels.Labels
hashes := map[uint64]struct{}{}
i := 0
2016-12-07 16:30:10 +00:00
for scanner.Scan() && i < n {
2017-01-16 20:29:53 +00:00
m := make(labels.Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, labels.Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
2017-01-16 20:29:53 +00:00
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
2016-12-07 16:30:10 +00:00
}
2017-01-16 20:29:53 +00:00
mets = append(mets, m)
hashes[h] = struct{}{}
i++
2016-12-07 16:30:10 +00:00
}
return mets, nil
2016-12-07 16:30:10 +00:00
}
func listBlocks(path string, humanReadable bool) error {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return err
}
defer func() {
err = tsdb_errors.NewMulti(err, db.Close()).Err()
}()
blocks, err := db.Blocks()
if err != nil {
return err
}
printBlocks(blocks, true, humanReadable)
return nil
}
func printBlocks(blocks []tsdb.BlockReader, writeHeader, humanReadable bool) {
promtool: Print block meta-data slightly more nicely I initially thought I could somehow rescue the current column layout by recycling the tabwriter, but flushing completely blanks it. However, by setting a minimum width of 13, we get a slightly broader DURATION column but otherwise nice formatting, unless numbers get really big, but that's OK, I guess. Before: ``` BLOCK ULID MIN TIME MAX TIME DURATION NUM SAMPLES NUM CHUNKS NUM SERIES SIZE 01ETN0KGNP5WWK9T5QMQGBG9F1 2020-11-19 07:39:17 +0000 UTC 2020-11-19 07:44:17 +0000 UTC 5m0.001s 8 2 2 624B 01ETN0KGQSFF0AB2QDZVQG3CWC 2020-11-19 10:25:57 +0000 UTC 2020-11-19 10:30:57 +0000 UTC 5m0.001s 8 2 2 622B 01ETN0KGSW8KYP3YPG4X20P60Z 2020-11-19 13:12:37 +0000 UTC 2020-11-19 13:17:37 +0000 UTC 5m0.001s 8 2 2 625B ``` After: ``` BLOCK ULID MIN TIME MAX TIME DURATION NUM SAMPLES NUM CHUNKS NUM SERIES SIZE 01ETN0R72SXN9A1FG732P7KFFN 2020-11-19 07:39:17 +0000 UTC 2020-11-19 07:44:17 +0000 UTC 5m0.001s 8 2 2 624B 01ETN0R74Y9AG1A1MKN4MZK7WM 2020-11-19 10:25:57 +0000 UTC 2020-11-19 10:30:57 +0000 UTC 5m0.001s 8 2 2 622B 01ETN0R76KXZ5VQECMDNES49J6 2020-11-19 13:12:37 +0000 UTC 2020-11-19 13:17:37 +0000 UTC 5m0.001s 8 2 2 625B ``` After without the `-r` flag: ``` BLOCK ULID MIN TIME MAX TIME DURATION NUM SAMPLES NUM CHUNKS NUM SERIES SIZE 01ETN0RFFJ42274NWR1GH0RTV6 1605771557000 1605771857001 5m0.001s 8 2 2 624 01ETN0RFJ1MZCHHS2SBZS8XC27 1605781557000 1605781857001 5m0.001s 8 2 2 622 01ETN0RFM98N3V4KD2DZXFGHGN 1605791557000 1605791857001 5m0.001s 8 2 2 625 ``` Signed-off-by: beorn7 <beorn@grafana.com>
2020-12-28 15:51:45 +00:00
tw := tabwriter.NewWriter(os.Stdout, 13, 0, 2, ' ', 0)
2017-10-02 20:48:47 +00:00
defer tw.Flush()
if writeHeader {
fmt.Fprintln(tw, "BLOCK ULID\tMIN TIME\tMAX TIME\tDURATION\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES\tSIZE")
}
2017-10-02 20:48:47 +00:00
for _, b := range blocks {
2017-10-11 09:02:57 +00:00
meta := b.Meta()
2017-10-02 20:48:47 +00:00
fmt.Fprintf(tw,
"%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
2017-10-11 09:02:57 +00:00
meta.ULID,
getFormatedTime(meta.MinTime, humanReadable),
getFormatedTime(meta.MaxTime, humanReadable),
time.Duration(meta.MaxTime-meta.MinTime)*time.Millisecond,
2017-10-11 09:02:57 +00:00
meta.Stats.NumSamples,
meta.Stats.NumChunks,
meta.Stats.NumSeries,
getFormatedBytes(b.Size(), humanReadable),
2017-10-02 20:48:47 +00:00
)
}
}
func getFormatedTime(timestamp int64, humanReadable bool) string {
if humanReadable {
return time.Unix(timestamp/1000, 0).UTC().String()
}
return strconv.FormatInt(timestamp, 10)
}
func getFormatedBytes(bytes int64, humanReadable bool) string {
if humanReadable {
return units.Base2Bytes(bytes).String()
}
return strconv.FormatInt(bytes, 10)
}
func openBlock(path, blockID string) (*tsdb.DBReadOnly, tsdb.BlockReader, error) {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return nil, nil, err
}
blocks, err := db.Blocks()
if err != nil {
return nil, nil, err
}
var block tsdb.BlockReader
if blockID != "" {
for _, b := range blocks {
if b.Meta().ULID.String() == blockID {
block = b
break
}
}
} else if len(blocks) > 0 {
block = blocks[len(blocks)-1]
}
if block == nil {
return nil, nil, fmt.Errorf("block %s not found", blockID)
}
return db, block, nil
}
func analyzeBlock(path, blockID string, limit int) error {
db, block, err := openBlock(path, blockID)
if err != nil {
return err
}
defer func() {
err = tsdb_errors.NewMulti(err, db.Close()).Err()
}()
meta := block.Meta()
fmt.Printf("Block ID: %s\n", meta.ULID)
// Presume 1ms resolution that Prometheus uses.
fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String())
fmt.Printf("Series: %d\n", meta.Stats.NumSeries)
ir, err := block.Index()
if err != nil {
return err
}
defer ir.Close()
allLabelNames, err := ir.LabelNames()
if err != nil {
return err
}
fmt.Printf("Label names: %d\n", len(allLabelNames))
type postingInfo struct {
key string
metric uint64
}
postingInfos := []postingInfo{}
printInfo := func(postingInfos []postingInfo) {
sort.Slice(postingInfos, func(i, j int) bool { return postingInfos[i].metric > postingInfos[j].metric })
for i, pc := range postingInfos {
if i >= limit {
break
}
fmt.Printf("%d %s\n", pc.metric, pc.key)
}
}
labelsUncovered := map[string]uint64{}
labelpairsUncovered := map[string]uint64{}
labelpairsCount := map[string]uint64{}
entries := 0
p, err := ir.Postings("", "") // The special all key.
if err != nil {
return err
}
lbls := labels.Labels{}
chks := []chunks.Meta{}
for p.Next() {
if err = ir.Series(p.At(), &lbls, &chks); err != nil {
return err
}
// Amount of the block time range not covered by this series.
uncovered := uint64(meta.MaxTime-meta.MinTime) - uint64(chks[len(chks)-1].MaxTime-chks[0].MinTime)
for _, lbl := range lbls {
key := lbl.Name + "=" + lbl.Value
labelsUncovered[lbl.Name] += uncovered
labelpairsUncovered[key] += uncovered
labelpairsCount[key]++
entries++
}
}
if p.Err() != nil {
return p.Err()
}
fmt.Printf("Postings (unique label pairs): %d\n", len(labelpairsUncovered))
fmt.Printf("Postings entries (total label pairs): %d\n", entries)
postingInfos = postingInfos[:0]
for k, m := range labelpairsUncovered {
postingInfos = append(postingInfos, postingInfo{k, uint64(float64(m) / float64(meta.MaxTime-meta.MinTime))})
}
fmt.Printf("\nLabel pairs most involved in churning:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
for k, m := range labelsUncovered {
postingInfos = append(postingInfos, postingInfo{k, uint64(float64(m) / float64(meta.MaxTime-meta.MinTime))})
}
fmt.Printf("\nLabel names most involved in churning:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
for k, m := range labelpairsCount {
postingInfos = append(postingInfos, postingInfo{k, m})
}
fmt.Printf("\nMost common label pairs:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
values, err := ir.SortedLabelValues(n)
if err != nil {
return err
}
var cumulativeLength uint64
Replace StringTuples with []string Benchmarks show slight cpu/allocs improvements. benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42% BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73% BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42% BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78% BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37% BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89% BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89% BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82% BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76% BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28% BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33% BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00% BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00% BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
for _, str := range values {
cumulativeLength += uint64(len(str))
}
postingInfos = append(postingInfos, postingInfo{n, cumulativeLength})
}
fmt.Printf("\nLabel names with highest cumulative label value length:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
for _, n := range allLabelNames {
lv, err := ir.SortedLabelValues(n)
if err != nil {
return err
}
Replace StringTuples with []string Benchmarks show slight cpu/allocs improvements. benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42% BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73% BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42% BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78% BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37% BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89% BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89% BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82% BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76% BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28% BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33% BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00% BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00% BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
postingInfos = append(postingInfos, postingInfo{n, uint64(len(lv))})
}
fmt.Printf("\nHighest cardinality labels:\n")
printInfo(postingInfos)
postingInfos = postingInfos[:0]
lv, err := ir.SortedLabelValues("__name__")
if err != nil {
return err
}
Replace StringTuples with []string Benchmarks show slight cpu/allocs improvements. benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42% BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73% BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42% BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78% BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37% BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89% BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89% BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82% BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76% BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28% BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33% BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00% BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00% BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
for _, n := range lv {
postings, err := ir.Postings("__name__", n)
if err != nil {
return err
}
Replace StringTuples with []string Benchmarks show slight cpu/allocs improvements. benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42% BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73% BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42% BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78% BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37% BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89% BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89% BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82% BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76% BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28% BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33% BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00% BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00% BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
count := 0
for postings.Next() {
count++
}
if postings.Err() != nil {
return postings.Err()
}
Replace StringTuples with []string Benchmarks show slight cpu/allocs improvements. benchmark old ns/op new ns/op delta BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42% BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73% BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42% BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78% BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37% BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11% BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89% BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89% BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82% BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19% benchmark old allocs new allocs delta BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76% BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28% BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33% BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67% BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00% benchmark old bytes new bytes delta BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00% BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00% BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00% BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00% BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00% BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00% BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00% BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00% BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18% BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00% BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00% BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00% BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01% BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00% BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00% Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
postingInfos = append(postingInfos, postingInfo{n, uint64(count)})
}
fmt.Printf("\nHighest cardinality metric names:\n")
printInfo(postingInfos)
return nil
}
func dumpSamples(path string, mint, maxt int64) (err error) {
db, err := tsdb.OpenDBReadOnly(path, nil)
if err != nil {
return err
}
defer func() {
err = tsdb_errors.NewMulti(err, db.Close()).Err()
}()
q, err := db.Querier(context.TODO(), mint, maxt)
if err != nil {
return err
}
defer q.Close()
*: Consistent Error/Warning handling for SeriesSet iterator: Allowing Async Select (#7251) * Add errors and Warnings to SeriesSet Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Change Querier interface and refactor accordingly Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor promql/engine to propagate warnings at eval stage Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Make sure all the series from all Selects are pre-advanced Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Separate merge series sets Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Clean Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor merge querier failure handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactored and simplified fanout with improvements from incoming chunk iterator PRs. * Secondary logic is hidden, instead of weird failed series set logic we had. * Fanout is well commented * Fanout closing record all errors * MergeQuerier improved API (clearer) * deferredGenericMergeSeriesSet is not needed as we return no samples anyway for failed series sets (next = false). Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fix formatting Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix CI issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Added final tests for error handling. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Brian's comments. * Moved hints in populate to be allocated only when needed. * Used sync.Once in secondary Querier to achieve all-or-nothing partial response logic. * Select after first Next is done will panic. NOTE: in lazySeriesSet in theory we could just panic, I think however we can totally just return error, it will panic in expand anyway. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Utilize errWithWarnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix recently introduced expansion issue Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add tests for secondary querier error handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Implement lazy merge Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add name to test cases Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Reorganize Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Remove redundant warnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix rebase mistake Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
2020-06-09 16:57:31 +00:00
ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*"))
for ss.Next() {
series := ss.At()
lbs := series.Labels()
it := series.Iterator()
for it.Next() {
ts, val := it.At()
fmt.Printf("%s %g %d\n", lbs, val, ts)
}
if it.Err() != nil {
return ss.Err()
}
}
*: Consistent Error/Warning handling for SeriesSet iterator: Allowing Async Select (#7251) * Add errors and Warnings to SeriesSet Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Change Querier interface and refactor accordingly Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor promql/engine to propagate warnings at eval stage Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Make sure all the series from all Selects are pre-advanced Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Separate merge series sets Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Clean Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor merge querier failure handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactored and simplified fanout with improvements from incoming chunk iterator PRs. * Secondary logic is hidden, instead of weird failed series set logic we had. * Fanout is well commented * Fanout closing record all errors * MergeQuerier improved API (clearer) * deferredGenericMergeSeriesSet is not needed as we return no samples anyway for failed series sets (next = false). Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fix formatting Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix CI issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Added final tests for error handling. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Brian's comments. * Moved hints in populate to be allocated only when needed. * Used sync.Once in secondary Querier to achieve all-or-nothing partial response logic. * Select after first Next is done will panic. NOTE: in lazySeriesSet in theory we could just panic, I think however we can totally just return error, it will panic in expand anyway. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Utilize errWithWarnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix recently introduced expansion issue Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add tests for secondary querier error handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Implement lazy merge Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add name to test cases Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Reorganize Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Remove redundant warnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix rebase mistake Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
2020-06-09 16:57:31 +00:00
if ws := ss.Warnings(); len(ws) > 0 {
return tsdb_errors.NewMulti(ws...).Err()
*: Consistent Error/Warning handling for SeriesSet iterator: Allowing Async Select (#7251) * Add errors and Warnings to SeriesSet Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Change Querier interface and refactor accordingly Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor promql/engine to propagate warnings at eval stage Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Make sure all the series from all Selects are pre-advanced Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Separate merge series sets Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Clean Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactor merge querier failure handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Refactored and simplified fanout with improvements from incoming chunk iterator PRs. * Secondary logic is hidden, instead of weird failed series set logic we had. * Fanout is well commented * Fanout closing record all errors * MergeQuerier improved API (clearer) * deferredGenericMergeSeriesSet is not needed as we return no samples anyway for failed series sets (next = false). Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fix formatting Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix CI issues Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Added final tests for error handling. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Brian's comments. * Moved hints in populate to be allocated only when needed. * Used sync.Once in secondary Querier to achieve all-or-nothing partial response logic. * Select after first Next is done will panic. NOTE: in lazySeriesSet in theory we could just panic, I think however we can totally just return error, it will panic in expand anyway. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Utilize errWithWarnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix recently introduced expansion issue Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add tests for secondary querier error handling Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Implement lazy merge Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Add name to test cases Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Reorganize Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Address review comments Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Remove redundant warnings Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> * Fix rebase mistake Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com>
2020-06-09 16:57:31 +00:00
}
if ss.Err() != nil {
return ss.Err()
}
return nil
}
func checkErr(err error) int {
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
return 0
}
Backfill from OpenMetrics format (#8084) * get parser working Signed-off-by: aSquare14 <atibhi.a@gmail.com> * import file created Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Find min and max ts Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make two passes over file and write to tsdb Signed-off-by: aSquare14 <atibhi.a@gmail.com> * print error messages Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Max and Min initializer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Start with unit tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reset file read Signed-off-by: aSquare14 <atibhi.a@gmail.com> * align blocks to two hour range Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add cleanup test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove .ds_store Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add license to import_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Circle CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Refactor code Move backfill from tsdb to promtool directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix gitignore Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove panic Rename ContenType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * adjust mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return statement Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go modules Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added unit test for backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix file handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close DB Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Error Handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * inline err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix command line flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add spaces before func fix pointers Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add defer'd calls Signed-off-by: aSquare14 <atibhi.a@gmail.com> * move openmetrics.go content to backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changed args to flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add tests for wrong OM files Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added additional tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add comment to warn of func reuse Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Make input required in main.go Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer blockwriter close Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove contentType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove defer from backfilltest Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix defer remove in backfill_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changes to fix CI errors Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go.mod Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change package name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * assert->require Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix format Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix createblock Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return Signed-off-by: aSquare14 <atibhi.a@gmail.com> * check err for anon func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * update comment Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix for the Flush Bug Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix formatting, comments, names Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * cleanup Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor test to take care of multiple samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * I dont know what I fixed Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests, add test description, print blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * commit after 5000 samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reviews part 1 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Series Count Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove extra func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make timestamp into sec Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Reviews 2 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add Todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fixes Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fixes reviews Signed-off-by: aSquare14 <atibhi.a@gmail.com> * =0 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove backfill.om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add global err var, remove stuff Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change var name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * sampleLimit pass as parameter Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add test when number of samples greater than batch size Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Change name of batchsize Signed-off-by: aSquare14 <atibhi.a@gmail.com> * revert export Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add comment, remove newline,consistent err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Modify comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * db.Querier Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add sanity check , get maxt and mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * ci error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * comment change Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * NoError Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com>
2020-11-26 05:07:06 +00:00
func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool) int {
inputFile, err := fileutil.OpenMmapFile(path)
Backfill from OpenMetrics format (#8084) * get parser working Signed-off-by: aSquare14 <atibhi.a@gmail.com> * import file created Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Find min and max ts Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make two passes over file and write to tsdb Signed-off-by: aSquare14 <atibhi.a@gmail.com> * print error messages Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Max and Min initializer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Start with unit tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reset file read Signed-off-by: aSquare14 <atibhi.a@gmail.com> * align blocks to two hour range Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add cleanup test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove .ds_store Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add license to import_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Circle CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Refactor code Move backfill from tsdb to promtool directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix gitignore Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove panic Rename ContenType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * adjust mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return statement Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go modules Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added unit test for backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix file handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close DB Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Error Handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * inline err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix command line flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add spaces before func fix pointers Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add defer'd calls Signed-off-by: aSquare14 <atibhi.a@gmail.com> * move openmetrics.go content to backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changed args to flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add tests for wrong OM files Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added additional tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add comment to warn of func reuse Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Make input required in main.go Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer blockwriter close Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove contentType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove defer from backfilltest Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix defer remove in backfill_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changes to fix CI errors Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go.mod Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change package name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * assert->require Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix format Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix createblock Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return Signed-off-by: aSquare14 <atibhi.a@gmail.com> * check err for anon func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * update comment Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix for the Flush Bug Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix formatting, comments, names Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * cleanup Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor test to take care of multiple samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * I dont know what I fixed Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests, add test description, print blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * commit after 5000 samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reviews part 1 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Series Count Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove extra func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make timestamp into sec Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Reviews 2 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add Todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fixes Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fixes reviews Signed-off-by: aSquare14 <atibhi.a@gmail.com> * =0 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove backfill.om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add global err var, remove stuff Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change var name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * sampleLimit pass as parameter Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add test when number of samples greater than batch size Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Change name of batchsize Signed-off-by: aSquare14 <atibhi.a@gmail.com> * revert export Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add comment, remove newline,consistent err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Modify comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * db.Querier Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add sanity check , get maxt and mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * ci error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * comment change Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * NoError Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com>
2020-11-26 05:07:06 +00:00
if err != nil {
return checkErr(err)
Backfill from OpenMetrics format (#8084) * get parser working Signed-off-by: aSquare14 <atibhi.a@gmail.com> * import file created Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Find min and max ts Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make two passes over file and write to tsdb Signed-off-by: aSquare14 <atibhi.a@gmail.com> * print error messages Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Max and Min initializer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Start with unit tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reset file read Signed-off-by: aSquare14 <atibhi.a@gmail.com> * align blocks to two hour range Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add cleanup test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove .ds_store Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add license to import_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Circle CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Refactor code Move backfill from tsdb to promtool directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix gitignore Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove panic Rename ContenType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * adjust mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return statement Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go modules Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added unit test for backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix file handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close DB Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Error Handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * inline err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix command line flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add spaces before func fix pointers Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add defer'd calls Signed-off-by: aSquare14 <atibhi.a@gmail.com> * move openmetrics.go content to backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changed args to flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add tests for wrong OM files Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added additional tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add comment to warn of func reuse Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Make input required in main.go Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer blockwriter close Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove contentType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove defer from backfilltest Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix defer remove in backfill_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changes to fix CI errors Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go.mod Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change package name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * assert->require Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix format Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix createblock Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return Signed-off-by: aSquare14 <atibhi.a@gmail.com> * check err for anon func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * update comment Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix for the Flush Bug Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix formatting, comments, names Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * cleanup Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor test to take care of multiple samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * I dont know what I fixed Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests, add test description, print blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * commit after 5000 samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reviews part 1 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Series Count Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove extra func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make timestamp into sec Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Reviews 2 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add Todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fixes Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fixes reviews Signed-off-by: aSquare14 <atibhi.a@gmail.com> * =0 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove backfill.om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add global err var, remove stuff Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change var name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * sampleLimit pass as parameter Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add test when number of samples greater than batch size Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Change name of batchsize Signed-off-by: aSquare14 <atibhi.a@gmail.com> * revert export Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add comment, remove newline,consistent err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Modify comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * db.Querier Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add sanity check , get maxt and mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * ci error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * comment change Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * NoError Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com>
2020-11-26 05:07:06 +00:00
}
defer inputFile.Close()
if err := os.MkdirAll(outputDir, 0777); err != nil {
return checkErr(errors.Wrap(err, "create output dir"))
}
return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet))
Backfill from OpenMetrics format (#8084) * get parser working Signed-off-by: aSquare14 <atibhi.a@gmail.com> * import file created Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Find min and max ts Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make two passes over file and write to tsdb Signed-off-by: aSquare14 <atibhi.a@gmail.com> * print error messages Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Max and Min initializer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Start with unit tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reset file read Signed-off-by: aSquare14 <atibhi.a@gmail.com> * align blocks to two hour range Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add cleanup test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove .ds_store Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add license to import_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix Circle CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Refactor code Move backfill from tsdb to promtool directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix gitignore Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove panic Rename ContenType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * adjust mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return statement Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go modules Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added unit test for backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix file handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close DB Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Close directory Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Error Handling Signed-off-by: aSquare14 <atibhi.a@gmail.com> * inline err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix command line flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add spaces before func fix pointers Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add defer'd calls Signed-off-by: aSquare14 <atibhi.a@gmail.com> * move openmetrics.go content to backfill Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changed args to flags Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add tests for wrong OM files Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Added additional tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add comment to warn of func reuse Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Make input required in main.go Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer blockwriter close Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Remove contentType Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove defer from backfilltest Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix defer remove in backfill_test Signed-off-by: aSquare14 <atibhi.a@gmail.com> * changes to fix CI errors Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix go.mod Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change package name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * assert->require Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix format Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix createblock Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix defer Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix return Signed-off-by: aSquare14 <atibhi.a@gmail.com> * check err for anon func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * update comment Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix for the Flush Bug Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix formatting, comments, names Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * cleanup Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor test to take care of multiple samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * refactor tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * I dont know what I fixed Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fix tests, add test description, print blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * commit after 5000 samples Signed-off-by: aSquare14 <atibhi.a@gmail.com> * reviews part 1 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Series Count Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix CI Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove extra func Signed-off-by: aSquare14 <atibhi.a@gmail.com> * make timestamp into sec Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Reviews 2 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add Todo Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Fixes Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fixes reviews Signed-off-by: aSquare14 <atibhi.a@gmail.com> * =0 Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove backfill.om Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add global err var, remove stuff Signed-off-by: aSquare14 <atibhi.a@gmail.com> * change var name Signed-off-by: aSquare14 <atibhi.a@gmail.com> * sampleLimit pass as parameter Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Add test when number of samples greater than batch size Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Change name of batchsize Signed-off-by: aSquare14 <atibhi.a@gmail.com> * revert export Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * remove Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add comment, remove newline,consistent err Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Print Blocks Signed-off-by: aSquare14 <atibhi.a@gmail.com> * Modify comments Signed-off-by: aSquare14 <atibhi.a@gmail.com> * db.Querier Signed-off-by: aSquare14 <atibhi.a@gmail.com> * add sanity check , get maxt and mint Signed-off-by: aSquare14 <atibhi.a@gmail.com> * ci error Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * comment change Signed-off-by: aSquare14 <atibhi.a@gmail.com> * nits Signed-off-by: aSquare14 <atibhi.a@gmail.com> * NoError Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com> * fix Signed-off-by: aSquare14 <atibhi.a@gmail.com>
2020-11-26 05:07:06 +00:00
}