2017-04-10 18:59:45 +00:00
// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2016-12-07 16:30:10 +00:00
package main
import (
2017-12-14 06:46:46 +00:00
"bufio"
2020-02-06 15:58:38 +00:00
"context"
2016-12-07 16:30:10 +00:00
"fmt"
"io"
"io/ioutil"
2019-02-25 13:51:33 +00:00
"math"
2016-12-07 16:30:10 +00:00
"os"
"path/filepath"
"runtime"
"runtime/pprof"
2017-12-18 23:32:39 +00:00
"sort"
2018-05-08 11:35:06 +00:00
"strconv"
2017-12-14 06:46:46 +00:00
"strings"
2016-12-07 16:30:10 +00:00
"sync"
2017-10-19 16:14:37 +00:00
"text/tabwriter"
2016-12-07 16:30:10 +00:00
"time"
2017-09-19 08:20:19 +00:00
"github.com/go-kit/kit/log"
2017-05-18 14:09:30 +00:00
"github.com/pkg/errors"
2019-11-18 19:53:33 +00:00
"github.com/prometheus/prometheus/pkg/labels"
2020-03-16 21:52:02 +00:00
"github.com/prometheus/prometheus/storage"
2019-08-13 08:34:14 +00:00
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunks"
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
2020-02-06 15:58:38 +00:00
"gopkg.in/alecthomas/kingpin.v2"
2016-12-07 16:30:10 +00:00
)
func main ( ) {
2019-07-23 08:04:48 +00:00
if err := execute ( ) ; err != nil {
fmt . Fprintln ( os . Stderr , err )
os . Exit ( 1 )
}
}
func execute ( ) ( err error ) {
2017-09-09 11:11:12 +00:00
var (
2019-02-25 13:51:33 +00:00
defaultDBPath = filepath . Join ( "benchout" , "storage" )
2017-09-09 11:11:12 +00:00
cli = kingpin . New ( filepath . Base ( os . Args [ 0 ] ) , "CLI tool for tsdb" )
benchCmd = cli . Command ( "bench" , "run benchmarks" )
benchWriteCmd = benchCmd . Command ( "write" , "run a write performance benchmark" )
2018-10-25 09:32:57 +00:00
benchWriteOutPath = benchWriteCmd . Flag ( "out" , "set the output path" ) . Default ( "benchout" ) . String ( )
2017-09-09 11:11:12 +00:00
benchWriteNumMetrics = benchWriteCmd . Flag ( "metrics" , "number of metrics to read" ) . Default ( "10000" ) . Int ( )
2019-03-27 14:33:59 +00:00
benchSamplesFile = benchWriteCmd . Arg ( "file" , "input file with samples data, default is (" + filepath . Join ( ".." , ".." , "testdata" , "20kseries.json" ) + ")" ) . Default ( filepath . Join ( ".." , ".." , "testdata" , "20kseries.json" ) ) . String ( )
2017-10-01 20:18:50 +00:00
listCmd = cli . Command ( "ls" , "list db blocks" )
2018-05-08 11:35:06 +00:00
listCmdHumanReadable = listCmd . Flag ( "human-readable" , "print human readable values" ) . Short ( 'h' ) . Bool ( )
2019-02-25 13:51:33 +00:00
listPath = listCmd . Arg ( "db path" , "database path (default is " + defaultDBPath + ")" ) . Default ( defaultDBPath ) . String ( )
2018-12-28 17:06:12 +00:00
analyzeCmd = cli . Command ( "analyze" , "analyze churn, label pair cardinality." )
2019-02-25 13:51:33 +00:00
analyzePath = analyzeCmd . Arg ( "db path" , "database path (default is " + defaultDBPath + ")" ) . Default ( defaultDBPath ) . String ( )
2019-01-02 16:48:42 +00:00
analyzeBlockID = analyzeCmd . Arg ( "block id" , "block to analyze (default is the last block)" ) . String ( )
2018-12-28 17:06:12 +00:00
analyzeLimit = analyzeCmd . Flag ( "limit" , "how many items to show in each list" ) . Default ( "20" ) . Int ( )
2019-02-25 13:51:33 +00:00
dumpCmd = cli . Command ( "dump" , "dump samples from a TSDB" )
dumpPath = dumpCmd . Arg ( "db path" , "database path (default is " + defaultDBPath + ")" ) . Default ( defaultDBPath ) . String ( )
dumpMinTime = dumpCmd . Flag ( "min-time" , "minimum timestamp to dump" ) . Default ( strconv . FormatInt ( math . MinInt64 , 10 ) ) . Int64 ( )
dumpMaxTime = dumpCmd . Flag ( "max-time" , "maximum timestamp to dump" ) . Default ( strconv . FormatInt ( math . MaxInt64 , 10 ) ) . Int64 ( )
2016-12-07 16:30:10 +00:00
)
2019-07-23 08:04:48 +00:00
logger := log . NewLogfmtLogger ( log . NewSyncWriter ( os . Stderr ) )
var merr tsdb_errors . MultiError
2019-02-25 13:51:33 +00:00
2017-09-09 11:11:12 +00:00
switch kingpin . MustParse ( cli . Parse ( os . Args [ 1 : ] ) ) {
case benchWriteCmd . FullCommand ( ) :
wb := & writeBenchmark {
outPath : * benchWriteOutPath ,
numMetrics : * benchWriteNumMetrics ,
samplesFile : * benchSamplesFile ,
2019-07-23 08:04:48 +00:00
logger : logger ,
2017-09-09 11:11:12 +00:00
}
2019-07-23 08:04:48 +00:00
return wb . run ( )
2017-10-01 20:18:50 +00:00
case listCmd . FullCommand ( ) :
2019-07-23 08:04:48 +00:00
db , err := tsdb . OpenDBReadOnly ( * listPath , nil )
2017-10-01 20:18:50 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2017-10-01 20:18:50 +00:00
}
2019-07-23 08:04:48 +00:00
defer func ( ) {
merr . Add ( err )
merr . Add ( db . Close ( ) )
err = merr . Err ( )
} ( )
blocks , err := db . Blocks ( )
if err != nil {
return err
}
printBlocks ( blocks , listCmdHumanReadable )
2018-12-28 17:06:12 +00:00
case analyzeCmd . FullCommand ( ) :
2019-07-23 08:04:48 +00:00
db , err := tsdb . OpenDBReadOnly ( * analyzePath , nil )
if err != nil {
return err
}
defer func ( ) {
merr . Add ( err )
merr . Add ( db . Close ( ) )
err = merr . Err ( )
} ( )
blocks , err := db . Blocks ( )
2018-12-28 17:06:12 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
2019-07-23 08:04:48 +00:00
var block tsdb . BlockReader
2019-01-02 16:48:42 +00:00
if * analyzeBlockID != "" {
2018-12-28 17:06:12 +00:00
for _ , b := range blocks {
2019-01-02 16:48:42 +00:00
if b . Meta ( ) . ULID . String ( ) == * analyzeBlockID {
2018-12-28 17:06:12 +00:00
block = b
break
}
}
} else if len ( blocks ) > 0 {
block = blocks [ len ( blocks ) - 1 ]
}
if block == nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "block not found" )
2018-12-28 17:06:12 +00:00
}
2019-07-23 08:04:48 +00:00
return analyzeBlock ( block , * analyzeLimit )
2019-02-25 13:51:33 +00:00
case dumpCmd . FullCommand ( ) :
2019-07-23 08:04:48 +00:00
db , err := tsdb . OpenDBReadOnly ( * dumpPath , nil )
2019-02-25 13:51:33 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2019-02-25 13:51:33 +00:00
}
2019-07-23 08:04:48 +00:00
defer func ( ) {
merr . Add ( err )
merr . Add ( db . Close ( ) )
err = merr . Err ( )
} ( )
return dumpSamples ( db , * dumpMinTime , * dumpMaxTime )
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
return nil
2016-12-07 16:30:10 +00:00
}
type writeBenchmark struct {
2017-09-09 11:11:12 +00:00
outPath string
samplesFile string
cleanup bool
numMetrics int
2016-12-07 16:30:10 +00:00
2017-02-19 15:04:37 +00:00
storage * tsdb . DB
2016-12-07 16:30:10 +00:00
cpuprof * os . File
memprof * os . File
blockprof * os . File
2017-05-14 09:51:56 +00:00
mtxprof * os . File
2019-07-23 08:04:48 +00:00
logger log . Logger
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
func ( b * writeBenchmark ) run ( ) error {
2016-12-07 16:30:10 +00:00
if b . outPath == "" {
dir , err := ioutil . TempDir ( "" , "tsdb_bench" )
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
b . outPath = dir
b . cleanup = true
}
if err := os . RemoveAll ( b . outPath ) ; err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
if err := os . MkdirAll ( b . outPath , 0777 ) ; err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
dir := filepath . Join ( b . outPath , "storage" )
2019-07-23 08:04:48 +00:00
l := log . With ( b . logger , "ts" , log . DefaultTimestampUTC , "caller" , log . DefaultCaller )
2017-09-19 08:20:19 +00:00
st , err := tsdb . Open ( dir , l , nil , & tsdb . Options {
2020-02-11 16:34:09 +00:00
RetentionDuration : int64 ( 15 * 24 * time . Hour / time . Millisecond ) ,
MinBlockDuration : int64 ( 2 * time . Hour / time . Millisecond ) ,
2017-02-10 01:54:26 +00:00
} )
2016-12-20 23:02:37 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
2016-12-20 23:02:37 +00:00
b . storage = st
2019-01-14 08:44:32 +00:00
var labels [ ] labels . Labels
2016-12-07 16:30:10 +00:00
2019-07-23 08:04:48 +00:00
_ , err = measureTime ( "readData" , func ( ) error {
2017-09-09 11:11:12 +00:00
f , err := os . Open ( b . samplesFile )
2016-12-07 16:30:10 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
defer f . Close ( )
2019-01-14 08:44:32 +00:00
labels , err = readPrometheusLabels ( f , b . numMetrics )
2016-12-07 16:30:10 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
return nil
2016-12-07 16:30:10 +00:00
} )
2019-07-23 08:04:48 +00:00
if err != nil {
return err
}
2016-12-07 16:30:10 +00:00
2017-02-01 14:29:48 +00:00
var total uint64
2019-07-23 08:04:48 +00:00
dur , err := measureTime ( "ingestScrapes" , func ( ) error {
2019-08-13 08:34:14 +00:00
if err := b . startProfiling ( ) ; err != nil {
return err
}
2019-01-14 08:44:32 +00:00
total , err = b . ingestScrapes ( labels , 3000 )
2017-02-01 14:29:48 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
return nil
2016-12-07 16:30:10 +00:00
} )
2019-07-23 08:04:48 +00:00
if err != nil {
return err
}
2017-02-01 14:29:48 +00:00
fmt . Println ( " > total samples:" , total )
fmt . Println ( " > samples/sec:" , float64 ( total ) / dur . Seconds ( ) )
2019-07-23 08:04:48 +00:00
_ , err = measureTime ( "stopStorage" , func ( ) error {
2016-12-20 23:02:37 +00:00
if err := b . storage . Close ( ) ; err != nil {
2019-07-23 08:04:48 +00:00
return err
2016-12-07 16:30:10 +00:00
}
2018-09-20 08:33:52 +00:00
if err := b . stopProfiling ( ) ; err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-09-20 08:33:52 +00:00
}
2019-07-23 08:04:48 +00:00
return nil
2016-12-07 16:30:10 +00:00
} )
2019-07-23 08:04:48 +00:00
if err != nil {
return err
}
return nil
2016-12-07 16:30:10 +00:00
}
2017-06-07 11:42:53 +00:00
const timeDelta = 30000
2017-02-01 14:29:48 +00:00
func ( b * writeBenchmark ) ingestScrapes ( lbls [ ] labels . Labels , scrapeCount int ) ( uint64 , error ) {
2017-01-10 10:17:37 +00:00
var mu sync . Mutex
var total uint64
2016-12-07 16:30:10 +00:00
2017-02-02 10:09:19 +00:00
for i := 0 ; i < scrapeCount ; i += 100 {
var wg sync . WaitGroup
2017-01-11 11:54:18 +00:00
lbls := lbls
for len ( lbls ) > 0 {
l := 1000
if len ( lbls ) < 1000 {
l = len ( lbls )
2016-12-07 16:30:10 +00:00
}
2017-01-11 11:54:18 +00:00
batch := lbls [ : l ]
lbls = lbls [ l : ]
wg . Add ( 1 )
go func ( ) {
2017-06-07 11:42:53 +00:00
n , err := b . ingestScrapesShard ( batch , 100 , int64 ( timeDelta * i ) )
2017-01-11 11:54:18 +00:00
if err != nil {
// exitWithError(err)
fmt . Println ( " err" , err )
}
mu . Lock ( )
total += n
mu . Unlock ( )
wg . Done ( )
} ( )
}
wg . Wait ( )
2016-12-07 16:30:10 +00:00
}
2017-07-13 14:15:13 +00:00
fmt . Println ( "ingestion completed" )
2017-01-10 10:17:37 +00:00
2017-02-01 14:29:48 +00:00
return total , nil
2016-12-07 16:30:10 +00:00
}
2019-01-14 08:44:32 +00:00
func ( b * writeBenchmark ) ingestScrapesShard ( lbls [ ] labels . Labels , scrapeCount int , baset int64 ) ( uint64 , error ) {
2017-01-11 11:54:18 +00:00
ts := baset
2016-12-07 16:30:10 +00:00
2016-12-08 09:04:24 +00:00
type sample struct {
2016-12-21 08:39:01 +00:00
labels labels . Labels
2016-12-08 09:04:24 +00:00
value int64
2017-09-05 09:45:18 +00:00
ref * uint64
2016-12-08 09:04:24 +00:00
}
2019-01-14 08:44:32 +00:00
scrape := make ( [ ] * sample , 0 , len ( lbls ) )
2016-12-08 09:04:24 +00:00
2019-01-14 08:44:32 +00:00
for _ , m := range lbls {
2017-01-11 11:54:18 +00:00
scrape = append ( scrape , & sample {
labels : m ,
2016-12-08 09:04:24 +00:00
value : 123456789 ,
2017-01-11 11:54:18 +00:00
} )
2016-12-08 09:04:24 +00:00
}
2017-01-10 10:17:37 +00:00
total := uint64 ( 0 )
2016-12-08 09:04:24 +00:00
2016-12-07 16:30:10 +00:00
for i := 0 ; i < scrapeCount ; i ++ {
2017-01-12 18:18:51 +00:00
app := b . storage . Appender ( )
2017-06-07 11:42:53 +00:00
ts += timeDelta
2016-12-07 16:30:10 +00:00
2016-12-08 09:04:24 +00:00
for _ , s := range scrape {
2016-12-09 09:00:14 +00:00
s . value += 1000
2017-01-12 18:18:51 +00:00
if s . ref == nil {
2017-02-01 14:29:48 +00:00
ref , err := app . Add ( s . labels , ts , float64 ( s . value ) )
2017-01-12 18:18:51 +00:00
if err != nil {
panic ( err )
}
s . ref = & ref
2017-02-01 14:29:48 +00:00
} else if err := app . AddFast ( * s . ref , ts , float64 ( s . value ) ) ; err != nil {
2017-02-02 10:09:19 +00:00
2020-03-16 21:52:02 +00:00
if errors . Cause ( err ) != storage . ErrNotFound {
2017-01-12 18:18:51 +00:00
panic ( err )
}
2017-02-01 14:29:48 +00:00
ref , err := app . Add ( s . labels , ts , float64 ( s . value ) )
2017-01-12 18:18:51 +00:00
if err != nil {
panic ( err )
}
s . ref = & ref
}
2017-01-10 10:17:37 +00:00
total ++
2016-12-07 16:30:10 +00:00
}
2016-12-20 23:02:37 +00:00
if err := app . Commit ( ) ; err != nil {
2017-01-10 10:17:37 +00:00
return total , err
2016-12-07 16:30:10 +00:00
}
}
2017-01-10 10:17:37 +00:00
return total , nil
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
func ( b * writeBenchmark ) startProfiling ( ) error {
2016-12-07 16:30:10 +00:00
var err error
// Start CPU profiling.
b . cpuprof , err = os . Create ( filepath . Join ( b . outPath , "cpu.prof" ) )
if err != nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "bench: could not create cpu profile: %v" , err )
2016-12-07 16:30:10 +00:00
}
2018-09-20 08:33:52 +00:00
if err := pprof . StartCPUProfile ( b . cpuprof ) ; err != nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "bench: could not start CPU profile: %v" , err )
2018-09-20 08:33:52 +00:00
}
2016-12-07 16:30:10 +00:00
// Start memory profiling.
b . memprof , err = os . Create ( filepath . Join ( b . outPath , "mem.prof" ) )
if err != nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "bench: could not create memory profile: %v" , err )
2016-12-07 16:30:10 +00:00
}
2017-05-14 09:51:56 +00:00
runtime . MemProfileRate = 64 * 1024
2016-12-07 16:30:10 +00:00
// Start fatal profiling.
b . blockprof , err = os . Create ( filepath . Join ( b . outPath , "block.prof" ) )
if err != nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "bench: could not create block profile: %v" , err )
2016-12-07 16:30:10 +00:00
}
2017-05-14 09:51:56 +00:00
runtime . SetBlockProfileRate ( 20 )
b . mtxprof , err = os . Create ( filepath . Join ( b . outPath , "mutex.prof" ) )
if err != nil {
2019-07-23 08:04:48 +00:00
return fmt . Errorf ( "bench: could not create mutex profile: %v" , err )
2017-05-14 09:51:56 +00:00
}
runtime . SetMutexProfileFraction ( 20 )
2019-07-23 08:04:48 +00:00
return nil
2016-12-07 16:30:10 +00:00
}
2018-09-20 08:33:52 +00:00
func ( b * writeBenchmark ) stopProfiling ( ) error {
2016-12-07 16:30:10 +00:00
if b . cpuprof != nil {
pprof . StopCPUProfile ( )
b . cpuprof . Close ( )
b . cpuprof = nil
}
if b . memprof != nil {
2018-09-20 08:33:52 +00:00
if err := pprof . Lookup ( "heap" ) . WriteTo ( b . memprof , 0 ) ; err != nil {
return fmt . Errorf ( "error writing mem profile: %v" , err )
}
2016-12-07 16:30:10 +00:00
b . memprof . Close ( )
b . memprof = nil
}
if b . blockprof != nil {
2018-09-20 08:33:52 +00:00
if err := pprof . Lookup ( "block" ) . WriteTo ( b . blockprof , 0 ) ; err != nil {
return fmt . Errorf ( "error writing block profile: %v" , err )
}
2016-12-07 16:30:10 +00:00
b . blockprof . Close ( )
b . blockprof = nil
runtime . SetBlockProfileRate ( 0 )
}
2017-05-14 09:51:56 +00:00
if b . mtxprof != nil {
2018-09-20 08:33:52 +00:00
if err := pprof . Lookup ( "mutex" ) . WriteTo ( b . mtxprof , 0 ) ; err != nil {
return fmt . Errorf ( "error writing mutex profile: %v" , err )
}
2017-05-14 09:51:56 +00:00
b . mtxprof . Close ( )
b . mtxprof = nil
runtime . SetMutexProfileFraction ( 0 )
}
2018-09-20 08:33:52 +00:00
return nil
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
func measureTime ( stage string , f func ( ) error ) ( time . Duration , error ) {
2016-12-07 16:30:10 +00:00
fmt . Printf ( ">> start stage=%s\n" , stage )
start := time . Now ( )
2019-07-23 08:04:48 +00:00
err := f ( )
if err != nil {
return 0 , err
}
2016-12-07 16:30:10 +00:00
fmt . Printf ( ">> completed stage=%s duration=%s\n" , stage , time . Since ( start ) )
2019-07-23 08:04:48 +00:00
return time . Since ( start ) , nil
2016-12-07 16:30:10 +00:00
}
2017-12-14 06:46:46 +00:00
func readPrometheusLabels ( r io . Reader , n int ) ( [ ] labels . Labels , error ) {
scanner := bufio . NewScanner ( r )
2016-12-07 16:30:10 +00:00
2017-01-16 20:29:53 +00:00
var mets [ ] labels . Labels
hashes := map [ uint64 ] struct { } { }
2017-12-14 06:46:46 +00:00
i := 0
2016-12-07 16:30:10 +00:00
2017-12-14 06:46:46 +00:00
for scanner . Scan ( ) && i < n {
2017-01-16 20:29:53 +00:00
m := make ( labels . Labels , 0 , 10 )
2017-12-14 06:46:46 +00:00
r := strings . NewReplacer ( "\"" , "" , "{" , "" , "}" , "" )
s := r . Replace ( scanner . Text ( ) )
labelChunks := strings . Split ( s , "," )
for _ , labelChunk := range labelChunks {
split := strings . Split ( labelChunk , ":" )
m = append ( m , labels . Label { Name : split [ 0 ] , Value : split [ 1 ] } )
}
2017-12-18 23:32:39 +00:00
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort . Sort ( m )
2017-01-16 20:29:53 +00:00
h := m . Hash ( )
if _ , ok := hashes [ h ] ; ok {
continue
2016-12-07 16:30:10 +00:00
}
2017-01-16 20:29:53 +00:00
mets = append ( mets , m )
hashes [ h ] = struct { } { }
i ++
2016-12-07 16:30:10 +00:00
}
2017-12-14 06:46:46 +00:00
return mets , nil
2016-12-07 16:30:10 +00:00
}
2019-07-23 08:04:48 +00:00
func printBlocks ( blocks [ ] tsdb . BlockReader , humanReadable * bool ) {
2017-10-19 16:14:37 +00:00
tw := tabwriter . NewWriter ( os . Stdout , 0 , 0 , 2 , ' ' , 0 )
2017-10-02 20:48:47 +00:00
defer tw . Flush ( )
fmt . Fprintln ( tw , "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES" )
for _ , b := range blocks {
2017-10-11 09:02:57 +00:00
meta := b . Meta ( )
2017-10-02 20:48:47 +00:00
fmt . Fprintf ( tw ,
"%v\t%v\t%v\t%v\t%v\t%v\n" ,
2017-10-11 09:02:57 +00:00
meta . ULID ,
2018-05-08 11:35:06 +00:00
getFormatedTime ( meta . MinTime , humanReadable ) ,
getFormatedTime ( meta . MaxTime , humanReadable ) ,
2017-10-11 09:02:57 +00:00
meta . Stats . NumSamples ,
meta . Stats . NumChunks ,
meta . Stats . NumSeries ,
2017-10-02 20:48:47 +00:00
)
}
}
2018-05-08 11:35:06 +00:00
func getFormatedTime ( timestamp int64 , humanReadable * bool ) string {
if * humanReadable {
return time . Unix ( timestamp / 1000 , 0 ) . String ( )
}
return strconv . FormatInt ( timestamp , 10 )
}
2018-12-28 17:06:12 +00:00
2019-07-23 08:04:48 +00:00
func analyzeBlock ( b tsdb . BlockReader , limit int ) error {
2018-12-28 17:06:12 +00:00
meta := b . Meta ( )
2019-07-23 08:04:48 +00:00
fmt . Printf ( "Block ID: %s\n" , meta . ULID )
2018-12-28 17:06:12 +00:00
// Presume 1ms resolution that Prometheus uses.
fmt . Printf ( "Duration: %s\n" , ( time . Duration ( meta . MaxTime - meta . MinTime ) * 1e6 ) . String ( ) )
fmt . Printf ( "Series: %d\n" , meta . Stats . NumSeries )
2020-03-25 19:13:47 +00:00
ir , err := b . Index ( )
2018-12-28 17:06:12 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
defer ir . Close ( )
allLabelNames , err := ir . LabelNames ( )
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
fmt . Printf ( "Label names: %d\n" , len ( allLabelNames ) )
type postingInfo struct {
key string
metric uint64
}
postingInfos := [ ] postingInfo { }
printInfo := func ( postingInfos [ ] postingInfo ) {
sort . Slice ( postingInfos , func ( i , j int ) bool { return postingInfos [ i ] . metric > postingInfos [ j ] . metric } )
for i , pc := range postingInfos {
fmt . Printf ( "%d %s\n" , pc . metric , pc . key )
if i >= limit {
break
}
}
}
labelsUncovered := map [ string ] uint64 { }
labelpairsUncovered := map [ string ] uint64 { }
labelpairsCount := map [ string ] uint64 { }
entries := 0
p , err := ir . Postings ( "" , "" ) // The special all key.
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
lbls := labels . Labels { }
chks := [ ] chunks . Meta { }
for p . Next ( ) {
2019-01-02 16:48:42 +00:00
if err = ir . Series ( p . At ( ) , & lbls , & chks ) ; err != nil {
2019-07-23 08:04:48 +00:00
return err
2019-01-02 16:48:42 +00:00
}
2018-12-28 17:06:12 +00:00
// Amount of the block time range not covered by this series.
uncovered := uint64 ( meta . MaxTime - meta . MinTime ) - uint64 ( chks [ len ( chks ) - 1 ] . MaxTime - chks [ 0 ] . MinTime )
for _ , lbl := range lbls {
key := lbl . Name + "=" + lbl . Value
labelsUncovered [ lbl . Name ] += uncovered
labelpairsUncovered [ key ] += uncovered
2019-01-02 16:48:42 +00:00
labelpairsCount [ key ] ++
entries ++
2018-12-28 17:06:12 +00:00
}
}
if p . Err ( ) != nil {
2019-07-23 08:04:48 +00:00
return p . Err ( )
2018-12-28 17:06:12 +00:00
}
fmt . Printf ( "Postings (unique label pairs): %d\n" , len ( labelpairsUncovered ) )
fmt . Printf ( "Postings entries (total label pairs): %d\n" , entries )
postingInfos = postingInfos [ : 0 ]
for k , m := range labelpairsUncovered {
postingInfos = append ( postingInfos , postingInfo { k , uint64 ( float64 ( m ) / float64 ( meta . MaxTime - meta . MinTime ) ) } )
}
fmt . Printf ( "\nLabel pairs most involved in churning:\n" )
printInfo ( postingInfos )
postingInfos = postingInfos [ : 0 ]
for k , m := range labelsUncovered {
postingInfos = append ( postingInfos , postingInfo { k , uint64 ( float64 ( m ) / float64 ( meta . MaxTime - meta . MinTime ) ) } )
}
fmt . Printf ( "\nLabel names most involved in churning:\n" )
printInfo ( postingInfos )
postingInfos = postingInfos [ : 0 ]
for k , m := range labelpairsCount {
postingInfos = append ( postingInfos , postingInfo { k , m } )
}
fmt . Printf ( "\nMost common label pairs:\n" )
printInfo ( postingInfos )
2019-06-21 08:07:50 +00:00
postingInfos = postingInfos [ : 0 ]
for _ , n := range allLabelNames {
values , err := ir . LabelValues ( n )
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2019-06-21 08:07:50 +00:00
}
var cumulativeLength uint64
Replace StringTuples with []string
Benchmarks show slight cpu/allocs improvements.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42%
BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78%
BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37%
BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89%
BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82%
BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
for _ , str := range values {
cumulativeLength += uint64 ( len ( str ) )
2019-06-21 08:07:50 +00:00
}
postingInfos = append ( postingInfos , postingInfo { n , cumulativeLength } )
}
fmt . Printf ( "\nLabel names with highest cumulative label value length:\n" )
printInfo ( postingInfos )
2018-12-28 17:06:12 +00:00
postingInfos = postingInfos [ : 0 ]
for _ , n := range allLabelNames {
lv , err := ir . LabelValues ( n )
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
Replace StringTuples with []string
Benchmarks show slight cpu/allocs improvements.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42%
BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78%
BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37%
BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89%
BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82%
BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
postingInfos = append ( postingInfos , postingInfo { n , uint64 ( len ( lv ) ) } )
2018-12-28 17:06:12 +00:00
}
fmt . Printf ( "\nHighest cardinality labels:\n" )
printInfo ( postingInfos )
postingInfos = postingInfos [ : 0 ]
lv , err := ir . LabelValues ( "__name__" )
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
Replace StringTuples with []string
Benchmarks show slight cpu/allocs improvements.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42%
BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78%
BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37%
BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89%
BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82%
BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
for _ , n := range lv {
postings , err := ir . Postings ( "__name__" , n )
2018-12-28 17:06:12 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2018-12-28 17:06:12 +00:00
}
Replace StringTuples with []string
Benchmarks show slight cpu/allocs improvements.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42%
BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78%
BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37%
BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89%
BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82%
BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
count := 0
for postings . Next ( ) {
count ++
}
if postings . Err ( ) != nil {
return postings . Err ( )
2018-12-28 17:06:12 +00:00
}
Replace StringTuples with []string
Benchmarks show slight cpu/allocs improvements.
benchmark old ns/op new ns/op delta
BenchmarkPostingsForMatchers/Head/n="1"-4 269978625 235305110 -12.84%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 129739974 121646193 -6.24%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 123826274 122056253 -1.43%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 126962188 130038235 +2.42%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 6423653989 5991126455 -6.73%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 6934647521 7033370634 +1.42%
BenchmarkPostingsForMatchers/Head/i=~""-4 1177781285 1121497736 -4.78%
BenchmarkPostingsForMatchers/Head/i!=""-4 7033680256 7246094991 +3.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 293702332 287440212 -2.13%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 307628268 307039964 -0.19%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 512247746 480003862 -6.29%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 361199794 367066917 +1.62%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 478863761 476037784 -0.59%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 103394659 102902098 -0.48%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 482552781 475453903 -1.47%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 559257389 589297047 +5.37%
BenchmarkPostingsForMatchers/Block/n="1"-4 36492 37012 +1.42%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 557788 611903 +9.70%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 554443 573814 +3.49%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 553227 553826 +0.11%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 113855090 111707221 -1.89%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 133994674 136520728 +1.89%
BenchmarkPostingsForMatchers/Block/i=~""-4 38138091 36299898 -4.82%
BenchmarkPostingsForMatchers/Block/i!=""-4 28861213 27396723 -5.07%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 112699941 110853868 -1.64%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 113198026 111389742 -1.60%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 28994069 27363804 -5.62%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 29709406 28589223 -3.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 134695119 135736971 +0.77%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 26783286 25826928 -3.57%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 134733254 134116739 -0.46%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 160713937 158802768 -1.19%
benchmark old allocs new allocs delta
BenchmarkPostingsForMatchers/Head/n="1"-4 36 36 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 38 38 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 42 40 -4.76%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 61 59 -3.28%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 100088 100087 -0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 100053 100051 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 100087 100085 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 44 42 -4.55%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 50 48 -4.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 100076 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 100077 100075 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 100077 100074 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 11167 11165 -0.02%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 100082 100080 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 111265 111261 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 6 6 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 11 11 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 15 13 -13.33%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 12 10 -16.67%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 100040 100038 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 100045 100043 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 100041 100039 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 17 15 -11.76%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 23 21 -8.70%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 100046 100044 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 100050 100048 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 100049 100047 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 11150 11148 -0.02%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 100055 100053 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 111238 111234 -0.00%
benchmark old bytes new bytes delta
BenchmarkPostingsForMatchers/Head/n="1"-4 10887816 10887817 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j="foo"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/j="foo",n="1"-4 5456648 5456648 +0.00%
BenchmarkPostingsForMatchers/Head/n="1",j!="foo"-4 5456792 5456712 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".*"-4 258254408 258254328 -0.00%
BenchmarkPostingsForMatchers/Head/i=~".+"-4 273912888 273912904 +0.00%
BenchmarkPostingsForMatchers/Head/i=~""-4 17266680 17266600 -0.00%
BenchmarkPostingsForMatchers/Head/i!=""-4 273912416 273912336 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",j="foo"-4 7062578 7062498 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".*",i!="2",j="foo"-4 7062770 7062690 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!=""-4 28152346 28152266 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i!="",j="foo"-4 22721178 22721098 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",j="foo"-4 22721336 22721224 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~"1.+",j="foo"-4 3623804 3623733 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!="2",j="foo"-4 22721480 22721400 -0.00%
BenchmarkPostingsForMatchers/Head/n="1",i=~".+",i!~"2.*",j="foo"-4 24816652 24816444 -0.00%
BenchmarkPostingsForMatchers/Block/n="1"-4 296 296 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j="foo"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/j="foo",n="1"-4 424 424 +0.00%
BenchmarkPostingsForMatchers/Block/n="1",j!="foo"-4 1544 1464 -5.18%
BenchmarkPostingsForMatchers/Block/i=~".*"-4 1606114 1606045 -0.00%
BenchmarkPostingsForMatchers/Block/i=~".+"-4 17264709 17264629 -0.00%
BenchmarkPostingsForMatchers/Block/i=~""-4 17264780 17264696 -0.00%
BenchmarkPostingsForMatchers/Block/i!=""-4 17264680 17264600 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",j="foo"-4 1606253 1606165 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i=~".*",i!="2",j="foo"-4 1606445 1606348 -0.01%
BenchmarkPostingsForMatchers/Block/n="1",i!=""-4 17264808 17264728 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i!="",j="foo"-4 17264936 17264856 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",j="foo"-4 17264965 17264885 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~"1.+",j="foo"-4 3148262 3148182 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!="2",j="foo"-4 17265141 17265061 -0.00%
BenchmarkPostingsForMatchers/Block/n="1",i=~".+",i!~"2.*",j="foo"-4 20416944 20416784 -0.00%
Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>
2020-01-01 11:38:01 +00:00
postingInfos = append ( postingInfos , postingInfo { n , uint64 ( count ) } )
2018-12-28 17:06:12 +00:00
}
fmt . Printf ( "\nHighest cardinality metric names:\n" )
printInfo ( postingInfos )
2019-07-23 08:04:48 +00:00
return nil
2018-12-28 17:06:12 +00:00
}
2019-02-25 13:51:33 +00:00
2019-07-23 08:04:48 +00:00
func dumpSamples ( db * tsdb . DBReadOnly , mint , maxt int64 ) ( err error ) {
2020-02-06 15:58:38 +00:00
q , err := db . Querier ( context . TODO ( ) , mint , maxt )
2019-02-25 13:51:33 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2019-02-25 13:51:33 +00:00
}
2019-07-23 08:04:48 +00:00
defer func ( ) {
var merr tsdb_errors . MultiError
merr . Add ( err )
merr . Add ( q . Close ( ) )
err = merr . Err ( )
} ( )
2019-02-25 13:51:33 +00:00
2020-03-12 09:36:09 +00:00
ss , ws , err := q . Select ( false , nil , labels . MustNewMatcher ( labels . MatchRegexp , "" , ".*" ) )
2019-02-25 13:51:33 +00:00
if err != nil {
2019-07-23 08:04:48 +00:00
return err
2019-02-25 13:51:33 +00:00
}
2020-02-06 15:58:38 +00:00
if len ( ws ) > 0 {
var merr tsdb_errors . MultiError
for _ , w := range ws {
merr . Add ( w )
}
return merr . Err ( )
}
2019-02-25 13:51:33 +00:00
for ss . Next ( ) {
series := ss . At ( )
labels := series . Labels ( )
it := series . Iterator ( )
for it . Next ( ) {
ts , val := it . At ( )
fmt . Printf ( "%s %g %d\n" , labels , val , ts )
}
if it . Err ( ) != nil {
2019-07-23 08:04:48 +00:00
return ss . Err ( )
2019-02-25 13:51:33 +00:00
}
}
if ss . Err ( ) != nil {
2019-07-23 08:04:48 +00:00
return ss . Err ( )
2019-02-25 13:51:33 +00:00
}
2019-07-23 08:04:48 +00:00
return nil
2019-02-25 13:51:33 +00:00
}