diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3f55b8ea9..9e78957ec 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,17 @@ \ No newline at end of file + --> diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c01f4ce1..d61f6e560 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.28.1 / 2021-07-01 + +* [BUGFIX]: HTTP SD: Allow `charset` specification in `Content-Type` header. #8981 +* [BUGFIX]: HTTP SD: Fix handling of disappeared target groups. #9019 +* [BUGFIX]: Fix incorrect log-level handling after moving to go-kit/log. #9021 + ## 2.28.0 / 2021-06-21 * [CHANGE] UI: Make the new experimental PromQL editor the default. #8925 diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index c22a2cd95..5d64ea918 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -131,9 +131,6 @@ type flagConfig struct { // setFeatureListOptions sets the corresponding options from the featureList. func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { - maxExemplars := c.tsdb.MaxExemplars - // Disabled at first. Value from the flag is used if exemplar-storage is set. - c.tsdb.MaxExemplars = 0 for _, f := range c.featureList { opts := strings.Split(f, ",") for _, o := range opts { @@ -151,8 +148,8 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { c.enableExpandExternalLabels = true level.Info(logger).Log("msg", "Experimental expand-external-labels enabled") case "exemplar-storage": - c.tsdb.MaxExemplars = maxExemplars - level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled", "maxExemplars", maxExemplars) + c.tsdb.EnableExemplarStorage = true + level.Info(logger).Log("msg", "Experimental in-memory exemplar storage enabled") case "": continue default: @@ -259,17 +256,17 @@ func main() { a.Flag("storage.tsdb.retention.time", "How long to retain samples in storage. When this flag is set it overrides \"storage.tsdb.retention\". If neither this flag nor \"storage.tsdb.retention\" nor \"storage.tsdb.retention.size\" is set, the retention time defaults to "+defaultRetentionString+". Units Supported: y, w, d, h, m, s, ms."). SetValue(&newFlagRetentionDuration) - a.Flag("storage.tsdb.retention.size", "[EXPERIMENTAL] Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases."). + a.Flag("storage.tsdb.retention.size", "Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: \"512MB\". This flag is experimental and can be changed in future releases."). BytesVar(&cfg.tsdb.MaxBytes) a.Flag("storage.tsdb.no-lockfile", "Do not create lockfile in data directory."). Default("false").BoolVar(&cfg.tsdb.NoLockfile) - a.Flag("storage.tsdb.allow-overlapping-blocks", "[EXPERIMENTAL] Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge."). + a.Flag("storage.tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge."). Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks) a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL."). - Default("true").BoolVar(&cfg.tsdb.WALCompression) + Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression) a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload."). Default("1m").PlaceHolder("").SetValue(&cfg.RemoteFlushDeadline) @@ -283,9 +280,6 @@ func main() { a.Flag("storage.remote.read-max-bytes-in-frame", "Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as recommended by protobuf by default."). Default("1048576").IntVar(&cfg.web.RemoteReadBytesInFrame) - a.Flag("storage.exemplars.exemplars-limit", "[EXPERIMENTAL] Maximum number of exemplars to store in in-memory exemplar storage total. 0 disables the exemplar storage. This flag is effective only with --enable-feature=exemplar-storage."). - Default("100000").IntVar(&cfg.tsdb.MaxExemplars) - a.Flag("rules.alert.for-outage-tolerance", "Max time to tolerate prometheus outage for restoring \"for\" state of alert."). Default("1h").SetValue(&cfg.outageTolerance) @@ -316,7 +310,7 @@ func main() { a.Flag("query.max-samples", "Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return."). Default("50000000").IntVar(&cfg.queryMaxSamples) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage, expand-external-labels. See https://prometheus.io/docs/prometheus/latest/disabled_features/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: promql-at-modifier, promql-negative-offset, remote-write-receiver, exemplar-storage, expand-external-labels. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) promlogflag.AddFlags(a, &cfg.promlogConfig) @@ -352,10 +346,18 @@ func main() { } // Throw error for invalid config before starting other components. - if _, err := config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil { + var cfgFile *config.Config + if cfgFile, err = config.LoadFile(cfg.configFile, false, log.NewNopLogger()); err != nil { level.Error(logger).Log("msg", fmt.Sprintf("Error loading config (--config.file=%s)", cfg.configFile), "err", err) os.Exit(2) } + if cfg.tsdb.EnableExemplarStorage { + if cfgFile.StorageConfig.ExemplarsConfig == nil { + cfgFile.StorageConfig.ExemplarsConfig = &config.DefaultExemplarsConfig + } + cfg.tsdb.MaxExemplars = int64(cfgFile.StorageConfig.ExemplarsConfig.MaxExemplars) + } + // Now that the validity of the config is established, set the config // success metrics accordingly, although the config isn't really loaded // yet. This will happen later (including setting these metrics again), @@ -533,6 +535,9 @@ func main() { reloaders := []reloader{ { + name: "db_storage", + reloader: localStorage.ApplyConfig, + }, { name: "remote_storage", reloader: remoteStorage.ApplyConfig, }, { @@ -734,11 +739,11 @@ func main() { for { select { case <-hup: - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { level.Error(logger).Log("msg", "Error reloading config", "err", err) } case rc := <-webHandler.Reload(): - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { level.Error(logger).Log("msg", "Error reloading config", "err", err) rc <- err } else { @@ -770,7 +775,7 @@ func main() { return nil } - if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, logger, noStepSubqueryInterval, reloaders...); err != nil { + if err := reloadConfig(cfg.configFile, cfg.enableExpandExternalLabels, cfg.tsdb.EnableExemplarStorage, logger, noStepSubqueryInterval, reloaders...); err != nil { return errors.Wrapf(err, "error loading config from %q", cfg.configFile) } @@ -959,7 +964,7 @@ type reloader struct { reloader func(*config.Config) error } -func reloadConfig(filename string, expandExternalLabels bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) { +func reloadConfig(filename string, expandExternalLabels bool, enableExemplarStorage bool, logger log.Logger, noStepSuqueryInterval *safePromQLNoStepSubqueryInterval, rls ...reloader) (err error) { start := time.Now() timings := []interface{}{} level.Info(logger).Log("msg", "Loading configuration file", "filename", filename) @@ -978,6 +983,12 @@ func reloadConfig(filename string, expandExternalLabels bool, logger log.Logger, return errors.Wrapf(err, "couldn't load configuration (--config.file=%q)", filename) } + if enableExemplarStorage { + if conf.StorageConfig.ExemplarsConfig == nil { + conf.StorageConfig.ExemplarsConfig = &config.DefaultExemplarsConfig + } + } + failed := false for _, rl := range rls { rstart := time.Now() @@ -1083,6 +1094,11 @@ type readyStorage struct { stats *tsdb.DBStats } +func (s *readyStorage) ApplyConfig(conf *config.Config) error { + db := s.get() + return db.ApplyConfig(conf) +} + // Set the storage. func (s *readyStorage) Set(db *tsdb.DB, startTimeMargin int64) { s.mtx.Lock() @@ -1262,7 +1278,8 @@ type tsdbOptions struct { StripeSize int MinBlockDuration model.Duration MaxBlockDuration model.Duration - MaxExemplars int + EnableExemplarStorage bool + MaxExemplars int64 } func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { @@ -1277,6 +1294,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { StripeSize: opts.StripeSize, MinBlockDuration: int64(time.Duration(opts.MinBlockDuration) / time.Millisecond), MaxBlockDuration: int64(time.Duration(opts.MaxBlockDuration) / time.Millisecond), + EnableExemplarStorage: opts.EnableExemplarStorage, MaxExemplars: opts.MaxExemplars, } } diff --git a/cmd/promtool/backfill.go b/cmd/promtool/backfill.go index 76e5a82df..3d05260e3 100644 --- a/cmd/promtool/backfill.go +++ b/cmd/promtool/backfill.go @@ -17,6 +17,7 @@ import ( "context" "io" "math" + "time" "github.com/go-kit/log" "github.com/pkg/errors" @@ -65,8 +66,19 @@ func getMinAndMaxTimestamps(p textparse.Parser) (int64, int64, error) { return maxt, mint, nil } -func createBlocks(input []byte, mint, maxt int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) { +func createBlocks(input []byte, mint, maxt, maxBlockDuration int64, maxSamplesInAppender int, outputDir string, humanReadable, quiet bool) (returnErr error) { blockDuration := tsdb.DefaultBlockDuration + if maxBlockDuration > tsdb.DefaultBlockDuration { + ranges := tsdb.ExponentialBlockRanges(tsdb.DefaultBlockDuration, 10, 3) + idx := len(ranges) - 1 // Use largest range if user asked for something enormous. + for i, v := range ranges { + if v > maxBlockDuration { + idx = i - 1 + break + } + } + blockDuration = ranges[idx] + } mint = blockDuration * (mint / blockDuration) db, err := tsdb.OpenDBReadOnly(outputDir, nil) @@ -199,11 +211,11 @@ func createBlocks(input []byte, mint, maxt int64, maxSamplesInAppender int, outp } -func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool) (err error) { +func backfill(maxSamplesInAppender int, input []byte, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) (err error) { p := textparse.NewOpenMetricsParser(input) maxt, mint, err := getMinAndMaxTimestamps(p) if err != nil { return errors.Wrap(err, "getting min and max timestamp") } - return errors.Wrap(createBlocks(input, mint, maxt, maxSamplesInAppender, outputDir, humanReadable, quiet), "block creation") + return errors.Wrap(createBlocks(input, mint, maxt, int64(maxBlockDuration/time.Millisecond), maxSamplesInAppender, outputDir, humanReadable, quiet), "block creation") } diff --git a/cmd/promtool/backfill_test.go b/cmd/promtool/backfill_test.go index bd0109da1..4bbb5602c 100644 --- a/cmd/promtool/backfill_test.go +++ b/cmd/promtool/backfill_test.go @@ -20,6 +20,7 @@ import ( "os" "sort" "testing" + "time" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" @@ -58,12 +59,12 @@ func queryAllSeries(t testing.TB, q storage.Querier, expectedMinTime, expectedMa return samples } -func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime int64, expectedSamples []backfillSample, expectedNumBlocks int) { +func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime, expectedBlockDuration int64, expectedSamples []backfillSample, expectedNumBlocks int) { blocks := db.Blocks() - require.Equal(t, expectedNumBlocks, len(blocks)) + require.Equal(t, expectedNumBlocks, len(blocks), "did not create correct number of blocks") - for _, block := range blocks { - require.Equal(t, true, block.MinTime()/tsdb.DefaultBlockDuration == (block.MaxTime()-1)/tsdb.DefaultBlockDuration) + for i, block := range blocks { + require.Equal(t, block.MinTime()/expectedBlockDuration, (block.MaxTime()-1)/expectedBlockDuration, "block %d contains data outside of one aligned block duration", i) } q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64) @@ -75,11 +76,11 @@ func testBlocks(t *testing.T, db *tsdb.DB, expectedMinTime, expectedMaxTime int6 allSamples := queryAllSeries(t, q, expectedMinTime, expectedMaxTime) sortSamples(allSamples) sortSamples(expectedSamples) - require.Equal(t, expectedSamples, allSamples) + require.Equal(t, expectedSamples, allSamples, "did not create correct samples") if len(allSamples) > 0 { - require.Equal(t, expectedMinTime, allSamples[0].Timestamp) - require.Equal(t, expectedMaxTime, allSamples[len(allSamples)-1].Timestamp) + require.Equal(t, expectedMinTime, allSamples[0].Timestamp, "timestamp of first sample is not the expected minimum time") + require.Equal(t, expectedMaxTime, allSamples[len(allSamples)-1].Timestamp, "timestamp of last sample is not the expected maximum time") } } @@ -89,11 +90,13 @@ func TestBackfill(t *testing.T) { IsOk bool Description string MaxSamplesInAppender int + MaxBlockDuration time.Duration Expected struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample } }{ { @@ -102,15 +105,17 @@ func TestBackfill(t *testing.T) { Description: "Empty file.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - NumBlocks: 0, - Samples: []backfillSample{}, + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + NumBlocks: 0, + BlockDuration: tsdb.DefaultBlockDuration, + Samples: []backfillSample{}, }, }, { @@ -124,14 +129,16 @@ http_requests_total{code="400"} 1 1565133713.990 Description: "Multiple samples with different timestamp for different series.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565133713990, - NumBlocks: 1, + MinTime: 1565133713989, + MaxTime: 1565133713990, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -158,14 +165,16 @@ http_requests_total{code="200"} 1023 1565652113.989 Description: "Multiple samples separated by 3 days.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565652113989, - NumBlocks: 3, + MinTime: 1565133713989, + MaxTime: 1565652113989, + NumBlocks: 3, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -196,14 +205,16 @@ http_requests_total{code="200"} 1021 1565133713.989 Description: "Unordered samples from multiple series, which end in different blocks.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565392913989, - NumBlocks: 2, + MinTime: 1565133713989, + MaxTime: 1565392913989, + NumBlocks: 2, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -230,14 +241,16 @@ http_requests_total{code="400"} 2 1565133715.989 Description: "Multiple samples with different timestamp for the same series.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565133715989, - NumBlocks: 1, + MinTime: 1565133713989, + MaxTime: 1565133715989, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -260,6 +273,132 @@ http_requests_total{code="400"} 2 1565133715.989 { ToParse: `# HELP http_requests_total The total number of HTTP requests. # TYPE http_requests_total counter +http_requests_total{code="200"} 1021 1624463088.000 +http_requests_total{code="200"} 1 1627055153.000 +http_requests_total{code="400"} 2 1627056153.000 +# EOF +`, + IsOk: true, + Description: "Long maximum block duration puts all data into one block.", + MaxSamplesInAppender: 5000, + MaxBlockDuration: 2048 * time.Hour, + Expected: struct { + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample + }{ + MinTime: 1624463088000, + MaxTime: 1627056153000, + NumBlocks: 1, + BlockDuration: int64(1458 * time.Hour / time.Millisecond), + Samples: []backfillSample{ + { + Timestamp: 1624463088000, + Value: 1021, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1627055153000, + Value: 1, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1627056153000, + Value: 2, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "400"), + }, + }, + }, + }, + { + ToParse: `# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{code="200"} 1 1624463088.000 +http_requests_total{code="200"} 2 1629503088.000 +http_requests_total{code="200"} 3 1629863088.000 +# EOF +`, + IsOk: true, + Description: "Long maximum block duration puts all data into two blocks.", + MaxSamplesInAppender: 5000, + MaxBlockDuration: 2048 * time.Hour, + Expected: struct { + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample + }{ + MinTime: 1624463088000, + MaxTime: 1629863088000, + NumBlocks: 2, + BlockDuration: int64(1458 * time.Hour / time.Millisecond), + Samples: []backfillSample{ + { + Timestamp: 1624463088000, + Value: 1, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1629503088000, + Value: 2, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1629863088000, + Value: 3, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + }, + }, + }, + { + ToParse: `# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter +http_requests_total{code="200"} 1 1624463088.000 +http_requests_total{code="200"} 2 1765943088.000 +http_requests_total{code="200"} 3 1768463088.000 +# EOF +`, + IsOk: true, + Description: "Maximum block duration longer than longest possible duration, uses largest duration, puts all data into two blocks.", + MaxSamplesInAppender: 5000, + MaxBlockDuration: 200000 * time.Hour, + Expected: struct { + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample + }{ + MinTime: 1624463088000, + MaxTime: 1768463088000, + NumBlocks: 2, + BlockDuration: int64(39366 * time.Hour / time.Millisecond), + Samples: []backfillSample{ + { + Timestamp: 1624463088000, + Value: 1, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1765943088000, + Value: 2, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + { + Timestamp: 1768463088000, + Value: 3, + Labels: labels.FromStrings("__name__", "http_requests_total", "code", "200"), + }, + }, + }, + }, + { + ToParse: `# HELP http_requests_total The total number of HTTP requests. +# TYPE http_requests_total counter http_requests_total{code="200"} 1021 1565133713.989 http_requests_total{code="200"} 1022 1565144513.989 http_requests_total{code="400"} 2 1565155313.989 @@ -270,14 +409,16 @@ http_requests_total{code="400"} 1 1565166113.989 Description: "Multiple samples that end up in different blocks.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565166113989, - NumBlocks: 4, + MinTime: 1565133713989, + MaxTime: 1565166113989, + NumBlocks: 4, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -318,14 +459,16 @@ http_requests_total{code="400"} 1 1565166113.989 Description: "Number of samples are greater than the sample batch size.", MaxSamplesInAppender: 2, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1565133713989, - MaxTime: 1565166113989, - NumBlocks: 4, + MinTime: 1565133713989, + MaxTime: 1565166113989, + NumBlocks: 4, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1565133713989, @@ -378,14 +521,16 @@ http_requests_total{code="400"} 1024 7199 Description: "One series spanning 2h in same block should not cause problems to other series.", MaxSamplesInAppender: 1, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 0, - MaxTime: 7199000, - NumBlocks: 1, + MinTime: 0, + MaxTime: 7199000, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 0, @@ -418,14 +563,16 @@ http_requests_total{code="400"} 1024 7199 Description: "Sample with no #HELP or #TYPE keyword.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 6900000, - MaxTime: 6900000, - NumBlocks: 1, + MinTime: 6900000, + MaxTime: 6900000, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 6900000, @@ -442,14 +589,16 @@ http_requests_total{code="400"} 1024 7199 Description: "Sample without newline after # EOF.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 6900000, - MaxTime: 6900000, - NumBlocks: 1, + MinTime: 6900000, + MaxTime: 6900000, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 6900000, @@ -467,14 +616,16 @@ http_requests_total{code="400"} 1024 7199 Description: "Bare sample.", MaxSamplesInAppender: 5000, Expected: struct { - MinTime int64 - MaxTime int64 - NumBlocks int - Samples []backfillSample + MinTime int64 + MaxTime int64 + NumBlocks int + BlockDuration int64 + Samples []backfillSample }{ - MinTime: 1001000, - MaxTime: 1001000, - NumBlocks: 1, + MinTime: 1001000, + MaxTime: 1001000, + NumBlocks: 1, + BlockDuration: tsdb.DefaultBlockDuration, Samples: []backfillSample{ { Timestamp: 1001000, @@ -532,28 +683,32 @@ after_eof 1 2 }, } for _, test := range tests { - t.Logf("Test:%s", test.Description) + t.Run(test.Description, func(t *testing.T) { + t.Logf("Test:%s", test.Description) - outputDir, err := ioutil.TempDir("", "myDir") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(outputDir)) - }() + outputDir, err := ioutil.TempDir("", "myDir") + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(outputDir)) + }() - err = backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false) + err = backfill(test.MaxSamplesInAppender, []byte(test.ToParse), outputDir, false, false, test.MaxBlockDuration) - if !test.IsOk { - require.Error(t, err, test.Description) - continue - } + if !test.IsOk { + require.Error(t, err, test.Description) + return + } - require.NoError(t, err) - db, err := tsdb.Open(outputDir, nil, nil, tsdb.DefaultOptions(), nil) - require.NoError(t, err) - defer func() { - require.NoError(t, db.Close()) - }() + require.NoError(t, err) + options := tsdb.DefaultOptions() + options.RetentionDuration = int64(10 * 365 * 24 * time.Hour / time.Millisecond) // maximum duration tests require a long retention + db, err := tsdb.Open(outputDir, nil, nil, options, nil) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() - testBlocks(t, db, test.Expected.MinTime, test.Expected.MaxTime, test.Expected.Samples, test.Expected.NumBlocks) + testBlocks(t, db, test.Expected.MinTime, test.Expected.MaxTime, test.Expected.BlockDuration, test.Expected.Samples, test.Expected.NumBlocks) + }) } } diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index c9e0aebe8..1267631aa 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "math" "net/http" "net/url" @@ -40,12 +41,15 @@ import ( "github.com/prometheus/common/version" "github.com/prometheus/exporter-toolkit/web" "gopkg.in/alecthomas/kingpin.v2" + yaml "gopkg.in/yaml.v2" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/file" _ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations. "github.com/prometheus/prometheus/discovery/kubernetes" + "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" ) func main() { @@ -128,7 +132,7 @@ func main() { benchWriteNumScrapes := tsdbBenchWriteCmd.Flag("scrapes", "Number of scrapes to simulate.").Default("3000").Int() benchSamplesFile := tsdbBenchWriteCmd.Arg("file", "Input file with samples data, default is ("+filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")+").").Default(filepath.Join("..", "..", "tsdb", "testdata", "20kseries.json")).String() - tsdbAnalyzeCmd := tsdbCmd.Command("analyze", "Analyze churn, label pair cardinality.") + tsdbAnalyzeCmd := tsdbCmd.Command("analyze", "Analyze churn, label pair cardinality and compaction efficiency.") analyzePath := tsdbAnalyzeCmd.Arg("db path", "Database path (default is "+defaultDBPath+").").Default(defaultDBPath).String() analyzeBlockID := tsdbAnalyzeCmd.Arg("block id", "Block to analyze (default is the last block).").String() analyzeLimit := tsdbAnalyzeCmd.Flag("limit", "How many items to show in each list.").Default("20").Int() @@ -145,8 +149,8 @@ func main() { importCmd := tsdbCmd.Command("create-blocks-from", "[Experimental] Import samples from input and produce TSDB blocks. Please refer to the storage docs for more details.") importHumanReadable := importCmd.Flag("human-readable", "Print human readable values.").Short('r').Bool() importQuiet := importCmd.Flag("quiet", "Do not print created blocks.").Short('q').Bool() + maxBlockDuration := importCmd.Flag("max-block-duration", "Maximum duration created blocks may span. Anything less than 2h is ignored.").Hidden().PlaceHolder("").Duration() openMetricsImportCmd := importCmd.Command("openmetrics", "Import samples from OpenMetrics input and produce TSDB blocks. Please refer to the storage docs for more details.") - // TODO(aSquare14): add flag to set default block duration importFilePath := openMetricsImportCmd.Arg("input file", "OpenMetrics file to read samples from.").Required().String() importDBPath := openMetricsImportCmd.Arg("output directory", "Output directory for generated blocks.").Default(defaultDBPath).String() importRulesCmd := importCmd.Command("rules", "Create blocks of data for new recording rules.") @@ -162,6 +166,8 @@ func main() { "A list of one or more files containing recording rules to be backfilled. All recording rules listed in the files will be backfilled. Alerting rules are not evaluated.", ).Required().ExistingFiles() + featureList := app.Flag("enable-feature", "Comma separated feature names to enable (only PromQL related). See https://prometheus.io/docs/prometheus/latest/feature_flags/ for the options and more details.").Default("").Strings() + parsedCmd := kingpin.MustParse(app.Parse(os.Args[1:])) var p printer @@ -172,6 +178,23 @@ func main() { p = &promqlPrinter{} } + var queryOpts promql.LazyLoaderOpts + for _, f := range *featureList { + opts := strings.Split(f, ",") + for _, o := range opts { + switch o { + case "promql-at-modifier": + queryOpts.EnableAtModifier = true + case "promql-negative-offset": + queryOpts.EnableNegativeOffset = true + case "": + continue + default: + fmt.Printf(" WARNING: Unknown option for --enable-feature: %q\n", o) + } + } + } + switch parsedCmd { case checkConfigCmd.FullCommand(): os.Exit(CheckConfig(*configFiles...)) @@ -207,7 +230,7 @@ func main() { os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p)) case testRulesCmd.FullCommand(): - os.Exit(RulesUnitTest(*testRulesFiles...)) + os.Exit(RulesUnitTest(queryOpts, *testRulesFiles...)) case tsdbBenchWriteCmd.FullCommand(): os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics, *benchWriteNumScrapes))) @@ -222,7 +245,7 @@ func main() { os.Exit(checkErr(dumpSamples(*dumpPath, *dumpMinTime, *dumpMaxTime))) //TODO(aSquare14): Work on adding support for custom block size. case openMetricsImportCmd.FullCommand(): - os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet)) + os.Exit(backfillOpenMetrics(*importFilePath, *importDBPath, *importHumanReadable, *importQuiet, *maxBlockDuration)) case importRulesCmd.FullCommand(): os.Exit(checkErr(importRules(*importRulesURL, *importRulesStart, *importRulesEnd, *importRulesOutputDir, *importRulesEvalInterval, *importRulesFiles...))) @@ -337,8 +360,12 @@ func checkConfig(filename string) ([]string, error) { return nil, err } if len(files) != 0 { - // There was at least one match for the glob and we can assume checkFileExists - // for all matches would pass, we can continue the loop. + for _, f := range files { + err = checkSDFile(f) + if err != nil { + return nil, errors.Errorf("checking SD file %q: %v", file, err) + } + } continue } fmt.Printf(" WARNING: file %q for file_sd in scrape job %q does not exist\n", file, scfg.JobName) @@ -368,6 +395,42 @@ func checkTLSConfig(tlsConfig config_util.TLSConfig) error { return nil } +func checkSDFile(filename string) error { + fd, err := os.Open(filename) + if err != nil { + return err + } + defer fd.Close() + + content, err := ioutil.ReadAll(fd) + if err != nil { + return err + } + + var targetGroups []*targetgroup.Group + + switch ext := filepath.Ext(filename); strings.ToLower(ext) { + case ".json": + if err := json.Unmarshal(content, &targetGroups); err != nil { + return err + } + case ".yml", ".yaml": + if err := yaml.UnmarshalStrict(content, &targetGroups); err != nil { + return err + } + default: + return errors.Errorf("invalid file extension: %q", ext) + } + + for i, tg := range targetGroups { + if tg == nil { + return errors.Errorf("nil target group item found (index %d)", i) + } + } + + return nil +} + // CheckRules validates rule files. func CheckRules(files ...string) int { failed := false diff --git a/cmd/promtool/main_test.go b/cmd/promtool/main_test.go index 112c387c9..4949c467c 100644 --- a/cmd/promtool/main_test.go +++ b/cmd/promtool/main_test.go @@ -77,3 +77,44 @@ func mockServer(code int, body string) (*httptest.Server, func() *http.Request) } return server, f } + +func TestCheckSDFile(t *testing.T) { + cases := []struct { + name string + file string + err string + }{ + { + name: "good .yml", + file: "./testdata/good-sd-file.yml", + }, + { + name: "good .yaml", + file: "./testdata/good-sd-file.yaml", + }, + { + name: "good .json", + file: "./testdata/good-sd-file.json", + }, + { + name: "bad file extension", + file: "./testdata/bad-sd-file-extension.nonexistant", + err: "invalid file extension: \".nonexistant\"", + }, + { + name: "bad format", + file: "./testdata/bad-sd-file-format.yml", + err: "yaml: unmarshal errors:\n line 1: field targats not found in type struct { Targets []string \"yaml:\\\"targets\\\"\"; Labels model.LabelSet \"yaml:\\\"labels\\\"\" }", + }, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + err := checkSDFile(test.file) + if test.err != "" { + require.Equalf(t, test.err, err.Error(), "Expected error %q, got %q", test.err, err.Error()) + return + } + require.NoError(t, err) + }) + } +} diff --git a/cmd/promtool/rules_test.go b/cmd/promtool/rules_test.go index ac326841f..4afae2c9f 100644 --- a/cmd/promtool/rules_test.go +++ b/cmd/promtool/rules_test.go @@ -91,7 +91,7 @@ func TestBackfillRuleIntegration(t *testing.T) { group1 := ruleImporter.groups[path1+";group0"] require.NotNil(t, group1) const defaultInterval = 60 - require.Equal(t, time.Duration(defaultInterval*time.Second), group1.Interval()) + require.Equal(t, defaultInterval*time.Second, group1.Interval()) gRules := group1.Rules() require.Equal(t, 1, len(gRules)) require.Equal(t, "rule1", gRules[0].Name()) @@ -100,7 +100,7 @@ func TestBackfillRuleIntegration(t *testing.T) { group2 := ruleImporter.groups[path2+";group2"] require.NotNil(t, group2) - require.Equal(t, time.Duration(defaultInterval*time.Second), group2.Interval()) + require.Equal(t, defaultInterval*time.Second, group2.Interval()) g2Rules := group2.Rules() require.Equal(t, 2, len(g2Rules)) require.Equal(t, "grp2_rule1", g2Rules[0].Name()) diff --git a/cmd/promtool/testdata/at-modifier-test.yml b/cmd/promtool/testdata/at-modifier-test.yml new file mode 100644 index 000000000..e493bf2bd --- /dev/null +++ b/cmd/promtool/testdata/at-modifier-test.yml @@ -0,0 +1,7 @@ +rule_files: + - at-modifier.yml + +tests: + - input_series: + - series: "requests{}" + values: 1 diff --git a/cmd/promtool/testdata/at-modifier.yml b/cmd/promtool/testdata/at-modifier.yml new file mode 100644 index 000000000..900469b4c --- /dev/null +++ b/cmd/promtool/testdata/at-modifier.yml @@ -0,0 +1,7 @@ +# This is the rules file for at-modifier-test.yml. + +groups: + - name: at-modifier + rules: + - record: x + expr: "requests @ 1000" diff --git a/cmd/promtool/testdata/bad-sd-file-extension.nonexistant b/cmd/promtool/testdata/bad-sd-file-extension.nonexistant new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/promtool/testdata/bad-sd-file-format.yml b/cmd/promtool/testdata/bad-sd-file-format.yml new file mode 100644 index 000000000..e79c5f3dc --- /dev/null +++ b/cmd/promtool/testdata/bad-sd-file-format.yml @@ -0,0 +1,2 @@ +- targats: + - localhost:9100 diff --git a/cmd/promtool/testdata/good-sd-file.json b/cmd/promtool/testdata/good-sd-file.json new file mode 100644 index 000000000..fbdd81432 --- /dev/null +++ b/cmd/promtool/testdata/good-sd-file.json @@ -0,0 +1,8 @@ +[ + { + "labels": { + "job": "node" + }, + "targets": ["localhost:9100"] + } +] diff --git a/cmd/promtool/testdata/good-sd-file.yaml b/cmd/promtool/testdata/good-sd-file.yaml new file mode 100644 index 000000000..f6f455f63 --- /dev/null +++ b/cmd/promtool/testdata/good-sd-file.yaml @@ -0,0 +1,4 @@ +- labels: + job: node +- targets: + - localhost:9100 diff --git a/cmd/promtool/testdata/good-sd-file.yml b/cmd/promtool/testdata/good-sd-file.yml new file mode 100644 index 000000000..f6f455f63 --- /dev/null +++ b/cmd/promtool/testdata/good-sd-file.yml @@ -0,0 +1,4 @@ +- labels: + job: node +- targets: + - localhost:9100 diff --git a/cmd/promtool/testdata/negative-offset-test.yml b/cmd/promtool/testdata/negative-offset-test.yml new file mode 100644 index 000000000..b5ff9c040 --- /dev/null +++ b/cmd/promtool/testdata/negative-offset-test.yml @@ -0,0 +1,7 @@ +rule_files: + - negative-offset.yml + +tests: + - input_series: + - series: "requests{}" + values: 1 diff --git a/cmd/promtool/testdata/negative-offset.yml b/cmd/promtool/testdata/negative-offset.yml new file mode 100644 index 000000000..6cf27e109 --- /dev/null +++ b/cmd/promtool/testdata/negative-offset.yml @@ -0,0 +1,7 @@ +# This is the rules file for negative-offset-test.yml. + +groups: + - name: negative-offset + rules: + - record: x + expr: "requests offset -5m" diff --git a/cmd/promtool/tsdb.go b/cmd/promtool/tsdb.go index bcf9f47e8..782ea7d54 100644 --- a/cmd/promtool/tsdb.go +++ b/cmd/promtool/tsdb.go @@ -17,8 +17,10 @@ import ( "bufio" "context" "fmt" + "github.com/prometheus/prometheus/tsdb/index" "io" "io/ioutil" + "math" "os" "path/filepath" "runtime" @@ -561,6 +563,60 @@ func analyzeBlock(path, blockID string, limit int) error { } fmt.Printf("\nHighest cardinality metric names:\n") printInfo(postingInfos) + + return analyzeCompaction(block, ir) +} + +func analyzeCompaction(block tsdb.BlockReader, indexr tsdb.IndexReader) (err error) { + postingsr, err := indexr.Postings(index.AllPostingsKey()) + if err != nil { + return err + } + chunkr, err := block.Chunks() + if err != nil { + return err + } + defer func() { + err = tsdb_errors.NewMulti(err, chunkr.Close()).Err() + }() + + const maxSamplesPerChunk = 120 + nBuckets := 10 + histogram := make([]int, nBuckets) + totalChunks := 0 + for postingsr.Next() { + var lbsl = labels.Labels{} + var chks []chunks.Meta + if err := indexr.Series(postingsr.At(), &lbsl, &chks); err != nil { + return err + } + + for _, chk := range chks { + // Load the actual data of the chunk. + chk, err := chunkr.Chunk(chk.Ref) + if err != nil { + return err + } + chunkSize := math.Min(float64(chk.NumSamples()), maxSamplesPerChunk) + // Calculate the bucket for the chunk and increment it in the histogram. + bucket := int(math.Ceil(float64(nBuckets)*chunkSize/maxSamplesPerChunk)) - 1 + histogram[bucket]++ + totalChunks++ + } + } + + fmt.Printf("\nCompaction analysis:\n") + fmt.Println("Fullness: Amount of samples in chunks (100% is 120 samples)") + // Normalize absolute counts to percentages and print them out. + for bucket, count := range histogram { + percentage := 100.0 * count / totalChunks + fmt.Printf("%7d%%: ", (bucket+1)*10) + for j := 0; j < percentage; j++ { + fmt.Printf("#") + } + fmt.Println() + } + return nil } @@ -611,7 +667,7 @@ func checkErr(err error) int { return 0 } -func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool) int { +func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet bool, maxBlockDuration time.Duration) int { inputFile, err := fileutil.OpenMmapFile(path) if err != nil { return checkErr(err) @@ -622,5 +678,5 @@ func backfillOpenMetrics(path string, outputDir string, humanReadable, quiet boo return checkErr(errors.Wrap(err, "create output dir")) } - return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet)) + return checkErr(backfill(5000, inputFile.Bytes(), outputDir, humanReadable, quiet, maxBlockDuration)) } diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 44cffd301..3ddec3152 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -39,11 +39,11 @@ import ( // RulesUnitTest does unit testing of rules based on the unit testing files provided. // More info about the file format can be found in the docs. -func RulesUnitTest(files ...string) int { +func RulesUnitTest(queryOpts promql.LazyLoaderOpts, files ...string) int { failed := false for _, f := range files { - if errs := ruleUnitTest(f); errs != nil { + if errs := ruleUnitTest(f, queryOpts); errs != nil { fmt.Fprintln(os.Stderr, " FAILED:") for _, e := range errs { fmt.Fprintln(os.Stderr, e.Error()) @@ -60,7 +60,7 @@ func RulesUnitTest(files ...string) int { return 0 } -func ruleUnitTest(filename string) []error { +func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts) []error { fmt.Println("Unit Testing: ", filename) b, err := ioutil.ReadFile(filename) @@ -95,7 +95,7 @@ func ruleUnitTest(filename string) []error { // Testing. var errs []error for _, t := range unitTestInp.Tests { - ers := t.test(evalInterval, groupOrderMap, unitTestInp.RuleFiles...) + ers := t.test(evalInterval, groupOrderMap, queryOpts, unitTestInp.RuleFiles...) if ers != nil { errs = append(errs, ers...) } @@ -151,9 +151,9 @@ type testGroup struct { } // test performs the unit tests. -func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, ruleFiles ...string) []error { +func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, ruleFiles ...string) []error { // Setup testing suite. - suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString()) + suite, err := promql.NewLazyLoader(nil, tg.seriesLoadingString(), queryOpts) if err != nil { return []error{err} } @@ -255,7 +255,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i for { if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) && - time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) { + time.Duration(alertEvalTimes[curr]) < ts.Add(evalInterval).Sub(mint)) { break } diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go index a4ed4f3ee..0f528471a 100644 --- a/cmd/promtool/unittest_test.go +++ b/cmd/promtool/unittest_test.go @@ -13,16 +13,21 @@ package main -import "testing" +import ( + "testing" + + "github.com/prometheus/prometheus/promql" +) func TestRulesUnitTest(t *testing.T) { type args struct { files []string } tests := []struct { - name string - args args - want int + name string + args args + queryOpts promql.LazyLoaderOpts + want int }{ { name: "Passing Unit Tests", @@ -66,10 +71,44 @@ func TestRulesUnitTest(t *testing.T) { }, want: 1, }, + { + name: "Disabled feature (@ modifier)", + args: args{ + files: []string{"./testdata/at-modifier-test.yml"}, + }, + want: 1, + }, + { + name: "Enabled feature (@ modifier)", + args: args{ + files: []string{"./testdata/at-modifier-test.yml"}, + }, + queryOpts: promql.LazyLoaderOpts{ + EnableAtModifier: true, + }, + want: 0, + }, + { + name: "Disabled feature (negative offset)", + args: args{ + files: []string{"./testdata/negative-offset-test.yml"}, + }, + want: 1, + }, + { + name: "Enabled feature (negative offset)", + args: args{ + files: []string{"./testdata/negative-offset-test.yml"}, + }, + queryOpts: promql.LazyLoaderOpts{ + EnableNegativeOffset: true, + }, + want: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := RulesUnitTest(tt.args.files...); got != tt.want { + if got := RulesUnitTest(tt.queryOpts, tt.args.files...); got != tt.want { t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want) } }) diff --git a/config/config.go b/config/config.go index af716d1f0..b560faace 100644 --- a/config/config.go +++ b/config/config.go @@ -183,6 +183,15 @@ var ( RemoteTimeout: model.Duration(1 * time.Minute), HTTPClientConfig: config.DefaultHTTPClientConfig, } + + // DefaultStorageConfig is the default TSDB/Exemplar storage configuration. + DefaultStorageConfig = StorageConfig{ + ExemplarsConfig: &DefaultExemplarsConfig, + } + + DefaultExemplarsConfig = ExemplarsConfig{ + MaxExemplars: 100000, + } ) // Config is the top-level configuration for Prometheus's config files. @@ -191,6 +200,7 @@ type Config struct { AlertingConfig AlertingConfig `yaml:"alerting,omitempty"` RuleFiles []string `yaml:"rule_files,omitempty"` ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` + StorageConfig StorageConfig `yaml:"storage,omitempty"` RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"` RemoteReadConfigs []*RemoteReadConfig `yaml:"remote_read,omitempty"` @@ -464,6 +474,18 @@ func (c *ScrapeConfig) MarshalYAML() (interface{}, error) { return discovery.MarshalYAMLWithInlineConfigs(c) } +// StorageConfig configures runtime reloadable configuration options. +type StorageConfig struct { + ExemplarsConfig *ExemplarsConfig `yaml:"exemplars,omitempty"` +} + +// ExemplarsConfig configures runtime reloadable configuration options. +type ExemplarsConfig struct { + // MaxExemplars sets the size, in # of exemplars stored, of the single circular buffer used to store exemplars in memory. + // Use a value of 0 or less than 0 to disable the storage without having to restart Prometheus. + MaxExemplars int64 `yaml:"max_exemplars,omitempty"` +} + // AlertingConfig configures alerting and alertmanager related configs. type AlertingConfig struct { AlertRelabelConfigs []*relabel.Config `yaml:"alert_relabel_configs,omitempty"` diff --git a/config/config_test.go b/config/config_test.go index afdfcbc52..d5cc092a2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -48,6 +48,7 @@ import ( "github.com/prometheus/prometheus/discovery/scaleway" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/discovery/triton" + "github.com/prometheus/prometheus/discovery/xds" "github.com/prometheus/prometheus/discovery/zookeeper" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/relabel" @@ -439,6 +440,26 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "service-kuma", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + + ServiceDiscoveryConfigs: discovery.Configs{ + &xds.KumaSDConfig{ + Server: "http://kuma-control-plane.kuma-system.svc:5676", + HTTPClientConfig: config.DefaultHTTPClientConfig, + RefreshInterval: model.Duration(15 * time.Second), + FetchTimeout: model.Duration(2 * time.Minute), + }, + }, + }, { JobName: "service-marathon", @@ -714,11 +735,12 @@ var expectedConf = &Config{ ServiceDiscoveryConfigs: discovery.Configs{ &moby.DockerSDConfig{ - Filters: []moby.Filter{}, - Host: "unix:///var/run/docker.sock", - Port: 80, - RefreshInterval: model.Duration(60 * time.Second), - HTTPClientConfig: config.DefaultHTTPClientConfig, + Filters: []moby.Filter{}, + Host: "unix:///var/run/docker.sock", + Port: 80, + HostNetworkingHost: "localhost", + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, }, }, }, diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index a72031de3..19fe0c9b2 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -109,7 +109,6 @@ scrape_configs: - second.dns.address.domain.com - names: - first.dns.address.domain.com - # refresh_interval defaults to 30s. relabel_configs: - source_labels: [job] @@ -193,6 +192,11 @@ scrape_configs: username: "myusername" password_file: valid_password_file + - job_name: service-kuma + + kuma_sd_configs: + - server: http://kuma-control-plane.kuma-system.svc:5676 + - job_name: service-marathon marathon_sd_configs: - servers: diff --git a/config/testdata/roundtrip.good.yml b/config/testdata/roundtrip.good.yml index 51b59dd81..f2634d257 100644 --- a/config/testdata/roundtrip.good.yml +++ b/config/testdata/roundtrip.good.yml @@ -106,6 +106,9 @@ scrape_configs: username: username password_file: valid_password_file + kuma_sd_configs: + - server: http://kuma-control-plane.kuma-system.svc:5676 + marathon_sd_configs: - servers: - https://marathon.example.com:443 diff --git a/consoles/node-disk.html b/consoles/node-disk.html index 6d0dcc05c..ffff41b79 100644 --- a/consoles/node-disk.html +++ b/consoles/node-disk.html @@ -1,5 +1,44 @@ {{ template "head" . }} +{{ template "prom_right_table_head" }} + + Disks + +{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} + {{ .Labels.device }} + + Utilization + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} + + + Throughput + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} + + + Avg Read Time + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} + + + Avg Write Time + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} + +{{ end }} + + Filesystem Fullness + +{{ define "roughlyNearZero" }} +{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} +{{ end }} +{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} + + {{ .Labels.mountpoint }} + {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} + +{{ end }} + + +{{ template "prom_right_table_tail" }} + {{ template "prom_content_head" . }}

Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

@@ -34,44 +73,6 @@ yTitle: 'Filesystem Fullness' }) - -{{ template "prom_right_table_head" }} - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - {{ .Labels.device }} - - Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - - - Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - - - Avg Read Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - - - Avg Write Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - -{{ end }} - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - - -{{ template "prom_right_table_tail" }} - {{ template "prom_content_tail" . }} {{ template "tail" }} diff --git a/consoles/node-overview.html b/consoles/node-overview.html index 092028d33..92f53ba87 100644 --- a/consoles/node-overview.html +++ b/consoles/node-overview.html @@ -1,5 +1,66 @@ {{ template "head" . }} +{{ template "prom_right_table_head" }} + Overview + + User CPU + {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} + + + System CPU + {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} + + + Memory Total + {{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} + + + Memory Free + {{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} + + + Network + +{{ range printf "node_network_receive_bytes_total{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} + + {{ .Labels.device }} Received + {{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} + + + {{ .Labels.device }} Transmitted + {{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} + +{{ end }} + + Disks + +{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} + + {{ .Labels.device }} Utilization + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} + +{{ end }} +{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} + + {{ .Labels.device }} Throughput + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} + +{{ end }} + + Filesystem Fullness + +{{ define "roughlyNearZero" }} +{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} +{{ end }} +{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} + + {{ .Labels.mountpoint }} + {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} + +{{ end }} + +{{ template "prom_right_table_tail" }} + {{ template "prom_content_head" . }}

Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

@@ -55,68 +116,6 @@ }) -{{ template "prom_right_table_head" }} - Overview - - User CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - System CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - Memory Total - {{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Memory Free - {{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Network - -{{ range printf "node_network_receive_bytes_total{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Received - {{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - - - {{ .Labels.device }} Transmitted - {{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - -{{ end }} -{{ range printf "node_disk_io_time_seconds_total{job='node',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - -{{ template "prom_right_table_tail" }} - {{ template "prom_content_tail" . }} {{ template "tail" }} diff --git a/consoles/node.html b/consoles/node.html index bf2b7672d..9a37544ee 100644 --- a/consoles/node.html +++ b/consoles/node.html @@ -27,6 +27,7 @@ {{ else }} No nodes found. {{ end }} + {{ template "prom_content_tail" . }} diff --git a/discovery/aws/ec2.go b/discovery/aws/ec2.go index c306c920b..d2ad9736a 100644 --- a/discovery/aws/ec2.go +++ b/discovery/aws/ec2.go @@ -28,6 +28,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -42,6 +43,7 @@ const ( ec2Label = model.MetaLabelPrefix + "ec2_" ec2LabelAMI = ec2Label + "ami" ec2LabelAZ = ec2Label + "availability_zone" + ec2LabelAZID = ec2Label + "availability_zone_id" ec2LabelArch = ec2Label + "architecture" ec2LabelIPv6Addresses = ec2Label + "ipv6_addresses" ec2LabelInstanceID = ec2Label + "instance_id" @@ -132,8 +134,14 @@ func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // the Discoverer interface. type EC2Discovery struct { *refresh.Discovery - cfg *EC2SDConfig - ec2 *ec2.EC2 + logger log.Logger + cfg *EC2SDConfig + ec2 *ec2.EC2 + + // azToAZID maps this account's availability zones to their underlying AZ + // ID, e.g. eu-west-2a -> euw2-az2. Refreshes are performed sequentially, so + // no locking is required. + azToAZID map[string]string } // NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. @@ -142,7 +150,8 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery { logger = log.NewNopLogger() } d := &EC2Discovery{ - cfg: conf, + logger: logger, + cfg: conf, } d.Discovery = refresh.NewDiscovery( logger, @@ -153,7 +162,7 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery { return d } -func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) { +func (d *EC2Discovery) ec2Client(ctx context.Context) (*ec2.EC2, error) { if d.ec2 != nil { return d.ec2, nil } @@ -185,8 +194,28 @@ func (d *EC2Discovery) ec2Client() (*ec2.EC2, error) { return d.ec2, nil } +func (d *EC2Discovery) azID(ctx context.Context, az string) (string, error) { + if azID, ok := d.azToAZID[az]; ok { + return azID, nil + } + + azs, err := d.ec2.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{}) + if err != nil { + return "", errors.Wrap(err, "could not describe availability zones") + } + d.azToAZID = make(map[string]string, len(azs.AvailabilityZones)) + for _, az := range azs.AvailabilityZones { + d.azToAZID[*az.ZoneName] = *az.ZoneId + } + + if azID, ok := d.azToAZID[az]; ok { + return azID, nil + } + return "", fmt.Errorf("no availability zone ID mapping found for %s", az) +} + func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - ec2Client, err := d.ec2Client() + ec2Client, err := d.ec2Client(ctx) if err != nil { return nil, err } @@ -211,6 +240,14 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error if inst.PrivateIpAddress == nil { continue } + azID, err := d.azID(ctx, *inst.Placement.AvailabilityZone) + if err != nil { + level.Warn(d.logger).Log( + "msg", "Unable to determine availability zone ID", + "az", *inst.Placement.AvailabilityZone, + "err", err) + } + labels := model.LabelSet{ ec2LabelInstanceID: model.LabelValue(*inst.InstanceId), } @@ -237,6 +274,7 @@ func (d *EC2Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId) labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone) + labels[ec2LabelAZID] = model.LabelValue(azID) labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name) labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType) diff --git a/discovery/azure/azure.go b/discovery/azure/azure.go index a23e80d8c..6de0995ef 100644 --- a/discovery/azure/azure.go +++ b/discovery/azure/azure.go @@ -46,6 +46,7 @@ const ( azureLabelMachineID = azureLabel + "machine_id" azureLabelMachineResourceGroup = azureLabel + "machine_resource_group" azureLabelMachineName = azureLabel + "machine_name" + azureLabelMachineComputerName = azureLabel + "machine_computer_name" azureLabelMachineOSType = azureLabel + "machine_os_type" azureLabelMachineLocation = azureLabel + "machine_location" azureLabelMachinePrivateIP = azureLabel + "machine_private_ip" @@ -226,6 +227,7 @@ type azureResource struct { type virtualMachine struct { ID string Name string + ComputerName string Type string Location string OsType string @@ -306,6 +308,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { azureLabelTenantID: model.LabelValue(d.cfg.TenantID), azureLabelMachineID: model.LabelValue(vm.ID), azureLabelMachineName: model.LabelValue(vm.Name), + azureLabelMachineComputerName: model.LabelValue(vm.ComputerName), azureLabelMachineOSType: model.LabelValue(vm.OsType), azureLabelMachineLocation: model.LabelValue(vm.Location), azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroup), @@ -449,6 +452,7 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine { osType := string(vm.StorageProfile.OsDisk.OsType) tags := map[string]*string{} networkInterfaces := []string{} + var computerName string if vm.Tags != nil { tags = vm.Tags @@ -460,9 +464,14 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine { } } + if vm.VirtualMachineProperties != nil && vm.VirtualMachineProperties.OsProfile != nil { + computerName = *(vm.VirtualMachineProperties.OsProfile.ComputerName) + } + return virtualMachine{ ID: *(vm.ID), Name: *(vm.Name), + ComputerName: computerName, Type: *(vm.Type), Location: *(vm.Location), OsType: osType, @@ -476,6 +485,7 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin osType := string(vm.StorageProfile.OsDisk.OsType) tags := map[string]*string{} networkInterfaces := []string{} + var computerName string if vm.Tags != nil { tags = vm.Tags @@ -487,9 +497,14 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin } } + if vm.VirtualMachineScaleSetVMProperties != nil && vm.VirtualMachineScaleSetVMProperties.OsProfile != nil { + computerName = *(vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName) + } + return virtualMachine{ ID: *(vm.ID), Name: *(vm.Name), + ComputerName: computerName, Type: *(vm.Type), Location: *(vm.Location), OsType: osType, diff --git a/discovery/azure/azure_test.go b/discovery/azure/azure_test.go index b6addbc8a..744d182de 100644 --- a/discovery/azure/azure_test.go +++ b/discovery/azure/azure_test.go @@ -30,10 +30,14 @@ func TestMapFromVMWithEmptyTags(t *testing.T) { name := "name" vmType := "type" location := "westeurope" + computerName := "computer_name" networkProfile := compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, } properties := &compute.VirtualMachineProperties{ + OsProfile: &compute.OSProfile{ + ComputerName: &computerName, + }, StorageProfile: &compute.StorageProfile{ OsDisk: &compute.OSDisk{ OsType: "Linux", @@ -54,6 +58,7 @@ func TestMapFromVMWithEmptyTags(t *testing.T) { expectedVM := virtualMachine{ ID: id, Name: name, + ComputerName: computerName, Type: vmType, Location: location, OsType: "Linux", @@ -71,6 +76,7 @@ func TestMapFromVMWithTags(t *testing.T) { name := "name" vmType := "type" location := "westeurope" + computerName := "computer_name" tags := map[string]*string{ "prometheus": new(string), } @@ -78,6 +84,9 @@ func TestMapFromVMWithTags(t *testing.T) { NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, } properties := &compute.VirtualMachineProperties{ + OsProfile: &compute.OSProfile{ + ComputerName: &computerName, + }, StorageProfile: &compute.StorageProfile{ OsDisk: &compute.OSDisk{ OsType: "Linux", @@ -98,6 +107,7 @@ func TestMapFromVMWithTags(t *testing.T) { expectedVM := virtualMachine{ ID: id, Name: name, + ComputerName: computerName, Type: vmType, Location: location, OsType: "Linux", @@ -115,10 +125,14 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { name := "name" vmType := "type" location := "westeurope" + computerName := "computer_name" networkProfile := compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, } properties := &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ + ComputerName: &computerName, + }, StorageProfile: &compute.StorageProfile{ OsDisk: &compute.OSDisk{ OsType: "Linux", @@ -140,6 +154,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) { expectedVM := virtualMachine{ ID: id, Name: name, + ComputerName: computerName, Type: vmType, Location: location, OsType: "Linux", @@ -158,6 +173,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) { name := "name" vmType := "type" location := "westeurope" + computerName := "computer_name" tags := map[string]*string{ "prometheus": new(string), } @@ -165,6 +181,9 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) { NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, } properties := &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ + ComputerName: &computerName, + }, StorageProfile: &compute.StorageProfile{ OsDisk: &compute.OSDisk{ OsType: "Linux", @@ -186,6 +205,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) { expectedVM := virtualMachine{ ID: id, Name: name, + ComputerName: computerName, Type: vmType, Location: location, OsType: "Linux", diff --git a/discovery/consul/consul.go b/discovery/consul/consul.go index 3416d86d6..19dc9ecca 100644 --- a/discovery/consul/consul.go +++ b/discovery/consul/consul.go @@ -199,7 +199,7 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { logger = log.NewNopLogger() } - wrapper, err := config.NewClientFromConfig(conf.HTTPClientConfig, "consul_sd", config.WithHTTP2Disabled(), config.WithIdleConnTimeout(2*time.Duration(watchTimeout))) + wrapper, err := config.NewClientFromConfig(conf.HTTPClientConfig, "consul_sd", config.WithHTTP2Disabled(), config.WithIdleConnTimeout(2*watchTimeout)) if err != nil { return nil, err } diff --git a/discovery/gce/gce.go b/discovery/gce/gce.go index 53ee9943d..e17c60f71 100644 --- a/discovery/gce/gce.go +++ b/discovery/gce/gce.go @@ -178,6 +178,12 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { addr := fmt.Sprintf("%s:%d", priIface.NetworkIP, d.port) labels[model.AddressLabel] = model.LabelValue(addr) + // Append named interface metadata for all interfaces + for _, iface := range inst.NetworkInterfaces { + gceLabelNetAddress := model.LabelName(fmt.Sprintf("%sinterface_ipv4_%s", gceLabel, strutil.SanitizeLabelName(iface.Name))) + labels[gceLabelNetAddress] = model.LabelValue(iface.NetworkIP) + } + // Tags in GCE are usually only used for networking rules. if inst.Tags != nil && len(inst.Tags.Items) > 0 { // We surround the separated list with the separator as well. This way regular expressions diff --git a/discovery/hetzner/hcloud.go b/discovery/hetzner/hcloud.go index 05471e453..494321fd5 100644 --- a/discovery/hetzner/hcloud.go +++ b/discovery/hetzner/hcloud.go @@ -47,6 +47,7 @@ const ( hetznerLabelHcloudDiskGB = hetznerHcloudLabelPrefix + "disk_size_gb" hetznerLabelHcloudType = hetznerHcloudLabelPrefix + "server_type" hetznerLabelHcloudLabel = hetznerHcloudLabelPrefix + "label_" + hetznerLabelHcloudLabelPresent = hetznerHcloudLabelPrefix + "labelpresent_" ) // Discovery periodically performs Hetzner Cloud requests. It implements @@ -124,6 +125,9 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er } } for labelKey, labelValue := range server.Labels { + presentLabel := model.LabelName(hetznerLabelHcloudLabelPresent + strutil.SanitizeLabelName(labelKey)) + labels[presentLabel] = model.LabelValue("true") + label := model.LabelName(hetznerLabelHcloudLabel + strutil.SanitizeLabelName(labelKey)) labels[label] = model.LabelValue(labelValue) } diff --git a/discovery/hetzner/hcloud_test.go b/discovery/hetzner/hcloud_test.go index b6b8300e8..a4f19cfdd 100644 --- a/discovery/hetzner/hcloud_test.go +++ b/discovery/hetzner/hcloud_test.go @@ -77,6 +77,7 @@ func TestHCloudSDRefresh(t *testing.T) { "__meta_hetzner_hcloud_disk_size_gb": model.LabelValue("25"), "__meta_hetzner_hcloud_server_type": model.LabelValue("cx11"), "__meta_hetzner_hcloud_private_ipv4_mynet": model.LabelValue("10.0.0.2"), + "__meta_hetzner_hcloud_labelpresent_my_key": model.LabelValue("true"), "__meta_hetzner_hcloud_label_my_key": model.LabelValue("my-value"), }, { @@ -99,6 +100,10 @@ func TestHCloudSDRefresh(t *testing.T) { "__meta_hetzner_hcloud_memory_size_gb": model.LabelValue("1"), "__meta_hetzner_hcloud_disk_size_gb": model.LabelValue("50"), "__meta_hetzner_hcloud_server_type": model.LabelValue("cpx11"), + "__meta_hetzner_hcloud_labelpresent_key": model.LabelValue("true"), + "__meta_hetzner_hcloud_labelpresent_other_key": model.LabelValue("true"), + "__meta_hetzner_hcloud_label_key": model.LabelValue(""), + "__meta_hetzner_hcloud_label_other_key": model.LabelValue("value"), }, { "__address__": model.LabelValue("1.2.3.6:80"), diff --git a/discovery/hetzner/mock_test.go b/discovery/hetzner/mock_test.go index 6c374b4bc..5936d5257 100644 --- a/discovery/hetzner/mock_test.go +++ b/discovery/hetzner/mock_test.go @@ -310,7 +310,10 @@ func (m *SDMock) HandleHcloudServers() { "delete": false, "rebuild": false }, - "labels": {}, + "labels": { + "key": "", + "other-key": "value" + }, "volumes": [], "load_balancers": [] }, diff --git a/discovery/hetzner/robot.go b/discovery/hetzner/robot.go index e6f103286..f7079d909 100644 --- a/discovery/hetzner/robot.go +++ b/discovery/hetzner/robot.go @@ -92,8 +92,13 @@ func (d *robotDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, err return nil, errors.Errorf("non 2xx status '%d' response during hetzner service discovery with role robot", resp.StatusCode) } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var servers serversList - err = json.NewDecoder(resp.Body).Decode(&servers) + err = json.Unmarshal(b, &servers) if err != nil { return nil, err } diff --git a/discovery/http/http.go b/discovery/http/http.go index 9c6db98a4..af2e99939 100644 --- a/discovery/http/http.go +++ b/discovery/http/http.go @@ -21,7 +21,9 @@ import ( "io/ioutil" "net/http" "net/url" + "regexp" "strconv" + "strings" "time" "github.com/go-kit/log" @@ -41,7 +43,8 @@ var ( RefreshInterval: model.Duration(60 * time.Second), HTTPClientConfig: config.DefaultHTTPClientConfig, } - userAgent = fmt.Sprintf("Prometheus/%s", version.Version) + userAgent = fmt.Sprintf("Prometheus/%s", version.Version) + matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`) ) func init() { @@ -101,6 +104,7 @@ type Discovery struct { url string client *http.Client refreshInterval time.Duration + tgLastLength int } // NewDiscovery returns a new HTTP discovery for the given config. @@ -152,7 +156,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { return nil, errors.Errorf("server returned HTTP status %s", resp.Status) } - if resp.Header.Get("Content-Type") != "application/json" { + if !matchContentType.MatchString(strings.TrimSpace(resp.Header.Get("Content-Type"))) { return nil, errors.Errorf("unsupported content type %q", resp.Header.Get("Content-Type")) } @@ -180,6 +184,13 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { tg.Labels[httpSDURLLabel] = model.LabelValue(d.url) } + // Generate empty updates for sources that disappeared. + l := len(targetGroups) + for i := l; i < d.tgLastLength; i++ { + targetGroups = append(targetGroups, &targetgroup.Group{Source: urlSource(d.url, i)}) + } + d.tgLastLength = l + return targetGroups, nil } diff --git a/discovery/http/http_test.go b/discovery/http/http_test.go index c248beb62..e9dc98b8f 100644 --- a/discovery/http/http_test.go +++ b/discovery/http/http_test.go @@ -104,3 +104,299 @@ func TestHTTPInvalidFormat(t *testing.T) { _, err = d.refresh(ctx) require.EqualError(t, err, `unsupported content type "text/plain; charset=utf-8"`) } + +func TestContentTypeRegex(t *testing.T) { + cases := []struct { + header string + match bool + }{ + { + header: "application/json;charset=utf-8", + match: true, + }, + { + header: "application/json;charset=UTF-8", + match: true, + }, + { + header: "Application/JSON;Charset=\"utf-8\"", + match: true, + }, + { + header: "application/json; charset=\"utf-8\"", + match: true, + }, + { + header: "application/json", + match: true, + }, + { + header: "application/jsonl; charset=\"utf-8\"", + match: false, + }, + { + header: "application/json;charset=UTF-9", + match: false, + }, + { + header: "application /json;charset=UTF-8", + match: false, + }, + { + header: "application/ json;charset=UTF-8", + match: false, + }, + { + header: "application/json;", + match: false, + }, + { + header: "charset=UTF-8", + match: false, + }, + } + + for _, test := range cases { + t.Run(test.header, func(t *testing.T) { + require.Equal(t, test.match, matchContentType.MatchString(test.header)) + }) + } +} + +func TestSourceDisappeared(t *testing.T) { + var stubResponse string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintln(w, stubResponse) + })) + t.Cleanup(ts.Close) + + cases := []struct { + responses []string + expectedTargets [][]*targetgroup.Group + }{ + { + responses: []string{ + `[]`, + `[]`, + }, + expectedTargets: [][]*targetgroup.Group{{}, {}}, + }, + { + responses: []string{ + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`, + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`, + }, + expectedTargets: [][]*targetgroup.Group{ + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + }, + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("2"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 1), + }, + }, + }, + }, + { + responses: []string{ + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`, + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`, + }, + expectedTargets: [][]*targetgroup.Group{ + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("2"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 1), + }, + }, + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: nil, + Labels: nil, + Source: urlSource(ts.URL, 1), + }, + }, + }, + }, + { + responses: []string{ + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "3"}, "targets": ["127.0.0.1"]}]`, + `[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`, + `[{"labels": {"k": "v"}, "targets": ["127.0.0.2"]}, {"labels": {"k": "vv"}, "targets": ["127.0.0.3"]}]`, + }, + expectedTargets: [][]*targetgroup.Group{ + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("2"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 1), + }, + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("3"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 2), + }, + }, + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.1"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("1"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: nil, + Labels: nil, + Source: urlSource(ts.URL, 1), + }, + { + Targets: nil, + Labels: nil, + Source: urlSource(ts.URL, 2), + }, + }, + { + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.2"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("v"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 0), + }, + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("127.0.0.3"), + }, + }, + Labels: model.LabelSet{ + model.LabelName("k"): model.LabelValue("vv"), + model.LabelName("__meta_url"): model.LabelValue(ts.URL), + }, + Source: urlSource(ts.URL, 1), + }, + }, + }, + }, + } + + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: ts.URL, + RefreshInterval: model.Duration(1 * time.Second), + } + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + for _, test := range cases { + ctx := context.Background() + for i, res := range test.responses { + stubResponse = res + tgs, err := d.refresh(ctx) + require.NoError(t, err) + require.Equal(t, test.expectedTargets[i], tgs) + } + } + +} diff --git a/discovery/install/install.go b/discovery/install/install.go index 6258d6fc0..34ccf3d0f 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -33,5 +33,6 @@ import ( _ "github.com/prometheus/prometheus/discovery/openstack" // register openstack _ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway _ "github.com/prometheus/prometheus/discovery/triton" // register triton + _ "github.com/prometheus/prometheus/discovery/xds" // register xds _ "github.com/prometheus/prometheus/discovery/zookeeper" // register zookeeper ) diff --git a/discovery/linode/linode.go b/discovery/linode/linode.go index 9c22eb684..21c856857 100644 --- a/discovery/linode/linode.go +++ b/discovery/linode/linode.go @@ -15,6 +15,7 @@ package linode import ( "context" + "errors" "fmt" "net" "net/http" @@ -56,6 +57,11 @@ const ( linodeLabelSpecsVCPUs = linodeLabel + "specs_vcpus" linodeLabelSpecsTransferBytes = linodeLabel + "specs_transfer_bytes" linodeLabelExtraIPs = linodeLabel + "extra_ips" + + // This is our events filter; when polling for changes, we care only about + // events since our last refresh. + // Docs: https://www.linode.com/docs/api/account/#events-list + filterTemplate = `{"created": {"+gte": "%s"}}` ) // DefaultSDConfig is the default Linode SD configuration. @@ -107,16 +113,23 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { // the Discoverer interface. type Discovery struct { *refresh.Discovery - client *linodego.Client - port int - tagSeparator string + client *linodego.Client + port int + tagSeparator string + lastRefreshTimestamp time.Time + pollCount int + lastResults []*targetgroup.Group + eventPollingEnabled bool } // NewDiscovery returns a new Discovery which periodically refreshes its targets. func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { d := &Discovery{ - port: conf.Port, - tagSeparator: conf.TagSeparator, + port: conf.Port, + tagSeparator: conf.TagSeparator, + pollCount: 0, + lastRefreshTimestamp: time.Now().UTC(), + eventPollingEnabled: true, } rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "linode_sd", config.WithHTTP2Disabled()) @@ -143,6 +156,50 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { } func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + needsRefresh := true + ts := time.Now().UTC() + + if d.lastResults != nil && d.eventPollingEnabled { + // Check to see if there have been any events. If so, refresh our data. + opts := linodego.NewListOptions(1, fmt.Sprintf(filterTemplate, d.lastRefreshTimestamp.Format("2006-01-02T15:04:05"))) + events, err := d.client.ListEvents(ctx, opts) + if err != nil { + var e *linodego.Error + if errors.As(err, &e) && e.Code == http.StatusUnauthorized { + // If we get a 401, the token doesn't have `events:read_only` scope. + // Disable event polling and fallback to doing a full refresh every interval. + d.eventPollingEnabled = false + } else { + return nil, err + } + } else { + // Event polling tells us changes the Linode API is aware of. Actions issued outside of the Linode API, + // such as issuing a `shutdown` at the VM's console instead of using the API to power off an instance, + // can potentially cause us to return stale data. Just in case, trigger a full refresh after ~10 polling + // intervals of no events. + d.pollCount++ + + if len(events) == 0 && d.pollCount < 10 { + needsRefresh = false + } + } + } + + if needsRefresh { + newData, err := d.refreshData(ctx) + if err != nil { + return nil, err + } + d.pollCount = 0 + d.lastResults = newData + } + + d.lastRefreshTimestamp = ts + + return d.lastResults, nil +} + +func (d *Discovery) refreshData(ctx context.Context) ([]*targetgroup.Group, error) { tg := &targetgroup.Group{ Source: "Linode", } diff --git a/discovery/linode/linode_test.go b/discovery/linode/linode_test.go index d222afffd..a201458a7 100644 --- a/discovery/linode/linode_test.go +++ b/discovery/linode/linode_test.go @@ -39,6 +39,7 @@ func (s *LinodeSDTestSuite) SetupTest(t *testing.T) { s.Mock.HandleLinodeInstancesList() s.Mock.HandleLinodeNeworkingIPs() + s.Mock.HandleLinodeAccountEvents() } func TestLinodeSDRefresh(t *testing.T) { diff --git a/discovery/linode/mock_test.go b/discovery/linode/mock_test.go index 62b9f7782..ade726ed7 100644 --- a/discovery/linode/mock_test.go +++ b/discovery/linode/mock_test.go @@ -413,3 +413,42 @@ func (m *SDMock) HandleLinodeNeworkingIPs() { ) }) } + +// HandleLinodeAccountEvents mocks linode the account/events endpoint. +func (m *SDMock) HandleLinodeAccountEvents() { + m.Mux.HandleFunc("/v4/account/events", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + if r.Header.Get("X-Filter") == "" { + // This should never happen; if the client sends an events request without + // a filter, cause it to fail. The error below is not a real response from + // the API, but should aid in debugging failed tests. + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, ` +{ + "errors": [ + { + "reason": "Request missing expected X-Filter headers" + } + ] +}`, + ) + return + } + + w.Header().Set("content-type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + fmt.Fprint(w, ` +{ + "data": [], + "results": 0, + "pages": 1, + "page": 1 +}`, + ) + }) +} diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index ff9c76961..586245bc4 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -339,8 +339,13 @@ func fetchApps(ctx context.Context, client *http.Client, url string) (*appList, return nil, errors.Errorf("non 2xx status '%v' response during marathon service discovery", resp.StatusCode) } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var apps appList - err = json.NewDecoder(resp.Body).Decode(&apps) + err = json.Unmarshal(b, &apps) if err != nil { return nil, errors.Wrapf(err, "%q", url) } diff --git a/discovery/moby/docker.go b/discovery/moby/docker.go index 34eea7d4c..98dc42acb 100644 --- a/discovery/moby/docker.go +++ b/discovery/moby/docker.go @@ -51,10 +51,11 @@ const ( // DefaultDockerSDConfig is the default Docker SD configuration. var DefaultDockerSDConfig = DockerSDConfig{ - RefreshInterval: model.Duration(60 * time.Second), - Port: 80, - Filters: []Filter{}, - HTTPClientConfig: config.DefaultHTTPClientConfig, + RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + Filters: []Filter{}, + HostNetworkingHost: "localhost", + HTTPClientConfig: config.DefaultHTTPClientConfig, } func init() { @@ -65,9 +66,10 @@ func init() { type DockerSDConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` - Host string `yaml:"host"` - Port int `yaml:"port"` - Filters []Filter `yaml:"filters"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Filters []Filter `yaml:"filters"` + HostNetworkingHost string `yaml:"host_networking_host"` RefreshInterval model.Duration `yaml:"refresh_interval"` } @@ -104,9 +106,10 @@ func (c *DockerSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error type DockerDiscovery struct { *refresh.Discovery - client *client.Client - port int - filters filters.Args + client *client.Client + port int + hostNetworkingHost string + filters filters.Args } // NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets. @@ -114,7 +117,8 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger) (*DockerDiscove var err error d := &DockerDiscovery{ - port: conf.Port, + port: conf.Port, + hostNetworkingHost: conf.HostNetworkingHost, } hostURL, err := url.Parse(conf.Host) @@ -245,7 +249,15 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er labels[model.LabelName(k)] = model.LabelValue(v) } - addr := net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(d.port), 10)) + // Containers in host networking mode don't have ports, + // so they only end up here, not in the previous loop. + var addr string + if c.HostConfig.NetworkMode != "host" { + addr = net.JoinHostPort(n.IPAddress, strconv.FormatUint(uint64(d.port), 10)) + } else { + addr = d.hostNetworkingHost + } + labels[model.AddressLabel] = model.LabelValue(addr) tg.Targets = append(tg.Targets, labels) } diff --git a/discovery/moby/docker_test.go b/discovery/moby/docker_test.go index 9ede9ab53..bb84b1571 100644 --- a/discovery/moby/docker_test.go +++ b/discovery/moby/docker_test.go @@ -49,7 +49,7 @@ host: %s tg := tgs[0] require.NotNil(t, tg) require.NotNil(t, tg.Targets) - require.Equal(t, 2, len(tg.Targets)) + require.Equal(t, 3, len(tg.Targets)) for i, lbls := range []model.LabelSet{ { @@ -93,6 +93,16 @@ host: %s "__meta_docker_network_name": "dockersd_default", "__meta_docker_network_scope": "local", }, + { + "__address__": "localhost", + "__meta_docker_container_id": "54ed6cc5c0988260436cb0e739b7b6c9cad6c439a93b4c4fdbe9753e1c94b189", + "__meta_docker_container_label_com_docker_compose_project": "dockersd", + "__meta_docker_container_label_com_docker_compose_service": "host_networking", + "__meta_docker_container_label_com_docker_compose_version": "1.25.0", + "__meta_docker_container_name": "/dockersd_host_networking_1", + "__meta_docker_container_network_mode": "host", + "__meta_docker_network_ip": "", + }, } { t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { require.Equal(t, lbls, tg.Targets[i]) diff --git a/discovery/moby/testdata/dockerprom/containers/json_limit_0.json b/discovery/moby/testdata/dockerprom/containers/json_limit_0.json index 6405a091c..37f575d22 100644 --- a/discovery/moby/testdata/dockerprom/containers/json_limit_0.json +++ b/discovery/moby/testdata/dockerprom/containers/json_limit_0.json @@ -89,5 +89,44 @@ } }, "Mounts": [] + }, + { + "Id": "54ed6cc5c0988260436cb0e739b7b6c9cad6c439a93b4c4fdbe9753e1c94b189", + "Names": [ + "/dockersd_host_networking_1" + ], + "Image": "httpd", + "ImageID": "sha256:73b8cfec11558fe86f565b4357f6d6c8560f4c49a5f15ae970a24da86c9adc93", + "Command": "httpd-foreground", + "Created": 1627440494, + "Ports": [], + "Labels": { + "com.docker.compose.project": "dockersd", + "com.docker.compose.service": "host_networking", + "com.docker.compose.version": "1.25.0" + }, + "State": "running", + "Status": "Up 3 minutes", + "HostConfig": { "NetworkMode": "host" }, + "NetworkSettings": { + "Networks": { + "host": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "80c4459fa193c5c8b57e90e117d2f899d1a86708e548738149d62e03df0ec35c", + "EndpointID": "e9bea4c499c34bd41609b0e1e9b38f9964c69180c1a22130f28b6af802c156d8", + "Gateway": "", + "IPAddress": "", + "IPPrefixLen": 0, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "", + "DriverOpts": null + } + } + }, + "Mounts": [] } ] diff --git a/discovery/openstack/hypervisor.go b/discovery/openstack/hypervisor.go index f02fe17e7..594f4e433 100644 --- a/discovery/openstack/hypervisor.go +++ b/discovery/openstack/hypervisor.go @@ -74,7 +74,7 @@ func (h *HypervisorDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group } // OpenStack API reference // https://developer.openstack.org/api-ref/compute/#list-hypervisors-details - pagerHypervisors := hypervisors.List(client) + pagerHypervisors := hypervisors.List(client, nil) err = pagerHypervisors.EachPage(func(page pagination.Page) (bool, error) { hypervisorList, err := hypervisors.ExtractHypervisors(page) if err != nil { diff --git a/discovery/xds/client.go b/discovery/xds/client.go new file mode 100644 index 000000000..cd8ffb017 --- /dev/null +++ b/discovery/xds/client.go @@ -0,0 +1,226 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "time" + + envoy_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/prometheus/common/config" + "github.com/prometheus/common/version" +) + +var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) + +// ResourceClient exposes the xDS protocol for a single resource type. +// See https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#rest-json-polling-subscriptions . +type ResourceClient interface { + // ResourceTypeURL is the type URL of the resource. + ResourceTypeURL() string + + // Server is the xDS Management server. + Server() string + + // Fetch requests the latest view of the entire resource state. + // If no updates have been made since the last request, the response will be nil. + Fetch(ctx context.Context) (*v3.DiscoveryResponse, error) + + // ID returns the ID of the client that is sent to the xDS server. + ID() string + + // Close releases currently held resources. + Close() +} + +type HTTPResourceClient struct { + client *http.Client + config *HTTPResourceClientConfig + + // endpoint is the fully-constructed xDS HTTP endpoint. + endpoint string + // Caching. + latestVersion string + latestNonce string +} + +type HTTPResourceClientConfig struct { + // HTTP config. + config.HTTPClientConfig + + Name string + + // ExtraQueryParams are extra query parameters to attach to the request URL. + ExtraQueryParams url.Values + + // General xDS config. + + // The timeout for a single fetch request. + Timeout time.Duration + + // Type is the xds type, e.g., clusters + // which is used in the discovery POST request. + ResourceType string + // ResourceTypeURL is the Google type url for the resource, e.g., type.googleapis.com/envoy.api.v2.Cluster. + ResourceTypeURL string + // Server is the xDS management server. + Server string + // ClientID is used to identify the client with the management server. + ClientID string +} + +func NewHTTPResourceClient(conf *HTTPResourceClientConfig, protocolVersion ProtocolVersion) (*HTTPResourceClient, error) { + if protocolVersion != ProtocolV3 { + return nil, errors.New("only the v3 protocol is supported") + } + + if len(conf.Server) == 0 { + return nil, errors.New("empty xDS server") + } + + serverURL, err := url.Parse(conf.Server) + if err != nil { + return nil, err + } + + endpointURL, err := makeXDSResourceHTTPEndpointURL(protocolVersion, serverURL, conf.ResourceType) + if err != nil { + return nil, err + } + + if conf.ExtraQueryParams != nil { + endpointURL.RawQuery = conf.ExtraQueryParams.Encode() + } + + client, err := config.NewClientFromConfig(conf.HTTPClientConfig, conf.Name, config.WithHTTP2Disabled(), config.WithIdleConnTimeout(conf.Timeout)) + if err != nil { + return nil, err + } + + client.Timeout = conf.Timeout + + return &HTTPResourceClient{ + client: client, + config: conf, + endpoint: endpointURL.String(), + latestVersion: "", + latestNonce: "", + }, nil +} + +func makeXDSResourceHTTPEndpointURL(protocolVersion ProtocolVersion, serverURL *url.URL, resourceType string) (*url.URL, error) { + if serverURL == nil { + return nil, errors.New("empty xDS server URL") + } + + if len(serverURL.Scheme) == 0 || len(serverURL.Host) == 0 { + return nil, errors.New("invalid xDS server URL") + } + + if serverURL.Scheme != "http" && serverURL.Scheme != "https" { + return nil, errors.New("invalid xDS server URL protocol. must be either 'http' or 'https'") + } + + serverURL.Path = path.Join(serverURL.Path, string(protocolVersion), fmt.Sprintf("discovery:%s", resourceType)) + + return serverURL, nil +} + +func (rc *HTTPResourceClient) Server() string { + return rc.config.Server +} + +func (rc *HTTPResourceClient) ResourceTypeURL() string { + return rc.config.ResourceTypeURL +} + +func (rc *HTTPResourceClient) ID() string { + return rc.config.ClientID +} + +func (rc *HTTPResourceClient) Close() { + rc.client.CloseIdleConnections() +} + +// Fetch requests the latest state of the resources from the xDS server and cache the version. +// Returns a nil response if the current local version is up to date. +func (rc *HTTPResourceClient) Fetch(ctx context.Context) (*v3.DiscoveryResponse, error) { + discoveryReq := &v3.DiscoveryRequest{ + VersionInfo: rc.latestVersion, + ResponseNonce: rc.latestNonce, + TypeUrl: rc.ResourceTypeURL(), + ResourceNames: []string{}, + Node: &envoy_core.Node{ + Id: rc.ID(), + }, + } + + reqBody, err := protoJSONMarshalOptions.Marshal(discoveryReq) + if err != nil { + return nil, err + } + + request, err := http.NewRequest("POST", rc.endpoint, bytes.NewBuffer(reqBody)) + if err != nil { + return nil, err + } + request = request.WithContext(ctx) + + request.Header.Add("User-Agent", userAgent) + request.Header.Add("Content-Type", "application/json") + request.Header.Add("Accept", "application/json") + + resp, err := rc.client.Do(request) + if err != nil { + return nil, err + } + defer func() { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode == http.StatusNotModified { + // Empty response, already have the latest. + return nil, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("non 200 status '%d' response during xDS fetch", resp.StatusCode) + } + + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + discoveryRes := &v3.DiscoveryResponse{} + if err = protoJSONUnmarshalOptions.Unmarshal(respBody, discoveryRes); err != nil { + return nil, err + } + + // Cache the latest nonce + version info. + rc.latestNonce = discoveryRes.Nonce + rc.latestVersion = discoveryRes.VersionInfo + + return discoveryRes, nil +} diff --git a/discovery/xds/client_test.go b/discovery/xds/client_test.go new file mode 100644 index 000000000..1c0e321d3 --- /dev/null +++ b/discovery/xds/client_test.go @@ -0,0 +1,161 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "context" + "errors" + "net/url" + "testing" + "time" + + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/prometheus/common/config" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" +) + +var ( + httpResourceConf = &HTTPResourceClientConfig{ + HTTPClientConfig: config.HTTPClientConfig{ + TLSConfig: config.TLSConfig{InsecureSkipVerify: true}, + }, + ResourceType: "monitoring", + // Some known type. + ResourceTypeURL: "type.googleapis.com/envoy.service.discovery.v3.DiscoveryRequest", + Server: "http://localhost", + ClientID: "test-id", + } +) + +func urlMustParse(str string) *url.URL { + parsed, err := url.Parse(str) + + if err != nil { + panic(err) + } + + return parsed +} + +func TestMakeXDSResourceHttpEndpointEmptyServerURLScheme(t *testing.T) { + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("127.0.0.1"), "monitoring") + + require.Empty(t, endpointURL) + require.Error(t, err) + require.Equal(t, err.Error(), "invalid xDS server URL") +} + +func TestMakeXDSResourceHttpEndpointEmptyServerURLHost(t *testing.T) { + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("grpc://127.0.0.1"), "monitoring") + + require.Empty(t, endpointURL) + require.NotNil(t, err) + require.Contains(t, err.Error(), "must be either 'http' or 'https'") +} + +func TestMakeXDSResourceHttpEndpoint(t *testing.T) { + endpointURL, err := makeXDSResourceHTTPEndpointURL(ProtocolV3, urlMustParse("http://127.0.0.1:5000"), "monitoring") + + require.NoError(t, err) + require.Equal(t, endpointURL.String(), "http://127.0.0.1:5000/v3/discovery:monitoring") +} + +func TestCreateNewHTTPResourceClient(t *testing.T) { + c := &HTTPResourceClientConfig{ + HTTPClientConfig: sdConf.HTTPClientConfig, + Name: "test", + ExtraQueryParams: url.Values{ + "param1": {"v1"}, + }, + Timeout: 1 * time.Minute, + ResourceType: "monitoring", + ResourceTypeURL: "type.googleapis.com/envoy.service.discovery.v3.DiscoveryRequest", + Server: "http://127.0.0.1:5000", + ClientID: "client", + } + + client, err := NewHTTPResourceClient(c, ProtocolV3) + + require.NoError(t, err) + + require.Equal(t, client.endpoint, "http://127.0.0.1:5000/v3/discovery:monitoring?param1=v1") + require.Equal(t, client.client.Timeout, 1*time.Minute) + +} + +func createTestHTTPResourceClient(t *testing.T, conf *HTTPResourceClientConfig, protocolVersion ProtocolVersion, responder discoveryResponder) (*HTTPResourceClient, func()) { + s := createTestHTTPServer(t, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + require.Equal(t, conf.ResourceTypeURL, request.TypeUrl) + require.Equal(t, conf.ClientID, request.Node.Id) + return responder(request) + }) + + conf.Server = s.URL + client, err := NewHTTPResourceClient(conf, protocolVersion) + require.NoError(t, err) + + return client, s.Close +} + +func TestHTTPResourceClientFetchEmptyResponse(t *testing.T) { + client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + return nil, nil + }) + defer cleanup() + + res, err := client.Fetch(context.Background()) + require.Nil(t, res) + require.NoError(t, err) +} + +func TestHTTPResourceClientFetchFullResponse(t *testing.T) { + client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + if request.VersionInfo == "1" { + return nil, nil + } + + return &v3.DiscoveryResponse{ + TypeUrl: request.TypeUrl, + VersionInfo: "1", + Nonce: "abc", + Resources: []*anypb.Any{}, + }, nil + }) + defer cleanup() + + res, err := client.Fetch(context.Background()) + require.NoError(t, err) + require.NotNil(t, res) + + require.Equal(t, client.ResourceTypeURL(), res.TypeUrl) + require.Len(t, res.Resources, 0) + require.Equal(t, "abc", client.latestNonce, "Nonce not cached") + require.Equal(t, "1", client.latestVersion, "Version not cached") + + res, err = client.Fetch(context.Background()) + require.Nil(t, res, "Update not expected") + require.NoError(t, err) +} + +func TestHTTPResourceClientServerError(t *testing.T) { + client, cleanup := createTestHTTPResourceClient(t, httpResourceConf, ProtocolV3, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + return nil, errors.New("server error") + }) + defer cleanup() + + res, err := client.Fetch(context.Background()) + require.Nil(t, res) + require.Error(t, err) +} diff --git a/discovery/xds/kuma.go b/discovery/xds/kuma.go new file mode 100644 index 000000000..d4071be9c --- /dev/null +++ b/discovery/xds/kuma.go @@ -0,0 +1,226 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "fmt" + "net/url" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/osutil" + "github.com/prometheus/prometheus/util/strutil" +) + +var ( + // DefaultKumaSDConfig is the default Kuma MADS SD configuration. + DefaultKumaSDConfig = KumaSDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + RefreshInterval: model.Duration(15 * time.Second), + FetchTimeout: model.Duration(2 * time.Minute), + } + + kumaFetchFailuresCount = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_failures_total", + Help: "The number of Kuma MADS fetch call failures.", + }) + kumaFetchSkipUpdateCount = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_skipped_updates_total", + Help: "The number of Kuma MADS fetch calls that result in no updates to the targets.", + }) + kumaFetchDuration = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "sd_kuma_fetch_duration_seconds", + Help: "The duration of a Kuma MADS fetch call.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + ) +) + +const ( + // kumaMetaLabelPrefix is the meta prefix used for all kuma meta labels. + kumaMetaLabelPrefix = model.MetaLabelPrefix + "kuma_" + + // kumaMeshLabel is the name of the label that holds the mesh name. + kumaMeshLabel = kumaMetaLabelPrefix + "mesh" + // kumaServiceLabel is the name of the label that holds the service name. + kumaServiceLabel = kumaMetaLabelPrefix + "service" + // kumaDataplaneLabel is the name of the label that holds the dataplane name. + kumaDataplaneLabel = kumaMetaLabelPrefix + "dataplane" + // kumaUserLabelPrefix is the name of the label that namespaces all user-defined labels. + kumaUserLabelPrefix = kumaMetaLabelPrefix + "label_" +) + +const ( + KumaMadsV1ResourceTypeURL = "type.googleapis.com/kuma.observability.v1.MonitoringAssignment" + KumaMadsV1ResourceType = "monitoringassignments" +) + +type KumaSDConfig = SDConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *KumaSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultKumaSDConfig + type plainKumaConf KumaSDConfig + err := unmarshal((*plainKumaConf)(c)) + if err != nil { + return err + } + + if len(c.Server) == 0 { + return errors.Errorf("kuma SD server must not be empty: %s", c.Server) + } + parsedURL, err := url.Parse(c.Server) + if err != nil { + return err + } + + if len(parsedURL.Scheme) == 0 || len(parsedURL.Host) == 0 { + return errors.Errorf("kuma SD server must not be empty and have a scheme: %s", c.Server) + } + + if err := c.HTTPClientConfig.Validate(); err != nil { + return err + } + + return nil +} + +func (c *KumaSDConfig) Name() string { + return "kuma" +} + +// SetDirectory joins any relative file paths with dir. +func (c *KumaSDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +func (c *KumaSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + logger := opts.Logger + if logger == nil { + logger = log.NewNopLogger() + } + + return NewKumaHTTPDiscovery(c, logger) +} + +func convertKumaV1MonitoringAssignment(assignment *MonitoringAssignment) *targetgroup.Group { + commonLabels := convertKumaUserLabels(assignment.Labels) + + commonLabels[kumaMeshLabel] = model.LabelValue(assignment.Mesh) + commonLabels[kumaServiceLabel] = model.LabelValue(assignment.Service) + + var targetLabelSets []model.LabelSet + + for _, target := range assignment.Targets { + targetLabels := convertKumaUserLabels(target.Labels) + + targetLabels[kumaDataplaneLabel] = model.LabelValue(target.Name) + targetLabels[model.InstanceLabel] = model.LabelValue(target.Name) + targetLabels[model.AddressLabel] = model.LabelValue(target.Address) + targetLabels[model.SchemeLabel] = model.LabelValue(target.Scheme) + targetLabels[model.MetricsPathLabel] = model.LabelValue(target.MetricsPath) + + targetLabelSets = append(targetLabelSets, targetLabels) + } + + return &targetgroup.Group{ + Labels: commonLabels, + Targets: targetLabelSets, + } +} + +func convertKumaUserLabels(labels map[string]string) model.LabelSet { + labelSet := model.LabelSet{} + for key, value := range labels { + name := kumaUserLabelPrefix + strutil.SanitizeLabelName(key) + labelSet[model.LabelName(name)] = model.LabelValue(value) + } + return labelSet +} + +// kumaMadsV1ResourceParser is an xds.resourceParser. +func kumaMadsV1ResourceParser(resources []*anypb.Any, typeURL string) ([]*targetgroup.Group, error) { + if typeURL != KumaMadsV1ResourceTypeURL { + return nil, errors.Errorf("recieved invalid typeURL for Kuma MADS v1 Resource: %s", typeURL) + } + + var groups []*targetgroup.Group + + for _, resource := range resources { + assignment := &MonitoringAssignment{} + + if err := anypb.UnmarshalTo(resource, assignment, protoUnmarshalOptions); err != nil { + return nil, err + } + + groups = append(groups, convertKumaV1MonitoringAssignment(assignment)) + } + + return groups, nil +} + +func NewKumaHTTPDiscovery(conf *KumaSDConfig, logger log.Logger) (discovery.Discoverer, error) { + // Default to "prometheus" if hostname is unavailable. + clientID, err := osutil.GetFQDN() + if err != nil { + level.Debug(logger).Log("msg", "error getting FQDN", "err", err) + clientID = "prometheus" + } + + clientConfig := &HTTPResourceClientConfig{ + HTTPClientConfig: conf.HTTPClientConfig, + ExtraQueryParams: url.Values{ + "fetch-timeout": {conf.FetchTimeout.String()}, + }, + // Allow 15s of buffer over the timeout sent to the xDS server for connection overhead. + Timeout: time.Duration(conf.FetchTimeout) + (15 * time.Second), + ResourceType: KumaMadsV1ResourceType, + ResourceTypeURL: KumaMadsV1ResourceTypeURL, + Server: conf.Server, + ClientID: clientID, + } + + client, err := NewHTTPResourceClient(clientConfig, ProtocolV3) + if err != nil { + return nil, fmt.Errorf("kuma_sd: %w", err) + } + + d := &fetchDiscovery{ + client: client, + logger: logger, + refreshInterval: time.Duration(conf.RefreshInterval), + source: "kuma", + parseResources: kumaMadsV1ResourceParser, + fetchFailuresCount: kumaFetchFailuresCount, + fetchSkipUpdateCount: kumaFetchSkipUpdateCount, + fetchDuration: kumaFetchDuration, + } + + return d, nil +} diff --git a/discovery/xds/kuma_mads.pb.go b/discovery/xds/kuma_mads.pb.go new file mode 100644 index 000000000..b1079bf23 --- /dev/null +++ b/discovery/xds/kuma_mads.pb.go @@ -0,0 +1,398 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.14.0 +// source: observability/v1/mads.proto + +// gRPC-removed vendored file from Kuma. + +package xds + +import ( + context "context" + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + _ "github.com/envoyproxy/protoc-gen-validate/validate" + _ "google.golang.org/genproto/googleapis/api/annotations" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// MADS resource type. +// +// Describes a group of targets on a single service that need to be monitored. +type MonitoringAssignment struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Mesh of the dataplane. + // + // E.g., `default` + Mesh string `protobuf:"bytes,2,opt,name=mesh,proto3" json:"mesh,omitempty"` + // Identifying service the dataplane is proxying. + // + // E.g., `backend` + Service string `protobuf:"bytes,3,opt,name=service,proto3" json:"service,omitempty"` + // List of targets that need to be monitored. + Targets []*MonitoringAssignment_Target `protobuf:"bytes,4,rep,name=targets,proto3" json:"targets,omitempty"` + // Arbitrary Labels associated with every target in the assignment. + // + // E.g., `{"zone" : "us-east-1", "team": "infra", "commit_hash": "620506a88"}`. + Labels map[string]string `protobuf:"bytes,5,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *MonitoringAssignment) Reset() { + *x = MonitoringAssignment{} + if protoimpl.UnsafeEnabled { + mi := &file_observability_v1_mads_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MonitoringAssignment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MonitoringAssignment) ProtoMessage() {} + +func (x *MonitoringAssignment) ProtoReflect() protoreflect.Message { + mi := &file_observability_v1_mads_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MonitoringAssignment.ProtoReflect.Descriptor instead. +func (*MonitoringAssignment) Descriptor() ([]byte, []int) { + return file_observability_v1_mads_proto_rawDescGZIP(), []int{0} +} + +func (x *MonitoringAssignment) GetMesh() string { + if x != nil { + return x.Mesh + } + return "" +} + +func (x *MonitoringAssignment) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *MonitoringAssignment) GetTargets() []*MonitoringAssignment_Target { + if x != nil { + return x.Targets + } + return nil +} + +func (x *MonitoringAssignment) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +// Describes a single target that needs to be monitored. +type MonitoringAssignment_Target struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // E.g., `backend-01` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Scheme on which to scrape the target. + //E.g., `http` + Scheme string `protobuf:"bytes,2,opt,name=scheme,proto3" json:"scheme,omitempty"` + // Address (preferably IP) for the service + // E.g., `backend.svc` or `10.1.4.32:9090` + Address string `protobuf:"bytes,3,opt,name=address,proto3" json:"address,omitempty"` + // Optional path to append to the address for scraping + //E.g., `/metrics` + MetricsPath string `protobuf:"bytes,4,opt,name=metrics_path,json=metricsPath,proto3" json:"metrics_path,omitempty"` + // Arbitrary labels associated with that particular target. + // + // E.g., + // `{ + // "commit_hash" : "620506a88", + // }`. + Labels map[string]string `protobuf:"bytes,5,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *MonitoringAssignment_Target) Reset() { + *x = MonitoringAssignment_Target{} + if protoimpl.UnsafeEnabled { + mi := &file_observability_v1_mads_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MonitoringAssignment_Target) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MonitoringAssignment_Target) ProtoMessage() {} + +func (x *MonitoringAssignment_Target) ProtoReflect() protoreflect.Message { + mi := &file_observability_v1_mads_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MonitoringAssignment_Target.ProtoReflect.Descriptor instead. +func (*MonitoringAssignment_Target) Descriptor() ([]byte, []int) { + return file_observability_v1_mads_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *MonitoringAssignment_Target) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *MonitoringAssignment_Target) GetScheme() string { + if x != nil { + return x.Scheme + } + return "" +} + +func (x *MonitoringAssignment_Target) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *MonitoringAssignment_Target) GetMetricsPath() string { + if x != nil { + return x.MetricsPath + } + return "" +} + +func (x *MonitoringAssignment_Target) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +var File_observability_v1_mads_proto protoreflect.FileDescriptor + +var file_observability_v1_mads_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x2f, + 0x76, 0x31, 0x2f, 0x6d, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x15, 0x6b, + 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x79, 0x2e, 0x76, 0x31, 0x1a, 0x2a, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x2f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2f, 0x76, 0x33, + 0x2f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, + 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd2, 0x04, 0x0a, 0x14, 0x4d, 0x6f, 0x6e, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x12, 0x1b, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, + 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x12, 0x21, 0x0a, + 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, + 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x4c, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x32, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x4f, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, + 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, + 0x9f, 0x02, 0x0a, 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, + 0x01, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x20, 0x01, + 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, + 0x20, 0x01, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x56, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, + 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xe6, 0x03, 0x0a, + 0x24, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x89, 0x01, 0x0a, 0x1a, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x4d, + 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x31, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, + 0x33, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, + 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, + 0x01, 0x12, 0x80, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x6f, 0x6e, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x12, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, + 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, + 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x69, 0x73, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x12, 0xae, 0x01, 0x0a, 0x1a, 0x46, 0x65, 0x74, 0x63, 0x68, 0x4d, 0x6f, + 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, + 0x2e, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x2e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2e, 0x76, 0x33, 0x2e, 0x44, + 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x33, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x24, 0x22, 0x22, 0x2f, 0x76, 0x33, 0x2f, 0x64, 0x69, + 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x3a, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, + 0x6e, 0x67, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x82, 0xd3, 0xe4, 0x93, + 0x02, 0x03, 0x3a, 0x01, 0x2a, 0x42, 0x04, 0x5a, 0x02, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_observability_v1_mads_proto_rawDescOnce sync.Once + file_observability_v1_mads_proto_rawDescData = file_observability_v1_mads_proto_rawDesc +) + +func file_observability_v1_mads_proto_rawDescGZIP() []byte { + file_observability_v1_mads_proto_rawDescOnce.Do(func() { + file_observability_v1_mads_proto_rawDescData = protoimpl.X.CompressGZIP(file_observability_v1_mads_proto_rawDescData) + }) + return file_observability_v1_mads_proto_rawDescData +} + +var file_observability_v1_mads_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_observability_v1_mads_proto_goTypes = []interface{}{ + (*MonitoringAssignment)(nil), // 0: kuma.observability.v1.MonitoringAssignment + (*MonitoringAssignment_Target)(nil), // 1: kuma.observability.v1.MonitoringAssignment.Target + nil, // 2: kuma.observability.v1.MonitoringAssignment.LabelsEntry + nil, // 3: kuma.observability.v1.MonitoringAssignment.Target.LabelsEntry + (*v3.DeltaDiscoveryRequest)(nil), // 4: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v3.DiscoveryRequest)(nil), // 5: envoy.service.discovery.v3.DiscoveryRequest + (*v3.DeltaDiscoveryResponse)(nil), // 6: envoy.service.discovery.v3.DeltaDiscoveryResponse + (*v3.DiscoveryResponse)(nil), // 7: envoy.service.discovery.v3.DiscoveryResponse +} +var file_observability_v1_mads_proto_depIdxs = []int32{ + 1, // 0: kuma.observability.v1.MonitoringAssignment.targets:type_name -> kuma.observability.v1.MonitoringAssignment.Target + 2, // 1: kuma.observability.v1.MonitoringAssignment.labels:type_name -> kuma.observability.v1.MonitoringAssignment.LabelsEntry + 3, // 2: kuma.observability.v1.MonitoringAssignment.Target.labels:type_name -> kuma.observability.v1.MonitoringAssignment.Target.LabelsEntry + 4, // 3: kuma.observability.v1.MonitoringAssignmentDiscoveryService.DeltaMonitoringAssignments:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 5, // 4: kuma.observability.v1.MonitoringAssignmentDiscoveryService.StreamMonitoringAssignments:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 5, // 5: kuma.observability.v1.MonitoringAssignmentDiscoveryService.FetchMonitoringAssignments:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 6, // 6: kuma.observability.v1.MonitoringAssignmentDiscoveryService.DeltaMonitoringAssignments:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 7, // 7: kuma.observability.v1.MonitoringAssignmentDiscoveryService.StreamMonitoringAssignments:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 7, // 8: kuma.observability.v1.MonitoringAssignmentDiscoveryService.FetchMonitoringAssignments:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_observability_v1_mads_proto_init() } +func file_observability_v1_mads_proto_init() { + if File_observability_v1_mads_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_observability_v1_mads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MonitoringAssignment); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_observability_v1_mads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MonitoringAssignment_Target); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_observability_v1_mads_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_observability_v1_mads_proto_goTypes, + DependencyIndexes: file_observability_v1_mads_proto_depIdxs, + MessageInfos: file_observability_v1_mads_proto_msgTypes, + }.Build() + File_observability_v1_mads_proto = out.File + file_observability_v1_mads_proto_rawDesc = nil + file_observability_v1_mads_proto_goTypes = nil + file_observability_v1_mads_proto_depIdxs = nil +} + +// MonitoringAssignmentDiscoveryServiceServer is the server API for MonitoringAssignmentDiscoveryService service. +type MonitoringAssignmentDiscoveryServiceServer interface { + // HTTP + FetchMonitoringAssignments(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) +} diff --git a/discovery/xds/kuma_test.go b/discovery/xds/kuma_test.go new file mode 100644 index 000000000..8db2ce443 --- /dev/null +++ b/discovery/xds/kuma_test.go @@ -0,0 +1,340 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "context" + "fmt" + "testing" + "time" + + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/pkg/errors" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +var ( + kumaConf KumaSDConfig = sdConf + + testKumaMadsV1Resources = []*MonitoringAssignment{ + { + Mesh: "metrics", + Service: "prometheus", + Targets: []*MonitoringAssignment_Target{ + { + Name: "prometheus-01", + Scheme: "http", + Address: "10.1.4.32:9090", + MetricsPath: "/custom-metrics", + Labels: map[string]string{ + "commit_hash": "620506a88", + }, + }, + { + Name: "prometheus-02", + Scheme: "http", + Address: "10.1.4.33:9090", + Labels: map[string]string{ + "commit_hash": "3513bba00", + }, + }, + }, + Labels: map[string]string{ + "kuma.io/zone": "us-east-1", + "team": "infra", + }, + }, + { + Mesh: "metrics", + Service: "grafana", + Targets: []*MonitoringAssignment_Target{}, + Labels: map[string]string{ + "kuma.io/zone": "us-east-1", + "team": "infra", + }, + }, + { + Mesh: "data", + Service: "elasticsearch", + Targets: []*MonitoringAssignment_Target{ + { + Name: "elasticsearch-01", + Scheme: "http", + Address: "10.1.1.1", + Labels: map[string]string{ + "role": "ml", + }, + }, + }, + }, + } +) + +func getKumaMadsV1DiscoveryResponse(resources ...*MonitoringAssignment) (*v3.DiscoveryResponse, error) { + serialized := make([]*anypb.Any, len(resources)) + for i, res := range resources { + data, err := proto.Marshal(res) + + if err != nil { + return nil, err + } + + serialized[i] = &anypb.Any{ + TypeUrl: KumaMadsV1ResourceTypeURL, + Value: data, + } + } + return &v3.DiscoveryResponse{ + TypeUrl: KumaMadsV1ResourceTypeURL, + Resources: serialized, + }, nil +} + +func newKumaTestHTTPDiscovery(c KumaSDConfig) (*fetchDiscovery, error) { + kd, err := NewKumaHTTPDiscovery(&c, nopLogger) + if err != nil { + return nil, err + } + + pd, ok := kd.(*fetchDiscovery) + if !ok { + return nil, errors.New("not a fetchDiscovery") + } + return pd, nil +} + +func TestKumaMadsV1ResourceParserInvalidTypeURL(t *testing.T) { + resources := make([]*anypb.Any, 0) + groups, err := kumaMadsV1ResourceParser(resources, "type.googleapis.com/some.api.v1.Monitoring") + require.Nil(t, groups) + require.Error(t, err) +} + +func TestKumaMadsV1ResourceParserEmptySlice(t *testing.T) { + resources := make([]*anypb.Any, 0) + groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL) + require.Len(t, groups, 0) + require.NoError(t, err) +} + +func TestKumaMadsV1ResourceParserValidResources(t *testing.T) { + res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...) + require.NoError(t, err) + + groups, err := kumaMadsV1ResourceParser(res.Resources, KumaMadsV1ResourceTypeURL) + require.NoError(t, err) + require.Len(t, groups, 3) + + expectedGroup1 := &targetgroup.Group{ + Targets: []model.LabelSet{ + { + "__address__": "10.1.4.32:9090", + "__meta_kuma_label_commit_hash": "620506a88", + "__meta_kuma_dataplane": "prometheus-01", + "__metrics_path__": "/custom-metrics", + "__scheme__": "http", + "instance": "prometheus-01", + }, + { + "__address__": "10.1.4.33:9090", + "__meta_kuma_label_commit_hash": "3513bba00", + "__meta_kuma_dataplane": "prometheus-02", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "prometheus-02", + }, + }, + Labels: model.LabelSet{ + "__meta_kuma_mesh": "metrics", + "__meta_kuma_service": "prometheus", + "__meta_kuma_label_team": "infra", + "__meta_kuma_label_kuma_io_zone": "us-east-1", + }, + } + require.Equal(t, expectedGroup1, groups[0]) + + expectedGroup2 := &targetgroup.Group{ + Labels: model.LabelSet{ + "__meta_kuma_mesh": "metrics", + "__meta_kuma_service": "grafana", + "__meta_kuma_label_team": "infra", + "__meta_kuma_label_kuma_io_zone": "us-east-1", + }, + } + require.Equal(t, expectedGroup2, groups[1]) + + expectedGroup3 := &targetgroup.Group{ + Targets: []model.LabelSet{ + { + "__address__": "10.1.1.1", + "__meta_kuma_label_role": "ml", + "__meta_kuma_dataplane": "elasticsearch-01", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "elasticsearch-01", + }, + }, + Labels: model.LabelSet{ + "__meta_kuma_mesh": "data", + "__meta_kuma_service": "elasticsearch", + }, + } + require.Equal(t, expectedGroup3, groups[2]) +} + +func TestKumaMadsV1ResourceParserInvalidResources(t *testing.T) { + data, err := protoJSONMarshalOptions.Marshal(&MonitoringAssignment_Target{}) + require.NoError(t, err) + + resources := []*anypb.Any{{ + TypeUrl: KumaMadsV1ResourceTypeURL, + Value: data, + }} + groups, err := kumaMadsV1ResourceParser(resources, KumaMadsV1ResourceTypeURL) + require.Nil(t, groups) + require.Error(t, err) + + require.Contains(t, err.Error(), "cannot parse") +} + +func TestNewKumaHTTPDiscovery(t *testing.T) { + kd, err := newKumaTestHTTPDiscovery(kumaConf) + require.NoError(t, err) + require.NotNil(t, kd) + + resClient, ok := kd.client.(*HTTPResourceClient) + require.True(t, ok) + require.Equal(t, kumaConf.Server, resClient.Server()) + require.Equal(t, KumaMadsV1ResourceTypeURL, resClient.ResourceTypeURL()) + require.NotEmpty(t, resClient.ID()) + require.Equal(t, KumaMadsV1ResourceType, resClient.config.ResourceType) +} + +func TestKumaHTTPDiscoveryRefresh(t *testing.T) { + s := createTestHTTPServer(t, func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { + if request.VersionInfo == "1" { + return nil, nil + } + + res, err := getKumaMadsV1DiscoveryResponse(testKumaMadsV1Resources...) + require.NoError(t, err) + + res.VersionInfo = "1" + res.Nonce = "abc" + + return res, nil + }) + defer s.Close() + + cfgString := fmt.Sprintf(` +--- +server: %s +refresh_interval: 10s +tls_config: + insecure_skip_verify: true +`, s.URL) + + var cfg KumaSDConfig + require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg)) + + kd, err := newKumaTestHTTPDiscovery(cfg) + require.NoError(t, err) + require.NotNil(t, kd) + + ch := make(chan []*targetgroup.Group, 1) + kd.poll(context.Background(), ch) + + groups := <-ch + require.Len(t, groups, 3) + + expectedGroup1 := &targetgroup.Group{ + Source: "kuma", + Targets: []model.LabelSet{ + { + "__address__": "10.1.4.32:9090", + "__meta_kuma_label_commit_hash": "620506a88", + "__meta_kuma_dataplane": "prometheus-01", + "__metrics_path__": "/custom-metrics", + "__scheme__": "http", + "instance": "prometheus-01", + }, + { + "__address__": "10.1.4.33:9090", + "__meta_kuma_label_commit_hash": "3513bba00", + "__meta_kuma_dataplane": "prometheus-02", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "prometheus-02", + }, + }, + Labels: model.LabelSet{ + "__meta_kuma_mesh": "metrics", + "__meta_kuma_service": "prometheus", + "__meta_kuma_label_team": "infra", + "__meta_kuma_label_kuma_io_zone": "us-east-1", + }, + } + require.Equal(t, expectedGroup1, groups[0]) + + expectedGroup2 := &targetgroup.Group{ + Source: "kuma", + Labels: model.LabelSet{ + "__meta_kuma_mesh": "metrics", + "__meta_kuma_service": "grafana", + "__meta_kuma_label_team": "infra", + "__meta_kuma_label_kuma_io_zone": "us-east-1", + }, + } + require.Equal(t, expectedGroup2, groups[1]) + + expectedGroup3 := &targetgroup.Group{ + Source: "kuma", + Targets: []model.LabelSet{ + { + "__address__": "10.1.1.1", + "__meta_kuma_label_role": "ml", + "__meta_kuma_dataplane": "elasticsearch-01", + "__metrics_path__": "", + "__scheme__": "http", + "instance": "elasticsearch-01", + }, + }, + Labels: model.LabelSet{ + "__meta_kuma_mesh": "data", + "__meta_kuma_service": "elasticsearch", + }, + } + require.Equal(t, expectedGroup3, groups[2]) + + // Should skip the next update. + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1 * time.Second) + cancel() + }() + + kd.poll(ctx, ch) + select { + case <-ctx.Done(): + return + case <-ch: + require.Fail(t, "no update expected") + } +} diff --git a/discovery/xds/xds.go b/discovery/xds/xds.go new file mode 100644 index 000000000..f99e03da9 --- /dev/null +++ b/discovery/xds/xds.go @@ -0,0 +1,176 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "context" + "github.com/prometheus/common/model" + "time" + + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + // Constants for instrumentation. + namespace = "prometheus" +) + +// ProtocolVersion is the xDS protocol version. +type ProtocolVersion string + +const ( + ProtocolV3 = ProtocolVersion("v3") +) + +type HTTPConfig struct { + config.HTTPClientConfig `yaml:",inline"` +} + +// SDConfig is a base config for xDS-based SD mechanisms. +type SDConfig struct { + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + FetchTimeout model.Duration `yaml:"fetch_timeout,omitempty"` + Server string `yaml:"server,omitempty"` +} + +// mustRegisterMessage registers the provided message type in the typeRegistry, and panics +// if there is an error. +func mustRegisterMessage(typeRegistry *protoregistry.Types, mt protoreflect.MessageType) { + if err := typeRegistry.RegisterMessage(mt); err != nil { + panic(err) + } +} + +func init() { + // Register top-level SD Configs. + discovery.RegisterConfig(&KumaSDConfig{}) + + // Register metrics. + prometheus.MustRegister(kumaFetchDuration, kumaFetchSkipUpdateCount, kumaFetchFailuresCount) + + // Register protobuf types that need to be marshalled/ unmarshalled. + mustRegisterMessage(protoTypes, (&v3.DiscoveryRequest{}).ProtoReflect().Type()) + mustRegisterMessage(protoTypes, (&v3.DiscoveryResponse{}).ProtoReflect().Type()) + mustRegisterMessage(protoTypes, (&MonitoringAssignment{}).ProtoReflect().Type()) +} + +var ( + protoTypes = new(protoregistry.Types) + protoUnmarshalOptions = proto.UnmarshalOptions{ + DiscardUnknown: true, // Only want known fields. + Merge: true, // Always using new messages. + Resolver: protoTypes, // Only want known types. + } + protoJSONUnmarshalOptions = protojson.UnmarshalOptions{ + DiscardUnknown: true, // Only want known fields. + Resolver: protoTypes, // Only want known types. + } + protoJSONMarshalOptions = protojson.MarshalOptions{ + UseProtoNames: true, + Resolver: protoTypes, // Only want known types. + } +) + +type resourceParser func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error) + +// fetchDiscovery implements long-polling via xDS Fetch REST-JSON. +type fetchDiscovery struct { + client ResourceClient + source string + + refreshInterval time.Duration + + parseResources resourceParser + logger log.Logger + + fetchDuration prometheus.Observer + fetchSkipUpdateCount prometheus.Counter + fetchFailuresCount prometheus.Counter +} + +func (d *fetchDiscovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { + defer d.client.Close() + + ticker := time.NewTicker(d.refreshInterval) + + for { + select { + case <-ctx.Done(): + ticker.Stop() + return + default: + d.poll(ctx, ch) + <-ticker.C + } + } +} + +func (d *fetchDiscovery) poll(ctx context.Context, ch chan<- []*targetgroup.Group) { + t0 := time.Now() + response, err := d.client.Fetch(ctx) + elapsed := time.Since(t0) + d.fetchDuration.Observe(elapsed.Seconds()) + + // Check the context before in order to exit early. + select { + case <-ctx.Done(): + return + default: + } + + if err != nil { + level.Error(d.logger).Log("msg", "error parsing resources", "err", err) + d.fetchFailuresCount.Inc() + return + } + + if response == nil { + // No update needed. + d.fetchSkipUpdateCount.Inc() + return + } + + parsedGroups, err := d.parseResources(response.Resources, response.TypeUrl) + if err != nil { + level.Error(d.logger).Log("msg", "error parsing resources", "err", err) + d.fetchFailuresCount.Inc() + return + } + + for _, group := range parsedGroups { + group.Source = d.source + } + + level.Debug(d.logger).Log("msg", "updated to version", "version", response.VersionInfo, "groups", len(parsedGroups)) + + // Check the context before sending an update on the channel. + select { + case <-ctx.Done(): + return + case ch <- parsedGroups: + } +} diff --git a/discovery/xds/xds_test.go b/discovery/xds/xds_test.go new file mode 100644 index 000000000..412cbda68 --- /dev/null +++ b/discovery/xds/xds_test.go @@ -0,0 +1,201 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xds + +import ( + "context" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +var ( + sdConf = SDConfig{ + Server: "http://127.0.0.1", + RefreshInterval: model.Duration(10 * time.Second), + } + + testFetchFailuresCount = prometheus.NewCounter( + prometheus.CounterOpts{}) + testFetchSkipUpdateCount = prometheus.NewCounter( + prometheus.CounterOpts{}) + testFetchDuration = prometheus.NewSummary( + prometheus.SummaryOpts{}, + ) +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +type discoveryResponder func(request *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) + +func createTestHTTPServer(t *testing.T, responder discoveryResponder) *httptest.Server { + return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Validate req MIME types. + require.Equal(t, "application/json", r.Header.Get("Content-Type")) + require.Equal(t, "application/json", r.Header.Get("Accept")) + + body, err := ioutil.ReadAll(r.Body) + defer func() { + _, _ = io.Copy(ioutil.Discard, r.Body) + _ = r.Body.Close() + }() + require.NotEmpty(t, body) + require.NoError(t, err) + + // Validate discovery request. + discoveryReq := &v3.DiscoveryRequest{} + err = protoJSONUnmarshalOptions.Unmarshal(body, discoveryReq) + require.NoError(t, err) + + discoveryRes, err := responder(discoveryReq) + if err != nil { + w.WriteHeader(500) + return + } + + if discoveryRes == nil { + w.WriteHeader(304) + return + } + + w.WriteHeader(200) + data, err := protoJSONMarshalOptions.Marshal(discoveryRes) + require.NoError(t, err) + + _, err = w.Write(data) + require.NoError(t, err) + })) +} + +func constantResourceParser(groups []*targetgroup.Group, err error) resourceParser { + return func(resources []*anypb.Any, typeUrl string) ([]*targetgroup.Group, error) { + return groups, err + } +} + +var nopLogger = log.NewNopLogger() + +type testResourceClient struct { + resourceTypeURL string + server string + protocolVersion ProtocolVersion + fetch func(ctx context.Context) (*v3.DiscoveryResponse, error) +} + +func (rc testResourceClient) ResourceTypeURL() string { + return rc.resourceTypeURL +} + +func (rc testResourceClient) Server() string { + return rc.server +} + +func (rc testResourceClient) Fetch(ctx context.Context) (*v3.DiscoveryResponse, error) { + return rc.fetch(ctx) +} + +func (rc testResourceClient) ID() string { + return "test-client" +} + +func (rc testResourceClient) Close() { +} + +func TestPollingRefreshSkipUpdate(t *testing.T) { + rc := &testResourceClient{ + fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + return nil, nil + }, + } + pd := &fetchDiscovery{ + client: rc, + logger: nopLogger, + fetchDuration: testFetchDuration, + fetchFailuresCount: testFetchFailuresCount, + fetchSkipUpdateCount: testFetchSkipUpdateCount, + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1 * time.Second) + cancel() + }() + + ch := make(chan []*targetgroup.Group, 1) + pd.poll(ctx, ch) + select { + case <-ctx.Done(): + return + case <-ch: + require.Fail(t, "no update expected") + } +} + +func TestPollingRefreshAttachesGroupMetadata(t *testing.T) { + server := "http://198.161.2.0" + source := "test" + rc := &testResourceClient{ + server: server, + protocolVersion: ProtocolV3, + fetch: func(ctx context.Context) (*v3.DiscoveryResponse, error) { + return &v3.DiscoveryResponse{}, nil + }, + } + pd := &fetchDiscovery{ + source: source, + client: rc, + logger: nopLogger, + fetchDuration: testFetchDuration, + fetchFailuresCount: testFetchFailuresCount, + fetchSkipUpdateCount: testFetchSkipUpdateCount, + parseResources: constantResourceParser([]*targetgroup.Group{ + {}, + { + Source: "a-custom-source", + Labels: model.LabelSet{ + "__meta_custom_xds_label": "a-value", + }, + }, + }, nil), + } + ch := make(chan []*targetgroup.Group, 1) + pd.poll(context.Background(), ch) + groups := <-ch + require.NotNil(t, groups) + + require.Len(t, groups, 2) + + for _, group := range groups { + require.Equal(t, source, group.Source) + } + + group2 := groups[1] + require.Contains(t, group2.Labels, model.LabelName("__meta_custom_xds_label")) + require.Equal(t, model.LabelValue("a-value"), group2.Labels["__meta_custom_xds_label"]) +} diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index ac1368b0c..36a22d14b 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -248,6 +248,10 @@ http_sd_configs: kubernetes_sd_configs: [ - ... ] +# List of Kuma service discovery configurations. +kuma_sd_configs: + [ - ... ] + # List of Lightsail service discovery configurations. lightsail_sd_configs: [ - ... ] @@ -382,6 +386,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_azure_machine_id`: the machine ID * `__meta_azure_machine_location`: the location the machine runs in * `__meta_azure_machine_name`: the machine name +* `__meta_azure_machine_computer_name`: the machine computer name * `__meta_azure_machine_os_type`: the machine operating system * `__meta_azure_machine_private_ip`: the machine's private IP * `__meta_azure_machine_public_ip`: the machine's public IP if it exists @@ -635,6 +640,9 @@ tls_config: # tasks and services that don't have published ports. [ port: | default = 80 ] +# The host to use if the container is in host networking mode. +[ host_networking_host: | default = "localhost" ] + # Optional filters to limit the discovery process to a subset of available # resources. # The available filters are listed in the upstream documentation: @@ -893,6 +901,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_ec2_ami`: the EC2 Amazon Machine Image * `__meta_ec2_architecture`: the architecture of the instance * `__meta_ec2_availability_zone`: the availability zone in which the instance is running +* `__meta_ec2_availability_zone_id`: the [availability zone ID](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) in which the instance is running * `__meta_ec2_instance_id`: the EC2 instance ID * `__meta_ec2_instance_lifecycle`: the lifecycle of the EC2 instance, set only for 'spot' or 'scheduled' instances, absent otherwise * `__meta_ec2_instance_state`: the state of the EC2 instance @@ -1131,6 +1140,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_gce_metadata_`: each metadata item of the instance * `__meta_gce_network`: the network URL of the instance * `__meta_gce_private_ip`: the private IP address of the instance +* `__meta_gce_interface_ipv4_`: IPv4 address of each named interface * `__meta_gce_project`: the GCP project in which the instance is running * `__meta_gce_public_ip`: the public IP address of the instance, if present * `__meta_gce_subnetwork`: the subnetwork URL of the instance @@ -1211,6 +1221,7 @@ The labels below are only available for targets with `role` set to `hcloud`: * `__meta_hetzner_hcloud_disk_size_gb`: the disk size of the server (in GB) * `__meta_hetzner_hcloud_private_ipv4_`: the private ipv4 address of the server within a given network * `__meta_hetzner_hcloud_label_`: each label of the server +* `__meta_hetzner_hcloud_labelpresent_`: `true` for each label of the server The labels below are only available for targets with `role` set to `robot`: @@ -1545,6 +1556,74 @@ for a detailed example of configuring Prometheus for Kubernetes. You may wish to check out the 3rd party [Prometheus Operator](https://github.com/coreos/prometheus-operator), which automates the Prometheus setup on top of Kubernetes. +### `` + +Kuma SD configurations allow retrieving scrape target from the [Kuma](https://kuma.io) control plane. + +This SD discovers "monitoring assignments" based on Kuma [Dataplane Proxies](https://kuma.io/docs/latest/documentation/dps-and-data-model), +via the MADS v1 (Monitoring Assignment Discovery Service) xDS API, and will create a target for each proxy +inside a Prometheus-enabled mesh. + +The following meta labels are available for each target: + +* `__meta_kuma_mesh`: the name of the proxy's Mesh +* `__meta_kuma_dataplane`: the name of the proxy +* `__meta_kuma_service`: the name of the proxy's associated Service +* `__meta_kuma_label_`: each tag of the proxy + +See below for the configuration options for Kuma MonitoringAssignment discovery: + +```yaml +# Address of the Kuma Control Plane's MADS xDS server. +server: + +# The time to wait between polling update requests. +[ refresh_interval: | default = 30s ] + +# The time after which the monitoring assignments are refreshed. +[ fetch_timeout: | default = 2m ] + +# Optional proxy URL. +[ proxy_url: ] + +# TLS configuration. +tls_config: + [ ] + +# Authentication information used to authenticate to the Docker daemon. +# Note that `basic_auth` and `authorization` options are +# mutually exclusive. +# password and password_file are mutually exclusive. + +# Optional HTTP basic authentication information. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional the `Authorization` header configuration. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials with the credentials read from the configured file. + # It is mutually exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration. +# Cannot be used at the same time as basic_auth or authorization. +oauth2: + [ ] + +# Configure whether HTTP requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] +``` + +The [relabeling phase](#relabel_config) is the preferred and more powerful way +to filter proxies and user-defined tags. + ### `` Lightsail SD configurations allow retrieving scrape targets from [AWS Lightsail](https://aws.amazon.com/lightsail/) @@ -1627,7 +1706,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ # Note that `basic_auth` and `authorization` options are # mutually exclusive. # password and password_file are mutually exclusive. -# Note: Linode APIv4 Token must be created with scopes: 'linodes:read_only' and 'ips:read_only' +# Note: Linode APIv4 Token must be created with scopes: 'linodes:read_only', 'ips:read_only', and 'events:read_only' # Optional HTTP basic authentication information, not currently supported by Linode APIv4. basic_auth: diff --git a/docs/configuration/https.md b/docs/configuration/https.md index b1e078cdd..1799739d0 100644 --- a/docs/configuration/https.md +++ b/docs/configuration/https.md @@ -36,6 +36,9 @@ tls_server_config: # Server policy for client authentication. Maps to ClientAuth Policies. # For more detail on clientAuth options: # https://golang.org/pkg/crypto/tls/#ClientAuthType + # + # NOTE: If you want to enable client authentication, you need to use + # RequireAndVerifyClientCert. Other values are insecure. [ client_auth_type: | default = "NoClientCert" ] # CA certificate for client certificate authentication to the server. diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 1cf87d6b3..d99255f01 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -78,9 +78,13 @@ series: # Expanding notation: # 'a+bxc' becomes 'a a+b a+(2*b) a+(3*b) … a+(c*b)' # 'a-bxc' becomes 'a a-b a-(2*b) a-(3*b) … a-(c*b)' +# There are special values to indicate missing and stale samples: +# '_' represents a missing sample from scrape +# 'stale' indicates a stale sample # Examples: # 1. '-2+4x3' becomes '-2 2 6 10' # 2. ' 1-2x4' becomes '1 -1 -3 -5 -7' +# 3. ' 1 _x3 stale' becomes '1 _ _ _ stale' values: ``` diff --git a/docs/disabled_features.md b/docs/feature_flags.md similarity index 98% rename from docs/disabled_features.md rename to docs/feature_flags.md index a9021383f..58d8189c3 100644 --- a/docs/disabled_features.md +++ b/docs/feature_flags.md @@ -1,9 +1,9 @@ --- -title: Disabled Features +title: Feature Flags sort_rank: 11 --- -# Disabled Features +# Feature Flags Here is a list of features that are disabled by default since they are breaking changes or are considered experimental. Their behaviour can change in future releases which will be communicated via the [release changelog](https://github.com/prometheus/prometheus/blob/main/CHANGELOG.md). diff --git a/docs/querying/api.md b/docs/querying/api.md index ba13a32b8..58d5c11eb 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -145,8 +145,8 @@ POST /api/v1/query_range URL query parameters: - `query=`: Prometheus expression query string. -- `start=`: Start timestamp. -- `end=`: End timestamp. +- `start=`: Start timestamp, inclusive. +- `end=`: End timestamp, inclusive. - `step=`: Query resolution step width in `duration` format or float number of seconds. - `timeout=`: Evaluation timeout. Optional. Defaults to and is capped by the value of the `-query.timeout` flag. diff --git a/docs/querying/functions.md b/docs/querying/functions.md index 848908ca9..079a1e320 100644 --- a/docs/querying/functions.md +++ b/docs/querying/functions.md @@ -430,6 +430,7 @@ over time and return an instant vector with per-series aggregation results: * `stddev_over_time(range-vector)`: the population standard deviation of the values in the specified interval. * `stdvar_over_time(range-vector)`: the population standard variance of the values in the specified interval. * `last_over_time(range-vector)`: the most recent point value in specified interval. +* `present_over_time(range-vector)`: the value 1 for any series in the specified interval. Note that all values in the specified interval have the same weight in the aggregation even if the values are not equally spaced throughout the interval. diff --git a/docs/storage.md b/docs/storage.md index c4907959d..85193ecf2 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -82,7 +82,7 @@ Prometheus has several flags that configure local storage. The most important ar * `--storage.tsdb.path`: Where Prometheus writes its database. Defaults to `data/`. * `--storage.tsdb.retention.time`: When to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default. -* `--storage.tsdb.retention.size`: [EXPERIMENTAL] The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. This flag is experimental and may change in future releases. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB" +* `--storage.tsdb.retention.size`: The maximum number of bytes of storage blocks to retain. The oldest data will be removed first. Defaults to `0` or disabled. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB" * `--storage.tsdb.retention`: Deprecated in favor of `storage.tsdb.retention.time`. * `--storage.tsdb.wal-compression`: Enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL. @@ -157,6 +157,16 @@ promtool tsdb create-blocks-from openmetrics [] After the creation of the blocks, move it to the data directory of Prometheus. If there is an overlap with the existing blocks in Prometheus, the flag `--storage.tsdb.allow-overlapping-blocks` needs to be set. Note that any backfilled data is subject to the retention configured for your Prometheus server (by time or size). +#### Longer Block Durations + +By default, the promtool will use the default block duration (2h) for the blocks; this behavior is the most generally applicable and correct. However, when backfilling data over a long range of times, it may be advantageous to use a larger value for the block duration to backfill faster and prevent additional compactions by TSDB later. + +The `--max-block-duration` flag allows the user to configure a maximum duration of blocks. The backfilling tool will pick a suitable block duration no larger than this. + +While larger blocks may improve the performance of backfilling large datasets, drawbacks exist as well. Time-based retention policies must keep the entire block around if even one sample of the (potentially large) block is still within the retention policy. Conversely, size-based retention policies will remove the entire block even if the TSDB only goes over the size limit in a minor way. + +Therefore, backfilling with few blocks, thereby choosing a larger block duration, must be done with care and is not recommended for any production instances. + ## Backfilling for Recording Rules ### Overview diff --git a/documentation/examples/custom-sd/adapter-usage/main.go b/documentation/examples/custom-sd/adapter-usage/main.go index 6a50c38ae..2214a823d 100644 --- a/documentation/examples/custom-sd/adapter-usage/main.go +++ b/documentation/examples/custom-sd/adapter-usage/main.go @@ -100,13 +100,17 @@ func (d *discovery) parseServiceNodes(resp *http.Response, name string) (*target Labels: make(model.LabelSet), } - dec := json.NewDecoder(resp.Body) defer func() { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() }() - err := dec.Decode(&nodes) + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(b, &nodes) if err != nil { return &tgroup, err } @@ -165,8 +169,8 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { continue } - dec := json.NewDecoder(resp.Body) - err = dec.Decode(&srvs) + b, err := ioutil.ReadAll(resp.Body) + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() if err != nil { level.Error(d.logger).Log("msg", "Error reading services list", "err", err) @@ -174,6 +178,14 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { continue } + err = json.Unmarshal(b, &srvs) + resp.Body.Close() + if err != nil { + level.Error(d.logger).Log("msg", "Error parsing services list", "err", err) + time.Sleep(time.Duration(d.refreshInterval) * time.Second) + continue + } + var tgs []*targetgroup.Group // Note that we treat errors when querying specific consul services as fatal for this // iteration of the time.Tick loop. It's better to have some stale targets than an incomplete @@ -191,6 +203,7 @@ func (d *discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { level.Error(d.logger).Log("msg", "Error getting services nodes", "service", name, "err", err) break } + tg, err := d.parseServiceNodes(resp, name) if err != nil { level.Error(d.logger).Log("msg", "Error parsing services nodes", "service", name, "err", err) diff --git a/documentation/examples/prometheus-linode.yml b/documentation/examples/prometheus-linode.yml index 315675071..a19b2c3a8 100644 --- a/documentation/examples/prometheus-linode.yml +++ b/documentation/examples/prometheus-linode.yml @@ -11,7 +11,7 @@ scrape_configs: - job_name: "node" linode_sd_configs: - authorization: - credentials: "" + credentials: "" relabel_configs: # Only scrape targets that have a tag 'monitoring'. - source_labels: [__meta_linode_tags] diff --git a/go.mod b/go.mod index 729add1fe..cce0344e8 100644 --- a/go.mod +++ b/go.mod @@ -3,83 +3,82 @@ module github.com/prometheus/prometheus go 1.14 require ( - github.com/Azure/azure-sdk-for-go v55.2.0+incompatible + github.com/Azure/azure-sdk-for-go v55.8.0+incompatible github.com/Azure/go-autorest/autorest v0.11.19 github.com/Azure/go-autorest/autorest/adal v0.9.14 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect - github.com/Microsoft/go-winio v0.4.16 // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 - github.com/aws/aws-sdk-go v1.38.60 + github.com/aws/aws-sdk-go v1.40.10 github.com/cespare/xxhash/v2 v2.1.1 - github.com/containerd/containerd v1.4.3 // indirect + github.com/containerd/containerd v1.5.4 // indirect + github.com/dennwc/varint v1.0.0 github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 - github.com/digitalocean/godo v1.62.0 - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/digitalocean/godo v1.64.2 github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/edsrzf/mmap-go v1.0.0 + github.com/envoyproxy/go-control-plane v0.9.9 + github.com/envoyproxy/protoc-gen-validate v0.6.1 github.com/go-kit/log v0.1.0 github.com/go-logfmt/logfmt v0.5.0 github.com/go-openapi/strfmt v0.20.1 github.com/go-zookeeper/zk v1.0.2 github.com/gogo/protobuf v1.3.2 - github.com/golang/snappy v0.0.3 - github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9 - github.com/gophercloud/gophercloud v0.18.0 + github.com/golang/snappy v0.0.4 + github.com/google/pprof v0.0.0-20210726183535-c50bf4fe5303 + github.com/gophercloud/gophercloud v0.19.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/hashicorp/consul/api v1.8.1 - github.com/hetznercloud/hcloud-go v1.26.2 - github.com/influxdata/influxdb v1.9.2 + github.com/hashicorp/consul/api v1.9.1 + github.com/hetznercloud/hcloud-go v1.28.0 + github.com/influxdata/influxdb v1.9.3 github.com/json-iterator/go v1.1.11 - github.com/linode/linodego v0.28.5 - github.com/miekg/dns v1.1.42 + github.com/linode/linodego v0.31.0 + github.com/miekg/dns v1.1.43 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.1.0 github.com/oklog/ulid v1.3.1 - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opentracing-contrib/go-stdlib v1.0.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/alertmanager v0.22.2 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.29.0 - github.com/prometheus/exporter-toolkit v0.5.1 + github.com/prometheus/common v0.30.0 + github.com/prometheus/exporter-toolkit v0.6.1 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 github.com/stretchr/testify v1.7.0 github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible - go.uber.org/atomic v1.8.0 + go.uber.org/atomic v1.9.0 go.uber.org/goleak v1.1.10 - golang.org/x/net v0.0.0-20210610132358-84b48f89b13b - golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 - golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 - golang.org/x/tools v0.1.3 - google.golang.org/api v0.48.0 + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac + golang.org/x/tools v0.1.5 + google.golang.org/api v0.51.0 + google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea + google.golang.org/protobuf v1.27.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/fsnotify/fsnotify.v1 v1.4.7 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - gotest.tools/v3 v3.0.3 // indirect - k8s.io/api v0.21.1 - k8s.io/apimachinery v0.21.1 - k8s.io/client-go v0.21.1 + k8s.io/api v0.21.3 + k8s.io/apimachinery v0.21.3 + k8s.io/client-go v0.21.3 k8s.io/klog v1.0.0 - k8s.io/klog/v2 v2.9.0 + k8s.io/klog/v2 v2.10.0 ) replace ( k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0 - k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v2 v2.1.0 + k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.0.0 ) // Exclude linodego v1.0.0 as it is no longer published on github. diff --git a/go.sum b/go.sum index c0e1b2894..1d9c4a7f0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -19,8 +20,10 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0 h1:bAMqZidYkmIsUqe6PtkEPT7Q+vfizScn+jfNA6jwK9c= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0 h1:8ZtzmY4a2JIO2sljMbpqkDYxA8aJQveYr3AMa+X40oc= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -42,17 +45,20 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v41.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v55.2.0+incompatible h1:TL2/vJWJEPOrmv97nHcbvjXES0Ntlb9P95hqGA1J2dU= -github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v55.8.0+incompatible h1:EuccMPzxu67cIE95/mrtwQivLv7ETmURi5IUgLNVug8= +github.com/Azure/azure-sdk-for-go v55.8.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.10.1/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.19 h1:7/IqD2fEYVha1EPeaiytVKhzmPV223pfkRIQUGOK2IE= github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= @@ -61,6 +67,7 @@ github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= @@ -74,6 +81,7 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= @@ -94,13 +102,30 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= -github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= +github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I= +github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -111,6 +136,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -125,6 +151,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -134,6 +161,7 @@ github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QN github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3 h1:a9F4rlj7EWWrbj7BYw8J8+x+ZZkJeqzNyRk8hdPF+ro= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -147,23 +175,34 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go v1.38.60 h1:MgyEsX0IMwivwth1VwEnesBpH0vxbjp5a0w1lurMOXY= -github.com/aws/aws-sdk-go v1.38.60/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.40.10 h1:h+xUINuuH/9CwxE7O8mAuW7Aj9E5agfE9jQ3DrJsnA8= +github.com/aws/aws-sdk-go v1.40.10/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0/go.mod h1:J4Y6YJm0qTWB9aFziB7cPeSyc6dOZFyJdteSeybVpXQ= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cactus/go-statsd-client/statsd v0.0.0-20191106001114-12b4e2b38748/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -176,9 +215,14 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -186,43 +230,157 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rnb5MREYqG8ixi5+WbeUsquF0c= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.4 h1:uPF0og3ByFzDnaStfiQj3fVGTEtaSNyU+bW7GR/nqGA= +github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245 h1:9cOfvEwjQxdwKuNDTQSaMKNRvwKwgZG+U4HrjeRKHso= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.62.0 h1:7Gw2KFsWkxl36qJa0s50tgXaE0Cgm51JdRP+MFQvNnM= -github.com/digitalocean/godo v1.62.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/digitalocean/godo v1.64.2 h1:lJEB2TVIkJydFWJMPtdYOPa2Xwib+smZqq/oUZF8/iA= +github.com/digitalocean/godo v1.64.2/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -236,6 +394,7 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -243,7 +402,12 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.9 h1:vQLjymTobffN2R0F8eTqw6q7iozfRO5Z0m+/4Vw+/uA= +github.com/envoyproxy/go-control-plane v0.9.9/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.1 h1:4CF52PCseTFt4bE+Yk3dIpdVi7XWuPVMhPtm4FaIJPM= +github.com/envoyproxy/protoc-gen-validate v0.6.1/go.mod h1:txg5va2Qkip90uYoSKH+nkAAmXrb2j3iq4FLwdrCbXQ= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -256,8 +420,13 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -266,9 +435,11 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -279,6 +450,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -413,14 +585,21 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -430,6 +609,7 @@ github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgR github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -442,6 +622,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -463,12 +644,15 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -504,13 +688,15 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9 h1:2tft2559dNwKl2znYB58oVTql0grRB+Ml3LWIBbc4WM= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210726183535-c50bf4fe5303 h1:UP+GUfRSErTzM7UtTMt4TjnfXKK3ybrGQasgSOp7vLY= +github.com/google/pprof v0.0.0-20210726183535-c50bf4fe5303/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -520,30 +706,37 @@ github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyyc github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= -github.com/gophercloud/gophercloud v0.18.0 h1:V6hcuMPmjXg+js9flU8T3RIHDCjV7F5CG5GD0MRhP/w= -github.com/gophercloud/gophercloud v0.18.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= +github.com/gophercloud/gophercloud v0.19.0 h1:zzaIh8W2K5M4AkJhPV1z6O4Sp0FOObzXm61NUmFz3Kw= +github.com/gophercloud/gophercloud v0.19.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= -github.com/hashicorp/consul/api v1.8.1 h1:BOEQaMWoGMhmQ29fC26bi0qb7/rId9JzZP2V0Xmx7m8= -github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk= +github.com/hashicorp/consul/api v1.9.1 h1:SngrdG2L62qqLsUz85qcPhFZ78rPf8tcD5qjMgs6MME= +github.com/hashicorp/consul/api v1.9.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= -github.com/hashicorp/consul/sdk v0.7.0 h1:H6R9d008jDcHPQPAqPNuydAshJ4v5/8URdFnUvK/+sc= -github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= +github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -557,6 +750,7 @@ github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5b github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= @@ -577,6 +771,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= @@ -590,37 +785,44 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.9.0/go.mod h1:YL0HO+FifKOW2u1ke99DGVu1zhcpZzNwrLIqBC7vbYU= github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hetznercloud/hcloud-go v1.26.2 h1:fI8BXAGJI4EFeCDd2a/I4EhqyK32cDdxGeWfYMGUi50= -github.com/hetznercloud/hcloud-go v1.26.2/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hetznercloud/hcloud-go v1.28.0 h1:T2a0CVGETf7BoWIdZ/TACqmTZAa/ROutcfdUHYiPAQ4= +github.com/hetznercloud/hcloud-go v1.28.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.0/go.mod h1:BwN2XG2lMszOoquQaFdPET8FRQfrXiZsWmcMO9rkaVY= -github.com/influxdata/flux v0.113.0/go.mod h1:3TJtvbm/Kwuo5/PEo5P6HUzwVg4bXWkb2wPQHPtQdlU= +github.com/influxdata/flux v0.120.1/go.mod h1:pGSAvyAA5d3et7SSzajaYShWYXmnRnJJq2qWi+WWZ2I= github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69/go.mod h1:pwymjR6SrP3gD3pRj9RJwdl1j5s3doEEV8gS4X9qSzA= github.com/influxdata/influxdb v1.8.0/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ= -github.com/influxdata/influxdb v1.9.2 h1:yDwzIu/R8Q8sZF5R7+69/7/Qie5DrMetbCsjS5BnRtw= -github.com/influxdata/influxdb v1.9.2/go.mod h1:UEe3MeD9AaP5rlPIes102IhYua3FhIWZuOXNHxDjSrI= +github.com/influxdata/influxdb v1.9.3 h1:60F7eqotCxogyuZAjNglNRG9D6WY65KR9mcmugBx6cs= +github.com/influxdata/influxdb v1.9.3/go.mod h1:xD4ZjAgEJQO9/bX3NhFrssKtdNPi+ki1kjrttJRDhGc= +github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= github.com/influxdata/influxql v1.1.1-0.20210223160523-b6ab99450c93/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/pkg-config v0.2.6/go.mod h1:EMS7Ll0S4qkzDk53XS3Z72/egBsPInt+BeRxb0WeSwk= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/pkg-config v0.2.7/go.mod h1:EMS7Ll0S4qkzDk53XS3Z72/egBsPInt+BeRxb0WeSwk= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -658,16 +860,19 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -675,12 +880,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linode/linodego v0.28.5 h1:JaCziTxHJ7a01MYyjHqskRc8zXQxXOddwrDeoQ2rBnw= -github.com/linode/linodego v0.28.5/go.mod h1:BR0gVkCJffEdIGJSl6bHR80Ty+Uvg/2jkjmrWaFectM= +github.com/linode/linodego v0.31.0 h1:99+t6GtTaIsL+ncz5BpCbkh8Y90rC7qW1Not8MFe0Gw= +github.com/linode/linodego v0.31.0/go.mod h1:BR0gVkCJffEdIGJSl6bHR80Ty+Uvg/2jkjmrWaFectM= +github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -692,30 +901,40 @@ github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY= -github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5/go.mod h1:JWhYAp2EXqUtsxTKdeGlY8Wp44M7VxThC9FEoNGi2IE= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -734,7 +953,13 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -746,8 +971,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -759,8 +986,11 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= @@ -768,21 +998,45 @@ github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DV github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= @@ -803,8 +1057,10 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= @@ -813,21 +1069,27 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg= github.com/prometheus/alertmanager v0.22.2 h1:JrDZalSEMb2/2bqGAhls6ZnvOxbC5jMIu29JV+uWTC0= github.com/prometheus/alertmanager v0.22.2/go.mod h1:rYinOWxFuCnNssc3iOjn2oMTlhLaPcUuqV5yk5JKUAE= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -836,6 +1098,7 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -843,7 +1106,10 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= @@ -853,20 +1119,28 @@ github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.23.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/exporter-toolkit v0.5.1 h1:9eqgis5er9xN613ZSADjypCJaDGj9ZlcWBvsIHa8/3c= +github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= +github.com/prometheus/exporter-toolkit v0.6.1 h1:Aqk75wQD92N9CqmTlZwjKwq6272nOGrWIbc8Z7+xQO0= +github.com/prometheus/exporter-toolkit v0.6.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/prometheus v0.0.0-20200609090129-a6600f564e3c/go.mod h1:S5n0C6tSgdnwWshBUceRx5G1OsjLv/EeZ9t3wIfEtsY= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -879,13 +1153,16 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44 h1:3egqo0Vut6daANFm7tOXdNAa8v5/uLU+sgCJrc88Meo= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -897,35 +1174,50 @@ github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvq github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/simonpasquier/klog-gokit v0.3.0 h1:TkFK21cbwDRS+CiystjqbAiq5ubJcVTk9hLUck5Ntcs= github.com/simonpasquier/klog-gokit v0.3.0/go.mod h1:+SUlDQNrhVtGt2FieaqNftzzk8P72zpWlACateWxA9k= -github.com/simonpasquier/klog-gokit/v2 v2.1.0 h1:EaXk8kgWawwPbRZwaEiijU/lE2s6eD3MCwy5usW3Jgw= -github.com/simonpasquier/klog-gokit/v2 v2.1.0/go.mod h1:VgeTFrwzYYcMH8edEfh3/ai2j/Yg8c/qIm1bkGkhuJg= +github.com/simonpasquier/klog-gokit/v3 v3.0.0 h1:J0QrVhAULISHWN05PeXX/xMqJBjnpl2fAuO8uHdQGsA= +github.com/simonpasquier/klog-gokit/v3 v3.0.0/go.mod h1:+WRhGy707Lp2Q4r727m9Oc7FxazOHgW76FIyCr23nus= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowflakedb/gosnowflake v1.3.4/go.mod h1:NsRq2QeiMUuoNUJhp5Q6xGC4uBrsS9g6LwZVEkTWgsE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -934,41 +1226,71 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/tally v3.3.15+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU= github.com/uber/athenadriver v1.1.4/go.mod h1:tQjho4NzXw55LGfSZEcETuYydpY1vtmixUabHkC1K/E= github.com/uber/jaeger-client-go v2.23.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.28.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.29.1+incompatible h1:R9ec3zO3sGpzs0abd43Y+fBZRJ9uiH6lXyR/+u6brW4= github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -980,6 +1302,7 @@ go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -990,12 +1313,14 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4= -go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1008,8 +1333,10 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1022,6 +1349,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1030,11 +1358,15 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1047,6 +1379,7 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1081,6 +1414,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1093,8 +1427,10 @@ golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1118,7 +1454,9 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1133,8 +1471,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1146,8 +1485,9 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1181,31 +1521,46 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1216,31 +1571,44 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 h1:faDu4veV+8pcThn4fewv6TVlNCezafGoC1gM/mxQLbQ= -golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -1260,9 +1628,11 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1325,6 +1695,7 @@ golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1338,8 +1709,10 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1348,9 +1721,13 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1372,8 +1749,10 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0 h1:RDAPWfNFY06dffEXfn7hZF5Fr1ZbnChzfQZAPyBd1+I= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0 h1:SQaA2Cx57B+iPw2MBgyjEkoeMkRK2IenSGoia0U3lCk= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1383,11 +1762,13 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -1400,6 +1781,7 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1418,6 +1800,7 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1428,8 +1811,13 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 h1:pc16UedxnxXXtGxHCSUhafAoVHQZ0yXl8ZelMH4EETc= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea h1:8ZyCcgugUqamxp/vZSEJw9CMy7VZlSWYJLLJPi/dSDA= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1447,6 +1835,7 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= @@ -1457,8 +1846,9 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1471,25 +1861,32 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1509,6 +1906,8 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -1521,19 +1920,40 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.17.5/go.mod h1:0zV5/ungglgy2Rlm3QK8fbxkXVs+BSJWpJP/+8gUVLY= -k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= -k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= +k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/apimachinery v0.17.5/go.mod h1:ioIo1G/a+uONV7Tv+ZmCbMG1/a3kVw5YcDdncd8ugQ0= -k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= -k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII= +k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/client-go v0.17.5/go.mod h1:S8uZpBpjJJdEH/fEyxcqg7Rn0P5jH+ilkgBHjriSmNo= -k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= -k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.21.3 h1:J9nxZTOmvkInRDCzcSNQmPJbDYN/PjlxXT9Mos3HcLg= +k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= @@ -1542,10 +1962,13 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index 0e66f826c..1e7854ba1 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -125,8 +125,16 @@ func TestHandlerSendAll(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) return } + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + err = errors.Errorf("error reading body: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + var alerts []*Alert - err = json.NewDecoder(r.Body).Decode(&alerts) + err = json.Unmarshal(b, &alerts) if err == nil { err = alertsEqual(expected, alerts) } @@ -322,7 +330,13 @@ func TestHandlerQueuing(t *testing.T) { select { case expected := <-expectedc: var alerts []*Alert - err := json.NewDecoder(r.Body).Decode(&alerts) + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + + err = json.Unmarshal(b, &alerts) if err == nil { err = alertsEqual(expected, alerts) } diff --git a/pkg/textparse/openmetricsparse.go b/pkg/textparse/openmetricsparse.go index b5925fa58..81a6bd7df 100644 --- a/pkg/textparse/openmetricsparse.go +++ b/pkg/textparse/openmetricsparse.go @@ -343,6 +343,9 @@ func (p *OpenMetricsParser) Next() (Entry, error) { if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { return EntryInvalid, err } + if math.IsNaN(ts) || math.IsInf(ts, 0) { + return EntryInvalid, errors.New("invalid timestamp") + } p.ts = int64(ts * 1000) switch t3 := p.nextToken(); t3 { case tLinebreak: @@ -399,6 +402,9 @@ func (p *OpenMetricsParser) parseComment() error { if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { return err } + if math.IsNaN(ts) || math.IsInf(ts, 0) { + return errors.New("invalid exemplar timestamp") + } p.exemplarTs = int64(ts * 1000) switch t3 := p.nextToken(); t3 { case tLinebreak: diff --git a/pkg/textparse/openmetricsparse_test.go b/pkg/textparse/openmetricsparse_test.go index 0ae06702b..5d8c94369 100644 --- a/pkg/textparse/openmetricsparse_test.go +++ b/pkg/textparse/openmetricsparse_test.go @@ -502,6 +502,30 @@ func TestOpenMetricsParseErrors(t *testing.T) { input: `{b="c",} 1`, err: `"INVALID" "{" is not a valid start token`, }, + { + input: `a 1 NaN`, + err: `invalid timestamp`, + }, + { + input: `a 1 -Inf`, + err: `invalid timestamp`, + }, + { + input: `a 1 Inf`, + err: `invalid timestamp`, + }, + { + input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 NaN", + err: `invalid exemplar timestamp`, + }, + { + input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 -Inf", + err: `invalid exemplar timestamp`, + }, + { + input: "# TYPE hhh histogram\nhhh_bucket{le=\"+Inf\"} 1 # {aa=\"bb\"} 4 Inf", + err: `invalid exemplar timestamp`, + }, } for i, c := range cases { diff --git a/promql/engine.go b/promql/engine.go index 46715fbb3..44290f163 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -117,6 +117,8 @@ type Query interface { Stats() *stats.QueryTimers // Cancel signals that a running query execution should be aborted. Cancel() + // String returns the original query string. + String() string } // query implements the Query interface. @@ -141,10 +143,17 @@ type query struct { type QueryOrigin struct{} // Statement implements the Query interface. +// Calling this after Exec may result in panic, +// see https://github.com/prometheus/prometheus/issues/8949. func (q *query) Statement() parser.Statement { return q.stmt } +// String implements the Query interface. +func (q *query) String() string { + return q.q +} + // Stats implements the Query interface. func (q *query) Stats() *stats.QueryTimers { return q.stats @@ -521,8 +530,6 @@ func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storag // Cancel when execution is done or an error was raised. defer q.cancel() - const env = "query execution" - evalSpanTimer, ctx := q.stats.GetSpanTimer(ctx, stats.EvalTotalTime) defer evalSpanTimer.Finish() diff --git a/promql/engine_test.go b/promql/engine_test.go index e6b00c84e..d5734439d 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -184,8 +184,10 @@ func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) stor func (*errQuerier) LabelValues(string, ...*labels.Matcher) ([]string, storage.Warnings, error) { return nil, nil, nil } -func (*errQuerier) LabelNames() ([]string, storage.Warnings, error) { return nil, nil, nil } -func (*errQuerier) Close() error { return nil } +func (*errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { + return nil, nil, nil +} +func (*errQuerier) Close() error { return nil } // errSeriesSet implements storage.SeriesSet which always returns error. type errSeriesSet struct { diff --git a/promql/functions.go b/promql/functions.go index e497be364..d72b4caf6 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -513,6 +513,13 @@ func funcAbsentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalN }) } +// === present_over_time(Vector parser.ValueTypeMatrix) Vector === +func funcPresentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + return aggrOverTime(vals, enh, func(values []Point) float64 { + return 1 + }) +} + func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float64) Vector { for _, el := range vals[0].(Vector) { enh.Out = append(enh.Out, Sample{ @@ -959,6 +966,7 @@ var FunctionCalls = map[string]FunctionCall{ "minute": funcMinute, "month": funcMonth, "predict_linear": funcPredictLinear, + "present_over_time": funcPresentOverTime, "quantile_over_time": funcQuantileOverTime, "rate": funcRate, "resets": funcResets, diff --git a/promql/parser/functions.go b/promql/parser/functions.go index a127cd28a..da5d279f3 100644 --- a/promql/parser/functions.go +++ b/promql/parser/functions.go @@ -39,6 +39,11 @@ var Functions = map[string]*Function{ ArgTypes: []ValueType{ValueTypeMatrix}, ReturnType: ValueTypeVector, }, + "present_over_time": { + Name: "present_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, "avg_over_time": { Name: "avg_over_time", ArgTypes: []ValueType{ValueTypeMatrix}, diff --git a/promql/test.go b/promql/test.go index acb1eb5ca..dc67b08bf 100644 --- a/promql/test.go +++ b/promql/test.go @@ -583,7 +583,7 @@ func (t *Test) exec(tc testCommand) error { err = cmd.compareResult(vec) } if err != nil { - return errors.Wrapf(err, "error in %s %s (line %d) rande mode", cmd, iq.expr, cmd.line) + return errors.Wrapf(err, "error in %s %s (line %d) range mode", cmd, iq.expr, cmd.line) } } @@ -675,12 +675,21 @@ type LazyLoader struct { queryEngine *Engine context context.Context cancelCtx context.CancelFunc + + opts LazyLoaderOpts +} + +// LazyLoaderOpts are options for the lazy loader. +type LazyLoaderOpts struct { + // Disabled PromQL engine features. + EnableAtModifier, EnableNegativeOffset bool } // NewLazyLoader returns an initialized empty LazyLoader. -func NewLazyLoader(t testutil.T, input string) (*LazyLoader, error) { +func NewLazyLoader(t testutil.T, input string, opts LazyLoaderOpts) (*LazyLoader, error) { ll := &LazyLoader{ - T: t, + T: t, + opts: opts, } err := ll.parse(input) ll.clear() @@ -728,7 +737,8 @@ func (ll *LazyLoader) clear() { MaxSamples: 10000, Timeout: 100 * time.Second, NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(ll.SubqueryInterval) }, - EnableAtModifier: true, + EnableAtModifier: ll.opts.EnableAtModifier, + EnableNegativeOffset: ll.opts.EnableNegativeOffset, } ll.queryEngine = NewEngine(opts) diff --git a/promql/test_test.go b/promql/test_test.go index 170037c30..ec50e5753 100644 --- a/promql/test_test.go +++ b/promql/test_test.go @@ -109,7 +109,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) { } for _, c := range cases { - suite, err := NewLazyLoader(t, c.loadString) + suite, err := NewLazyLoader(t, c.loadString, LazyLoaderOpts{}) require.NoError(t, err) defer suite.Close() diff --git a/promql/testdata/functions.test b/promql/testdata/functions.test index 63b67d181..b216c42c7 100644 --- a/promql/testdata/functions.test +++ b/promql/testdata/functions.test @@ -901,6 +901,66 @@ eval instant at 10m absent_over_time({job="ingress"}[4m]) clear +# Testdata for present_over_time() +eval instant at 1m present_over_time(http_requests[5m]) + +eval instant at 1m present_over_time(http_requests{handler="/foo"}[5m]) + +eval instant at 1m present_over_time(http_requests{handler!="/foo"}[5m]) + +eval instant at 1m present_over_time(http_requests{handler="/foo", handler="/bar", handler="/foobar"}[5m]) + +eval instant at 1m present_over_time(rate(nonexistant[5m])[5m:]) + +eval instant at 1m present_over_time(http_requests{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m]) + +load 1m + http_requests{path="/foo",instance="127.0.0.1",job="httpd"} 1+1x10 + http_requests{path="/bar",instance="127.0.0.1",job="httpd"} 1+1x10 + httpd_handshake_failures_total{instance="127.0.0.1",job="node"} 1+1x15 + httpd_log_lines_total{instance="127.0.0.1",job="node"} 1 + ssl_certificate_expiry_seconds{job="ingress"} NaN NaN NaN NaN NaN + +eval instant at 5m present_over_time(http_requests[5m]) + {instance="127.0.0.1", job="httpd", path="/bar"} 1 + {instance="127.0.0.1", job="httpd", path="/foo"} 1 + +eval instant at 5m present_over_time(rate(http_requests[5m])[5m:1m]) + {instance="127.0.0.1", job="httpd", path="/bar"} 1 + {instance="127.0.0.1", job="httpd", path="/foo"} 1 + +eval instant at 0m present_over_time(httpd_log_lines_total[30s]) + {instance="127.0.0.1",job="node"} 1 + +eval instant at 1m present_over_time(httpd_log_lines_total[30s]) + +eval instant at 15m present_over_time(http_requests[5m]) + {instance="127.0.0.1", job="httpd", path="/bar"} 1 + {instance="127.0.0.1", job="httpd", path="/foo"} 1 + +eval instant at 16m present_over_time(http_requests[5m]) + +eval instant at 16m present_over_time(http_requests[6m]) + {instance="127.0.0.1", job="httpd", path="/bar"} 1 + {instance="127.0.0.1", job="httpd", path="/foo"} 1 + +eval instant at 16m present_over_time(httpd_handshake_failures_total[1m]) + {instance="127.0.0.1", job="node"} 1 + +eval instant at 16m present_over_time({instance="127.0.0.1"}[5m]) + {instance="127.0.0.1",job="node"} 1 + +eval instant at 21m present_over_time({job="grok"}[20m]) + +eval instant at 30m present_over_time({instance="127.0.0.1"}[5m:5s]) + +eval instant at 5m present_over_time({job="ingress"}[4m]) + {job="ingress"} 1 + +eval instant at 10m present_over_time({job="ingress"}[4m]) + +clear + # Testing exp() sqrt() log2() log10() ln() load 5m exp_root_log{l="x"} 10 diff --git a/promql/value.go b/promql/value.go index 50642f439..0936bb6c7 100644 --- a/promql/value.go +++ b/promql/value.go @@ -171,8 +171,8 @@ func (m Matrix) Len() int { return len(m) } func (m Matrix) Less(i, j int) bool { return labels.Compare(m[i].Metric, m[j].Metric) < 0 } func (m Matrix) Swap(i, j int) { m[i], m[j] = m[j], m[i] } -// ContainsSameLabelset checks if a matrix has samples with the same labelset -// Such a behavior is semantically undefined +// ContainsSameLabelset checks if a matrix has samples with the same labelset. +// Such a behavior is semantically undefined. // https://github.com/prometheus/prometheus/issues/4562 func (m Matrix) ContainsSameLabelset() bool { l := make(map[uint64]struct{}, len(m)) diff --git a/scrape/manager.go b/scrape/manager.go index a33c7e179..e86221c66 100644 --- a/scrape/manager.go +++ b/scrape/manager.go @@ -14,11 +14,8 @@ package scrape import ( - "encoding" "fmt" "hash/fnv" - "net" - "os" "reflect" "sync" "time" @@ -32,6 +29,7 @@ import ( "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/osutil" ) var targetMetadataCache = newMetadataMetricsCollector() @@ -119,7 +117,7 @@ func NewManager(logger log.Logger, app storage.Appendable) *Manager { } // Manager maintains a set of scrape pools and manages start/stop cycles -// when receiving new target groups form the discovery manager. +// when receiving new target groups from the discovery manager. type Manager struct { logger log.Logger append storage.Appendable @@ -206,7 +204,7 @@ func (m *Manager) reload() { // setJitterSeed calculates a global jitterSeed per server relying on extra label set. func (m *Manager) setJitterSeed(labels labels.Labels) error { h := fnv.New64a() - hostname, err := getFqdn() + hostname, err := osutil.GetFQDN() if err != nil { return err } @@ -319,46 +317,3 @@ func (m *Manager) TargetsDropped() map[string][]*Target { } return targets } - -// getFqdn returns a FQDN if it's possible, otherwise falls back to hostname. -func getFqdn() (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", err - } - - ips, err := net.LookupIP(hostname) - if err != nil { - // Return the system hostname if we can't look up the IP address. - return hostname, nil - } - - lookup := func(ipStr encoding.TextMarshaler) (string, error) { - ip, err := ipStr.MarshalText() - if err != nil { - return "", err - } - hosts, err := net.LookupAddr(string(ip)) - if err != nil || len(hosts) == 0 { - return "", err - } - return hosts[0], nil - } - - for _, addr := range ips { - if ip := addr.To4(); ip != nil { - if fqdn, err := lookup(ip); err == nil { - return fqdn, nil - } - - } - - if ip := addr.To16(); ip != nil { - if fqdn, err := lookup(ip); err == nil { - return fqdn, nil - } - - } - } - return hostname, nil -} diff --git a/scrape/scrape.go b/scrape/scrape.go index 109d89dc3..865f01e3d 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -226,11 +226,10 @@ type scrapePool struct { cancel context.CancelFunc // mtx must not be taken after targetMtx. - mtx sync.Mutex - config *config.ScrapeConfig - client *http.Client - loops map[uint64]loop - targetLimitHit bool // Internal state to speed up the target_limit checks. + mtx sync.Mutex + config *config.ScrapeConfig + client *http.Client + loops map[uint64]loop targetMtx sync.Mutex // activeTargets and loops must always be synchronized to have the same @@ -575,20 +574,14 @@ func (sp *scrapePool) sync(targets []*Target) { // refreshTargetLimitErr returns an error that can be passed to the scrape loops // if the number of targets exceeds the configured limit. func (sp *scrapePool) refreshTargetLimitErr() error { - if sp.config == nil || sp.config.TargetLimit == 0 && !sp.targetLimitHit { + if sp.config == nil || sp.config.TargetLimit == 0 { return nil } - l := len(sp.activeTargets) - if l <= int(sp.config.TargetLimit) && !sp.targetLimitHit { - return nil - } - var err error - sp.targetLimitHit = l > int(sp.config.TargetLimit) - if sp.targetLimitHit { + if l := len(sp.activeTargets); l > int(sp.config.TargetLimit) { targetScrapePoolExceededTargetLimit.Inc() - err = fmt.Errorf("target_limit exceeded (number of targets: %d, limit: %d)", l, sp.config.TargetLimit) + return fmt.Errorf("target_limit exceeded (number of targets: %d, limit: %d)", l, sp.config.TargetLimit) } - return err + return nil } func verifyLabelLimits(lset labels.Labels, limits *labelLimits) error { diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index d87ed1fae..e462ffcec 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -423,6 +423,10 @@ func TestScrapePoolTargetLimit(t *testing.T) { validateIsRunning() validateErrorMessage(true) + reloadWithLimit(0) + validateIsRunning() + validateErrorMessage(false) + reloadWithLimit(51) validateIsRunning() validateErrorMessage(false) diff --git a/storage/fanout_test.go b/storage/fanout_test.go index 486f60f2f..725ffec49 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -234,7 +234,7 @@ func (errQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]strin return nil, nil, errors.New("label values error") } -func (errQuerier) LabelNames() ([]string, storage.Warnings, error) { +func (errQuerier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { return nil, nil, errors.New("label names error") } diff --git a/storage/interface.go b/storage/interface.go index f90ded1b1..bd4e0c6c7 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -34,6 +34,7 @@ var ( ErrOutOfOrderExemplar = errors.New("out of order exemplar") ErrDuplicateExemplar = errors.New("duplicate exemplar") ErrExemplarLabelLength = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength) + ErrExemplarsDisabled = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0") ) // Appendable allows creating appenders. @@ -113,8 +114,9 @@ type LabelQuerier interface { LabelValues(name string, matchers ...*labels.Matcher) ([]string, Warnings, error) // LabelNames returns all the unique label names present in the block in sorted order. - // TODO(yeya24): support matchers or hints. - LabelNames() ([]string, Warnings, error) + // If matchers are specified the returned result set is reduced + // to label names of metrics matching the matchers. + LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) // Close releases the resources of the Querier. Close() error diff --git a/storage/merge.go b/storage/merge.go index cda1259f3..1ff041c40 100644 --- a/storage/merge.go +++ b/storage/merge.go @@ -219,13 +219,13 @@ func mergeStrings(a, b []string) []string { } // LabelNames returns all the unique label names present in all queriers in sorted order. -func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) { +func (q *mergeGenericQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) { var ( labelNamesMap = make(map[string]struct{}) warnings Warnings ) for _, querier := range q.queriers { - names, wrn, err := querier.LabelNames() + names, wrn, err := querier.LabelNames(matchers...) if wrn != nil { // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) diff --git a/storage/merge_test.go b/storage/merge_test.go index 90f5725e1..d44ffce7c 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -778,7 +778,7 @@ func (m *mockGenericQuerier) LabelValues(name string, matchers ...*labels.Matche return m.resp, m.warnings, m.err } -func (m *mockGenericQuerier) LabelNames() ([]string, Warnings, error) { +func (m *mockGenericQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { m.mtx.Lock() m.labelNamesCalls++ m.mtx.Unlock() diff --git a/storage/noop.go b/storage/noop.go index 3f800e76c..c63353b92 100644 --- a/storage/noop.go +++ b/storage/noop.go @@ -32,7 +32,7 @@ func (noopQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warnings, return nil, nil, nil } -func (noopQuerier) LabelNames() ([]string, Warnings, error) { +func (noopQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { return nil, nil, nil } @@ -55,7 +55,7 @@ func (noopChunkQuerier) LabelValues(string, ...*labels.Matcher) ([]string, Warni return nil, nil, nil } -func (noopChunkQuerier) LabelNames() ([]string, Warnings, error) { +func (noopChunkQuerier) LabelNames(...*labels.Matcher) ([]string, Warnings, error) { return nil, nil, nil } diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index 655886b46..9631de1c9 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -352,10 +352,12 @@ type QueueManager struct { clientMtx sync.RWMutex storeClient WriteClient - seriesMtx sync.Mutex - seriesLabels map[uint64]labels.Labels + seriesMtx sync.Mutex // Covers seriesLabels and droppedSeries. + seriesLabels map[uint64]labels.Labels + droppedSeries map[uint64]struct{} + + seriesSegmentMtx sync.Mutex // Covers seriesSegmentIndexes - if you also lock seriesMtx, take seriesMtx first. seriesSegmentIndexes map[uint64]int - droppedSeries map[uint64]struct{} shards *shards numShards int @@ -642,6 +644,8 @@ func (t *QueueManager) Stop() { func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) { t.seriesMtx.Lock() defer t.seriesMtx.Unlock() + t.seriesSegmentMtx.Lock() + defer t.seriesSegmentMtx.Unlock() for _, s := range series { // Just make sure all the Refs of Series will insert into seriesSegmentIndexes map for tracking. t.seriesSegmentIndexes[s.Ref] = index @@ -664,12 +668,23 @@ func (t *QueueManager) StoreSeries(series []record.RefSeries, index int) { } } +// Update the segment number held against the series, so we can trim older ones in SeriesReset. +func (t *QueueManager) UpdateSeriesSegment(series []record.RefSeries, index int) { + t.seriesSegmentMtx.Lock() + defer t.seriesSegmentMtx.Unlock() + for _, s := range series { + t.seriesSegmentIndexes[s.Ref] = index + } +} + // SeriesReset is used when reading a checkpoint. WAL Watcher should have // stored series records with the checkpoints index number, so we can now // delete any ref ID's lower than that # from the two maps. func (t *QueueManager) SeriesReset(index int) { t.seriesMtx.Lock() defer t.seriesMtx.Unlock() + t.seriesSegmentMtx.Lock() + defer t.seriesSegmentMtx.Unlock() // Check for series that are in segments older than the checkpoint // that were not also present in the checkpoint. for k, v := range t.seriesSegmentIndexes { diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 94546fb1d..ccca000df 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -295,10 +295,10 @@ func TestShutdown(t *testing.T) { // be at least equal to deadline, otherwise the flush deadline // was not respected. duration := time.Since(start) - if duration > time.Duration(deadline+(deadline/10)) { + if duration > deadline+(deadline/10) { t.Errorf("Took too long to shutdown: %s > %s", duration, deadline) } - if duration < time.Duration(deadline) { + if duration < deadline { t.Errorf("Shutdown occurred before flush deadline: %s < %s", duration, deadline) } } @@ -474,7 +474,7 @@ func TestShouldReshard(t *testing.T) { } } -func createTimeseries(numSamples, numSeries int) ([]record.RefSample, []record.RefSeries) { +func createTimeseries(numSamples, numSeries int, extraLabels ...labels.Label) ([]record.RefSample, []record.RefSeries) { samples := make([]record.RefSample, 0, numSamples) series := make([]record.RefSeries, 0, numSeries) for i := 0; i < numSeries; i++ { @@ -488,7 +488,7 @@ func createTimeseries(numSamples, numSeries int) ([]record.RefSample, []record.R } series = append(series, record.RefSeries{ Ref: uint64(i), - Labels: labels.Labels{{Name: "__name__", Value: name}}, + Labels: append(labels.Labels{{Name: "__name__", Value: name}}, extraLabels...), }) } return samples, series @@ -709,10 +709,29 @@ func (c *TestBlockingWriteClient) Endpoint() string { } func BenchmarkSampleDelivery(b *testing.B) { - // Let's create an even number of send batches so we don't run into the - // batch timeout case. - n := config.DefaultQueueConfig.MaxSamplesPerSend * 10 - samples, series := createTimeseries(n, n) + // Send one sample per series, which is the typical remote_write case + const numSamples = 1 + const numSeries = 10000 + + // Extra labels to make a more realistic workload - taken from Kubernetes' embedded cAdvisor metrics. + var extraLabels = labels.Labels{ + {Name: "kubernetes_io_arch", Value: "amd64"}, + {Name: "kubernetes_io_instance_type", Value: "c3.somesize"}, + {Name: "kubernetes_io_os", Value: "linux"}, + {Name: "container_name", Value: "some-name"}, + {Name: "failure_domain_kubernetes_io_region", Value: "somewhere-1"}, + {Name: "failure_domain_kubernetes_io_zone", Value: "somewhere-1b"}, + {Name: "id", Value: "/kubepods/burstable/pod6e91c467-e4c5-11e7-ace3-0a97ed59c75e/a3c8498918bd6866349fed5a6f8c643b77c91836427fb6327913276ebc6bde28"}, + {Name: "image", Value: "registry/organisation/name@sha256:dca3d877a80008b45d71d7edc4fd2e44c0c8c8e7102ba5cbabec63a374d1d506"}, + {Name: "instance", Value: "ip-111-11-1-11.ec2.internal"}, + {Name: "job", Value: "kubernetes-cadvisor"}, + {Name: "kubernetes_io_hostname", Value: "ip-111-11-1-11"}, + {Name: "monitor", Value: "prod"}, + {Name: "name", Value: "k8s_some-name_some-other-name-5j8s8_kube-system_6e91c467-e4c5-11e7-ace3-0a97ed59c75e_0"}, + {Name: "namespace", Value: "kube-system"}, + {Name: "pod_name", Value: "some-other-name-5j8s8"}, + } + samples, series := createTimeseries(numSamples, numSeries, extraLabels...) c := NewTestWriteClient() @@ -736,7 +755,9 @@ func BenchmarkSampleDelivery(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { c.expectDataCount(len(samples)) - m.Append(samples) + go m.Append(samples) + m.UpdateSeriesSegment(series, i+1) // simulate what wal.Watcher.garbageCollectSeries does + m.SeriesReset(i + 1) c.waitForExpectedDataCount() } // Do not include shutdown diff --git a/storage/remote/read.go b/storage/remote/read.go index 94eab01cf..7f1d749e6 100644 --- a/storage/remote/read.go +++ b/storage/remote/read.go @@ -212,7 +212,7 @@ func (q *querier) LabelValues(string, ...*labels.Matcher) ([]string, storage.War } // LabelNames implements storage.Querier and is a noop. -func (q *querier) LabelNames() ([]string, storage.Warnings, error) { +func (q *querier) LabelNames(...*labels.Matcher) ([]string, storage.Warnings, error) { // TODO: Implement: https://github.com/prometheus/prometheus/issues/3351 return nil, nil, errors.New("not implemented") } diff --git a/storage/secondary.go b/storage/secondary.go index 2586a7744..64a83b5e7 100644 --- a/storage/secondary.go +++ b/storage/secondary.go @@ -55,8 +55,8 @@ func (s *secondaryQuerier) LabelValues(name string, matchers ...*labels.Matcher) return vals, w, nil } -func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) { - names, w, err := s.genericQuerier.LabelNames() +func (s *secondaryQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, Warnings, error) { + names, w, err := s.genericQuerier.LabelNames(matchers...) if err != nil { return nil, append([]error{err}, w...), nil } diff --git a/tsdb/block.go b/tsdb/block.go index 23a075c4f..42a91ff59 100644 --- a/tsdb/block.go +++ b/tsdb/block.go @@ -85,13 +85,17 @@ type IndexReader interface { Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error // LabelNames returns all the unique label names present in the index in sorted order. - LabelNames() ([]string, error) + LabelNames(matchers ...*labels.Matcher) ([]string, error) // LabelValueFor returns label value for the given label name in the series referred to by ID. // If the series couldn't be found or the series doesn't have the requested label a // storage.ErrNotFound is returned as error. LabelValueFor(id uint64, label string) (string, error) + // LabelNamesFor returns all the label names for the series referred to by IDs. + // The names returned are sorted. + LabelNamesFor(ids ...uint64) ([]string, error) + // Close releases the underlying resources of the reader. Close() error } @@ -443,7 +447,15 @@ func (r blockIndexReader) LabelValues(name string, matchers ...*labels.Matcher) return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID) } - return labelValuesWithMatchers(r, name, matchers...) + return labelValuesWithMatchers(r.ir, name, matchers...) +} + +func (r blockIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { + if len(matchers) == 0 { + return r.b.LabelNames() + } + + return labelNamesWithMatchers(r.ir, matchers...) } func (r blockIndexReader) Postings(name string, values ...string) (index.Postings, error) { @@ -465,10 +477,6 @@ func (r blockIndexReader) Series(ref uint64, lset *labels.Labels, chks *[]chunks return nil } -func (r blockIndexReader) LabelNames() ([]string, error) { - return r.b.LabelNames() -} - func (r blockIndexReader) Close() error { r.b.pendingReaders.Done() return nil @@ -479,6 +487,12 @@ func (r blockIndexReader) LabelValueFor(id uint64, label string) (string, error) return r.ir.LabelValueFor(id, label) } +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (r blockIndexReader) LabelNamesFor(ids ...uint64) ([]string, error) { + return r.ir.LabelNamesFor(ids...) +} + type blockTombstoneReader struct { tombstones.Reader b *Block diff --git a/tsdb/block_test.go b/tsdb/block_test.go index fc53d02dc..54cfbc2c4 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -418,6 +418,82 @@ func BenchmarkLabelValuesWithMatchers(b *testing.B) { } } +func TestLabelNamesWithMatchers(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "test_block_label_names_with_matchers") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpdir)) }) + + var seriesEntries []storage.Series + for i := 0; i < 100; i++ { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + }, []tsdbutil.Sample{sample{100, 0}})) + + if i%10 == 0 { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + }, []tsdbutil.Sample{sample{100, 0}})) + } + + if i%20 == 0 { + seriesEntries = append(seriesEntries, storage.NewListSeries(labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + {Name: "twenties", Value: fmt.Sprintf("value%d", i/20)}, + }, []tsdbutil.Sample{sample{100, 0}})) + } + + } + + blockDir := createBlock(t, tmpdir, seriesEntries) + files, err := sequenceFiles(chunkDir(blockDir)) + require.NoError(t, err) + require.Greater(t, len(files), 0, "No chunk created.") + + // Check open err. + block, err := OpenBlock(nil, blockDir, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, block.Close()) }) + + indexReader, err := block.Index() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, indexReader.Close()) }) + + testCases := []struct { + name string + labelName string + matchers []*labels.Matcher + expectedNames []string + }{ + { + name: "get with non-empty unique: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get with unique ending in 1: only unique", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value.*1")}, + expectedNames: []string{"unique"}, + }, { + name: "get with unique = value20: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "unique", "value20")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get tens = 1: unique & tens", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "tens", "value1")}, + expectedNames: []string{"tens", "unique"}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + actualNames, err := indexReader.LabelNames(tt.matchers...) + require.NoError(t, err) + require.Equal(t, tt.expectedNames, actualNames) + }) + } +} + // createBlock creates a block with given set of series and returns its dir. func createBlock(tb testing.TB, dir string, series []storage.Series) string { blockDir, err := CreateBlock(series, dir, 0, log.NewNopLogger()) diff --git a/tsdb/chunkenc/bstream.go b/tsdb/chunkenc/bstream.go index ad8077c27..c8efeab1d 100644 --- a/tsdb/chunkenc/bstream.go +++ b/tsdb/chunkenc/bstream.go @@ -95,7 +95,7 @@ func (b *bstream) writeByte(byt byte) { } func (b *bstream) writeBits(u uint64, nbits int) { - u <<= (64 - uint(nbits)) + u <<= 64 - uint(nbits) for nbits >= 8 { byt := byte(u >> 56) b.writeByte(byt) diff --git a/tsdb/chunkenc/chunk_test.go b/tsdb/chunkenc/chunk_test.go index edcd6d2cc..419f5d301 100644 --- a/tsdb/chunkenc/chunk_test.go +++ b/tsdb/chunkenc/chunk_test.go @@ -110,12 +110,13 @@ func testChunk(t *testing.T, c Chunk) { } func benchmarkIterator(b *testing.B, newChunk func() Chunk) { + const samplesPerChunk = 250 var ( t = int64(1234123324) v = 1243535.123 exp []pair ) - for i := 0; i < b.N; i++ { + for i := 0; i < samplesPerChunk; i++ { // t += int64(rand.Intn(10000) + 1) t += int64(1000) // v = rand.Float64() @@ -123,11 +124,9 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) { exp = append(exp, pair{t: t, v: v}) } - var chunks []Chunk - for i := 0; i < b.N; { - c := newChunk() - - a, err := c.Appender() + chunk := newChunk() + { + a, err := chunk.Appender() if err != nil { b.Fatalf("get appender: %s", err) } @@ -137,32 +136,27 @@ func benchmarkIterator(b *testing.B, newChunk func() Chunk) { break } a.Append(p.t, p.v) - i++ j++ } - chunks = append(chunks, c) } b.ReportAllocs() b.ResetTimer() - b.Log("num", b.N, "created chunks", len(chunks)) - - res := make([]float64, 0, 1024) - + var res float64 var it Iterator - for i := 0; i < len(chunks); i++ { - c := chunks[i] - it := c.Iterator(it) + for i := 0; i < b.N; { + it := chunk.Iterator(it) for it.Next() { _, v := it.At() - res = append(res, v) + res = v + i++ } if it.Err() != io.EOF { require.NoError(b, it.Err()) } - res = res[:0] + _ = res } } diff --git a/tsdb/chunkenc/xor.go b/tsdb/chunkenc/xor.go index 99be253e1..24cec61cb 100644 --- a/tsdb/chunkenc/xor.go +++ b/tsdb/chunkenc/xor.go @@ -185,16 +185,16 @@ func (a *xorAppender) Append(t int64, v float64) { case dod == 0: a.b.writeBit(zero) case bitRange(dod, 14): - a.b.writeBits(0x02, 2) // '10' + a.b.writeBits(0b10, 2) a.b.writeBits(uint64(dod), 14) case bitRange(dod, 17): - a.b.writeBits(0x06, 3) // '110' + a.b.writeBits(0b110, 3) a.b.writeBits(uint64(dod), 17) case bitRange(dod, 20): - a.b.writeBits(0x0e, 4) // '1110' + a.b.writeBits(0b1110, 4) a.b.writeBits(uint64(dod), 20) default: - a.b.writeBits(0x0f, 4) // '1111' + a.b.writeBits(0b1111, 4) a.b.writeBits(uint64(dod), 64) } @@ -359,15 +359,15 @@ func (it *xorIterator) Next() bool { var sz uint8 var dod int64 switch d { - case 0x00: + case 0b0: // dod == 0 - case 0x02: + case 0b10: sz = 14 - case 0x06: + case 0b110: sz = 17 - case 0x0e: + case 0b1110: sz = 20 - case 0x0f: + case 0b1111: // Do not use fast because it's very unlikely it will succeed. bits, err := it.br.readBits(64) if err != nil { diff --git a/tsdb/chunks/head_chunks.go b/tsdb/chunks/head_chunks.go index bc6915c80..1443e0bd4 100644 --- a/tsdb/chunks/head_chunks.go +++ b/tsdb/chunks/head_chunks.go @@ -386,7 +386,7 @@ func (cdm *ChunkDiskMapper) cut() (returnErr error) { cdm.readPathMtx.Unlock() } - mmapFile, err := fileutil.OpenMmapFileWithSize(newFile.Name(), int(MaxHeadChunkFileSize)) + mmapFile, err := fileutil.OpenMmapFileWithSize(newFile.Name(), MaxHeadChunkFileSize) if err != nil { return err } diff --git a/tsdb/db.go b/tsdb/db.go index 6507c25f8..9d17b406b 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -35,6 +35,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -147,9 +148,12 @@ type Options struct { // mainly meant for external users who import TSDB. BlocksToDelete BlocksToDeleteFunc + // Enables the in memory exemplar storage,. + EnableExemplarStorage bool + // MaxExemplars sets the size, in # of exemplars stored, of the single circular buffer used to store exemplars in memory. // See tsdb/exemplar.go, specifically the CircularExemplarStorage struct and it's constructor NewCircularExemplarStorage. - MaxExemplars int + MaxExemplars int64 } type BlocksToDeleteFunc func(blocks []*Block) map[ulid.ULID]struct{} @@ -716,7 +720,8 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs headOpts.ChunkWriteBufferSize = opts.HeadChunksWriteBufferSize headOpts.StripeSize = opts.StripeSize headOpts.SeriesCallback = opts.SeriesLifecycleCallback - headOpts.NumExemplars = opts.MaxExemplars + headOpts.EnableExemplarStorage = opts.EnableExemplarStorage + headOpts.MaxExemplars.Store(opts.MaxExemplars) db.head, err = NewHead(r, l, wlog, headOpts, stats.Head) if err != nil { return nil, err @@ -838,6 +843,10 @@ func (db *DB) Appender(ctx context.Context) storage.Appender { return dbAppender{db: db, Appender: db.head.Appender(ctx)} } +func (db *DB) ApplyConfig(conf *config.Config) error { + return db.head.ApplyConfig(conf) +} + // dbAppender wraps the DB's head appender and triggers compactions on commit // if necessary. type dbAppender struct { @@ -1516,8 +1525,32 @@ func (db *DB) Querier(_ context.Context, mint, maxt int64) (storage.Querier, err blocks = append(blocks, b) } } + var headQuerier storage.Querier if maxt >= db.head.MinTime() { - blocks = append(blocks, NewRangeHead(db.head, mint, maxt)) + rh := NewRangeHead(db.head, mint, maxt) + var err error + headQuerier, err = NewBlockQuerier(rh, mint, maxt) + if err != nil { + return nil, errors.Wrapf(err, "open querier for head %s", rh) + } + + // Getting the querier above registers itself in the queue that the truncation waits on. + // So if the querier is currently not colliding with any truncation, we can continue to use it and still + // won't run into a race later since any truncation that comes after will wait on this querier if it overlaps. + shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt) + if shouldClose { + if err := headQuerier.Close(); err != nil { + return nil, errors.Wrapf(err, "closing head querier %s", rh) + } + headQuerier = nil + } + if getNew { + rh := NewRangeHead(db.head, newMint, maxt) + headQuerier, err = NewBlockQuerier(rh, newMint, maxt) + if err != nil { + return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh) + } + } } blockQueriers := make([]storage.Querier, 0, len(blocks)) @@ -1534,6 +1567,9 @@ func (db *DB) Querier(_ context.Context, mint, maxt int64) (storage.Querier, err } return nil, errors.Wrapf(err, "open querier for block %s", b) } + if headQuerier != nil { + blockQueriers = append(blockQueriers, headQuerier) + } return storage.NewMergeQuerier(blockQueriers, nil, storage.ChainedSeriesMerge), nil } @@ -1549,8 +1585,32 @@ func (db *DB) ChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQu blocks = append(blocks, b) } } + var headQuerier storage.ChunkQuerier if maxt >= db.head.MinTime() { - blocks = append(blocks, NewRangeHead(db.head, mint, maxt)) + rh := NewRangeHead(db.head, mint, maxt) + var err error + headQuerier, err = NewBlockChunkQuerier(rh, mint, maxt) + if err != nil { + return nil, errors.Wrapf(err, "open querier for head %s", rh) + } + + // Getting the querier above registers itself in the queue that the truncation waits on. + // So if the querier is currently not colliding with any truncation, we can continue to use it and still + // won't run into a race later since any truncation that comes after will wait on this querier if it overlaps. + shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(mint, maxt) + if shouldClose { + if err := headQuerier.Close(); err != nil { + return nil, errors.Wrapf(err, "closing head querier %s", rh) + } + headQuerier = nil + } + if getNew { + rh := NewRangeHead(db.head, newMint, maxt) + headQuerier, err = NewBlockChunkQuerier(rh, newMint, maxt) + if err != nil { + return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh) + } + } } blockQueriers := make([]storage.ChunkQuerier, 0, len(blocks)) @@ -1567,6 +1627,9 @@ func (db *DB) ChunkQuerier(_ context.Context, mint, maxt int64) (storage.ChunkQu } return nil, errors.Wrapf(err, "open querier for block %s", b) } + if headQuerier != nil { + blockQueriers = append(blockQueriers, headQuerier) + } return storage.NewMergeChunkQuerier(blockQueriers, nil, storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)), nil } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 677e65da4..876a13920 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -3444,3 +3444,18 @@ func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChun require.NoError(t, err) } } + +func newTestDB(t *testing.T) *DB { + dir, err := ioutil.TempDir("", "test") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(dir)) + }) + + db, err := Open(dir, nil, nil, DefaultOptions(), nil) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + return db +} diff --git a/tsdb/encoding/encoding.go b/tsdb/encoding/encoding.go index 82270ce86..54ab8ff36 100644 --- a/tsdb/encoding/encoding.go +++ b/tsdb/encoding/encoding.go @@ -19,6 +19,7 @@ import ( "hash/crc32" "unsafe" + "github.com/dennwc/varint" "github.com/pkg/errors" ) @@ -139,7 +140,7 @@ func NewDecbufUvarintAt(bs ByteSlice, off int, castagnoliTable *crc32.Table) Dec } b := bs.Range(off, off+binary.MaxVarintLen32) - l, n := binary.Uvarint(b) + l, n := varint.Uvarint(b) if n <= 0 || n > binary.MaxVarintLen32 { return Decbuf{E: errors.Errorf("invalid uvarint %d", n)} } @@ -207,11 +208,17 @@ func (d *Decbuf) Varint64() int64 { if d.E != nil { return 0 } - x, n := binary.Varint(d.B) + // Decode as unsigned first, since that's what the varint library implements. + ux, n := varint.Uvarint(d.B) if n < 1 { d.E = ErrInvalidSize return 0 } + // Now decode "ZigZag encoding" https://developers.google.com/protocol-buffers/docs/encoding#signed_integers. + x := int64(ux >> 1) + if ux&1 != 0 { + x = ^x + } d.B = d.B[n:] return x } @@ -220,7 +227,7 @@ func (d *Decbuf) Uvarint64() uint64 { if d.E != nil { return 0 } - x, n := binary.Uvarint(d.B) + x, n := varint.Uvarint(d.B) if n < 1 { d.E = ErrInvalidSize return 0 diff --git a/tsdb/exemplar.go b/tsdb/exemplar.go index d5b7be57f..3aa9607bb 100644 --- a/tsdb/exemplar.go +++ b/tsdb/exemplar.go @@ -20,21 +20,20 @@ import ( "unicode/utf8" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" ) -type CircularExemplarStorage struct { - exemplarsAppended prometheus.Counter - exemplarsInStorage prometheus.Gauge - seriesWithExemplarsInStorage prometheus.Gauge - lastExemplarsTs prometheus.Gauge - outOfOrderExemplars prometheus.Counter +// Indicates that there is no index entry for an exmplar. +const noExemplar = -1 +type CircularExemplarStorage struct { lock sync.RWMutex exemplars []*circularBufferEntry nextIndex int + metrics *ExemplarMetrics // Map of series labels as a string to index entry, which points to the first // and last exemplar for the series in the exemplars circular buffer. @@ -53,17 +52,17 @@ type circularBufferEntry struct { ref *indexEntry } -// NewCircularExemplarStorage creates an circular in memory exemplar storage. -// If we assume the average case 95 bytes per exemplar we can fit 5651272 exemplars in -// 1GB of extra memory, accounting for the fact that this is heap allocated space. -// If len < 1, then the exemplar storage is disabled. -func NewCircularExemplarStorage(len int, reg prometheus.Registerer) (ExemplarStorage, error) { - if len < 1 { - return &noopExemplarStorage{}, nil - } - c := &CircularExemplarStorage{ - exemplars: make([]*circularBufferEntry, len), - index: make(map[string]*indexEntry), +type ExemplarMetrics struct { + exemplarsAppended prometheus.Counter + exemplarsInStorage prometheus.Gauge + seriesWithExemplarsInStorage prometheus.Gauge + lastExemplarsTs prometheus.Gauge + maxExemplars prometheus.Gauge + outOfOrderExemplars prometheus.Counter +} + +func NewExemplarMetrics(reg prometheus.Registerer) *ExemplarMetrics { + m := ExemplarMetrics{ exemplarsAppended: prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_exemplar_exemplars_appended_total", Help: "Total number of appended exemplars.", @@ -86,20 +85,51 @@ func NewCircularExemplarStorage(len int, reg prometheus.Registerer) (ExemplarSto Name: "prometheus_tsdb_exemplar_out_of_order_exemplars_total", Help: "Total number of out of order exemplar ingestion failed attempts.", }), + maxExemplars: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_tsdb_exemplar_max_exemplars", + Help: "Total number of exemplars the exemplar storage can store, resizeable.", + }), } + if reg != nil { reg.MustRegister( - c.exemplarsAppended, - c.exemplarsInStorage, - c.seriesWithExemplarsInStorage, - c.lastExemplarsTs, - c.outOfOrderExemplars, + m.exemplarsAppended, + m.exemplarsInStorage, + m.seriesWithExemplarsInStorage, + m.lastExemplarsTs, + m.outOfOrderExemplars, + m.maxExemplars, ) } + return &m +} + +// NewCircularExemplarStorage creates an circular in memory exemplar storage. +// If we assume the average case 95 bytes per exemplar we can fit 5651272 exemplars in +// 1GB of extra memory, accounting for the fact that this is heap allocated space. +// If len <= 0, then the exemplar storage is essentially a noop storage but can later be +// resized to store exemplars. +func NewCircularExemplarStorage(len int64, m *ExemplarMetrics) (ExemplarStorage, error) { + if len < 0 { + len = 0 + } + c := &CircularExemplarStorage{ + exemplars: make([]*circularBufferEntry, len), + index: make(map[string]*indexEntry), + metrics: m, + } + + c.metrics.maxExemplars.Set(float64(len)) + return c, nil } +func (ce *CircularExemplarStorage) ApplyConfig(cfg *config.Config) error { + ce.Resize(cfg.StorageConfig.ExemplarsConfig.MaxExemplars) + return nil +} + func (ce *CircularExemplarStorage) Appender() *CircularExemplarStorage { return ce } @@ -116,6 +146,10 @@ func (ce *CircularExemplarStorage) Querier(_ context.Context) (storage.ExemplarQ func (ce *CircularExemplarStorage) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { ret := make([]exemplar.QueryResult, 0) + if len(ce.exemplars) <= 0 { + return ret, nil + } + ce.lock.RLock() defer ce.lock.RUnlock() @@ -136,7 +170,7 @@ func (ce *CircularExemplarStorage) Select(start, end int64, matchers ...[]*label if e.exemplar.Ts >= start { se.Exemplars = append(se.Exemplars, e.exemplar) } - if e.next == -1 { + if e.next == noExemplar { break } e = ce.exemplars[e.next] @@ -179,6 +213,10 @@ func (ce *CircularExemplarStorage) ValidateExemplar(l labels.Labels, e exemplar. // Not thread safe. The append parameters tells us whether this is an external validation, or internal // as a result of an AddExemplar call, in which case we should update any relevant metrics. func (ce *CircularExemplarStorage) validateExemplar(l string, e exemplar.Exemplar, append bool) error { + if len(ce.exemplars) <= 0 { + return storage.ErrExemplarsDisabled + } + // Exemplar label length does not include chars involved in text rendering such as quotes // equals sign, or commas. See definition of const ExemplarMaxLabelLength. labelSetLen := 0 @@ -204,14 +242,92 @@ func (ce *CircularExemplarStorage) validateExemplar(l string, e exemplar.Exempla if e.Ts <= ce.exemplars[idx.newest].exemplar.Ts { if append { - ce.outOfOrderExemplars.Inc() + ce.metrics.outOfOrderExemplars.Inc() } return storage.ErrOutOfOrderExemplar } return nil } +// Resize changes the size of exemplar buffer by allocating a new buffer and migrating data to it. +// Exemplars are kept when possible. Shrinking will discard oldest data (in order of ingest) as needed. +func (ce *CircularExemplarStorage) Resize(l int64) int { + // Accept negative values as just 0 size. + if l <= 0 { + l = 0 + } + + if l == int64(len(ce.exemplars)) { + return 0 + } + + ce.lock.Lock() + defer ce.lock.Unlock() + + oldBuffer := ce.exemplars + oldNextIndex := int64(ce.nextIndex) + + ce.exemplars = make([]*circularBufferEntry, l) + ce.index = make(map[string]*indexEntry) + ce.nextIndex = 0 + + // Replay as many entries as needed, starting with oldest first. + count := int64(len(oldBuffer)) + if l < count { + count = l + } + + migrated := 0 + + if l > 0 { + // Rewind previous next index by count with wrap-around. + // This math is essentially looking at nextIndex, where we would write the next exemplar to, + // and find the index in the old exemplar buffer that we should start migrating exemplars from. + // This way we don't migrate exemplars that would just be overwritten when migrating later exemplars. + var startIndex int64 = (oldNextIndex - count + int64(len(oldBuffer))) % int64(len(oldBuffer)) + + for i := int64(0); i < count; i++ { + idx := (startIndex + i) % int64(len(oldBuffer)) + if entry := oldBuffer[idx]; entry != nil { + ce.migrate(entry) + migrated++ + } + } + } + + ce.computeMetrics() + ce.metrics.maxExemplars.Set(float64(l)) + + return migrated +} + +// migrate is like AddExemplar but reuses existing structs. Expected to be called in batch and requires +// external lock and does not compute metrics. +func (ce *CircularExemplarStorage) migrate(entry *circularBufferEntry) { + seriesLabels := entry.ref.seriesLabels.String() + + idx, ok := ce.index[seriesLabels] + if !ok { + idx = entry.ref + idx.oldest = ce.nextIndex + ce.index[seriesLabels] = idx + } else { + entry.ref = idx + ce.exemplars[idx.newest].next = ce.nextIndex + } + idx.newest = ce.nextIndex + + entry.next = noExemplar + ce.exemplars[ce.nextIndex] = entry + + ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars) +} + func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error { + if len(ce.exemplars) <= 0 { + return storage.ErrExemplarsDisabled + } + seriesLabels := l.String() // TODO(bwplotka): This lock can lock all scrapers, there might high contention on this on scale. @@ -241,7 +357,7 @@ func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemp // There exists exemplar already on this ce.nextIndex entry, drop it, to make place // for others. prevLabels := prev.ref.seriesLabels.String() - if prev.next == -1 { + if prev.next == noExemplar { // Last item for this series, remove index entry. delete(ce.index, prevLabels) } else { @@ -251,43 +367,36 @@ func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemp // Default the next value to -1 (which we use to detect that we've iterated through all exemplars for a series in Select) // since this is the first exemplar stored for this series. + ce.exemplars[ce.nextIndex].next = noExemplar ce.exemplars[ce.nextIndex].exemplar = e - ce.exemplars[ce.nextIndex].next = -1 ce.exemplars[ce.nextIndex].ref = ce.index[seriesLabels] ce.index[seriesLabels].newest = ce.nextIndex ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars) - ce.exemplarsAppended.Inc() - ce.seriesWithExemplarsInStorage.Set(float64(len(ce.index))) + ce.metrics.exemplarsAppended.Inc() + ce.computeMetrics() + return nil +} + +func (ce *CircularExemplarStorage) computeMetrics() { + ce.metrics.seriesWithExemplarsInStorage.Set(float64(len(ce.index))) + + if len(ce.exemplars) == 0 { + ce.metrics.exemplarsInStorage.Set(float64(0)) + ce.metrics.lastExemplarsTs.Set(float64(0)) + return + } + if next := ce.exemplars[ce.nextIndex]; next != nil { - ce.exemplarsInStorage.Set(float64(len(ce.exemplars))) - ce.lastExemplarsTs.Set(float64(next.exemplar.Ts) / 1000) - return nil + ce.metrics.exemplarsInStorage.Set(float64(len(ce.exemplars))) + ce.metrics.lastExemplarsTs.Set(float64(next.exemplar.Ts) / 1000) + return } // We did not yet fill the buffer. - ce.exemplarsInStorage.Set(float64(ce.nextIndex)) - ce.lastExemplarsTs.Set(float64(ce.exemplars[0].exemplar.Ts) / 1000) - return nil -} - -type noopExemplarStorage struct{} - -func (noopExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error { - return nil -} - -func (noopExemplarStorage) ValidateExemplar(l labels.Labels, e exemplar.Exemplar) error { - return nil -} - -func (noopExemplarStorage) ExemplarQuerier(context.Context) (storage.ExemplarQuerier, error) { - return &noopExemplarQuerier{}, nil -} - -type noopExemplarQuerier struct{} - -func (noopExemplarQuerier) Select(_, _ int64, _ ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { - return nil, nil + ce.metrics.exemplarsInStorage.Set(float64(ce.nextIndex)) + if ce.exemplars[0] != nil { + ce.metrics.lastExemplarsTs.Set(float64(ce.exemplars[0].exemplar.Ts) / 1000) + } } diff --git a/tsdb/exemplar_test.go b/tsdb/exemplar_test.go index 55e6fd375..c81b3ffce 100644 --- a/tsdb/exemplar_test.go +++ b/tsdb/exemplar_test.go @@ -14,6 +14,9 @@ package tsdb import ( + "context" + "fmt" + "math" "reflect" "strconv" "strings" @@ -21,14 +24,17 @@ import ( "github.com/stretchr/testify/require" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" ) +var eMetrics = NewExemplarMetrics(prometheus.DefaultRegisterer) + // Tests the same exemplar cases as AddExemplar, but specifically the ValidateExemplar function so it can be relied on externally. func TestValidateExemplar(t *testing.T) { - exs, err := NewCircularExemplarStorage(2, nil) + exs, err := NewCircularExemplarStorage(2, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -87,7 +93,7 @@ func TestValidateExemplar(t *testing.T) { } func TestAddExemplar(t *testing.T) { - exs, err := NewCircularExemplarStorage(2, nil) + exs, err := NewCircularExemplarStorage(2, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -150,7 +156,7 @@ func TestStorageOverflow(t *testing.T) { // Test that circular buffer index and assignment // works properly, adding more exemplars than can // be stored and then querying for them. - exs, err := NewCircularExemplarStorage(5, nil) + exs, err := NewCircularExemplarStorage(5, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -185,7 +191,7 @@ func TestStorageOverflow(t *testing.T) { } func TestSelectExemplar(t *testing.T) { - exs, err := NewCircularExemplarStorage(5, nil) + exs, err := NewCircularExemplarStorage(5, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -216,7 +222,7 @@ func TestSelectExemplar(t *testing.T) { } func TestSelectExemplar_MultiSeries(t *testing.T) { - exs, err := NewCircularExemplarStorage(5, nil) + exs, err := NewCircularExemplarStorage(5, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -273,8 +279,8 @@ func TestSelectExemplar_MultiSeries(t *testing.T) { } func TestSelectExemplar_TimeRange(t *testing.T) { - lenEs := 5 - exs, err := NewCircularExemplarStorage(lenEs, nil) + var lenEs int64 = 5 + exs, err := NewCircularExemplarStorage(lenEs, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -282,7 +288,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) { {Name: "service", Value: "asdf"}, } - for i := 0; i < lenEs; i++ { + for i := 0; int64(i) < lenEs; i++ { err := es.AddExemplar(l, exemplar.Exemplar{ Labels: labels.Labels{ labels.Label{ @@ -308,7 +314,7 @@ func TestSelectExemplar_TimeRange(t *testing.T) { // Test to ensure that even though a series matches more than one matcher from the // query that it's exemplars are only included in the result a single time. func TestSelectExemplar_DuplicateSeries(t *testing.T) { - exs, err := NewCircularExemplarStorage(4, nil) + exs, err := NewCircularExemplarStorage(4, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -349,7 +355,7 @@ func TestSelectExemplar_DuplicateSeries(t *testing.T) { } func TestIndexOverwrite(t *testing.T) { - exs, err := NewCircularExemplarStorage(2, nil) + exs, err := NewCircularExemplarStorage(2, eMetrics) require.NoError(t, err) es := exs.(*CircularExemplarStorage) @@ -380,3 +386,162 @@ func TestIndexOverwrite(t *testing.T) { i := es.index[l2.String()] require.Equal(t, &indexEntry{0, 0, l2}, i) } + +func TestResize(t *testing.T) { + testCases := []struct { + name string + startSize int64 + newCount int64 + expectedSeries []int + notExpectedSeries []int + expectedMigrated int + }{ + { + name: "Grow", + startSize: 100, + newCount: 200, + expectedSeries: []int{99, 98, 1, 0}, + notExpectedSeries: []int{100}, + expectedMigrated: 100, + }, + { + name: "Shrink", + startSize: 100, + newCount: 50, + expectedSeries: []int{99, 98, 50}, + notExpectedSeries: []int{49, 1, 0}, + expectedMigrated: 50, + }, + { + name: "Zero", + startSize: 100, + newCount: 0, + expectedSeries: []int{}, + notExpectedSeries: []int{}, + expectedMigrated: 0, + }, + { + name: "Negative", + startSize: 100, + newCount: -1, + expectedSeries: []int{}, + notExpectedSeries: []int{}, + expectedMigrated: 0, + }, + { + name: "NegativeToNegative", + startSize: -1, + newCount: -2, + expectedSeries: []int{}, + notExpectedSeries: []int{}, + expectedMigrated: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + exs, err := NewCircularExemplarStorage(tc.startSize, eMetrics) + require.NoError(t, err) + es := exs.(*CircularExemplarStorage) + + for i := 0; int64(i) < tc.startSize; i++ { + err = es.AddExemplar(labels.FromStrings("service", strconv.Itoa(i)), exemplar.Exemplar{ + Value: float64(i), + Ts: int64(i)}) + require.NoError(t, err) + } + + resized := es.Resize(tc.newCount) + require.Equal(t, tc.expectedMigrated, resized) + + q, err := es.Querier(context.TODO()) + require.NoError(t, err) + + matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "service", "")} + + for _, expected := range tc.expectedSeries { + matchers[0].Value = strconv.Itoa(expected) + ex, err := q.Select(0, math.MaxInt64, matchers) + require.NoError(t, err) + require.NotEmpty(t, ex) + } + + for _, notExpected := range tc.notExpectedSeries { + matchers[0].Value = strconv.Itoa(notExpected) + ex, err := q.Select(0, math.MaxInt64, matchers) + require.NoError(t, err) + require.Empty(t, ex) + } + }) + } +} + +func BenchmarkAddExemplar(t *testing.B) { + exs, err := NewCircularExemplarStorage(int64(t.N), eMetrics) + require.NoError(t, err) + es := exs.(*CircularExemplarStorage) + + for i := 0; i < t.N; i++ { + l := labels.FromStrings("service", strconv.Itoa(i)) + + err = es.AddExemplar(l, exemplar.Exemplar{Value: float64(i), Ts: int64(i)}) + require.NoError(t, err) + } +} + +func BenchmarkResizeExemplars(b *testing.B) { + testCases := []struct { + name string + startSize int64 + endSize int64 + numExemplars int + }{ + { + name: "grow", + startSize: 100000, + endSize: 200000, + numExemplars: 150000, + }, + { + name: "shrink", + startSize: 100000, + endSize: 50000, + numExemplars: 100000, + }, + { + name: "grow", + startSize: 1000000, + endSize: 2000000, + numExemplars: 1500000, + }, + { + name: "shrink", + startSize: 1000000, + endSize: 500000, + numExemplars: 1000000, + }, + } + + for _, tc := range testCases { + exs, err := NewCircularExemplarStorage(tc.startSize, eMetrics) + require.NoError(b, err) + es := exs.(*CircularExemplarStorage) + + for i := 0; i < int(float64(tc.startSize)*float64(1.5)); i++ { + l := labels.FromStrings("service", strconv.Itoa(i)) + + err = es.AddExemplar(l, exemplar.Exemplar{Value: float64(i), Ts: int64(i)}) + require.NoError(b, err) + } + saveIndex := es.index + saveExemplars := es.exemplars + + b.Run(fmt.Sprintf("%s-%d-to-%d", tc.name, tc.startSize, tc.endSize), func(t *testing.B) { + es.index = saveIndex + es.exemplars = saveExemplars + b.ResetTimer() + es.Resize(tc.endSize) + }) + } +} diff --git a/tsdb/head.go b/tsdb/head.go index 1475635e7..42caca3a3 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -14,12 +14,9 @@ package tsdb import ( - "context" "fmt" "math" "path/filepath" - "runtime" - "sort" "sync" "time" @@ -30,6 +27,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" @@ -56,34 +54,30 @@ var ( ErrAppenderClosed = errors.New("appender closed") ) -type ExemplarStorage interface { - storage.ExemplarQueryable - AddExemplar(labels.Labels, exemplar.Exemplar) error - ValidateExemplar(labels.Labels, exemplar.Exemplar) error -} - // Head handles reads and writes of time series data within a time window. type Head struct { - chunkRange atomic.Int64 - numSeries atomic.Uint64 - minTime, maxTime atomic.Int64 // Current min and max of the samples included in the head. - minValidTime atomic.Int64 // Mint allowed to be added to the head. It shouldn't be lower than the maxt of the last persisted block. - lastWALTruncationTime atomic.Int64 - lastSeriesID atomic.Uint64 + chunkRange atomic.Int64 + numSeries atomic.Uint64 + minTime, maxTime atomic.Int64 // Current min and max of the samples included in the head. + minValidTime atomic.Int64 // Mint allowed to be added to the head. It shouldn't be lower than the maxt of the last persisted block. + lastWALTruncationTime atomic.Int64 + lastMemoryTruncationTime atomic.Int64 + lastSeriesID atomic.Uint64 // hasHistograms this is used to m-map all chunks in case there are histograms. // A hack to avoid updating all the failing tests. hasHistograms atomic.Bool - metrics *headMetrics - opts *HeadOptions - wal *wal.WAL - exemplars ExemplarStorage - logger log.Logger - appendPool sync.Pool - exemplarsPool sync.Pool - seriesPool sync.Pool - bytesPool sync.Pool - memChunkPool sync.Pool + metrics *headMetrics + opts *HeadOptions + wal *wal.WAL + exemplarMetrics *ExemplarMetrics + exemplars ExemplarStorage + logger log.Logger + appendPool sync.Pool + exemplarsPool sync.Pool + seriesPool sync.Pool + bytesPool sync.Pool + memChunkPool sync.Pool // All series addressable by their ID or hash. series *stripeSeries @@ -111,6 +105,15 @@ type Head struct { closed bool stats *HeadStats + reg prometheus.Registerer + + memTruncationInProcess atomic.Bool +} + +type ExemplarStorage interface { + storage.ExemplarQueryable + AddExemplar(labels.Labels, exemplar.Exemplar) error + ValidateExemplar(labels.Labels, exemplar.Exemplar) error } // HeadOptions are parameters for the Head block. @@ -123,9 +126,12 @@ type HeadOptions struct { // StripeSize sets the number of entries in the hash map, it must be a power of 2. // A larger StripeSize will allocate more memory up-front, but will increase performance when handling a large number of series. // A smaller StripeSize reduces the memory allocated, but can decrease performance with large number of series. - StripeSize int - SeriesCallback SeriesLifecycleCallback - NumExemplars int + StripeSize int + SeriesCallback SeriesLifecycleCallback + EnableExemplarStorage bool + + // Runtime reloadable options. + MaxExemplars atomic.Int64 } func DefaultHeadOptions() *HeadOptions { @@ -139,6 +145,87 @@ func DefaultHeadOptions() *HeadOptions { } } +// SeriesLifecycleCallback specifies a list of callbacks that will be called during a lifecycle of a series. +// It is always a no-op in Prometheus and mainly meant for external users who import TSDB. +// All the callbacks should be safe to be called concurrently. +// It is up to the user to implement soft or hard consistency by making the callbacks +// atomic or non-atomic. Atomic callbacks can cause degradation performance. +type SeriesLifecycleCallback interface { + // PreCreation is called before creating a series to indicate if the series can be created. + // A non nil error means the series should not be created. + PreCreation(labels.Labels) error + // PostCreation is called after creating a series to indicate a creation of series. + PostCreation(labels.Labels) + // PostDeletion is called after deletion of series. + PostDeletion(...labels.Labels) +} + +// NewHead opens the head block in dir. +func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOptions, stats *HeadStats) (*Head, error) { + var err error + if l == nil { + l = log.NewNopLogger() + } + if opts.ChunkRange < 1 { + return nil, errors.Errorf("invalid chunk range %d", opts.ChunkRange) + } + if opts.SeriesCallback == nil { + opts.SeriesCallback = &noopSeriesLifecycleCallback{} + } + + em := NewExemplarMetrics(r) + es, err := NewCircularExemplarStorage(opts.MaxExemplars.Load(), em) + if err != nil { + return nil, err + } + + if stats == nil { + stats = NewHeadStats() + } + + h := &Head{ + wal: wal, + logger: l, + opts: opts, + exemplarMetrics: em, + exemplars: es, + series: newStripeSeries(opts.StripeSize, opts.SeriesCallback), + symbols: map[string]struct{}{}, + postings: index.NewUnorderedMemPostings(), + tombstones: tombstones.NewMemTombstones(), + iso: newIsolation(), + deleted: map[uint64]int{}, + memChunkPool: sync.Pool{ + New: func() interface{} { + return &memChunk{} + }, + }, + stats: stats, + reg: r, + } + h.chunkRange.Store(opts.ChunkRange) + h.minTime.Store(math.MaxInt64) + h.maxTime.Store(math.MinInt64) + h.lastWALTruncationTime.Store(math.MinInt64) + h.lastMemoryTruncationTime.Store(math.MinInt64) + h.metrics = newHeadMetrics(h, r) + + if opts.ChunkPool == nil { + opts.ChunkPool = chunkenc.NewPool() + } + + h.chunkDiskMapper, err = chunks.NewChunkDiskMapper( + mmappedChunksDir(opts.ChunkDirRoot), + opts.ChunkPool, + opts.ChunkWriteBufferSize, + ) + if err != nil { + return nil, err + } + + return h, nil +} + type headMetrics struct { activeAppenders prometheus.Gauge series prometheus.GaugeFunc @@ -313,6 +400,8 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { return m } +func mmappedChunksDir(dir string) string { return filepath.Join(dir, "chunks_head") } + // HeadStats are the statistics for the head component of the DB. type HeadStats struct { WALReplayStatus *WALReplayStatus @@ -347,432 +436,6 @@ func (s *WALReplayStatus) GetWALReplayStatus() WALReplayStatus { const cardinalityCacheExpirationTime = time.Duration(30) * time.Second -// PostingsCardinalityStats returns top 10 highest cardinality stats By label and value names. -func (h *Head) PostingsCardinalityStats(statsByLabelName string) *index.PostingsStats { - h.cardinalityMutex.Lock() - defer h.cardinalityMutex.Unlock() - currentTime := time.Duration(time.Now().Unix()) * time.Second - seconds := currentTime - h.lastPostingsStatsCall - if seconds > cardinalityCacheExpirationTime { - h.cardinalityCache = nil - } - if h.cardinalityCache != nil { - return h.cardinalityCache - } - h.cardinalityCache = h.postings.Stats(statsByLabelName) - h.lastPostingsStatsCall = time.Duration(time.Now().Unix()) * time.Second - - return h.cardinalityCache -} - -// NewHead opens the head block in dir. -func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOptions, stats *HeadStats) (*Head, error) { - if l == nil { - l = log.NewNopLogger() - } - if opts.ChunkRange < 1 { - return nil, errors.Errorf("invalid chunk range %d", opts.ChunkRange) - } - if opts.SeriesCallback == nil { - opts.SeriesCallback = &noopSeriesLifecycleCallback{} - } - - es, err := NewCircularExemplarStorage(opts.NumExemplars, r) - if err != nil { - return nil, err - } - - if stats == nil { - stats = NewHeadStats() - } - - h := &Head{ - wal: wal, - logger: l, - opts: opts, - exemplars: es, - series: newStripeSeries(opts.StripeSize, opts.SeriesCallback), - symbols: map[string]struct{}{}, - postings: index.NewUnorderedMemPostings(), - tombstones: tombstones.NewMemTombstones(), - iso: newIsolation(), - deleted: map[uint64]int{}, - memChunkPool: sync.Pool{ - New: func() interface{} { - return &memChunk{} - }, - }, - stats: stats, - } - h.chunkRange.Store(opts.ChunkRange) - h.minTime.Store(math.MaxInt64) - h.maxTime.Store(math.MinInt64) - h.lastWALTruncationTime.Store(math.MinInt64) - h.metrics = newHeadMetrics(h, r) - - if opts.ChunkPool == nil { - opts.ChunkPool = chunkenc.NewPool() - } - - h.chunkDiskMapper, err = chunks.NewChunkDiskMapper( - mmappedChunksDir(opts.ChunkDirRoot), - opts.ChunkPool, - opts.ChunkWriteBufferSize, - ) - if err != nil { - return nil, err - } - - return h, nil -} - -func mmappedChunksDir(dir string) string { return filepath.Join(dir, "chunks_head") } - -func (h *Head) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { - return h.exemplars.ExemplarQuerier(ctx) -} - -// processWALSamples adds a partition of samples it receives to the head and passes -// them on to other workers. -// Samples before the mint timestamp are discarded. -func (h *Head) processWALSamples( - minValidTime int64, - input <-chan []record.RefSample, output chan<- []record.RefSample, -) (unknownRefs uint64) { - defer close(output) - - // Mitigate lock contention in getByID. - refSeries := map[uint64]*memSeries{} - - mint, maxt := int64(math.MaxInt64), int64(math.MinInt64) - - for samples := range input { - for _, s := range samples { - if s.T < minValidTime { - continue - } - ms := refSeries[s.Ref] - if ms == nil { - ms = h.series.getByID(s.Ref) - if ms == nil { - unknownRefs++ - continue - } - refSeries[s.Ref] = ms - } - if _, chunkCreated := ms.append(s.T, s.V, 0, h.chunkDiskMapper); chunkCreated { - h.metrics.chunksCreated.Inc() - h.metrics.chunks.Inc() - } - if s.T > maxt { - maxt = s.T - } - if s.T < mint { - mint = s.T - } - } - output <- samples - } - h.updateMinMaxTime(mint, maxt) - - return unknownRefs -} - -func (h *Head) updateMinMaxTime(mint, maxt int64) { - for { - lt := h.MinTime() - if mint >= lt { - break - } - if h.minTime.CAS(lt, mint) { - break - } - } - for { - ht := h.MaxTime() - if maxt <= ht { - break - } - if h.maxTime.CAS(ht, maxt) { - break - } - } -} - -func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks map[uint64][]*mmappedChunk) (err error) { - // Track number of samples that referenced a series we don't know about - // for error reporting. - var unknownRefs atomic.Uint64 - var unknownExemplarRefs atomic.Uint64 - - // Start workers that each process samples for a partition of the series ID space. - // They are connected through a ring of channels which ensures that all sample batches - // read from the WAL are processed in order. - var ( - wg sync.WaitGroup - n = runtime.GOMAXPROCS(0) - inputs = make([]chan []record.RefSample, n) - outputs = make([]chan []record.RefSample, n) - exemplarsInput chan record.RefExemplar - - dec record.Decoder - shards = make([][]record.RefSample, n) - - decoded = make(chan interface{}, 10) - decodeErr, seriesCreationErr error - seriesPool = sync.Pool{ - New: func() interface{} { - return []record.RefSeries{} - }, - } - samplesPool = sync.Pool{ - New: func() interface{} { - return []record.RefSample{} - }, - } - tstonesPool = sync.Pool{ - New: func() interface{} { - return []tombstones.Stone{} - }, - } - exemplarsPool = sync.Pool{ - New: func() interface{} { - return []record.RefExemplar{} - }, - } - ) - - defer func() { - // For CorruptionErr ensure to terminate all workers before exiting. - _, ok := err.(*wal.CorruptionErr) - if ok || seriesCreationErr != nil { - for i := 0; i < n; i++ { - close(inputs[i]) - for range outputs[i] { - } - } - close(exemplarsInput) - wg.Wait() - } - }() - - wg.Add(n) - for i := 0; i < n; i++ { - outputs[i] = make(chan []record.RefSample, 300) - inputs[i] = make(chan []record.RefSample, 300) - - go func(input <-chan []record.RefSample, output chan<- []record.RefSample) { - unknown := h.processWALSamples(h.minValidTime.Load(), input, output) - unknownRefs.Add(unknown) - wg.Done() - }(inputs[i], outputs[i]) - } - - wg.Add(1) - exemplarsInput = make(chan record.RefExemplar, 300) - go func(input <-chan record.RefExemplar) { - defer wg.Done() - for e := range input { - ms := h.series.getByID(e.Ref) - if ms == nil { - unknownExemplarRefs.Inc() - continue - } - - if e.T < h.minValidTime.Load() { - continue - } - // At the moment the only possible error here is out of order exemplars, which we shouldn't see when - // replaying the WAL, so lets just log the error if it's not that type. - err = h.exemplars.AddExemplar(ms.lset, exemplar.Exemplar{Ts: e.T, Value: e.V, Labels: e.Labels}) - if err != nil && err == storage.ErrOutOfOrderExemplar { - level.Warn(h.logger).Log("msg", "Unexpected error when replaying WAL on exemplar record", "err", err) - } - } - }(exemplarsInput) - - go func() { - defer close(decoded) - for r.Next() { - rec := r.Record() - switch dec.Type(rec) { - case record.Series: - series := seriesPool.Get().([]record.RefSeries)[:0] - series, err = dec.Series(rec, series) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode series"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- series - case record.Samples: - samples := samplesPool.Get().([]record.RefSample)[:0] - samples, err = dec.Samples(rec, samples) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode samples"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- samples - case record.Tombstones: - tstones := tstonesPool.Get().([]tombstones.Stone)[:0] - tstones, err = dec.Tombstones(rec, tstones) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode tombstones"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- tstones - case record.Exemplars: - exemplars := exemplarsPool.Get().([]record.RefExemplar)[:0] - exemplars, err = dec.Exemplars(rec, exemplars) - if err != nil { - decodeErr = &wal.CorruptionErr{ - Err: errors.Wrap(err, "decode exemplars"), - Segment: r.Segment(), - Offset: r.Offset(), - } - return - } - decoded <- exemplars - default: - // Noop. - } - } - }() - -Outer: - for d := range decoded { - switch v := d.(type) { - case []record.RefSeries: - for _, s := range v { - series, created, err := h.getOrCreateWithID(s.Ref, s.Labels.Hash(), s.Labels) - if err != nil { - seriesCreationErr = err - break Outer - } - - if created { - // If this series gets a duplicate record, we don't restore its mmapped chunks, - // and instead restore everything from WAL records. - series.mmappedChunks = mmappedChunks[series.ref] - - h.metrics.chunks.Add(float64(len(series.mmappedChunks))) - h.metrics.chunksCreated.Add(float64(len(series.mmappedChunks))) - - if len(series.mmappedChunks) > 0 { - h.updateMinMaxTime(series.minTime(), series.maxTime()) - } - } else { - // TODO(codesome) Discard old samples and mmapped chunks and use mmap chunks for the new series ID. - - // There's already a different ref for this series. - multiRef[s.Ref] = series.ref - } - - if h.lastSeriesID.Load() < s.Ref { - h.lastSeriesID.Store(s.Ref) - } - } - //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. - seriesPool.Put(v) - case []record.RefSample: - samples := v - // We split up the samples into chunks of 5000 samples or less. - // With O(300 * #cores) in-flight sample batches, large scrapes could otherwise - // cause thousands of very large in flight buffers occupying large amounts - // of unused memory. - for len(samples) > 0 { - m := 5000 - if len(samples) < m { - m = len(samples) - } - for i := 0; i < n; i++ { - var buf []record.RefSample - select { - case buf = <-outputs[i]: - default: - } - shards[i] = buf[:0] - } - for _, sam := range samples[:m] { - if r, ok := multiRef[sam.Ref]; ok { - sam.Ref = r - } - mod := sam.Ref % uint64(n) - shards[mod] = append(shards[mod], sam) - } - for i := 0; i < n; i++ { - inputs[i] <- shards[i] - } - samples = samples[m:] - } - //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. - samplesPool.Put(v) - case []tombstones.Stone: - for _, s := range v { - for _, itv := range s.Intervals { - if itv.Maxt < h.minValidTime.Load() { - continue - } - if m := h.series.getByID(s.Ref); m == nil { - unknownRefs.Inc() - continue - } - h.tombstones.AddInterval(s.Ref, itv) - } - } - //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. - tstonesPool.Put(v) - case []record.RefExemplar: - for _, e := range v { - exemplarsInput <- e - } - //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. - exemplarsPool.Put(v) - default: - panic(fmt.Errorf("unexpected decoded type: %T", d)) - } - } - - if decodeErr != nil { - return decodeErr - } - if seriesCreationErr != nil { - // Drain the channel to unblock the goroutine. - for range decoded { - } - return seriesCreationErr - } - - // Signal termination to each worker and wait for it to close its output channel. - for i := 0; i < n; i++ { - close(inputs[i]) - for range outputs[i] { - } - } - close(exemplarsInput) - wg.Wait() - - if r.Err() != nil { - return errors.Wrap(r.Err(), "read records") - } - - if unknownRefs.Load() > 0 || unknownExemplarRefs.Load() > 0 { - level.Warn(h.logger).Log("msg", "Unknown series references", "samples", unknownRefs.Load(), "exemplars", unknownExemplarRefs.Load()) - } - return nil -} - // Init loads data from the write ahead log and prepares the head for writes. // It should be called before using an appender so that it // limits the ingested samples to the head min valid time. @@ -809,6 +472,15 @@ func (h *Head) Init(minValidTime int64) error { if err != nil && err != record.ErrNotFound { return errors.Wrap(err, "find last checkpoint") } + + // Find the last segment. + _, endAt, e := wal.Segments(h.wal.Dir()) + if e != nil { + return errors.Wrap(e, "finding WAL segments") + } + + h.startWALReplayStatus(startFrom, endAt) + multiRef := map[uint64]uint64{} if err == nil { sr, err := wal.NewSegmentsReader(dir) @@ -826,22 +498,16 @@ func (h *Head) Init(minValidTime int64) error { if err := h.loadWAL(wal.NewReader(sr), multiRef, mmappedChunks); err != nil { return errors.Wrap(err, "backfill checkpoint") } + h.updateWALReplayStatusRead(startFrom) startFrom++ level.Info(h.logger).Log("msg", "WAL checkpoint loaded") } checkpointReplayDuration := time.Since(checkpointReplayStart) walReplayStart := time.Now() - // Find the last segment. - _, last, err := wal.Segments(h.wal.Dir()) - if err != nil { - return errors.Wrap(err, "finding WAL segments") - } - - h.startWALReplayStatus(startFrom, last) // Backfill segments from the most recent checkpoint onwards. - for i := startFrom; i <= last; i++ { + for i := startFrom; i <= endAt; i++ { s, err := wal.OpenReadSegment(wal.SegmentName(h.wal.Dir(), i)) if err != nil { return errors.Wrap(err, fmt.Sprintf("open WAL segment: %d", i)) @@ -855,7 +521,7 @@ func (h *Head) Init(minValidTime int64) error { if err != nil { return err } - level.Info(h.logger).Log("msg", "WAL segment loaded", "segment", i, "maxSegment", last) + level.Info(h.logger).Log("msg", "WAL segment loaded", "segment", i, "maxSegment", endAt) h.updateWALReplayStatusRead(i) } @@ -871,11 +537,6 @@ func (h *Head) Init(minValidTime int64) error { return nil } -// SetMinValidTime sets the minimum timestamp the head can ingest. -func (h *Head) SetMinValidTime(minValidTime int64) { - h.minValidTime.Store(minValidTime) -} - func (h *Head) loadMmappedChunks() (map[uint64][]*mmappedChunk, error) { mmappedChunks := map[uint64][]*mmappedChunk{} if err := h.chunkDiskMapper.IterateAllChunks(func(seriesRef, chunkRef uint64, mint, maxt int64, numSamples uint16) error { @@ -926,6 +587,70 @@ func (h *Head) removeCorruptedMmappedChunks(err error) map[uint64][]*mmappedChun return mmappedChunks } +func (h *Head) ApplyConfig(cfg *config.Config) error { + if !h.opts.EnableExemplarStorage { + return nil + } + + // Head uses opts.MaxExemplars in combination with opts.EnableExemplarStorage + // to decide if it should pass exemplars along to it's exemplar storage, so we + // need to update opts.MaxExemplars here. + prevSize := h.opts.MaxExemplars.Load() + h.opts.MaxExemplars.Store(cfg.StorageConfig.ExemplarsConfig.MaxExemplars) + + if prevSize == h.opts.MaxExemplars.Load() { + return nil + } + + migrated := h.exemplars.(*CircularExemplarStorage).Resize(h.opts.MaxExemplars.Load()) + level.Info(h.logger).Log("msg", "Exemplar storage resized", "from", prevSize, "to", h.opts.MaxExemplars, "migrated", migrated) + return nil +} + +// PostingsCardinalityStats returns top 10 highest cardinality stats By label and value names. +func (h *Head) PostingsCardinalityStats(statsByLabelName string) *index.PostingsStats { + h.cardinalityMutex.Lock() + defer h.cardinalityMutex.Unlock() + currentTime := time.Duration(time.Now().Unix()) * time.Second + seconds := currentTime - h.lastPostingsStatsCall + if seconds > cardinalityCacheExpirationTime { + h.cardinalityCache = nil + } + if h.cardinalityCache != nil { + return h.cardinalityCache + } + h.cardinalityCache = h.postings.Stats(statsByLabelName) + h.lastPostingsStatsCall = time.Duration(time.Now().Unix()) * time.Second + + return h.cardinalityCache +} + +func (h *Head) updateMinMaxTime(mint, maxt int64) { + for { + lt := h.MinTime() + if mint >= lt { + break + } + if h.minTime.CAS(lt, mint) { + break + } + } + for { + ht := h.MaxTime() + if maxt <= ht { + break + } + if h.maxTime.CAS(ht, maxt) { + break + } + } +} + +// SetMinValidTime sets the minimum timestamp the head can ingest. +func (h *Head) SetMinValidTime(minValidTime int64) { + h.minValidTime.Store(minValidTime) +} + // Truncate removes old data before mint from the head and WAL. func (h *Head) Truncate(mint int64) (err error) { initialize := h.MinTime() == math.MaxInt64 @@ -945,11 +670,24 @@ func (h *Head) truncateMemory(mint int64) (err error) { h.metrics.headTruncateFail.Inc() } }() + initialize := h.MinTime() == math.MaxInt64 if h.MinTime() >= mint && !initialize { return nil } + + // The order of these two Store() should not be changed, + // i.e. truncation time is set before in-process boolean. + h.lastMemoryTruncationTime.Store(mint) + h.memTruncationInProcess.Store(true) + defer h.memTruncationInProcess.Store(false) + + // We wait for pending queries to end that overlap with this truncation. + if !initialize { + h.WaitForPendingReadersInTimeRange(h.MinTime(), mint) + } + h.minTime.Store(mint) h.minValidTime.Store(mint) @@ -991,6 +729,75 @@ func (h *Head) truncateMemory(mint int64) (err error) { return nil } +// WaitForPendingReadersInTimeRange waits for queries overlapping with given range to finish querying. +// The query timeout limits the max wait time of this function implicitly. +// The mint is inclusive and maxt is the truncation time hence exclusive. +func (h *Head) WaitForPendingReadersInTimeRange(mint, maxt int64) { + maxt-- // Making it inclusive before checking overlaps. + overlaps := func() bool { + o := false + h.iso.TraverseOpenReads(func(s *isolationState) bool { + if s.mint <= maxt && mint <= s.maxt { + // Overlaps with the truncation range. + o = true + return false + } + return true + }) + return o + } + for overlaps() { + time.Sleep(500 * time.Millisecond) + } +} + +// IsQuerierCollidingWithTruncation returns if the current querier needs to be closed and if a new querier +// has to be created. In the latter case, the method also returns the new mint to be used for creating the +// new range head and the new querier. This methods helps preventing races with the truncation of in-memory data. +// +// NOTE: The querier should already be taken before calling this. +func (h *Head) IsQuerierCollidingWithTruncation(querierMint, querierMaxt int64) (shouldClose bool, getNew bool, newMint int64) { + if !h.memTruncationInProcess.Load() { + return false, false, 0 + } + // Head truncation is in process. It also means that the block that was + // created for this truncation range is also available. + // Check if we took a querier that overlaps with this truncation. + memTruncTime := h.lastMemoryTruncationTime.Load() + if querierMaxt < memTruncTime { + // Head compaction has happened and this time range is being truncated. + // This query doesn't overlap with the Head any longer. + // We should close this querier to avoid races and the data would be + // available with the blocks below. + // Cases: + // 1. |------truncation------| + // |---query---| + // 2. |------truncation------| + // |---query---| + return true, false, 0 + } + if querierMint < memTruncTime { + // The truncation time is not same as head mint that we saw above but the + // query still overlaps with the Head. + // The truncation started after we got the querier. So it is not safe + // to use this querier and/or might block truncation. We should get + // a new querier for the new Head range while remaining will be available + // in the blocks below. + // Case: + // |------truncation------| + // |----query----| + // Turns into + // |------truncation------| + // |---qu---| + return true, true, memTruncTime + } + + // Other case is this, which is a no-op + // |------truncation------| + // |---query---| + return false, false, 0 +} + // truncateWAL removes old data before mint from the WAL. func (h *Head) truncateWAL(mint int64) error { if h.wal == nil || mint <= h.lastWALTruncationTime.Load() { @@ -1071,17 +878,6 @@ func (h *Head) truncateWAL(mint int64) error { return nil } -// initTime initializes a head with the first timestamp. This only needs to be called -// for a completely fresh head with an empty WAL. -func (h *Head) initTime(t int64) { - if !h.minTime.CAS(math.MaxInt64, t) { - return - } - // Ensure that max time is initialized to at least the min time we just set. - // Concurrent appenders may already have set it to a higher value. - h.maxTime.CAS(math.MinInt64, t) -} - type Stats struct { NumSeries uint64 MinTime, MaxTime int64 @@ -1118,7 +914,7 @@ func (h *RangeHead) Index() (IndexReader, error) { } func (h *RangeHead) Chunks() (ChunkReader, error) { - return h.head.chunksRange(h.mint, h.maxt, h.head.iso.State()) + return h.head.chunksRange(h.mint, h.maxt, h.head.iso.State(h.mint, h.maxt)) } func (h *RangeHead) Tombstones() (tombstones.Reader, error) { @@ -1164,526 +960,6 @@ func (h *RangeHead) String() string { return fmt.Sprintf("range head (mint: %d, maxt: %d)", h.MinTime(), h.MaxTime()) } -// initAppender is a helper to initialize the time bounds of the head -// upon the first sample it receives. -type initAppender struct { - app storage.Appender - head *Head -} - -func (a *initAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) { - if a.app != nil { - return a.app.Append(ref, lset, t, v) - } - - a.head.initTime(t) - a.app = a.head.appender() - return a.app.Append(ref, lset, t, v) -} - -func (a *initAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { - // Check if exemplar storage is enabled. - if a.head.opts.NumExemplars <= 0 { - return 0, nil - } - - if a.app != nil { - return a.app.AppendExemplar(ref, l, e) - } - // We should never reach here given we would call Append before AppendExemplar - // and we probably want to always base head/WAL min time on sample times. - a.head.initTime(e.Ts) - a.app = a.head.appender() - - return a.app.AppendExemplar(ref, l, e) -} - -func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { - if a.app != nil { - return a.app.AppendHistogram(ref, l, t, sh) - } - a.head.initTime(t) - a.app = a.head.appender() - - return a.app.AppendHistogram(ref, l, t, sh) -} - -var _ storage.GetRef = &initAppender{} - -func (a *initAppender) GetRef(lset labels.Labels) (uint64, labels.Labels) { - if g, ok := a.app.(storage.GetRef); ok { - return g.GetRef(lset) - } - return 0, nil -} - -func (a *initAppender) Commit() error { - if a.app == nil { - return nil - } - return a.app.Commit() -} - -func (a *initAppender) Rollback() error { - if a.app == nil { - return nil - } - return a.app.Rollback() -} - -// Appender returns a new Appender on the database. -func (h *Head) Appender(_ context.Context) storage.Appender { - h.metrics.activeAppenders.Inc() - - // The head cache might not have a starting point yet. The init appender - // picks up the first appended timestamp as the base. - if h.MinTime() == math.MaxInt64 { - return &initAppender{ - head: h, - } - } - return h.appender() -} - -func (h *Head) appender() *headAppender { - appendID := h.iso.newAppendID() - cleanupAppendIDsBelow := h.iso.lowWatermark() - - // Allocate the exemplars buffer only if exemplars are enabled. - var exemplarsBuf []exemplarWithSeriesRef - if h.opts.NumExemplars > 0 { - exemplarsBuf = h.getExemplarBuffer() - } - - return &headAppender{ - head: h, - minValidTime: h.appendableMinValidTime(), - mint: math.MaxInt64, - maxt: math.MinInt64, - samples: h.getAppendBuffer(), - sampleSeries: h.getSeriesBuffer(), - exemplars: exemplarsBuf, - appendID: appendID, - cleanupAppendIDsBelow: cleanupAppendIDsBelow, - exemplarAppender: h.exemplars, - } -} - -func (h *Head) appendableMinValidTime() int64 { - // Setting the minimum valid time to whichever is greater, the head min valid time or the compaction window, - // ensures that no samples will be added within the compaction window to avoid races. - return max(h.minValidTime.Load(), h.MaxTime()-h.chunkRange.Load()/2) -} - -func max(a, b int64) int64 { - if a > b { - return a - } - return b -} - -func (h *Head) ExemplarAppender() storage.ExemplarAppender { - h.metrics.activeAppenders.Inc() - - // The head cache might not have a starting point yet. The init appender - // picks up the first appended timestamp as the base. - if h.MinTime() == math.MaxInt64 { - return &initAppender{ - head: h, - } - } - return h.appender() -} - -func (h *Head) getAppendBuffer() []record.RefSample { - b := h.appendPool.Get() - if b == nil { - return make([]record.RefSample, 0, 512) - } - return b.([]record.RefSample) -} - -func (h *Head) putAppendBuffer(b []record.RefSample) { - //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. - h.appendPool.Put(b[:0]) -} - -func (h *Head) getExemplarBuffer() []exemplarWithSeriesRef { - b := h.exemplarsPool.Get() - if b == nil { - return make([]exemplarWithSeriesRef, 0, 512) - } - return b.([]exemplarWithSeriesRef) -} - -func (h *Head) putExemplarBuffer(b []exemplarWithSeriesRef) { - if b == nil { - return - } - - //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. - h.exemplarsPool.Put(b[:0]) -} - -func (h *Head) getSeriesBuffer() []*memSeries { - b := h.seriesPool.Get() - if b == nil { - return make([]*memSeries, 0, 512) - } - return b.([]*memSeries) -} - -func (h *Head) putSeriesBuffer(b []*memSeries) { - //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. - h.seriesPool.Put(b[:0]) -} - -func (h *Head) getBytesBuffer() []byte { - b := h.bytesPool.Get() - if b == nil { - return make([]byte, 0, 1024) - } - return b.([]byte) -} - -func (h *Head) putBytesBuffer(b []byte) { - //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. - h.bytesPool.Put(b[:0]) -} - -type exemplarWithSeriesRef struct { - ref uint64 - exemplar exemplar.Exemplar -} - -type headAppender struct { - head *Head - minValidTime int64 // No samples below this timestamp are allowed. - mint, maxt int64 - exemplarAppender ExemplarStorage - - series []record.RefSeries - samples []record.RefSample - exemplars []exemplarWithSeriesRef - sampleSeries []*memSeries - histograms []record.RefHistogram - histogramSeries []*memSeries - - appendID, cleanupAppendIDsBelow uint64 - closed bool -} - -func (a *headAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) { - if t < a.minValidTime { - a.head.metrics.outOfBoundSamples.Inc() - return 0, storage.ErrOutOfBounds - } - - s := a.head.series.getByID(ref) - if s == nil { - // Ensure no empty labels have gotten through. - lset = lset.WithoutEmpty() - if len(lset) == 0 { - return 0, errors.Wrap(ErrInvalidSample, "empty labelset") - } - - if l, dup := lset.HasDuplicateLabelNames(); dup { - return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l)) - } - - var created bool - var err error - s, created, err = a.head.getOrCreate(lset.Hash(), lset) - if err != nil { - return 0, err - } - if created { - a.series = append(a.series, record.RefSeries{ - Ref: s.ref, - Labels: lset, - }) - } - } - - s.Lock() - if err := s.appendable(t, v); err != nil { - s.Unlock() - if err == storage.ErrOutOfOrderSample { - a.head.metrics.outOfOrderSamples.Inc() - } - return 0, err - } - s.pendingCommit = true - s.Unlock() - - if t < a.mint { - a.mint = t - } - if t > a.maxt { - a.maxt = t - } - - a.samples = append(a.samples, record.RefSample{ - Ref: s.ref, - T: t, - V: v, - }) - a.sampleSeries = append(a.sampleSeries, s) - return s.ref, nil -} - -// AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't -// use getOrCreate or make any of the lset sanity checks that Append does. -func (a *headAppender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exemplar) (uint64, error) { - // Check if exemplar storage is enabled. - if a.head.opts.NumExemplars <= 0 { - return 0, nil - } - - s := a.head.series.getByID(ref) - if s == nil { - return 0, fmt.Errorf("unknown series ref. when trying to add exemplar: %d", ref) - } - - // Ensure no empty labels have gotten through. - e.Labels = e.Labels.WithoutEmpty() - - err := a.exemplarAppender.ValidateExemplar(s.lset, e) - if err != nil { - if err == storage.ErrDuplicateExemplar { - // Duplicate, don't return an error but don't accept the exemplar. - return 0, nil - } - return 0, err - } - - a.exemplars = append(a.exemplars, exemplarWithSeriesRef{ref, e}) - - return s.ref, nil -} - -func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { - if t < a.minValidTime { - a.head.metrics.outOfBoundSamples.Inc() - return 0, storage.ErrOutOfBounds - } - - s := a.head.series.getByID(ref) - if s == nil { - // Ensure no empty labels have gotten through. - lset = lset.WithoutEmpty() - if len(lset) == 0 { - return 0, errors.Wrap(ErrInvalidSample, "empty labelset") - } - - if l, dup := lset.HasDuplicateLabelNames(); dup { - return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l)) - } - - var created bool - var err error - s, created, err = a.head.getOrCreate(lset.Hash(), lset) - if err != nil { - return 0, err - } - if created { - a.series = append(a.series, record.RefSeries{ - Ref: s.ref, - Labels: lset, - }) - } - } - - s.Lock() - if err := s.appendableHistogram(t, sh); err != nil { - s.Unlock() - if err == storage.ErrOutOfOrderSample { - a.head.metrics.outOfOrderSamples.Inc() - } - return 0, err - } - s.pendingCommit = true - s.Unlock() - - if t < a.mint { - a.mint = t - } - if t > a.maxt { - a.maxt = t - } - - a.histograms = append(a.histograms, record.RefHistogram{ - Ref: s.ref, - T: t, - H: sh, - }) - a.histogramSeries = append(a.histogramSeries, s) - return s.ref, nil -} - -var _ storage.GetRef = &headAppender{} - -func (a *headAppender) GetRef(lset labels.Labels) (uint64, labels.Labels) { - s := a.head.series.getByHash(lset.Hash(), lset) - if s == nil { - return 0, nil - } - // returned labels must be suitable to pass to Append() - return s.ref, s.lset -} - -func (a *headAppender) log() error { - if a.head.wal == nil { - return nil - } - - buf := a.head.getBytesBuffer() - defer func() { a.head.putBytesBuffer(buf) }() - - var rec []byte - var enc record.Encoder - - if len(a.series) > 0 { - rec = enc.Series(a.series, buf) - buf = rec[:0] - - if err := a.head.wal.Log(rec); err != nil { - return errors.Wrap(err, "log series") - } - } - if len(a.samples) > 0 { - rec = enc.Samples(a.samples, buf) - buf = rec[:0] - - if err := a.head.wal.Log(rec); err != nil { - return errors.Wrap(err, "log samples") - } - } - if len(a.exemplars) > 0 { - rec = enc.Exemplars(exemplarsForEncoding(a.exemplars), buf) - buf = rec[:0] - - if err := a.head.wal.Log(rec); err != nil { - return errors.Wrap(err, "log exemplars") - } - } - return nil -} - -func exemplarsForEncoding(es []exemplarWithSeriesRef) []record.RefExemplar { - ret := make([]record.RefExemplar, 0, len(es)) - for _, e := range es { - ret = append(ret, record.RefExemplar{ - Ref: e.ref, - T: e.exemplar.Ts, - V: e.exemplar.Value, - Labels: e.exemplar.Labels, - }) - } - return ret -} - -func (a *headAppender) Commit() (err error) { - if a.closed { - return ErrAppenderClosed - } - defer func() { a.closed = true }() - - if err := a.log(); err != nil { - _ = a.Rollback() // Most likely the same error will happen again. - return errors.Wrap(err, "write to WAL") - } - - // No errors logging to WAL, so pass the exemplars along to the in memory storage. - for _, e := range a.exemplars { - s := a.head.series.getByID(e.ref) - // We don't instrument exemplar appends here, all is instrumented by storage. - if err := a.exemplarAppender.AddExemplar(s.lset, e.exemplar); err != nil { - if err == storage.ErrOutOfOrderExemplar { - continue - } - level.Debug(a.head.logger).Log("msg", "Unknown error while adding exemplar", "err", err) - } - } - - defer a.head.metrics.activeAppenders.Dec() - defer a.head.putAppendBuffer(a.samples) - defer a.head.putSeriesBuffer(a.sampleSeries) - defer a.head.putExemplarBuffer(a.exemplars) - defer a.head.iso.closeAppend(a.appendID) - - total := len(a.samples) - var series *memSeries - for i, s := range a.samples { - series = a.sampleSeries[i] - series.Lock() - ok, chunkCreated := series.append(s.T, s.V, a.appendID, a.head.chunkDiskMapper) - series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) - series.pendingCommit = false - series.Unlock() - - if !ok { - total-- - a.head.metrics.outOfOrderSamples.Inc() - } - if chunkCreated { - a.head.metrics.chunks.Inc() - a.head.metrics.chunksCreated.Inc() - } - } - total += len(a.histograms) // TODO: different metric? - for i, s := range a.histograms { - series = a.histogramSeries[i] - series.Lock() - a.head.hasHistograms.Store(true) - ok, chunkCreated := series.appendHistogram(s.T, s.H, a.appendID, a.head.chunkDiskMapper) - series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) - series.pendingCommit = false - series.Unlock() - - if !ok { - total-- - a.head.metrics.outOfOrderSamples.Inc() - } - if chunkCreated { - a.head.metrics.chunks.Inc() - a.head.metrics.chunksCreated.Inc() - } - } - - a.head.metrics.samplesAppended.Add(float64(total)) - a.head.updateMinMaxTime(a.mint, a.maxt) - - return nil -} - -func (a *headAppender) Rollback() (err error) { - if a.closed { - return ErrAppenderClosed - } - defer func() { a.closed = true }() - defer a.head.metrics.activeAppenders.Dec() - defer a.head.iso.closeAppend(a.appendID) - defer a.head.putSeriesBuffer(a.sampleSeries) - - var series *memSeries - for i := range a.samples { - series = a.sampleSeries[i] - series.Lock() - series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) - series.pendingCommit = false - series.Unlock() - } - a.head.putAppendBuffer(a.samples) - a.head.putExemplarBuffer(a.exemplars) - a.samples = nil - a.exemplars = nil - - // Series are created in the head memory regardless of rollback. Thus we have - // to log them to the WAL in any case. - return a.log() -} - // Delete all samples in the range of [mint, maxt] for series that satisfy the given // label matchers. func (h *Head) Delete(mint, maxt int64, ms ...*labels.Matcher) error { @@ -1785,40 +1061,6 @@ func (h *Head) Tombstones() (tombstones.Reader, error) { return h.tombstones, nil } -// Index returns an IndexReader against the block. -func (h *Head) Index() (IndexReader, error) { - return h.indexRange(math.MinInt64, math.MaxInt64), nil -} - -func (h *Head) indexRange(mint, maxt int64) *headIndexReader { - if hmin := h.MinTime(); hmin > mint { - mint = hmin - } - return &headIndexReader{head: h, mint: mint, maxt: maxt} -} - -// Chunks returns a ChunkReader against the block. -func (h *Head) Chunks() (ChunkReader, error) { - return h.chunksRange(math.MinInt64, math.MaxInt64, h.iso.State()) -} - -func (h *Head) chunksRange(mint, maxt int64, is *isolationState) (*headChunkReader, error) { - h.closedMtx.Lock() - defer h.closedMtx.Unlock() - if h.closed { - return nil, errors.New("can't read from a closed head") - } - if hmin := h.MinTime(); hmin > mint { - mint = hmin - } - return &headChunkReader{ - head: h, - mint: mint, - maxt: maxt, - isoState: is, - }, nil -} - // NumSeries returns the number of active series in the head. func (h *Head) NumSeries() uint64 { return h.numSeries.Load() @@ -1886,245 +1128,6 @@ func (h *Head) String() string { return "head" } -type headChunkReader struct { - head *Head - mint, maxt int64 - isoState *isolationState -} - -func (h *headChunkReader) Close() error { - h.isoState.Close() - return nil -} - -// packChunkID packs a seriesID and a chunkID within it into a global 8 byte ID. -// It panicks if the seriesID exceeds 5 bytes or the chunk ID 3 bytes. -func packChunkID(seriesID, chunkID uint64) uint64 { - if seriesID > (1<<40)-1 { - panic("series ID exceeds 5 bytes") - } - if chunkID > (1<<24)-1 { - panic("chunk ID exceeds 3 bytes") - } - return (seriesID << 24) | chunkID -} - -func unpackChunkID(id uint64) (seriesID, chunkID uint64) { - return id >> 24, (id << 40) >> 40 -} - -// Chunk returns the chunk for the reference number. -func (h *headChunkReader) Chunk(ref uint64) (chunkenc.Chunk, error) { - sid, cid := unpackChunkID(ref) - - s := h.head.series.getByID(sid) - // This means that the series has been garbage collected. - if s == nil { - return nil, storage.ErrNotFound - } - - s.Lock() - c, garbageCollect, err := s.chunk(int(cid), h.head.chunkDiskMapper) - if err != nil { - s.Unlock() - return nil, err - } - defer func() { - if garbageCollect { - // Set this to nil so that Go GC can collect it after it has been used. - c.chunk = nil - s.memChunkPool.Put(c) - } - }() - - // This means that the chunk is outside the specified range. - if !c.OverlapsClosedInterval(h.mint, h.maxt) { - s.Unlock() - return nil, storage.ErrNotFound - } - s.Unlock() - - return &safeChunk{ - Chunk: c.chunk, - s: s, - cid: int(cid), - isoState: h.isoState, - chunkDiskMapper: h.head.chunkDiskMapper, - }, nil -} - -type safeChunk struct { - chunkenc.Chunk - s *memSeries - cid int - isoState *isolationState - chunkDiskMapper *chunks.ChunkDiskMapper -} - -func (c *safeChunk) Iterator(reuseIter chunkenc.Iterator) chunkenc.Iterator { - c.s.Lock() - it := c.s.iterator(c.cid, c.isoState, c.chunkDiskMapper, reuseIter) - c.s.Unlock() - return it -} - -type headIndexReader struct { - head *Head - mint, maxt int64 -} - -func (h *headIndexReader) Close() error { - return nil -} - -func (h *headIndexReader) Symbols() index.StringIter { - h.head.symMtx.RLock() - res := make([]string, 0, len(h.head.symbols)) - - for s := range h.head.symbols { - res = append(res, s) - } - h.head.symMtx.RUnlock() - - sort.Strings(res) - return index.NewStringListIter(res) -} - -// SortedLabelValues returns label values present in the head for the -// specific label name that are within the time range mint to maxt. -// If matchers are specified the returned result set is reduced -// to label values of metrics matching the matchers. -func (h *headIndexReader) SortedLabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { - values, err := h.LabelValues(name, matchers...) - if err == nil { - sort.Strings(values) - } - return values, err -} - -// LabelValues returns label values present in the head for the -// specific label name that are within the time range mint to maxt. -// If matchers are specified the returned result set is reduced -// to label values of metrics matching the matchers. -func (h *headIndexReader) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { - if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { - return []string{}, nil - } - - if len(matchers) == 0 { - h.head.symMtx.RLock() - defer h.head.symMtx.RUnlock() - return h.head.postings.LabelValues(name), nil - } - - return labelValuesWithMatchers(h, name, matchers...) -} - -// LabelNames returns all the unique label names present in the head -// that are within the time range mint to maxt. -func (h *headIndexReader) LabelNames() ([]string, error) { - h.head.symMtx.RLock() - if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { - h.head.symMtx.RUnlock() - return []string{}, nil - } - - labelNames := h.head.postings.LabelNames() - h.head.symMtx.RUnlock() - - sort.Strings(labelNames) - return labelNames, nil -} - -// Postings returns the postings list iterator for the label pairs. -func (h *headIndexReader) Postings(name string, values ...string) (index.Postings, error) { - res := make([]index.Postings, 0, len(values)) - for _, value := range values { - res = append(res, h.head.postings.Get(name, value)) - } - return index.Merge(res...), nil -} - -func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings { - series := make([]*memSeries, 0, 128) - - // Fetch all the series only once. - for p.Next() { - s := h.head.series.getByID(p.At()) - if s == nil { - level.Debug(h.head.logger).Log("msg", "Looked up series not found") - } else { - series = append(series, s) - } - } - if err := p.Err(); err != nil { - return index.ErrPostings(errors.Wrap(err, "expand postings")) - } - - sort.Slice(series, func(i, j int) bool { - return labels.Compare(series[i].lset, series[j].lset) < 0 - }) - - // Convert back to list. - ep := make([]uint64, 0, len(series)) - for _, p := range series { - ep = append(ep, p.ref) - } - return index.NewListPostings(ep) -} - -// Series returns the series for the given reference. -func (h *headIndexReader) Series(ref uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { - s := h.head.series.getByID(ref) - - if s == nil { - h.head.metrics.seriesNotFound.Inc() - return storage.ErrNotFound - } - *lbls = append((*lbls)[:0], s.lset...) - - s.Lock() - defer s.Unlock() - - *chks = (*chks)[:0] - - for i, c := range s.mmappedChunks { - // Do not expose chunks that are outside of the specified range. - if !c.OverlapsClosedInterval(h.mint, h.maxt) { - continue - } - *chks = append(*chks, chunks.Meta{ - MinTime: c.minTime, - MaxTime: c.maxTime, - Ref: packChunkID(s.ref, uint64(s.chunkID(i))), - }) - } - if s.headChunk != nil && s.headChunk.OverlapsClosedInterval(h.mint, h.maxt) { - *chks = append(*chks, chunks.Meta{ - MinTime: s.headChunk.minTime, - MaxTime: math.MaxInt64, // Set the head chunks as open (being appended to). - Ref: packChunkID(s.ref, uint64(s.chunkID(len(s.mmappedChunks)))), - }) - } - - return nil -} - -// LabelValueFor returns label value for the given label name in the series referred to by ID. -func (h *headIndexReader) LabelValueFor(id uint64, label string) (string, error) { - memSeries := h.head.series.getByID(id) - if memSeries == nil { - return "", storage.ErrNotFound - } - - value := memSeries.lset.Get(label) - if value == "" { - return "", storage.ErrNotFound - } - - return value, nil -} - func (h *Head) getOrCreate(hash uint64, lset labels.Labels) (*memSeries, bool, error) { // Just using `getOrCreateWithID` below would be semantically sufficient, but we'd create // a new series on every sample inserted via Add(), which causes allocations @@ -2442,135 +1445,6 @@ func (s *memSeries) maxTime() int64 { return c.maxTime } -func (s *memSeries) cutNewHeadChunk(mint int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper) *memChunk { - s.mmapCurrentHeadChunk(chunkDiskMapper) - - s.headChunk = &memChunk{ - minTime: mint, - maxTime: math.MinInt64, - } - - if chunkenc.IsValidEncoding(e) { - var err error - s.headChunk.chunk, err = chunkenc.NewEmptyChunk(e) - if err != nil { - panic(err) // This should never happen. - } - } else { - s.headChunk.chunk = chunkenc.NewXORChunk() - } - - // Set upper bound on when the next chunk must be started. An earlier timestamp - // may be chosen dynamically at a later point. - s.nextAt = rangeForTimestamp(mint, s.chunkRange) - - app, err := s.headChunk.chunk.Appender() - if err != nil { - panic(err) - } - s.app = app - return s.headChunk -} - -func (s *memSeries) mmapCurrentHeadChunk(chunkDiskMapper *chunks.ChunkDiskMapper) { - if s.headChunk == nil || s.headChunk.chunk.NumSamples() == 0 { - // There is no head chunk, so nothing to m-map here. - return - } - - chunkRef, err := chunkDiskMapper.WriteChunk(s.ref, s.headChunk.minTime, s.headChunk.maxTime, s.headChunk.chunk) - if err != nil { - if err != chunks.ErrChunkDiskMapperClosed { - panic(err) - } - } - s.mmappedChunks = append(s.mmappedChunks, &mmappedChunk{ - ref: chunkRef, - numSamples: uint16(s.headChunk.chunk.NumSamples()), - minTime: s.headChunk.minTime, - maxTime: s.headChunk.maxTime, - }) -} - -// appendable checks whether the given sample is valid for appending to the series. -func (s *memSeries) appendable(t int64, v float64) error { - c := s.head() - if c == nil { - return nil - } - - if t > c.maxTime { - return nil - } - if t < c.maxTime { - return storage.ErrOutOfOrderSample - } - // We are allowing exact duplicates as we can encounter them in valid cases - // like federation and erroring out at that time would be extremely noisy. - if math.Float64bits(s.sampleBuf[3].v) != math.Float64bits(v) { - return storage.ErrDuplicateSampleForTimestamp - } - return nil -} - -// appendableHistogram checks whether the given sample is valid for appending to the series. -func (s *memSeries) appendableHistogram(t int64, sh histogram.SparseHistogram) error { - c := s.head() - if c == nil { - return nil - } - - if t > c.maxTime { - return nil - } - if t < c.maxTime { - return storage.ErrOutOfOrderSample - } - // TODO: do it for histogram. - // We are allowing exact duplicates as we can encounter them in valid cases - // like federation and erroring out at that time would be extremely noisy. - //if math.Float64bits(s.sampleBuf[3].v) != math.Float64bits(v) { - // return storage.ErrDuplicateSampleForTimestamp - //} - return nil -} - -// chunk returns the chunk for the chunk id from memory or by m-mapping it from the disk. -// If garbageCollect is true, it means that the returned *memChunk -// (and not the chunkenc.Chunk inside it) can be garbage collected after it's usage. -func (s *memSeries) chunk(id int, chunkDiskMapper *chunks.ChunkDiskMapper) (chunk *memChunk, garbageCollect bool, err error) { - // ix represents the index of chunk in the s.mmappedChunks slice. The chunk id's are - // incremented by 1 when new chunk is created, hence (id - firstChunkID) gives the slice index. - // The max index for the s.mmappedChunks slice can be len(s.mmappedChunks)-1, hence if the ix - // is len(s.mmappedChunks), it represents the next chunk, which is the head chunk. - ix := id - s.firstChunkID - if ix < 0 || ix > len(s.mmappedChunks) { - return nil, false, storage.ErrNotFound - } - if ix == len(s.mmappedChunks) { - if s.headChunk == nil { - return nil, false, errors.New("invalid head chunk") - } - return s.headChunk, false, nil - } - chk, err := chunkDiskMapper.Chunk(s.mmappedChunks[ix].ref) - if err != nil { - if _, ok := err.(*chunks.CorruptionErr); ok { - panic(err) - } - return nil, false, err - } - mc := s.memChunkPool.Get().(*memChunk) - mc.chunk = chk - mc.minTime = s.mmappedChunks[ix].minTime - mc.maxTime = s.mmappedChunks[ix].maxTime - return mc, true, nil -} - -func (s *memSeries) chunkID(pos int) int { - return pos + s.firstChunkID -} - // truncateChunksBefore removes all chunks from the series that // have no timestamp at or after mint. // Chunk IDs remain unchanged. @@ -2596,241 +1470,12 @@ func (s *memSeries) truncateChunksBefore(mint int64) (removed int) { return removed } -// append adds the sample (t, v) to the series. The caller also has to provide -// the appendID for isolation. (The appendID can be zero, which results in no -// isolation for this append.) -// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. -func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { - c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncXOR, chunkDiskMapper) - if !sampleInOrder { - return sampleInOrder, chunkCreated - } - - s.app.Append(t, v) - - c.maxTime = t - - s.sampleBuf[0] = s.sampleBuf[1] - s.sampleBuf[1] = s.sampleBuf[2] - s.sampleBuf[2] = s.sampleBuf[3] - s.sampleBuf[3] = sample{t: t, v: v} - - if appendID > 0 { - s.txs.add(appendID) - } - - return true, chunkCreated -} - -// appendHistogram adds the sparse histogram. -// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. -func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { - c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncSHS, chunkDiskMapper) - if !sampleInOrder { - return sampleInOrder, chunkCreated - } - - if !chunkCreated { - // Head controls the execution of recoding, so that we own the proper chunk reference afterwards - app, _ := s.app.(*chunkenc.HistoAppender) - posInterjections, negInterjections, ok := app.Appendable(sh) - // we have 3 cases here - // !ok -> we need to cut a new chunk - // ok but we have interjections -> existing chunk needs recoding before we can append our histogram - // ok and no interjections -> chunk is ready to support our histogram - if !ok { - c = s.cutNewHeadChunk(t, chunkenc.EncSHS, chunkDiskMapper) - chunkCreated = true - } else if len(posInterjections) > 0 || len(negInterjections) > 0 { - // new buckets have appeared. we need to recode all prior histograms within the chunk before we can process this one. - chunk, app := app.Recode(posInterjections, negInterjections, sh.PositiveSpans, sh.NegativeSpans) - s.headChunk = &memChunk{ - minTime: s.headChunk.minTime, - maxTime: s.headChunk.maxTime, - chunk: chunk, - } - s.app = app - } - } - - s.app.AppendHistogram(t, sh) - - c.maxTime = t - - s.histBuf[0] = s.histBuf[1] - s.histBuf[1] = s.histBuf[2] - s.histBuf[2] = s.histBuf[3] - s.histBuf[3] = hist{t: t, h: sh} - - if appendID > 0 { - s.txs.add(appendID) - } - - return true, chunkCreated -} - -// appendPreprocessor takes care of cutting new chunks and m-mapping old chunks. -// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. -// This should be called only when appending data. -func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper) (c *memChunk, sampleInOrder, chunkCreated bool) { - // Based on Gorilla white papers this offers near-optimal compression ratio - // so anything bigger that this has diminishing returns and increases - // the time range within which we have to decompress all samples. - const samplesPerChunk = 120 - - c = s.head() - - if c == nil { - if len(s.mmappedChunks) > 0 && s.mmappedChunks[len(s.mmappedChunks)-1].maxTime >= t { - // Out of order sample. Sample timestamp is already in the mmaped chunks, so ignore it. - return c, false, false - } - // There is no chunk in this series yet, create the first chunk for the sample. - c = s.cutNewHeadChunk(t, e, chunkDiskMapper) - chunkCreated = true - } - numSamples := c.chunk.NumSamples() - - // Out of order sample. - if c.maxTime >= t { - return c, false, chunkCreated - } - // If we reach 25% of a chunk's desired sample count, set a definitive time - // at which to start the next chunk. - // At latest it must happen at the timestamp set when the chunk was cut. - if numSamples == samplesPerChunk/4 { - s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt) - } - if t >= s.nextAt { - c = s.cutNewHeadChunk(t, e, chunkDiskMapper) - chunkCreated = true - } - return c, true, chunkCreated -} - // cleanupAppendIDsBelow cleans up older appendIDs. Has to be called after // acquiring lock. func (s *memSeries) cleanupAppendIDsBelow(bound uint64) { s.txs.cleanupAppendIDsBelow(bound) } -// computeChunkEndTime estimates the end timestamp based the beginning of a -// chunk, its current timestamp and the upper bound up to which we insert data. -// It assumes that the time range is 1/4 full. -func computeChunkEndTime(start, cur, max int64) int64 { - a := (max - start) / ((cur - start + 1) * 4) - if a == 0 { - return max - } - return start + (max-start)/a -} - -// iterator returns a chunk iterator. -// It is unsafe to call this concurrently with s.append(...) without holding the series lock. -func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *chunks.ChunkDiskMapper, it chunkenc.Iterator) (ttt chunkenc.Iterator) { - c, garbageCollect, err := s.chunk(id, chunkDiskMapper) - // TODO(fabxc): Work around! An error will be returns when a querier have retrieved a pointer to a - // series's chunk, which got then garbage collected before it got - // accessed. We must ensure to not garbage collect as long as any - // readers still hold a reference. - if err != nil { - return chunkenc.NewNopIterator() - } - defer func() { - if garbageCollect { - // Set this to nil so that Go GC can collect it after it has been used. - // This should be done always at the end. - c.chunk = nil - s.memChunkPool.Put(c) - } - }() - - ix := id - s.firstChunkID - - numSamples := c.chunk.NumSamples() - stopAfter := numSamples - - if isoState != nil { - totalSamples := 0 // Total samples in this series. - previousSamples := 0 // Samples before this chunk. - - for j, d := range s.mmappedChunks { - totalSamples += int(d.numSamples) - if j < ix { - previousSamples += int(d.numSamples) - } - } - - if s.headChunk != nil { - totalSamples += s.headChunk.chunk.NumSamples() - } - - // Removing the extra transactionIDs that are relevant for samples that - // come after this chunk, from the total transactionIDs. - appendIDsToConsider := s.txs.txIDCount - (totalSamples - (previousSamples + numSamples)) - - // Iterate over the appendIDs, find the first one that the isolation state says not - // to return. - it := s.txs.iterator() - for index := 0; index < appendIDsToConsider; index++ { - appendID := it.At() - if appendID <= isoState.maxAppendID { // Easy check first. - if _, ok := isoState.incompleteAppends[appendID]; !ok { - it.Next() - continue - } - } - stopAfter = numSamples - (appendIDsToConsider - index) - if stopAfter < 0 { - stopAfter = 0 // Stopped in a previous chunk. - } - break - } - } - - if stopAfter == 0 { - return chunkenc.NewNopIterator() - } - - if id-s.firstChunkID < len(s.mmappedChunks) { - if stopAfter == numSamples { - return c.chunk.Iterator(it) - } - if msIter, ok := it.(*stopIterator); ok { - msIter.Iterator = c.chunk.Iterator(msIter.Iterator) - msIter.i = -1 - msIter.stopAfter = stopAfter - return msIter - } - return &stopIterator{ - Iterator: c.chunk.Iterator(it), - i: -1, - stopAfter: stopAfter, - } - } - // Serve the last 4 samples for the last chunk from the sample buffer - // as their compressed bytes may be mutated by added samples. - if msIter, ok := it.(*memSafeIterator); ok { - msIter.Iterator = c.chunk.Iterator(msIter.Iterator) - msIter.i = -1 - msIter.total = numSamples - msIter.stopAfter = stopAfter - msIter.buf = s.sampleBuf - msIter.histBuf = s.histBuf - return msIter - } - return &memSafeIterator{ - stopIterator: stopIterator{ - Iterator: c.chunk.Iterator(it), - i: -1, - stopAfter: stopAfter, - }, - total: numSamples, - buf: s.sampleBuf, - histBuf: s.histBuf, - } -} - func (s *memSeries) head() *memChunk { return s.headChunk } @@ -2842,73 +1487,11 @@ type memChunk struct { // OverlapsClosedInterval returns true if the chunk overlaps [mint, maxt]. func (mc *memChunk) OverlapsClosedInterval(mint, maxt int64) bool { - return mc.minTime <= maxt && mint <= mc.maxTime + return overlapsClosedInterval(mc.minTime, mc.maxTime, mint, maxt) } -type stopIterator struct { - chunkenc.Iterator - - i, stopAfter int -} - -func (it *stopIterator) Next() bool { - if it.i+1 >= it.stopAfter { - return false - } - it.i++ - return it.Iterator.Next() -} - -type memSafeIterator struct { - stopIterator - - total int - buf [4]sample - histBuf [4]hist -} - -func (it *memSafeIterator) Seek(t int64) bool { - if it.Err() != nil { - return false - } - - ts, _ := it.At() - - for t > ts || it.i == -1 { - if !it.Next() { - return false - } - ts, _ = it.At() - } - - return true -} - -func (it *memSafeIterator) Next() bool { - if it.i+1 >= it.stopAfter { - return false - } - it.i++ - if it.total-it.i > 4 { - return it.Iterator.Next() - } - return true -} - -func (it *memSafeIterator) At() (int64, float64) { - if it.total-it.i > 4 { - return it.Iterator.At() - } - s := it.buf[4-(it.total-it.i)] - return s.t, s.v -} - -func (it *memSafeIterator) AtHistogram() (int64, histogram.SparseHistogram) { - if it.total-it.i > 4 { - return it.Iterator.AtHistogram() - } - s := it.histBuf[4-(it.total-it.i)] - return s.t, s.h +func overlapsClosedInterval(mint1, maxt1, mint2, maxt2 int64) bool { + return mint1 <= maxt2 && mint2 <= maxt1 } type mmappedChunk struct { @@ -2919,22 +1502,7 @@ type mmappedChunk struct { // Returns true if the chunk overlaps [mint, maxt]. func (mc *mmappedChunk) OverlapsClosedInterval(mint, maxt int64) bool { - return mc.minTime <= maxt && mint <= mc.maxTime -} - -// SeriesLifecycleCallback specifies a list of callbacks that will be called during a lifecycle of a series. -// It is always a no-op in Prometheus and mainly meant for external users who import TSDB. -// All the callbacks should be safe to be called concurrently. -// It is up to the user to implement soft or hard consistency by making the callbacks -// atomic or non-atomic. Atomic callbacks can cause degradation performance. -type SeriesLifecycleCallback interface { - // PreCreation is called before creating a series to indicate if the series can be created. - // A non nil error means the series should not be created. - PreCreation(labels.Labels) error - // PostCreation is called after creating a series to indicate a creation of series. - PostCreation(labels.Labels) - // PostDeletion is called after deletion of series. - PostDeletion(...labels.Labels) + return overlapsClosedInterval(mc.minTime, mc.maxTime, mint, maxt) } type noopSeriesLifecycleCallback struct{} diff --git a/tsdb/head_append.go b/tsdb/head_append.go new file mode 100644 index 000000000..8f7296bf2 --- /dev/null +++ b/tsdb/head_append.go @@ -0,0 +1,764 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdb + +import ( + "context" + "fmt" + "math" + + "github.com/go-kit/log/level" + "github.com/pkg/errors" + + "github.com/prometheus/prometheus/pkg/exemplar" + "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/record" +) + +// initAppender is a helper to initialize the time bounds of the head +// upon the first sample it receives. +type initAppender struct { + app storage.Appender + head *Head +} + +var _ storage.GetRef = &initAppender{} + +func (a *initAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) { + if a.app != nil { + return a.app.Append(ref, lset, t, v) + } + + a.head.initTime(t) + a.app = a.head.appender() + return a.app.Append(ref, lset, t, v) +} + +func (a *initAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { + // Check if exemplar storage is enabled. + if !a.head.opts.EnableExemplarStorage || a.head.opts.MaxExemplars.Load() <= 0 { + return 0, nil + } + + if a.app != nil { + return a.app.AppendExemplar(ref, l, e) + } + // We should never reach here given we would call Append before AppendExemplar + // and we probably want to always base head/WAL min time on sample times. + a.head.initTime(e.Ts) + a.app = a.head.appender() + + return a.app.AppendExemplar(ref, l, e) +} + +func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { + if a.app != nil { + return a.app.AppendHistogram(ref, l, t, sh) + } + a.head.initTime(t) + a.app = a.head.appender() + + return a.app.AppendHistogram(ref, l, t, sh) +} + +// initTime initializes a head with the first timestamp. This only needs to be called +// for a completely fresh head with an empty WAL. +func (h *Head) initTime(t int64) { + if !h.minTime.CAS(math.MaxInt64, t) { + return + } + // Ensure that max time is initialized to at least the min time we just set. + // Concurrent appenders may already have set it to a higher value. + h.maxTime.CAS(math.MinInt64, t) +} + +func (a *initAppender) GetRef(lset labels.Labels) (uint64, labels.Labels) { + if g, ok := a.app.(storage.GetRef); ok { + return g.GetRef(lset) + } + return 0, nil +} + +func (a *initAppender) Commit() error { + if a.app == nil { + return nil + } + return a.app.Commit() +} + +func (a *initAppender) Rollback() error { + if a.app == nil { + return nil + } + return a.app.Rollback() +} + +// Appender returns a new Appender on the database. +func (h *Head) Appender(_ context.Context) storage.Appender { + h.metrics.activeAppenders.Inc() + + // The head cache might not have a starting point yet. The init appender + // picks up the first appended timestamp as the base. + if h.MinTime() == math.MaxInt64 { + return &initAppender{ + head: h, + } + } + return h.appender() +} + +func (h *Head) appender() *headAppender { + appendID, cleanupAppendIDsBelow := h.iso.newAppendID() + + // Allocate the exemplars buffer only if exemplars are enabled. + var exemplarsBuf []exemplarWithSeriesRef + if h.opts.EnableExemplarStorage { + exemplarsBuf = h.getExemplarBuffer() + } + + return &headAppender{ + head: h, + minValidTime: h.appendableMinValidTime(), + mint: math.MaxInt64, + maxt: math.MinInt64, + samples: h.getAppendBuffer(), + sampleSeries: h.getSeriesBuffer(), + exemplars: exemplarsBuf, + appendID: appendID, + cleanupAppendIDsBelow: cleanupAppendIDsBelow, + } +} + +func (h *Head) appendableMinValidTime() int64 { + // Setting the minimum valid time to whichever is greater, the head min valid time or the compaction window, + // ensures that no samples will be added within the compaction window to avoid races. + return max(h.minValidTime.Load(), h.MaxTime()-h.chunkRange.Load()/2) +} + +func max(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func (h *Head) getAppendBuffer() []record.RefSample { + b := h.appendPool.Get() + if b == nil { + return make([]record.RefSample, 0, 512) + } + return b.([]record.RefSample) +} + +func (h *Head) putAppendBuffer(b []record.RefSample) { + //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. + h.appendPool.Put(b[:0]) +} + +func (h *Head) getExemplarBuffer() []exemplarWithSeriesRef { + b := h.exemplarsPool.Get() + if b == nil { + return make([]exemplarWithSeriesRef, 0, 512) + } + return b.([]exemplarWithSeriesRef) +} + +func (h *Head) putExemplarBuffer(b []exemplarWithSeriesRef) { + if b == nil { + return + } + + //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. + h.exemplarsPool.Put(b[:0]) +} + +func (h *Head) getSeriesBuffer() []*memSeries { + b := h.seriesPool.Get() + if b == nil { + return make([]*memSeries, 0, 512) + } + return b.([]*memSeries) +} + +func (h *Head) putSeriesBuffer(b []*memSeries) { + //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. + h.seriesPool.Put(b[:0]) +} + +func (h *Head) getBytesBuffer() []byte { + b := h.bytesPool.Get() + if b == nil { + return make([]byte, 0, 1024) + } + return b.([]byte) +} + +func (h *Head) putBytesBuffer(b []byte) { + //nolint:staticcheck // Ignore SA6002 safe to ignore and actually fixing it has some performance penalty. + h.bytesPool.Put(b[:0]) +} + +type exemplarWithSeriesRef struct { + ref uint64 + exemplar exemplar.Exemplar +} + +type headAppender struct { + head *Head + minValidTime int64 // No samples below this timestamp are allowed. + mint, maxt int64 + + series []record.RefSeries + samples []record.RefSample + exemplars []exemplarWithSeriesRef + sampleSeries []*memSeries + histograms []record.RefHistogram + histogramSeries []*memSeries + + appendID, cleanupAppendIDsBelow uint64 + closed bool +} + +func (a *headAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) { + if t < a.minValidTime { + a.head.metrics.outOfBoundSamples.Inc() + return 0, storage.ErrOutOfBounds + } + + s := a.head.series.getByID(ref) + if s == nil { + // Ensure no empty labels have gotten through. + lset = lset.WithoutEmpty() + if len(lset) == 0 { + return 0, errors.Wrap(ErrInvalidSample, "empty labelset") + } + + if l, dup := lset.HasDuplicateLabelNames(); dup { + return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l)) + } + + var created bool + var err error + s, created, err = a.head.getOrCreate(lset.Hash(), lset) + if err != nil { + return 0, err + } + if created { + a.series = append(a.series, record.RefSeries{ + Ref: s.ref, + Labels: lset, + }) + } + } + + s.Lock() + if err := s.appendable(t, v); err != nil { + s.Unlock() + if err == storage.ErrOutOfOrderSample { + a.head.metrics.outOfOrderSamples.Inc() + } + return 0, err + } + s.pendingCommit = true + s.Unlock() + + if t < a.mint { + a.mint = t + } + if t > a.maxt { + a.maxt = t + } + + a.samples = append(a.samples, record.RefSample{ + Ref: s.ref, + T: t, + V: v, + }) + a.sampleSeries = append(a.sampleSeries, s) + return s.ref, nil +} + +// appendable checks whether the given sample is valid for appending to the series. +func (s *memSeries) appendable(t int64, v float64) error { + c := s.head() + if c == nil { + return nil + } + + if t > c.maxTime { + return nil + } + if t < c.maxTime { + return storage.ErrOutOfOrderSample + } + // We are allowing exact duplicates as we can encounter them in valid cases + // like federation and erroring out at that time would be extremely noisy. + if math.Float64bits(s.sampleBuf[3].v) != math.Float64bits(v) { + return storage.ErrDuplicateSampleForTimestamp + } + return nil +} + +// appendableHistogram checks whether the given sample is valid for appending to the series. +func (s *memSeries) appendableHistogram(t int64, sh histogram.SparseHistogram) error { + c := s.head() + if c == nil { + return nil + } + + if t > c.maxTime { + return nil + } + if t < c.maxTime { + return storage.ErrOutOfOrderSample + } + // TODO: do it for histogram. + // We are allowing exact duplicates as we can encounter them in valid cases + // like federation and erroring out at that time would be extremely noisy. + //if math.Float64bits(s.sampleBuf[3].v) != math.Float64bits(v) { + // return storage.ErrDuplicateSampleForTimestamp + //} + return nil +} + +// AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't +// use getOrCreate or make any of the lset sanity checks that Append does. +func (a *headAppender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exemplar) (uint64, error) { + // Check if exemplar storage is enabled. + if !a.head.opts.EnableExemplarStorage || a.head.opts.MaxExemplars.Load() <= 0 { + return 0, nil + } + s := a.head.series.getByID(ref) + if s == nil { + return 0, fmt.Errorf("unknown series ref. when trying to add exemplar: %d", ref) + } + + // Ensure no empty labels have gotten through. + e.Labels = e.Labels.WithoutEmpty() + + err := a.head.exemplars.ValidateExemplar(s.lset, e) + if err != nil { + if err == storage.ErrDuplicateExemplar || err == storage.ErrExemplarsDisabled { + // Duplicate, don't return an error but don't accept the exemplar. + return 0, nil + } + return 0, err + } + + a.exemplars = append(a.exemplars, exemplarWithSeriesRef{ref, e}) + + return s.ref, nil +} + +func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) { + if t < a.minValidTime { + a.head.metrics.outOfBoundSamples.Inc() + return 0, storage.ErrOutOfBounds + } + + s := a.head.series.getByID(ref) + if s == nil { + // Ensure no empty labels have gotten through. + lset = lset.WithoutEmpty() + if len(lset) == 0 { + return 0, errors.Wrap(ErrInvalidSample, "empty labelset") + } + + if l, dup := lset.HasDuplicateLabelNames(); dup { + return 0, errors.Wrap(ErrInvalidSample, fmt.Sprintf(`label name "%s" is not unique`, l)) + } + + var created bool + var err error + s, created, err = a.head.getOrCreate(lset.Hash(), lset) + if err != nil { + return 0, err + } + if created { + a.series = append(a.series, record.RefSeries{ + Ref: s.ref, + Labels: lset, + }) + } + } + + s.Lock() + if err := s.appendableHistogram(t, sh); err != nil { + s.Unlock() + if err == storage.ErrOutOfOrderSample { + a.head.metrics.outOfOrderSamples.Inc() + } + return 0, err + } + s.pendingCommit = true + s.Unlock() + + if t < a.mint { + a.mint = t + } + if t > a.maxt { + a.maxt = t + } + + a.histograms = append(a.histograms, record.RefHistogram{ + Ref: s.ref, + T: t, + H: sh, + }) + a.histogramSeries = append(a.histogramSeries, s) + return s.ref, nil +} + +var _ storage.GetRef = &headAppender{} + +func (a *headAppender) GetRef(lset labels.Labels) (uint64, labels.Labels) { + s := a.head.series.getByHash(lset.Hash(), lset) + if s == nil { + return 0, nil + } + // returned labels must be suitable to pass to Append() + return s.ref, s.lset +} + +func (a *headAppender) log() error { + if a.head.wal == nil { + return nil + } + + buf := a.head.getBytesBuffer() + defer func() { a.head.putBytesBuffer(buf) }() + + var rec []byte + var enc record.Encoder + + if len(a.series) > 0 { + rec = enc.Series(a.series, buf) + buf = rec[:0] + + if err := a.head.wal.Log(rec); err != nil { + return errors.Wrap(err, "log series") + } + } + if len(a.samples) > 0 { + rec = enc.Samples(a.samples, buf) + buf = rec[:0] + + if err := a.head.wal.Log(rec); err != nil { + return errors.Wrap(err, "log samples") + } + } + if len(a.exemplars) > 0 { + rec = enc.Exemplars(exemplarsForEncoding(a.exemplars), buf) + buf = rec[:0] + + if err := a.head.wal.Log(rec); err != nil { + return errors.Wrap(err, "log exemplars") + } + } + return nil +} + +func exemplarsForEncoding(es []exemplarWithSeriesRef) []record.RefExemplar { + ret := make([]record.RefExemplar, 0, len(es)) + for _, e := range es { + ret = append(ret, record.RefExemplar{ + Ref: e.ref, + T: e.exemplar.Ts, + V: e.exemplar.Value, + Labels: e.exemplar.Labels, + }) + } + return ret +} + +func (a *headAppender) Commit() (err error) { + if a.closed { + return ErrAppenderClosed + } + defer func() { a.closed = true }() + + if err := a.log(); err != nil { + _ = a.Rollback() // Most likely the same error will happen again. + return errors.Wrap(err, "write to WAL") + } + + // No errors logging to WAL, so pass the exemplars along to the in memory storage. + for _, e := range a.exemplars { + s := a.head.series.getByID(e.ref) + // We don't instrument exemplar appends here, all is instrumented by storage. + if err := a.head.exemplars.AddExemplar(s.lset, e.exemplar); err != nil { + if err == storage.ErrOutOfOrderExemplar { + continue + } + level.Debug(a.head.logger).Log("msg", "Unknown error while adding exemplar", "err", err) + } + } + + defer a.head.metrics.activeAppenders.Dec() + defer a.head.putAppendBuffer(a.samples) + defer a.head.putSeriesBuffer(a.sampleSeries) + defer a.head.putExemplarBuffer(a.exemplars) + defer a.head.iso.closeAppend(a.appendID) + + total := len(a.samples) + var series *memSeries + for i, s := range a.samples { + series = a.sampleSeries[i] + series.Lock() + ok, chunkCreated := series.append(s.T, s.V, a.appendID, a.head.chunkDiskMapper) + series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) + series.pendingCommit = false + series.Unlock() + + if !ok { + total-- + a.head.metrics.outOfOrderSamples.Inc() + } + if chunkCreated { + a.head.metrics.chunks.Inc() + a.head.metrics.chunksCreated.Inc() + } + } + + total += len(a.histograms) // TODO: different metric? + for i, s := range a.histograms { + series = a.histogramSeries[i] + series.Lock() + a.head.hasHistograms.Store(true) + ok, chunkCreated := series.appendHistogram(s.T, s.H, a.appendID, a.head.chunkDiskMapper) + series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) + series.pendingCommit = false + series.Unlock() + + if !ok { + total-- + a.head.metrics.outOfOrderSamples.Inc() + } + if chunkCreated { + a.head.metrics.chunks.Inc() + a.head.metrics.chunksCreated.Inc() + } + } + + a.head.metrics.samplesAppended.Add(float64(total)) + a.head.updateMinMaxTime(a.mint, a.maxt) + + return nil +} + +// append adds the sample (t, v) to the series. The caller also has to provide +// the appendID for isolation. (The appendID can be zero, which results in no +// isolation for this append.) +// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. +func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { + c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncXOR, chunkDiskMapper) + if !sampleInOrder { + return sampleInOrder, chunkCreated + } + + s.app.Append(t, v) + + c.maxTime = t + + s.sampleBuf[0] = s.sampleBuf[1] + s.sampleBuf[1] = s.sampleBuf[2] + s.sampleBuf[2] = s.sampleBuf[3] + s.sampleBuf[3] = sample{t: t, v: v} + + if appendID > 0 { + s.txs.add(appendID) + } + + return true, chunkCreated +} + +// appendHistogram adds the sparse histogram. +// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. +func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) { + c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncSHS, chunkDiskMapper) + if !sampleInOrder { + return sampleInOrder, chunkCreated + } + + if !chunkCreated { + // Head controls the execution of recoding, so that we own the proper chunk reference afterwards + app, _ := s.app.(*chunkenc.HistoAppender) + posInterjections, negInterjections, ok := app.Appendable(sh) + // we have 3 cases here + // !ok -> we need to cut a new chunk + // ok but we have interjections -> existing chunk needs recoding before we can append our histogram + // ok and no interjections -> chunk is ready to support our histogram + if !ok { + c = s.cutNewHeadChunk(t, chunkenc.EncSHS, chunkDiskMapper) + chunkCreated = true + } else if len(posInterjections) > 0 || len(negInterjections) > 0 { + // new buckets have appeared. we need to recode all prior histograms within the chunk before we can process this one. + chunk, app := app.Recode(posInterjections, negInterjections, sh.PositiveSpans, sh.NegativeSpans) + s.headChunk = &memChunk{ + minTime: s.headChunk.minTime, + maxTime: s.headChunk.maxTime, + chunk: chunk, + } + s.app = app + } + } + + s.app.AppendHistogram(t, sh) + + c.maxTime = t + + s.histBuf[0] = s.histBuf[1] + s.histBuf[1] = s.histBuf[2] + s.histBuf[2] = s.histBuf[3] + s.histBuf[3] = hist{t: t, h: sh} + + if appendID > 0 { + s.txs.add(appendID) + } + + return true, chunkCreated +} + +// appendPreprocessor takes care of cutting new chunks and m-mapping old chunks. +// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock. +// This should be called only when appending data. +func (s *memSeries) appendPreprocessor(t int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper) (c *memChunk, sampleInOrder, chunkCreated bool) { + // Based on Gorilla white papers this offers near-optimal compression ratio + // so anything bigger that this has diminishing returns and increases + // the time range within which we have to decompress all samples. + const samplesPerChunk = 120 + + c = s.head() + + if c == nil { + if len(s.mmappedChunks) > 0 && s.mmappedChunks[len(s.mmappedChunks)-1].maxTime >= t { + // Out of order sample. Sample timestamp is already in the mmaped chunks, so ignore it. + return c, false, false + } + // There is no chunk in this series yet, create the first chunk for the sample. + c = s.cutNewHeadChunk(t, e, chunkDiskMapper) + chunkCreated = true + } + numSamples := c.chunk.NumSamples() + + // Out of order sample. + if c.maxTime >= t { + return c, false, chunkCreated + } + // If we reach 25% of a chunk's desired sample count, set a definitive time + // at which to start the next chunk. + // At latest it must happen at the timestamp set when the chunk was cut. + if numSamples == samplesPerChunk/4 { + s.nextAt = computeChunkEndTime(c.minTime, c.maxTime, s.nextAt) + } + if t >= s.nextAt { + c = s.cutNewHeadChunk(t, e, chunkDiskMapper) + chunkCreated = true + } + return c, true, chunkCreated +} + +// computeChunkEndTime estimates the end timestamp based the beginning of a +// chunk, its current timestamp and the upper bound up to which we insert data. +// It assumes that the time range is 1/4 full. +// Assuming that the samples will keep arriving at the same rate, it will make the +// remaining n chunks within this chunk range (before max) equally sized. +func computeChunkEndTime(start, cur, max int64) int64 { + n := (max - start) / ((cur - start + 1) * 4) + if n <= 1 { + return max + } + return start + (max-start)/n +} + +func (s *memSeries) cutNewHeadChunk(mint int64, e chunkenc.Encoding, chunkDiskMapper *chunks.ChunkDiskMapper) *memChunk { + s.mmapCurrentHeadChunk(chunkDiskMapper) + + s.headChunk = &memChunk{ + minTime: mint, + maxTime: math.MinInt64, + } + + if chunkenc.IsValidEncoding(e) { + var err error + s.headChunk.chunk, err = chunkenc.NewEmptyChunk(e) + if err != nil { + panic(err) // This should never happen. + } + } else { + s.headChunk.chunk = chunkenc.NewXORChunk() + } + + // Set upper bound on when the next chunk must be started. An earlier timestamp + // may be chosen dynamically at a later point. + s.nextAt = rangeForTimestamp(mint, s.chunkRange) + + app, err := s.headChunk.chunk.Appender() + if err != nil { + panic(err) + } + s.app = app + return s.headChunk +} + +func (s *memSeries) mmapCurrentHeadChunk(chunkDiskMapper *chunks.ChunkDiskMapper) { + if s.headChunk == nil || s.headChunk.chunk.NumSamples() == 0 { + // There is no head chunk, so nothing to m-map here. + return + } + + chunkRef, err := chunkDiskMapper.WriteChunk(s.ref, s.headChunk.minTime, s.headChunk.maxTime, s.headChunk.chunk) + if err != nil { + if err != chunks.ErrChunkDiskMapperClosed { + panic(err) + } + } + s.mmappedChunks = append(s.mmappedChunks, &mmappedChunk{ + ref: chunkRef, + numSamples: uint16(s.headChunk.chunk.NumSamples()), + minTime: s.headChunk.minTime, + maxTime: s.headChunk.maxTime, + }) +} + +func (a *headAppender) Rollback() (err error) { + if a.closed { + return ErrAppenderClosed + } + defer func() { a.closed = true }() + defer a.head.metrics.activeAppenders.Dec() + defer a.head.iso.closeAppend(a.appendID) + defer a.head.putSeriesBuffer(a.sampleSeries) + + var series *memSeries + for i := range a.samples { + series = a.sampleSeries[i] + series.Lock() + series.cleanupAppendIDsBelow(a.cleanupAppendIDsBelow) + series.pendingCommit = false + series.Unlock() + } + a.head.putAppendBuffer(a.samples) + a.head.putExemplarBuffer(a.exemplars) + a.samples = nil + a.exemplars = nil + + // Series are created in the head memory regardless of rollback. Thus we have + // to log them to the WAL in any case. + return a.log() +} diff --git a/tsdb/head_read.go b/tsdb/head_read.go new file mode 100644 index 000000000..a5fc2df1e --- /dev/null +++ b/tsdb/head_read.go @@ -0,0 +1,539 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdb + +import ( + "context" + "math" + "sort" + + "github.com/go-kit/log/level" + "github.com/pkg/errors" + + "github.com/prometheus/prometheus/pkg/histogram" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/index" +) + +func (h *Head) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { + return h.exemplars.ExemplarQuerier(ctx) +} + +// Index returns an IndexReader against the block. +func (h *Head) Index() (IndexReader, error) { + return h.indexRange(math.MinInt64, math.MaxInt64), nil +} + +func (h *Head) indexRange(mint, maxt int64) *headIndexReader { + if hmin := h.MinTime(); hmin > mint { + mint = hmin + } + return &headIndexReader{head: h, mint: mint, maxt: maxt} +} + +type headIndexReader struct { + head *Head + mint, maxt int64 +} + +func (h *headIndexReader) Close() error { + return nil +} + +func (h *headIndexReader) Symbols() index.StringIter { + h.head.symMtx.RLock() + res := make([]string, 0, len(h.head.symbols)) + + for s := range h.head.symbols { + res = append(res, s) + } + h.head.symMtx.RUnlock() + + sort.Strings(res) + return index.NewStringListIter(res) +} + +// SortedLabelValues returns label values present in the head for the +// specific label name that are within the time range mint to maxt. +// If matchers are specified the returned result set is reduced +// to label values of metrics matching the matchers. +func (h *headIndexReader) SortedLabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { + values, err := h.LabelValues(name, matchers...) + if err == nil { + sort.Strings(values) + } + return values, err +} + +// LabelValues returns label values present in the head for the +// specific label name that are within the time range mint to maxt. +// If matchers are specified the returned result set is reduced +// to label values of metrics matching the matchers. +func (h *headIndexReader) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { + if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { + return []string{}, nil + } + + if len(matchers) == 0 { + h.head.symMtx.RLock() + defer h.head.symMtx.RUnlock() + return h.head.postings.LabelValues(name), nil + } + + return labelValuesWithMatchers(h, name, matchers...) +} + +// LabelNames returns all the unique label names present in the head +// that are within the time range mint to maxt. +func (h *headIndexReader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { + if h.maxt < h.head.MinTime() || h.mint > h.head.MaxTime() { + return []string{}, nil + } + + if len(matchers) == 0 { + h.head.symMtx.RLock() + labelNames := h.head.postings.LabelNames() + h.head.symMtx.RUnlock() + + sort.Strings(labelNames) + return labelNames, nil + } + + return labelNamesWithMatchers(h, matchers...) +} + +// Postings returns the postings list iterator for the label pairs. +func (h *headIndexReader) Postings(name string, values ...string) (index.Postings, error) { + res := make([]index.Postings, 0, len(values)) + for _, value := range values { + res = append(res, h.head.postings.Get(name, value)) + } + return index.Merge(res...), nil +} + +func (h *headIndexReader) SortedPostings(p index.Postings) index.Postings { + series := make([]*memSeries, 0, 128) + + // Fetch all the series only once. + for p.Next() { + s := h.head.series.getByID(p.At()) + if s == nil { + level.Debug(h.head.logger).Log("msg", "Looked up series not found") + } else { + series = append(series, s) + } + } + if err := p.Err(); err != nil { + return index.ErrPostings(errors.Wrap(err, "expand postings")) + } + + sort.Slice(series, func(i, j int) bool { + return labels.Compare(series[i].lset, series[j].lset) < 0 + }) + + // Convert back to list. + ep := make([]uint64, 0, len(series)) + for _, p := range series { + ep = append(ep, p.ref) + } + return index.NewListPostings(ep) +} + +// Series returns the series for the given reference. +func (h *headIndexReader) Series(ref uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { + s := h.head.series.getByID(ref) + + if s == nil { + h.head.metrics.seriesNotFound.Inc() + return storage.ErrNotFound + } + *lbls = append((*lbls)[:0], s.lset...) + + s.Lock() + defer s.Unlock() + + *chks = (*chks)[:0] + + for i, c := range s.mmappedChunks { + // Do not expose chunks that are outside of the specified range. + if !c.OverlapsClosedInterval(h.mint, h.maxt) { + continue + } + *chks = append(*chks, chunks.Meta{ + MinTime: c.minTime, + MaxTime: c.maxTime, + Ref: packChunkID(s.ref, uint64(s.chunkID(i))), + }) + } + if s.headChunk != nil && s.headChunk.OverlapsClosedInterval(h.mint, h.maxt) { + *chks = append(*chks, chunks.Meta{ + MinTime: s.headChunk.minTime, + MaxTime: math.MaxInt64, // Set the head chunks as open (being appended to). + Ref: packChunkID(s.ref, uint64(s.chunkID(len(s.mmappedChunks)))), + }) + } + + return nil +} + +func (s *memSeries) chunkID(pos int) int { + return pos + s.firstChunkID +} + +// LabelValueFor returns label value for the given label name in the series referred to by ID. +func (h *headIndexReader) LabelValueFor(id uint64, label string) (string, error) { + memSeries := h.head.series.getByID(id) + if memSeries == nil { + return "", storage.ErrNotFound + } + + value := memSeries.lset.Get(label) + if value == "" { + return "", storage.ErrNotFound + } + + return value, nil +} + +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (h *headIndexReader) LabelNamesFor(ids ...uint64) ([]string, error) { + namesMap := make(map[string]struct{}) + for _, id := range ids { + memSeries := h.head.series.getByID(id) + if memSeries == nil { + return nil, storage.ErrNotFound + } + for _, lbl := range memSeries.lset { + namesMap[lbl.Name] = struct{}{} + } + } + names := make([]string, 0, len(namesMap)) + for name := range namesMap { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// Chunks returns a ChunkReader against the block. +func (h *Head) Chunks() (ChunkReader, error) { + return h.chunksRange(math.MinInt64, math.MaxInt64, h.iso.State(math.MinInt64, math.MaxInt64)) +} + +func (h *Head) chunksRange(mint, maxt int64, is *isolationState) (*headChunkReader, error) { + h.closedMtx.Lock() + defer h.closedMtx.Unlock() + if h.closed { + return nil, errors.New("can't read from a closed head") + } + if hmin := h.MinTime(); hmin > mint { + mint = hmin + } + return &headChunkReader{ + head: h, + mint: mint, + maxt: maxt, + isoState: is, + }, nil +} + +type headChunkReader struct { + head *Head + mint, maxt int64 + isoState *isolationState +} + +func (h *headChunkReader) Close() error { + h.isoState.Close() + return nil +} + +// packChunkID packs a seriesID and a chunkID within it into a global 8 byte ID. +// It panicks if the seriesID exceeds 5 bytes or the chunk ID 3 bytes. +func packChunkID(seriesID, chunkID uint64) uint64 { + if seriesID > (1<<40)-1 { + panic("series ID exceeds 5 bytes") + } + if chunkID > (1<<24)-1 { + panic("chunk ID exceeds 3 bytes") + } + return (seriesID << 24) | chunkID +} + +func unpackChunkID(id uint64) (seriesID, chunkID uint64) { + return id >> 24, (id << 40) >> 40 +} + +// Chunk returns the chunk for the reference number. +func (h *headChunkReader) Chunk(ref uint64) (chunkenc.Chunk, error) { + sid, cid := unpackChunkID(ref) + + s := h.head.series.getByID(sid) + // This means that the series has been garbage collected. + if s == nil { + return nil, storage.ErrNotFound + } + + s.Lock() + c, garbageCollect, err := s.chunk(int(cid), h.head.chunkDiskMapper) + if err != nil { + s.Unlock() + return nil, err + } + defer func() { + if garbageCollect { + // Set this to nil so that Go GC can collect it after it has been used. + c.chunk = nil + s.memChunkPool.Put(c) + } + }() + + // This means that the chunk is outside the specified range. + if !c.OverlapsClosedInterval(h.mint, h.maxt) { + s.Unlock() + return nil, storage.ErrNotFound + } + s.Unlock() + + return &safeChunk{ + Chunk: c.chunk, + s: s, + cid: int(cid), + isoState: h.isoState, + chunkDiskMapper: h.head.chunkDiskMapper, + }, nil +} + +// chunk returns the chunk for the chunk id from memory or by m-mapping it from the disk. +// If garbageCollect is true, it means that the returned *memChunk +// (and not the chunkenc.Chunk inside it) can be garbage collected after it's usage. +func (s *memSeries) chunk(id int, chunkDiskMapper *chunks.ChunkDiskMapper) (chunk *memChunk, garbageCollect bool, err error) { + // ix represents the index of chunk in the s.mmappedChunks slice. The chunk id's are + // incremented by 1 when new chunk is created, hence (id - firstChunkID) gives the slice index. + // The max index for the s.mmappedChunks slice can be len(s.mmappedChunks)-1, hence if the ix + // is len(s.mmappedChunks), it represents the next chunk, which is the head chunk. + ix := id - s.firstChunkID + if ix < 0 || ix > len(s.mmappedChunks) { + return nil, false, storage.ErrNotFound + } + if ix == len(s.mmappedChunks) { + if s.headChunk == nil { + return nil, false, errors.New("invalid head chunk") + } + return s.headChunk, false, nil + } + chk, err := chunkDiskMapper.Chunk(s.mmappedChunks[ix].ref) + if err != nil { + if _, ok := err.(*chunks.CorruptionErr); ok { + panic(err) + } + return nil, false, err + } + mc := s.memChunkPool.Get().(*memChunk) + mc.chunk = chk + mc.minTime = s.mmappedChunks[ix].minTime + mc.maxTime = s.mmappedChunks[ix].maxTime + return mc, true, nil +} + +type safeChunk struct { + chunkenc.Chunk + s *memSeries + cid int + isoState *isolationState + chunkDiskMapper *chunks.ChunkDiskMapper +} + +func (c *safeChunk) Iterator(reuseIter chunkenc.Iterator) chunkenc.Iterator { + c.s.Lock() + it := c.s.iterator(c.cid, c.isoState, c.chunkDiskMapper, reuseIter) + c.s.Unlock() + return it +} + +// iterator returns a chunk iterator. +// It is unsafe to call this concurrently with s.append(...) without holding the series lock. +func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *chunks.ChunkDiskMapper, it chunkenc.Iterator) chunkenc.Iterator { + c, garbageCollect, err := s.chunk(id, chunkDiskMapper) + // TODO(fabxc): Work around! An error will be returns when a querier have retrieved a pointer to a + // series's chunk, which got then garbage collected before it got + // accessed. We must ensure to not garbage collect as long as any + // readers still hold a reference. + if err != nil { + return chunkenc.NewNopIterator() + } + defer func() { + if garbageCollect { + // Set this to nil so that Go GC can collect it after it has been used. + // This should be done always at the end. + c.chunk = nil + s.memChunkPool.Put(c) + } + }() + + ix := id - s.firstChunkID + + numSamples := c.chunk.NumSamples() + stopAfter := numSamples + + if isoState != nil { + totalSamples := 0 // Total samples in this series. + previousSamples := 0 // Samples before this chunk. + + for j, d := range s.mmappedChunks { + totalSamples += int(d.numSamples) + if j < ix { + previousSamples += int(d.numSamples) + } + } + + if s.headChunk != nil { + totalSamples += s.headChunk.chunk.NumSamples() + } + + // Removing the extra transactionIDs that are relevant for samples that + // come after this chunk, from the total transactionIDs. + appendIDsToConsider := s.txs.txIDCount - (totalSamples - (previousSamples + numSamples)) + + // Iterate over the appendIDs, find the first one that the isolation state says not + // to return. + it := s.txs.iterator() + for index := 0; index < appendIDsToConsider; index++ { + appendID := it.At() + if appendID <= isoState.maxAppendID { // Easy check first. + if _, ok := isoState.incompleteAppends[appendID]; !ok { + it.Next() + continue + } + } + stopAfter = numSamples - (appendIDsToConsider - index) + if stopAfter < 0 { + stopAfter = 0 // Stopped in a previous chunk. + } + break + } + } + + if stopAfter == 0 { + return chunkenc.NewNopIterator() + } + + if id-s.firstChunkID < len(s.mmappedChunks) { + if stopAfter == numSamples { + return c.chunk.Iterator(it) + } + if msIter, ok := it.(*stopIterator); ok { + msIter.Iterator = c.chunk.Iterator(msIter.Iterator) + msIter.i = -1 + msIter.stopAfter = stopAfter + return msIter + } + return &stopIterator{ + Iterator: c.chunk.Iterator(it), + i: -1, + stopAfter: stopAfter, + } + } + // Serve the last 4 samples for the last chunk from the sample buffer + // as their compressed bytes may be mutated by added samples. + if msIter, ok := it.(*memSafeIterator); ok { + msIter.Iterator = c.chunk.Iterator(msIter.Iterator) + msIter.i = -1 + msIter.total = numSamples + msIter.stopAfter = stopAfter + msIter.buf = s.sampleBuf + msIter.histBuf = s.histBuf + return msIter + } + return &memSafeIterator{ + stopIterator: stopIterator{ + Iterator: c.chunk.Iterator(it), + i: -1, + stopAfter: stopAfter, + }, + total: numSamples, + buf: s.sampleBuf, + histBuf: s.histBuf, + } +} + +type memSafeIterator struct { + stopIterator + + total int + buf [4]sample + histBuf [4]hist +} + +func (it *memSafeIterator) Seek(t int64) bool { + if it.Err() != nil { + return false + } + + ts, _ := it.At() + + for t > ts || it.i == -1 { + if !it.Next() { + return false + } + ts, _ = it.At() + } + + return true +} + +func (it *memSafeIterator) Next() bool { + if it.i+1 >= it.stopAfter { + return false + } + it.i++ + if it.total-it.i > 4 { + return it.Iterator.Next() + } + return true +} + +func (it *memSafeIterator) At() (int64, float64) { + if it.total-it.i > 4 { + return it.Iterator.At() + } + s := it.buf[4-(it.total-it.i)] + return s.t, s.v +} + +func (it *memSafeIterator) AtHistogram() (int64, histogram.SparseHistogram) { + if it.total-it.i > 4 { + return it.Iterator.AtHistogram() + } + s := it.histBuf[4-(it.total-it.i)] + return s.t, s.h +} + +type stopIterator struct { + chunkenc.Iterator + + i, stopAfter int +} + +func (it *stopIterator) Next() bool { + if it.i+1 >= it.stopAfter { + return false + } + it.i++ + return it.Iterator.Next() +} diff --git a/tsdb/head_test.go b/tsdb/head_test.go index b5dd45d32..01f0ee1ba 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -16,6 +16,7 @@ package tsdb import ( "context" "fmt" + "io" "io/ioutil" "math" "math/rand" @@ -25,11 +26,14 @@ import ( "strconv" "sync" "testing" + "time" "github.com/pkg/errors" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "go.uber.org/atomic" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/histogram" "github.com/prometheus/prometheus/pkg/labels" @@ -52,7 +56,8 @@ func newTestHead(t testing.TB, chunkRange int64, compressWAL bool) (*Head, *wal. opts := DefaultHeadOptions() opts.ChunkRange = chunkRange opts.ChunkDirRoot = dir - opts.NumExemplars = 10 + opts.EnableExemplarStorage = true + opts.MaxExemplars.Store(config.DefaultExemplarsConfig.MaxExemplars) h, err := NewHead(nil, nil, wlog, opts, nil) require.NoError(t, err) @@ -302,7 +307,9 @@ func TestHead_ReadWAL(t *testing.T) { } require.Equal(t, []sample{{100, 2}, {101, 5}}, expandChunk(s10.iterator(0, nil, head.chunkDiskMapper, nil))) require.Equal(t, []sample{{101, 6}}, expandChunk(s50.iterator(0, nil, head.chunkDiskMapper, nil))) - require.Equal(t, []sample{{100, 3}, {101, 7}}, expandChunk(s100.iterator(0, nil, head.chunkDiskMapper, nil))) + // The samples before the new series record should be discarded since a duplicate record + // is only possible when old samples were compacted. + require.Equal(t, []sample{{101, 7}}, expandChunk(s100.iterator(0, nil, head.chunkDiskMapper, nil))) q, err := head.ExemplarQuerier(context.Background()) require.NoError(t, err) @@ -365,9 +372,9 @@ func TestHead_WALMultiRef(t *testing.T) { q, err := NewBlockQuerier(head, 0, 2100) require.NoError(t, err) series := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + // The samples before the new ref should be discarded since Head truncation + // happens only after compacting the Head. require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: { - sample{100, 1}, - sample{1500, 2}, sample{1700, 3}, sample{2000, 4}, }}, series) @@ -1229,11 +1236,11 @@ func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) { q, err := NewBlockQuerier(h, 1500, 2500) require.NoError(t, err) - defer q.Close() ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1")) require.Equal(t, false, ss.Next()) require.Equal(t, 0, len(ss.Warnings())) + require.NoError(t, q.Close()) // Truncate again, this time the series should be deleted require.NoError(t, h.Truncate(2050)) @@ -1489,7 +1496,7 @@ func TestMemSeriesIsolation(t *testing.T) { require.NoError(t, err) - iso := h.iso.State() + iso := h.iso.State(math.MinInt64, math.MaxInt64) iso.maxAppendID = maxAppendID chunks, err := h.chunksRange(math.MinInt64, math.MaxInt64, iso) @@ -1704,7 +1711,7 @@ func TestIsolationLowWatermarkMonotonous(t *testing.T) { require.NoError(t, app2.Commit()) require.Equal(t, uint64(2), hb.iso.lowWatermark(), "Low watermark should stay two because app1 is not committed yet.") - is := hb.iso.State() + is := hb.iso.State(math.MinInt64, math.MaxInt64) require.Equal(t, uint64(2), hb.iso.lowWatermark(), "After simulated read (iso state retrieved), low watermark should stay at 2.") require.NoError(t, app1.Commit()) @@ -1930,9 +1937,7 @@ func TestHeadLabelNamesValuesWithMinMaxRange(t *testing.T) { func TestHeadLabelValuesWithMatchers(t *testing.T) { head, _ := newTestHead(t, 1000, false) - defer func() { - require.NoError(t, head.Close()) - }() + t.Cleanup(func() { require.NoError(t, head.Close()) }) app := head.Appender(context.Background()) for i := 0; i < 100; i++ { @@ -1989,6 +1994,74 @@ func TestHeadLabelValuesWithMatchers(t *testing.T) { } } +func TestHeadLabelNamesWithMatchers(t *testing.T) { + head, _ := newTestHead(t, 1000, false) + defer func() { + require.NoError(t, head.Close()) + }() + + app := head.Appender(context.Background()) + for i := 0; i < 100; i++ { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + }, 100, 0) + require.NoError(t, err) + + if i%10 == 0 { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + }, 100, 0) + require.NoError(t, err) + } + + if i%20 == 0 { + _, err := app.Append(0, labels.Labels{ + {Name: "unique", Value: fmt.Sprintf("value%d", i)}, + {Name: "tens", Value: fmt.Sprintf("value%d", i/10)}, + {Name: "twenties", Value: fmt.Sprintf("value%d", i/20)}, + }, 100, 0) + require.NoError(t, err) + } + } + require.NoError(t, app.Commit()) + + testCases := []struct { + name string + labelName string + matchers []*labels.Matcher + expectedNames []string + }{ + { + name: "get with non-empty unique: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "unique", "")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get with unique ending in 1: only unique", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "unique", "value.*1")}, + expectedNames: []string{"unique"}, + }, { + name: "get with unique = value20: all", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "unique", "value20")}, + expectedNames: []string{"tens", "twenties", "unique"}, + }, { + name: "get tens = 1: unique & tens", + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "tens", "value1")}, + expectedNames: []string{"tens", "unique"}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + headIdxReader := head.indexRange(0, 200) + + actualNames, err := headIdxReader.LabelNames(tt.matchers...) + require.NoError(t, err) + require.Equal(t, tt.expectedNames, actualNames) + }) + } +} + func TestErrReuseAppender(t *testing.T) { head, _ := newTestHead(t, 1000, false) defer func() { @@ -2179,6 +2252,221 @@ func TestMemSafeIteratorSeekIntoBuffer(t *testing.T) { require.False(t, ok) } +// Tests https://github.com/prometheus/prometheus/issues/8221. +func TestChunkNotFoundHeadGCRace(t *testing.T) { + db := newTestDB(t) + db.DisableCompactions() + + var ( + app = db.Appender(context.Background()) + ref = uint64(0) + mint, maxt = int64(0), int64(0) + err error + ) + + // Appends samples to span over 1.5 block ranges. + // 7 chunks with 15s scrape interval. + for i := int64(0); i <= 120*7; i++ { + ts := i * DefaultBlockDuration / (4 * 120) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + require.NoError(t, err) + maxt = ts + } + require.NoError(t, app.Commit()) + + // Get a querier before compaction (or when compaction is about to begin). + q, err := db.Querier(context.Background(), mint, maxt) + require.NoError(t, err) + + // Query the compacted range and get the first series before compaction. + ss := q.Select(true, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) + require.True(t, ss.Next()) + s := ss.At() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // Compacting head while the querier spans the compaction time. + require.NoError(t, db.Compact()) + require.Greater(t, len(db.Blocks()), 0) + }() + + // Give enough time for compaction to finish. + // We expect it to be blocked until querier is closed. + <-time.After(3 * time.Second) + + // Now consume after compaction when it's gone. + it := s.Iterator() + for it.Next() { + _, _ = it.At() + } + // It should error here without any fix for the mentioned issue. + require.NoError(t, it.Err()) + for ss.Next() { + s = ss.At() + it := s.Iterator() + for it.Next() { + _, _ = it.At() + } + require.NoError(t, it.Err()) + } + require.NoError(t, ss.Err()) + + require.NoError(t, q.Close()) + wg.Wait() +} + +// Tests https://github.com/prometheus/prometheus/issues/9079. +func TestDataMissingOnQueryDuringCompaction(t *testing.T) { + db := newTestDB(t) + db.DisableCompactions() + + var ( + app = db.Appender(context.Background()) + ref = uint64(0) + mint, maxt = int64(0), int64(0) + err error + ) + + // Appends samples to span over 1.5 block ranges. + expSamples := make([]tsdbutil.Sample, 0) + // 7 chunks with 15s scrape interval. + for i := int64(0); i <= 120*7; i++ { + ts := i * DefaultBlockDuration / (4 * 120) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + require.NoError(t, err) + maxt = ts + expSamples = append(expSamples, sample{ts, float64(i)}) + } + require.NoError(t, app.Commit()) + + // Get a querier before compaction (or when compaction is about to begin). + q, err := db.Querier(context.Background(), mint, maxt) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // Compacting head while the querier spans the compaction time. + require.NoError(t, db.Compact()) + require.Greater(t, len(db.Blocks()), 0) + }() + + // Give enough time for compaction to finish. + // We expect it to be blocked until querier is closed. + <-time.After(3 * time.Second) + + // Querying the querier that was got before compaction. + series := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) + require.Equal(t, map[string][]tsdbutil.Sample{`{a="b"}`: expSamples}, series) + + wg.Wait() +} + +func TestIsQuerierCollidingWithTruncation(t *testing.T) { + db := newTestDB(t) + db.DisableCompactions() + + var ( + app = db.Appender(context.Background()) + ref = uint64(0) + err error + ) + + for i := int64(0); i <= 3000; i++ { + ref, err = app.Append(ref, labels.FromStrings("a", "b"), i, float64(i)) + require.NoError(t, err) + } + require.NoError(t, app.Commit()) + + // This mocks truncation. + db.head.memTruncationInProcess.Store(true) + db.head.lastMemoryTruncationTime.Store(2000) + + // Test that IsQuerierValid suggests correct querier ranges. + cases := []struct { + mint, maxt int64 // For the querier. + expShouldClose, expGetNew bool + expNewMint int64 + }{ + {-200, -100, true, false, 0}, + {-200, 300, true, false, 0}, + {100, 1900, true, false, 0}, + {1900, 2200, true, true, 2000}, + {2000, 2500, false, false, 0}, + } + + for _, c := range cases { + t.Run(fmt.Sprintf("mint=%d,maxt=%d", c.mint, c.maxt), func(t *testing.T) { + shouldClose, getNew, newMint := db.head.IsQuerierCollidingWithTruncation(c.mint, c.maxt) + require.Equal(t, c.expShouldClose, shouldClose) + require.Equal(t, c.expGetNew, getNew) + if getNew { + require.Equal(t, c.expNewMint, newMint) + } + }) + } +} + +func TestWaitForPendingReadersInTimeRange(t *testing.T) { + db := newTestDB(t) + db.DisableCompactions() + + sampleTs := func(i int64) int64 { return i * DefaultBlockDuration / (4 * 120) } + + var ( + app = db.Appender(context.Background()) + ref = uint64(0) + err error + ) + + for i := int64(0); i <= 3000; i++ { + ts := sampleTs(i) + ref, err = app.Append(ref, labels.FromStrings("a", "b"), ts, float64(i)) + require.NoError(t, err) + } + require.NoError(t, app.Commit()) + + truncMint, truncMaxt := int64(1000), int64(2000) + cases := []struct { + mint, maxt int64 + shouldWait bool + }{ + {0, 500, false}, // Before truncation range. + {500, 1500, true}, // Overlaps with truncation at the start. + {1200, 1700, true}, // Within truncation range. + {1800, 2500, true}, // Overlaps with truncation at the end. + {2000, 2500, false}, // After truncation range. + {2100, 2500, false}, // After truncation range. + } + for _, c := range cases { + t.Run(fmt.Sprintf("mint=%d,maxt=%d,shouldWait=%t", c.mint, c.maxt, c.shouldWait), func(t *testing.T) { + checkWaiting := func(cl io.Closer) { + var waitOver atomic.Bool + go func() { + db.head.WaitForPendingReadersInTimeRange(truncMint, truncMaxt) + waitOver.Store(true) + }() + <-time.After(550 * time.Millisecond) + require.Equal(t, !c.shouldWait, waitOver.Load()) + require.NoError(t, cl.Close()) + <-time.After(550 * time.Millisecond) + require.True(t, waitOver.Load()) + } + + q, err := db.Querier(context.Background(), c.mint, c.maxt) + require.NoError(t, err) + checkWaiting(q) + + cq, err := db.ChunkQuerier(context.Background(), c.mint, c.maxt) + require.NoError(t, err) + checkWaiting(cq) + }) + } +} + func TestAppendHistogram(t *testing.T) { l := labels.Labels{{Name: "a", Value: "b"}} for _, numHistograms := range []int{1, 10, 150, 200, 250, 300} { diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go new file mode 100644 index 000000000..056bd288a --- /dev/null +++ b/tsdb/head_wal.go @@ -0,0 +1,388 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdb + +import ( + "fmt" + "math" + "runtime" + "sync" + "time" + + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "go.uber.org/atomic" + + "github.com/prometheus/prometheus/pkg/exemplar" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/record" + "github.com/prometheus/prometheus/tsdb/tombstones" + "github.com/prometheus/prometheus/tsdb/wal" +) + +func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks map[uint64][]*mmappedChunk) (err error) { + // Track number of samples that referenced a series we don't know about + // for error reporting. + var unknownRefs atomic.Uint64 + var unknownExemplarRefs atomic.Uint64 + + // Start workers that each process samples for a partition of the series ID space. + // They are connected through a ring of channels which ensures that all sample batches + // read from the WAL are processed in order. + var ( + wg sync.WaitGroup + n = runtime.GOMAXPROCS(0) + inputs = make([]chan []record.RefSample, n) + outputs = make([]chan []record.RefSample, n) + exemplarsInput chan record.RefExemplar + + dec record.Decoder + shards = make([][]record.RefSample, n) + + decoded = make(chan interface{}, 10) + decodeErr, seriesCreationErr error + seriesPool = sync.Pool{ + New: func() interface{} { + return []record.RefSeries{} + }, + } + samplesPool = sync.Pool{ + New: func() interface{} { + return []record.RefSample{} + }, + } + tstonesPool = sync.Pool{ + New: func() interface{} { + return []tombstones.Stone{} + }, + } + exemplarsPool = sync.Pool{ + New: func() interface{} { + return []record.RefExemplar{} + }, + } + ) + + defer func() { + // For CorruptionErr ensure to terminate all workers before exiting. + _, ok := err.(*wal.CorruptionErr) + if ok || seriesCreationErr != nil { + for i := 0; i < n; i++ { + close(inputs[i]) + for range outputs[i] { + } + } + close(exemplarsInput) + wg.Wait() + } + }() + + wg.Add(n) + for i := 0; i < n; i++ { + outputs[i] = make(chan []record.RefSample, 300) + inputs[i] = make(chan []record.RefSample, 300) + + go func(input <-chan []record.RefSample, output chan<- []record.RefSample) { + unknown := h.processWALSamples(h.minValidTime.Load(), input, output) + unknownRefs.Add(unknown) + wg.Done() + }(inputs[i], outputs[i]) + } + + wg.Add(1) + exemplarsInput = make(chan record.RefExemplar, 300) + go func(input <-chan record.RefExemplar) { + defer wg.Done() + for e := range input { + ms := h.series.getByID(e.Ref) + if ms == nil { + unknownExemplarRefs.Inc() + continue + } + + if e.T < h.minValidTime.Load() { + continue + } + // At the moment the only possible error here is out of order exemplars, which we shouldn't see when + // replaying the WAL, so lets just log the error if it's not that type. + err = h.exemplars.AddExemplar(ms.lset, exemplar.Exemplar{Ts: e.T, Value: e.V, Labels: e.Labels}) + if err != nil && err == storage.ErrOutOfOrderExemplar { + level.Warn(h.logger).Log("msg", "Unexpected error when replaying WAL on exemplar record", "err", err) + } + } + }(exemplarsInput) + + go func() { + defer close(decoded) + for r.Next() { + rec := r.Record() + switch dec.Type(rec) { + case record.Series: + series := seriesPool.Get().([]record.RefSeries)[:0] + series, err = dec.Series(rec, series) + if err != nil { + decodeErr = &wal.CorruptionErr{ + Err: errors.Wrap(err, "decode series"), + Segment: r.Segment(), + Offset: r.Offset(), + } + return + } + decoded <- series + case record.Samples: + samples := samplesPool.Get().([]record.RefSample)[:0] + samples, err = dec.Samples(rec, samples) + if err != nil { + decodeErr = &wal.CorruptionErr{ + Err: errors.Wrap(err, "decode samples"), + Segment: r.Segment(), + Offset: r.Offset(), + } + return + } + decoded <- samples + case record.Tombstones: + tstones := tstonesPool.Get().([]tombstones.Stone)[:0] + tstones, err = dec.Tombstones(rec, tstones) + if err != nil { + decodeErr = &wal.CorruptionErr{ + Err: errors.Wrap(err, "decode tombstones"), + Segment: r.Segment(), + Offset: r.Offset(), + } + return + } + decoded <- tstones + case record.Exemplars: + exemplars := exemplarsPool.Get().([]record.RefExemplar)[:0] + exemplars, err = dec.Exemplars(rec, exemplars) + if err != nil { + decodeErr = &wal.CorruptionErr{ + Err: errors.Wrap(err, "decode exemplars"), + Segment: r.Segment(), + Offset: r.Offset(), + } + return + } + decoded <- exemplars + default: + // Noop. + } + } + }() + + // The records are always replayed from the oldest to the newest. +Outer: + for d := range decoded { + switch v := d.(type) { + case []record.RefSeries: + for _, walSeries := range v { + mSeries, created, err := h.getOrCreateWithID(walSeries.Ref, walSeries.Labels.Hash(), walSeries.Labels) + if err != nil { + seriesCreationErr = err + break Outer + } + + if h.lastSeriesID.Load() < walSeries.Ref { + h.lastSeriesID.Store(walSeries.Ref) + } + + mmc := mmappedChunks[walSeries.Ref] + + if created { + // This is the first WAL series record for this series. + h.metrics.chunksCreated.Add(float64(len(mmc))) + h.metrics.chunks.Add(float64(len(mmc))) + mSeries.mmappedChunks = mmc + continue + } + + // There's already a different ref for this series. + // A duplicate series record is only possible when the old samples were already compacted into a block. + // Hence we can discard all the samples and m-mapped chunks replayed till now for this series. + + multiRef[walSeries.Ref] = mSeries.ref + + idx := mSeries.ref % uint64(n) + // It is possible that some old sample is being processed in processWALSamples that + // could cause race below. So we wait for the goroutine to empty input the buffer and finish + // processing all old samples after emptying the buffer. + inputs[idx] <- []record.RefSample{} + for len(inputs[idx]) != 0 { + time.Sleep(1 * time.Millisecond) + } + + // Checking if the new m-mapped chunks overlap with the already existing ones. + // This should never happen, but we have a check anyway to detect any + // edge cases that we might have missed. + if len(mSeries.mmappedChunks) > 0 && len(mmc) > 0 { + if overlapsClosedInterval( + mSeries.mmappedChunks[0].minTime, + mSeries.mmappedChunks[len(mSeries.mmappedChunks)-1].maxTime, + mmc[0].minTime, + mmc[len(mmc)-1].maxTime, + ) { + // The m-map chunks for the new series ref overlaps with old m-map chunks. + seriesCreationErr = errors.Errorf("overlapping m-mapped chunks for series %s", mSeries.lset.String()) + break Outer + } + } + + // Replacing m-mapped chunks with the new ones (could be empty). + h.metrics.chunksCreated.Add(float64(len(mmc))) + h.metrics.chunksRemoved.Add(float64(len(mSeries.mmappedChunks))) + h.metrics.chunks.Add(float64(len(mmc) - len(mSeries.mmappedChunks))) + mSeries.mmappedChunks = mmc + + // Any samples replayed till now would already be compacted. Resetting the head chunk. + mSeries.nextAt = 0 + mSeries.headChunk = nil + mSeries.app = nil + h.updateMinMaxTime(mSeries.minTime(), mSeries.maxTime()) + } + //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. + seriesPool.Put(v) + case []record.RefSample: + samples := v + // We split up the samples into chunks of 5000 samples or less. + // With O(300 * #cores) in-flight sample batches, large scrapes could otherwise + // cause thousands of very large in flight buffers occupying large amounts + // of unused memory. + for len(samples) > 0 { + m := 5000 + if len(samples) < m { + m = len(samples) + } + for i := 0; i < n; i++ { + var buf []record.RefSample + select { + case buf = <-outputs[i]: + default: + } + shards[i] = buf[:0] + } + for _, sam := range samples[:m] { + if r, ok := multiRef[sam.Ref]; ok { + sam.Ref = r + } + mod := sam.Ref % uint64(n) + shards[mod] = append(shards[mod], sam) + } + for i := 0; i < n; i++ { + inputs[i] <- shards[i] + } + samples = samples[m:] + } + //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. + samplesPool.Put(v) + case []tombstones.Stone: + for _, s := range v { + for _, itv := range s.Intervals { + if itv.Maxt < h.minValidTime.Load() { + continue + } + if m := h.series.getByID(s.Ref); m == nil { + unknownRefs.Inc() + continue + } + h.tombstones.AddInterval(s.Ref, itv) + } + } + //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. + tstonesPool.Put(v) + case []record.RefExemplar: + for _, e := range v { + exemplarsInput <- e + } + //nolint:staticcheck // Ignore SA6002 relax staticcheck verification. + exemplarsPool.Put(v) + default: + panic(fmt.Errorf("unexpected decoded type: %T", d)) + } + } + + if decodeErr != nil { + return decodeErr + } + if seriesCreationErr != nil { + // Drain the channel to unblock the goroutine. + for range decoded { + } + return seriesCreationErr + } + + // Signal termination to each worker and wait for it to close its output channel. + for i := 0; i < n; i++ { + close(inputs[i]) + for range outputs[i] { + } + } + close(exemplarsInput) + wg.Wait() + + if r.Err() != nil { + return errors.Wrap(r.Err(), "read records") + } + + if unknownRefs.Load() > 0 || unknownExemplarRefs.Load() > 0 { + level.Warn(h.logger).Log("msg", "Unknown series references", "samples", unknownRefs.Load(), "exemplars", unknownExemplarRefs.Load()) + } + return nil +} + +// processWALSamples adds a partition of samples it receives to the head and passes +// them on to other workers. +// Samples before the mint timestamp are discarded. +func (h *Head) processWALSamples( + minValidTime int64, + input <-chan []record.RefSample, output chan<- []record.RefSample, +) (unknownRefs uint64) { + defer close(output) + + // Mitigate lock contention in getByID. + refSeries := map[uint64]*memSeries{} + + mint, maxt := int64(math.MaxInt64), int64(math.MinInt64) + + for samples := range input { + for _, s := range samples { + if s.T < minValidTime { + continue + } + ms := refSeries[s.Ref] + if ms == nil { + ms = h.series.getByID(s.Ref) + if ms == nil { + unknownRefs++ + continue + } + refSeries[s.Ref] = ms + } + if _, chunkCreated := ms.append(s.T, s.V, 0, h.chunkDiskMapper); chunkCreated { + h.metrics.chunksCreated.Inc() + h.metrics.chunks.Inc() + } + if s.T > maxt { + maxt = s.T + } + if s.T < mint { + mint = s.T + } + } + output <- samples + } + h.updateMinMaxTime(mint, maxt) + + return unknownRefs +} diff --git a/tsdb/index/index.go b/tsdb/index/index.go index a6ade9455..a7b9e57ef 100644 --- a/tsdb/index/index.go +++ b/tsdb/index/index.go @@ -523,9 +523,15 @@ func (w *Writer) AddSymbol(sym string) error { } func (w *Writer) finishSymbols() error { + symbolTableSize := w.f.pos - w.toc.Symbols - 4 + // The symbol table's part is 4 bytes. So the total symbol table size must be less than or equal to 2^32-1 + if symbolTableSize > 4294967295 { + return errors.Errorf("symbol table size exceeds 4 bytes: %d", symbolTableSize) + } + // Write out the length and symbol count. w.buf1.Reset() - w.buf1.PutBE32int(int(w.f.pos - w.toc.Symbols - 4)) + w.buf1.PutBE32int(int(symbolTableSize)) w.buf1.PutBE32int(int(w.numSymbols)) if err := w.writeAt(w.buf1.Get(), w.toc.Symbols); err != nil { return err @@ -1511,6 +1517,49 @@ func (r *Reader) LabelValues(name string, matchers ...*labels.Matcher) ([]string return values, nil } +// LabelNamesFor returns all the label names for the series referred to by IDs. +// The names returned are sorted. +func (r *Reader) LabelNamesFor(ids ...uint64) ([]string, error) { + // Gather offsetsMap the name offsetsMap in the symbol table first + offsetsMap := make(map[uint32]struct{}) + for _, id := range ids { + offset := id + // In version 2 series IDs are no longer exact references but series are 16-byte padded + // and the ID is the multiple of 16 of the actual position. + if r.version == FormatV2 { + offset = id * 16 + } + + d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable) + buf := d.Get() + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "get buffer for series") + } + + offsets, err := r.dec.LabelNamesOffsetsFor(buf) + if err != nil { + return nil, errors.Wrap(err, "get label name offsets") + } + for _, off := range offsets { + offsetsMap[off] = struct{}{} + } + } + + // Lookup the unique symbols. + names := make([]string, 0, len(offsetsMap)) + for off := range offsetsMap { + name, err := r.lookupSymbol(off) + if err != nil { + return nil, errors.Wrap(err, "lookup symbol in LabelNamesFor") + } + names = append(names, name) + } + + sort.Strings(names) + + return names, nil +} + // LabelValueFor returns label value for the given label name in the series referred to by ID. func (r *Reader) LabelValueFor(id uint64, label string) (string, error) { offset := id @@ -1664,7 +1713,12 @@ func (r *Reader) Size() int64 { } // LabelNames returns all the unique label names present in the index. -func (r *Reader) LabelNames() ([]string, error) { +// TODO(twilkie) implement support for matchers +func (r *Reader) LabelNames(matchers ...*labels.Matcher) ([]string, error) { + if len(matchers) > 0 { + return nil, errors.Errorf("matchers parameter is not implemented: %+v", matchers) + } + labelNames := make([]string, 0, len(r.postings)) for name := range r.postings { if name == allPostingsKey.Name { @@ -1715,6 +1769,25 @@ func (dec *Decoder) Postings(b []byte) (int, Postings, error) { return n, newBigEndianPostings(l), d.Err() } +// LabelNamesOffsetsFor decodes the offsets of the name symbols for a given series. +// They are returned in the same order they're stored, which should be sorted lexicographically. +func (dec *Decoder) LabelNamesOffsetsFor(b []byte) ([]uint32, error) { + d := encoding.Decbuf{B: b} + k := d.Uvarint() + + offsets := make([]uint32, k) + for i := 0; i < k; i++ { + offsets[i] = uint32(d.Uvarint()) + _ = d.Uvarint() // skip the label value + + if d.Err() != nil { + return nil, errors.Wrap(d.Err(), "read series label offsets") + } + } + + return offsets, d.Err() +} + // LabelValueFor decodes a label for a given series. func (dec *Decoder) LabelValueFor(b []byte, label string) (string, error) { d := encoding.Decbuf{B: b} diff --git a/tsdb/isolation.go b/tsdb/isolation.go index cee6899dd..ca6ce1980 100644 --- a/tsdb/isolation.go +++ b/tsdb/isolation.go @@ -24,6 +24,7 @@ type isolationState struct { incompleteAppends map[uint64]struct{} lowWatermark uint64 // Lowest of incompleteAppends/maxAppendID. isolation *isolation + mint, maxt int64 // Time ranges of the read. // Doubly linked list of active reads. next *isolationState @@ -86,6 +87,10 @@ func newIsolation() *isolation { func (i *isolation) lowWatermark() uint64 { i.appendMtx.RLock() // Take appendMtx first. defer i.appendMtx.RUnlock() + return i.lowWatermarkLocked() +} + +func (i *isolation) lowWatermarkLocked() uint64 { i.readMtx.RLock() defer i.readMtx.RUnlock() if i.readsOpen.prev != i.readsOpen { @@ -98,7 +103,7 @@ func (i *isolation) lowWatermark() uint64 { // State returns an object used to control isolation // between a query and appends. Must be closed when complete. -func (i *isolation) State() *isolationState { +func (i *isolation) State(mint, maxt int64) *isolationState { i.appendMtx.RLock() // Take append mutex before read mutex. defer i.appendMtx.RUnlock() isoState := &isolationState{ @@ -106,6 +111,8 @@ func (i *isolation) State() *isolationState { lowWatermark: i.appendsOpenList.next.appendID, // Lowest appendID from appenders, or lastAppendId. incompleteAppends: make(map[uint64]struct{}, len(i.appendsOpen)), isolation: i, + mint: mint, + maxt: maxt, } for k := range i.appendsOpen { isoState.incompleteAppends[k] = struct{}{} @@ -120,9 +127,25 @@ func (i *isolation) State() *isolationState { return isoState } +// TraverseOpenReads iterates through the open reads and runs the given +// function on those states. The given function MUST NOT mutate the isolationState. +// The iteration is stopped when the function returns false or once all reads have been iterated. +func (i *isolation) TraverseOpenReads(f func(s *isolationState) bool) { + i.readMtx.RLock() + defer i.readMtx.RUnlock() + s := i.readsOpen.next + for s != i.readsOpen { + if !f(s) { + return + } + s = s.next + } +} + // newAppendID increments the transaction counter and returns a new transaction // ID. The first ID returned is 1. -func (i *isolation) newAppendID() uint64 { +// Also returns the low watermark, to keep lock/unlock operations down. +func (i *isolation) newAppendID() (uint64, uint64) { i.appendMtx.Lock() defer i.appendMtx.Unlock() @@ -138,7 +161,7 @@ func (i *isolation) newAppendID() uint64 { i.appendsOpenList.prev = app i.appendsOpen[app.appendID] = app - return app.appendID + return app.appendID, i.lowWatermarkLocked() } func (i *isolation) lastAppendID() uint64 { diff --git a/tsdb/isolation_test.go b/tsdb/isolation_test.go index 3a90547b2..e41e3f5a2 100644 --- a/tsdb/isolation_test.go +++ b/tsdb/isolation_test.go @@ -14,6 +14,7 @@ package tsdb import ( + "math" "strconv" "sync" "testing" @@ -35,8 +36,7 @@ func BenchmarkIsolation(b *testing.B) { <-start for i := 0; i < b.N; i++ { - appendID := iso.newAppendID() - _ = iso.lowWatermark() + appendID, _ := iso.newAppendID() iso.closeAppend(appendID) } @@ -66,8 +66,7 @@ func BenchmarkIsolationWithState(b *testing.B) { <-start for i := 0; i < b.N; i++ { - appendID := iso.newAppendID() - _ = iso.lowWatermark() + appendID, _ := iso.newAppendID() iso.closeAppend(appendID) } @@ -87,7 +86,7 @@ func BenchmarkIsolationWithState(b *testing.B) { <-start for i := 0; i < b.N; i++ { - s := iso.State() + s := iso.State(math.MinInt64, math.MaxInt64) s.Close() } }() diff --git a/tsdb/querier.go b/tsdb/querier.go index c9daf2384..c5d5f2bae 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -89,8 +89,8 @@ func (q *blockBaseQuerier) LabelValues(name string, matchers ...*labels.Matcher) return res, nil, err } -func (q *blockBaseQuerier) LabelNames() ([]string, storage.Warnings, error) { - res, err := q.index.LabelNames() +func (q *blockBaseQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { + res, err := q.index.LabelNames(matchers...) return res, nil, err } @@ -408,6 +408,23 @@ func labelValuesWithMatchers(r IndexReader, name string, matchers ...*labels.Mat return values, nil } +func labelNamesWithMatchers(r IndexReader, matchers ...*labels.Matcher) ([]string, error) { + p, err := PostingsForMatchers(r, matchers...) + if err != nil { + return nil, err + } + + var postings []uint64 + for p.Next() { + postings = append(postings, p.At()) + } + if p.Err() != nil { + return nil, errors.Wrapf(p.Err(), "postings for label names with matchers") + } + + return r.LabelNamesFor(postings...) +} + // blockBaseSeriesSet allows to iterate over all series in the single block. // Iterated series are trimmed with given min and max time as well as tombstones. // See newBlockSeriesSet and newBlockChunkSeriesSet to use it for either sample or chunk iterating. diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index 8f82d1d55..514b1f5b9 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -1154,7 +1154,7 @@ func (m mockIndex) SortedLabelValues(name string, matchers ...*labels.Matcher) ( } func (m mockIndex) LabelValues(name string, matchers ...*labels.Matcher) ([]string, error) { - values := []string{} + var values []string if len(matchers) == 0 { for l := range m.postings { @@ -1168,6 +1168,7 @@ func (m mockIndex) LabelValues(name string, matchers ...*labels.Matcher) ([]stri for _, series := range m.series { for _, matcher := range matchers { if matcher.Matches(series.l.Get(matcher.Name)) { + // TODO(colega): shouldn't we check all the matchers before adding this to the values? values = append(values, series.l.Get(name)) } } @@ -1180,6 +1181,20 @@ func (m mockIndex) LabelValueFor(id uint64, label string) (string, error) { return m.series[id].l.Get(label), nil } +func (m mockIndex) LabelNamesFor(ids ...uint64) ([]string, error) { + namesMap := make(map[string]bool) + for _, id := range ids { + for _, lbl := range m.series[id].l { + namesMap[lbl.Name] = true + } + } + names := make([]string, 0, len(namesMap)) + for name := range namesMap { + names = append(names, name) + } + return names, nil +} + func (m mockIndex) Postings(name string, values ...string) (index.Postings, error) { res := make([]index.Postings, 0, len(values)) for _, value := range values { @@ -1212,10 +1227,27 @@ func (m mockIndex) Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) return nil } -func (m mockIndex) LabelNames() ([]string, error) { +func (m mockIndex) LabelNames(matchers ...*labels.Matcher) ([]string, error) { names := map[string]struct{}{} - for l := range m.postings { - names[l.Name] = struct{}{} + if len(matchers) == 0 { + for l := range m.postings { + names[l.Name] = struct{}{} + } + } else { + for _, series := range m.series { + matches := true + for _, matcher := range matchers { + matches = matches || matcher.Matches(series.l.Get(matcher.Name)) + if !matches { + break + } + } + if matches { + for _, lbl := range series.l { + names[lbl.Name] = struct{}{} + } + } + } } l := make([]string, 0, len(names)) for name := range names { @@ -2007,6 +2039,10 @@ func (m mockMatcherIndex) LabelValueFor(id uint64, label string) (string, error) return "", errors.New("label value for called") } +func (m mockMatcherIndex) LabelNamesFor(ids ...uint64) ([]string, error) { + return nil, errors.New("label names for for called") +} + func (m mockMatcherIndex) Postings(name string, values ...string) (index.Postings, error) { return index.EmptyPostings(), nil } @@ -2019,7 +2055,9 @@ func (m mockMatcherIndex) Series(ref uint64, lset *labels.Labels, chks *[]chunks return nil } -func (m mockMatcherIndex) LabelNames() ([]string, error) { return []string{}, nil } +func (m mockMatcherIndex) LabelNames(...*labels.Matcher) ([]string, error) { + return []string{}, nil +} func TestPostingsForMatcher(t *testing.T) { cases := []struct { diff --git a/tsdb/wal/watcher.go b/tsdb/wal/watcher.go index fda5ad1e6..accefa7aa 100644 --- a/tsdb/wal/watcher.go +++ b/tsdb/wal/watcher.go @@ -48,7 +48,10 @@ type WriteTo interface { Append([]record.RefSample) bool AppendExemplars([]record.RefExemplar) bool StoreSeries([]record.RefSeries, int) - // SeriesReset is called after reading a checkpoint to allow the deletion + // Next two methods are intended for garbage-collection: first we call + // UpdateSeriesSegment on all current series + UpdateSeriesSegment([]record.RefSeries, int) + // Then SeriesReset is called to allow the deletion // of all series created in a segment lower than the argument. SeriesReset(int) } @@ -234,7 +237,7 @@ func (w *Watcher) Run() error { } if err == nil { - if err = w.readCheckpoint(lastCheckpoint); err != nil { + if err = w.readCheckpoint(lastCheckpoint, (*Watcher).readSegment); err != nil { return errors.Wrap(err, "readCheckpoint") } } @@ -454,7 +457,7 @@ func (w *Watcher) garbageCollectSeries(segmentNum int) error { level.Debug(w.logger).Log("msg", "New checkpoint detected", "new", dir, "currentSegment", segmentNum) - if err = w.readCheckpoint(dir); err != nil { + if err = w.readCheckpoint(dir, (*Watcher).readSegmentForGC); err != nil { return errors.Wrap(err, "readCheckpoint") } @@ -463,6 +466,8 @@ func (w *Watcher) garbageCollectSeries(segmentNum int) error { return nil } +// Read from a segment and pass the details to w.writer. +// Also used with readCheckpoint - implements segmentReadFn. func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error { var ( dec record.Decoder @@ -538,6 +543,39 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error { return errors.Wrapf(r.Err(), "segment %d: %v", segmentNum, r.Err()) } +// Go through all series in a segment updating the segmentNum, so we can delete older series. +// Used with readCheckpoint - implements segmentReadFn. +func (w *Watcher) readSegmentForGC(r *LiveReader, segmentNum int, _ bool) error { + var ( + dec record.Decoder + series []record.RefSeries + ) + for r.Next() && !isClosed(w.quit) { + rec := r.Record() + w.recordsReadMetric.WithLabelValues(recordType(dec.Type(rec))).Inc() + + switch dec.Type(rec) { + case record.Series: + series, err := dec.Series(rec, series[:0]) + if err != nil { + w.recordDecodeFailsMetric.Inc() + return err + } + w.writer.UpdateSeriesSegment(series, segmentNum) + + // Ignore these; we're only interested in series. + case record.Samples: + case record.Exemplars: + case record.Tombstones: + + default: + // Could be corruption, or reading from a WAL from a newer Prometheus. + w.recordDecodeFailsMetric.Inc() + } + } + return errors.Wrapf(r.Err(), "segment %d: %v", segmentNum, r.Err()) +} + func (w *Watcher) SetStartTime(t time.Time) { w.startTime = t w.startTimestamp = timestamp.FromTime(t) @@ -556,8 +594,10 @@ func recordType(rt record.Type) string { } } +type segmentReadFn func(w *Watcher, r *LiveReader, segmentNum int, tail bool) error + // Read all the series records from a Checkpoint directory. -func (w *Watcher) readCheckpoint(checkpointDir string) error { +func (w *Watcher) readCheckpoint(checkpointDir string, readFn segmentReadFn) error { level.Debug(w.logger).Log("msg", "Reading checkpoint", "dir", checkpointDir) index, err := checkpointNum(checkpointDir) if err != nil { @@ -582,7 +622,7 @@ func (w *Watcher) readCheckpoint(checkpointDir string) error { defer sr.Close() r := NewLiveReader(w.logger, w.readerMetrics, sr) - if err := w.readSegment(r, index, false); errors.Cause(err) != io.EOF && err != nil { + if err := readFn(w, r, index, false); errors.Cause(err) != io.EOF && err != nil { return errors.Wrap(err, "readSegment") } diff --git a/tsdb/wal/watcher_test.go b/tsdb/wal/watcher_test.go index dcde251a2..3ff7d1ad2 100644 --- a/tsdb/wal/watcher_test.go +++ b/tsdb/wal/watcher_test.go @@ -66,6 +66,10 @@ func (wtm *writeToMock) AppendExemplars(e []record.RefExemplar) bool { } func (wtm *writeToMock) StoreSeries(series []record.RefSeries, index int) { + wtm.UpdateSeriesSegment(series, index) +} + +func (wtm *writeToMock) UpdateSeriesSegment(series []record.RefSeries, index int) { wtm.seriesLock.Lock() defer wtm.seriesLock.Unlock() for _, s := range series { @@ -496,7 +500,7 @@ func TestReadCheckpointMultipleSegments(t *testing.T) { lastCheckpoint, _, err := LastCheckpoint(watcher.walDir) require.NoError(t, err) - err = watcher.readCheckpoint(lastCheckpoint) + err = watcher.readCheckpoint(lastCheckpoint, (*Watcher).readSegment) require.NoError(t, err) }) } diff --git a/util/osutil/hostname.go b/util/osutil/hostname.go new file mode 100644 index 000000000..224dffe7c --- /dev/null +++ b/util/osutil/hostname.go @@ -0,0 +1,63 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package osutil + +import ( + "encoding" + "net" + "os" +) + +// GetFQDN returns a FQDN if it's possible, otherwise falls back to hostname. +func GetFQDN() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + ips, err := net.LookupIP(hostname) + if err != nil { + // Return the system hostname if we can't look up the IP address. + return hostname, nil + } + + lookup := func(ipStr encoding.TextMarshaler) (string, error) { + ip, err := ipStr.MarshalText() + if err != nil { + return "", err + } + hosts, err := net.LookupAddr(string(ip)) + if err != nil || len(hosts) == 0 { + return "", err + } + return hosts[0], nil + } + + for _, addr := range ips { + if ip := addr.To4(); ip != nil { + if fqdn, err := lookup(ip); err == nil { + return fqdn, nil + } + + } + + if ip := addr.To16(); ip != nil { + if fqdn, err := lookup(ip); err == nil { + return fqdn, nil + } + + } + } + return hostname, nil +} diff --git a/util/teststorage/storage.go b/util/teststorage/storage.go index 75c179c52..53c4ece13 100644 --- a/util/teststorage/storage.go +++ b/util/teststorage/storage.go @@ -18,6 +18,7 @@ import ( "os" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" @@ -38,12 +39,14 @@ func New(t testutil.T) *TestStorage { opts := tsdb.DefaultOptions() opts.MinBlockDuration = int64(24 * time.Hour / time.Millisecond) opts.MaxBlockDuration = int64(24 * time.Hour / time.Millisecond) - opts.MaxExemplars = 10 db, err := tsdb.Open(dir, nil, nil, opts, tsdb.NewDBStats()) if err != nil { t.Fatalf("Opening test storage failed: %s", err) } - es, err := tsdb.NewCircularExemplarStorage(10, nil) + reg := prometheus.NewRegistry() + eMetrics := tsdb.NewExemplarMetrics(reg) + + es, err := tsdb.NewCircularExemplarStorage(10, eMetrics) if err != nil { t.Fatalf("Opening test exemplar storage failed: %s", err) } diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 70fcae7d2..d3b9580b6 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -703,26 +703,18 @@ func (api *API) labelNames(r *http.Request) apiFuncResult { warnings storage.Warnings ) if len(matcherSets) > 0 { - hints := &storage.SelectHints{ - Start: timestamp.FromTime(start), - End: timestamp.FromTime(end), - Func: "series", // There is no series function, this token is used for lookups that don't need samples. - } - labelNamesSet := make(map[string]struct{}) - // Get all series which match matchers. - for _, mset := range matcherSets { - s := q.Select(false, hints, mset...) - for s.Next() { - series := s.At() - for _, lb := range series.Labels() { - labelNamesSet[lb.Name] = struct{}{} - } - } - warnings = append(warnings, s.Warnings()...) - if err := s.Err(); err != nil { + + for _, matchers := range matcherSets { + vals, callWarnings, err := q.LabelNames(matchers...) + if err != nil { return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil} } + + warnings = append(warnings, callWarnings...) + for _, val := range vals { + labelNamesSet[val] = struct{}{} + } } // Convert the map to an array. diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 505daf042..f5a44ed9e 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -465,6 +465,7 @@ func TestLabelNames(t *testing.T) { test_metric1{foo2="boo"} 1+0x100 test_metric2{foo="boo"} 1+0x100 test_metric2{foo="boo", xyz="qwerty"} 1+0x100 + test_metric2{foo="baz", abc="qwerty"} 1+0x100 `) require.NoError(t, err) defer suite.Close() @@ -473,21 +474,57 @@ func TestLabelNames(t *testing.T) { api := &API{ Queryable: suite.Storage(), } - request := func(m string) (*http.Request, error) { - if m == http.MethodPost { - r, err := http.NewRequest(m, "http://example.com", nil) - r.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return r, err - } - return http.NewRequest(m, "http://example.com", nil) - } - for _, method := range []string{http.MethodGet, http.MethodPost} { - ctx := context.Background() - req, err := request(method) + request := func(method string, matchers ...string) (*http.Request, error) { + u, err := url.Parse("http://example.com") require.NoError(t, err) - res := api.labelNames(req.WithContext(ctx)) - assertAPIError(t, res.err, "") - assertAPIResponse(t, res.data, []string{"__name__", "baz", "foo", "foo1", "foo2", "xyz"}) + q := u.Query() + for _, matcher := range matchers { + q.Add("match[]", matcher) + } + u.RawQuery = q.Encode() + + r, err := http.NewRequest(method, u.String(), nil) + if method == http.MethodPost { + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + return r, err + } + + for _, tc := range []struct { + name string + matchers []string + expected []string + }{ + { + name: "no matchers", + expected: []string{"__name__", "abc", "baz", "foo", "foo1", "foo2", "xyz"}, + }, + { + name: "non empty label matcher", + matchers: []string{`{foo=~".+"}`}, + expected: []string{"__name__", "abc", "foo", "xyz"}, + }, + { + name: "exact label matcher", + matchers: []string{`{foo="boo"}`}, + expected: []string{"__name__", "foo", "xyz"}, + }, + { + name: "two matchers", + matchers: []string{`{foo="boo"}`, `{foo="baz"}`}, + expected: []string{"__name__", "abc", "foo", "xyz"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, method := range []string{http.MethodGet, http.MethodPost} { + ctx := context.Background() + req, err := request(method, tc.matchers...) + require.NoError(t, err) + res := api.labelNames(req.WithContext(ctx)) + assertAPIError(t, res.err, "") + assertAPIResponse(t, res.data, tc.expected) + } + }) } } diff --git a/web/federate.go b/web/federate.go index 7319b34e8..f97579ad4 100644 --- a/web/federate.go +++ b/web/federate.go @@ -95,7 +95,7 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { var sets []storage.SeriesSet for _, mset := range matcherSets { - s := q.Select(false, hints, mset...) + s := q.Select(true, hints, mset...) sets = append(sets, s) } diff --git a/web/federate_test.go b/web/federate_test.go index b7f42b908..35a02bb44 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -113,6 +113,14 @@ test_metric1{foo="bar",instance="i"} 10000 6000000 test_metric1{foo="boo",instance="i"} 1 6000000 # TYPE test_metric2 untyped test_metric2{foo="boo",instance="i"} 1 6000000 +`, + }, + "two matchers with overlap": { + params: "match[]={__name__=~'test_metric1'}&match[]={foo='bar'}", + code: 200, + body: `# TYPE test_metric1 untyped +test_metric1{foo="bar",instance="i"} 10000 6000000 +test_metric1{foo="boo",instance="i"} 1 6000000 `, }, "everything": { diff --git a/web/ui/react-app/package.json b/web/ui/react-app/package.json index 25d27cef8..9bca4ab16 100644 --- a/web/ui/react-app/package.json +++ b/web/ui/react-app/package.json @@ -19,13 +19,13 @@ "@fortawesome/fontawesome-svg-core": "^1.2.14", "@fortawesome/free-solid-svg-icons": "^5.7.1", "@fortawesome/react-fontawesome": "^0.1.4", + "@nexucis/fuzzy": "^0.2.2", "@reach/router": "^1.2.1", "bootstrap": "^4.6.0", "codemirror-promql": "^0.16.0", "css.escape": "^1.5.1", "downshift": "^3.4.8", "enzyme-to-json": "^3.4.3", - "fuzzy": "^0.1.3", "i": "^0.3.6", "jquery": "^3.5.1", "jquery.flot.tooltip": "^0.9.0", diff --git a/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx b/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx index 72edfbab7..8a5197aa7 100644 --- a/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx +++ b/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx @@ -28,81 +28,85 @@ const CollapsibleAlertPanel: FC = ({ rule, showAnnot {rule.name} ({`${rule.alerts.length} active`}) - 
-          
-            

-              name: {rule.name}
-            

-            

-              expr: {rule.query}
-            

-            {rule.duration > 0 && (
-              

-                
for: {formatDuration(rule.duration * 1000)}

-              

-            )}
-            {rule.labels && Object.keys(rule.labels).length > 0 && (
-              

-                
labels:

-                {Object.entries(rule.labels).map(([key, value]) => (
-                  

-                    {key}: {value}
+        {open && (
+          <>
+            
+              
+                

+                  name: {rule.name}
+                

+                

+                  expr: {rule.query}
+                

+                {rule.duration > 0 && (
+                  

+                    
for: {formatDuration(rule.duration * 1000)}

                   

-                ))}
-              

-            )}
-            {rule.annotations && Object.keys(rule.annotations).length > 0 && (
-              

-                
annotations:

-                {Object.entries(rule.annotations).map(([key, value]) => (
-                  

-                    {key}: {value}
+                )}
+                {rule.labels && Object.keys(rule.labels).length > 0 && (
+                  

+                    
labels:

+                    {Object.entries(rule.labels).map(([key, value]) => (
+                      

+                        {key}: {value}
+                      

+                    ))}
                   

-                ))}
-              

+                )}
+                {rule.annotations && Object.keys(rule.annotations).length > 0 && (
+                  

+                    
annotations:

+                    {Object.entries(rule.annotations).map(([key, value]) => (
+                      

+                        {key}: {value}
+                      

+                    ))}
+                  

+                )}
+              
+
+ {rule.alerts.length > 0 && ( + + + + + + + + + + + {rule.alerts.map((alert, i) => { + return ( + + + + + + + + {showAnnotations && } + + ); + })} + +
LabelsStateActive SinceValue
+ {Object.entries(alert.labels).map(([k, v], j) => { + return ( + + {k}={v} + + ); + })} + +
+ + {alert.state} + +
+ 		{alert.activeAt}{parsePrometheusFloat(alert.value)}
)} - - - {rule.alerts.length > 0 && ( - - - - - - - - - - - {rule.alerts.map((alert, i) => { - return ( - - - - - - - - {showAnnotations && } - - ); - })} - -
LabelsStateActive SinceValue
- {Object.entries(alert.labels).map(([k, v], j) => { - return ( - - {k}={v} - - ); - })} - -
- - {alert.state} - -
- 		{alert.activeAt}{parsePrometheusFloat(alert.value)}
+ )} diff --git a/web/ui/react-app/src/pages/flags/Flags.test.tsx b/web/ui/react-app/src/pages/flags/Flags.test.tsx index 6c8c254c7..4fdc8519f 100644 --- a/web/ui/react-app/src/pages/flags/Flags.test.tsx +++ b/web/ui/react-app/src/pages/flags/Flags.test.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; import { shallow } from 'enzyme'; import { FlagsContent } from './Flags'; -import { Table } from 'reactstrap'; +import { Input, Table } from 'reactstrap'; import toJson from 'enzyme-to-json'; const sampleFlagsResponse = { @@ -55,11 +55,60 @@ describe('Flags', () => { striped: true, }); }); + it('should not fail if data is missing', () => { expect(shallow()).toHaveLength(1); }); + it('should match snapshot', () => { const w = shallow(); expect(toJson(w)).toMatchSnapshot(); }); + + it('is sorted by flag by default', (): void => { + const w = shallow(); + const td = w + .find('tbody') + .find('td') + .find('span') + .first(); + expect(td.html()).toBe('--alertmanager.notification-queue-capacity'); + }); + + it('sorts', (): void => { + const w = shallow(); + const th = w + .find('thead') + .find('td') + .filterWhere((td): boolean => td.hasClass('Flag')); + th.simulate('click'); + const td = w + .find('tbody') + .find('td') + .find('span') + .first(); + expect(td.html()).toBe('--web.user-assets'); + }); + + it('filters by flag name', (): void => { + const w = shallow(); + const input = w.find(Input); + input.simulate('change', { target: { value: 'timeout' } }); + const tds = w + .find('tbody') + .find('td') + .filterWhere(code => code.hasClass('flag-item')); + expect(tds.length).toEqual(3); + }); + + it('filters by flag value', (): void => { + const w = shallow(); + const input = w.find(Input); + input.simulate('change', { target: { value: '10s' } }); + const tds = w + .find('tbody') + .find('td') + .filterWhere(code => code.hasClass('flag-value')); + expect(tds.length).toEqual(1); + }); }); diff --git a/web/ui/react-app/src/pages/flags/Flags.tsx b/web/ui/react-app/src/pages/flags/Flags.tsx index 60b67ae77..06e2434b8 100644 --- a/web/ui/react-app/src/pages/flags/Flags.tsx +++ b/web/ui/react-app/src/pages/flags/Flags.tsx @@ -1,10 +1,18 @@ -import React, { FC } from 'react'; +import React, { ChangeEvent, FC, useState } from 'react'; import { RouteComponentProps } from '@reach/router'; -import { Table } from 'reactstrap'; +import { Input, InputGroup, Table } from 'reactstrap'; import { withStatusIndicator } from '../../components/withStatusIndicator'; import { useFetch } from '../../hooks/useFetch'; import { usePathPrefix } from '../../contexts/PathPrefixContext'; import { API_PATH } from '../../constants/constants'; +import { faSort, faSortDown, faSortUp } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { IconDefinition } from '@fortawesome/fontawesome-common-types'; +import sanitizeHTML from 'sanitize-html'; +import { Fuzzy, FuzzyResult } from '@nexucis/fuzzy'; + +const fuz = new Fuzzy({ pre: '', post: '', shouldSort: true }); +const flagSeparator = '||'; interface FlagMap { [key: string]: string; @@ -14,18 +22,99 @@ interface FlagsProps { data?: FlagMap; } +const compareAlphaFn = (keys: boolean, reverse: boolean) => ( + [k1, v1]: [string, string], + [k2, v2]: [string, string] +): number => { + const a = keys ? k1 : v1; + const b = keys ? k2 : v2; + const reverser = reverse ? -1 : 1; + return reverser * a.localeCompare(b); +}; + +const getSortIcon = (b: boolean | undefined): IconDefinition => { + if (b === undefined) { + return faSort; + } + if (b) { + return faSortDown; + } + return faSortUp; +}; + +interface SortState { + name: string; + alpha: boolean; + focused: boolean; +} + export const FlagsContent: FC = ({ data = {} }) => { + const initialSearch = ''; + const [searchState, setSearchState] = useState(initialSearch); + const initialSort: SortState = { + name: 'Flag', + alpha: true, + focused: true, + }; + const [sortState, setSortState] = useState(initialSort); + const searchable = Object.entries(data) + .sort(compareAlphaFn(sortState.name === 'Flag', !sortState.alpha)) + .map(([flag, value]) => `--${flag}${flagSeparator}${value}`); + let filtered = searchable; + if (searchState.length > 0) { + filtered = fuz.filter(searchState, searchable).map((value: FuzzyResult) => value.rendered); + } return ( <>

Command-Line Flags

- + + ): void => { + setSearchState(target.value); + }} + /> + +
+ + + {['Flag', 'Value'].map((col: string) => ( + + ))} + + - {Object.keys(data).map(key => ( - - - - - ))} + {filtered.map((result: string) => { + const [flagMatchStr, valueMatchStr] = result.split(flagSeparator); + const sanitizeOpts = { allowedTags: ['strong'] }; + return ( + + + + + ); + })}
+ setSortState({ + name: col, + focused: true, + alpha: sortState.name === col ? !sortState.alpha : true, + }) + } + > + {col} + +
{key}{data[key]}
+ + + +
diff --git a/web/ui/react-app/src/pages/flags/__snapshots__/Flags.test.tsx.snap b/web/ui/react-app/src/pages/flags/__snapshots__/Flags.test.tsx.snap index ae3825986..487205ea2 100644 --- a/web/ui/react-app/src/pages/flags/__snapshots__/Flags.test.tsx.snap +++ b/web/ui/react-app/src/pages/flags/__snapshots__/Flags.test.tsx.snap @@ -5,389 +5,1112 @@ exports[`Flags should match snapshot 1`] = `

Command-Line Flags

+ + + + + + + + + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - + - - - - - + - - + - - + - - + - - + - - + + + + +
+ + Flag + + + + + Value + + +
- alertmanager.notification-queue-capacity - - 10000 + + + +
- alertmanager.timeout - - 10s + + + +
- config.file - - ./documentation/examples/prometheus.yml + + + +
- log.format - - logfmt + + + +
- log.level - - info + + + +
- query.lookback-delta - - 5m + + + +
- query.max-concurrency - - 20 + + + +
- query.max-samples - - 50000000 + + + +
- query.timeout - - 2m + + + +
- rules.alert.for-grace-period - - 10m + + + +
- rules.alert.for-outage-tolerance - - 1h + + + +
- rules.alert.resend-delay - - 1m + + + +
- storage.remote.flush-deadline - - 1m + + + +
- storage.remote.read-concurrent-limit - - 10 + + + +
- storage.remote.read-max-bytes-in-frame - - 1048576 + + + +
- storage.remote.read-sample-limit - - 50000000 + + + +
- storage.tsdb.allow-overlapping-blocks - - false + + + +
- storage.tsdb.max-block-duration - - 36h + + + +
- storage.tsdb.min-block-duration - - 2h + + + +
- storage.tsdb.no-lockfile - - false + + + +
- storage.tsdb.path - - data/ + + + +
- storage.tsdb.retention - - 0s + + + +
- storage.tsdb.retention.size - - 0B + + + +
- storage.tsdb.retention.time - - 0s + + + +
- storage.tsdb.wal-compression - - false + + + +
- storage.tsdb.wal-segment-size - - 0B + + + +
- web.console.libraries - - console_libraries + + + +
- web.console.templates - - consoles + + + +
- web.cors.origin - - .* + + + +
- web.enable-admin-api - - false + + + +
- web.enable-lifecycle - - false + + + +
- web.external-url - -
- web.listen-address - - 0.0.0.0:9090 + + + +
- web.max-connections - - 512 + + + +
- web.page-title - - Prometheus Time Series Collection and Processing Server + + + +
- web.read-timeout - - 5m + + + +
- web.route-prefix - - / + + + +
- web.user-assets - + + + + +
+ + + +
diff --git a/web/ui/react-app/src/pages/graph/ExpressionInput.tsx b/web/ui/react-app/src/pages/graph/ExpressionInput.tsx index 38200b650..bc6f52293 100644 --- a/web/ui/react-app/src/pages/graph/ExpressionInput.tsx +++ b/web/ui/react-app/src/pages/graph/ExpressionInput.tsx @@ -2,12 +2,12 @@ import React, { Component } from 'react'; import { Button, InputGroup, InputGroupAddon, InputGroupText, Input } from 'reactstrap'; import Downshift, { ControllerStateAndHelpers } from 'downshift'; -import fuzzy from 'fuzzy'; import sanitizeHTML from 'sanitize-html'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faSearch, faSpinner, faGlobeEurope } from '@fortawesome/free-solid-svg-icons'; import MetricsExplorer from './MetricsExplorer'; +import { Fuzzy, FuzzyResult } from '@nexucis/fuzzy'; interface ExpressionInputProps { value: string; @@ -24,6 +24,8 @@ interface ExpressionInputState { showMetricsExplorer: boolean; } +const fuz = new Fuzzy({ pre: '', post: '', shouldSort: true }); + class ExpressionInput extends Component { private exprInputRef = React.createRef(); @@ -71,10 +73,7 @@ class ExpressionInput extends Component { - return fuzzy.filter(input.replace(/ /g, ''), expressions, { - pre: '', - post: '', - }); + return fuz.filter(input.replace(/ /g, ''), expressions); }; createAutocompleteSection = (downshift: ControllerStateAndHelpers) => { @@ -96,11 +95,11 @@ class ExpressionInput extends Component{title} {matches .slice(0, 100) // Limit DOM rendering to 100 results, as DOM rendering is sloooow. - .map(({ original, string: text }) => { + .map((result: FuzzyResult) => { const itemProps = downshift.getItemProps({ - key: original, + key: result.original, index, - item: original, + item: result.original, style: { backgroundColor: highlightedIndex === index++ ? 'lightgray' : 'white', }, @@ -109,7 +108,7 @@ class ExpressionInput extends Component ); })} diff --git a/web/ui/react-app/src/pages/graph/Graph.tsx b/web/ui/react-app/src/pages/graph/Graph.tsx index 4a68e1d81..fd1b9f78c 100644 --- a/web/ui/react-app/src/pages/graph/Graph.tsx +++ b/web/ui/react-app/src/pages/graph/Graph.tsx @@ -14,6 +14,7 @@ require('../../vendor/flot/jquery.flot'); require('../../vendor/flot/jquery.flot.stack'); require('../../vendor/flot/jquery.flot.time'); require('../../vendor/flot/jquery.flot.crosshair'); +require('../../vendor/flot/jquery.flot.selection'); require('jquery.flot.tooltip'); export interface GraphProps { @@ -25,6 +26,7 @@ export interface GraphProps { stacked: boolean; useLocalTime: boolean; showExemplars: boolean; + handleTimeRangeSelection: (startTime: number, endTime: number) => void; queryParams: QueryParams | null; id: string; } @@ -117,6 +119,15 @@ class Graph extends PureComponent { }); } }); + + $(`.graph-${this.props.id}`).bind('plotselected', (_, ranges) => { + if (isPresent(this.$chart)) { + // eslint-disable-next-line + // @ts-ignore Typescript doesn't think this method exists although it actually does. + this.$chart.clearSelection(); + this.props.handleTimeRangeSelection(ranges.xaxis.from, ranges.xaxis.to); + } + }); } componentWillUnmount() { diff --git a/web/ui/react-app/src/pages/graph/GraphHelpers.test.ts b/web/ui/react-app/src/pages/graph/GraphHelpers.test.ts index d4d2df704..cb1cea8bf 100644 --- a/web/ui/react-app/src/pages/graph/GraphHelpers.test.ts +++ b/web/ui/react-app/src/pages/graph/GraphHelpers.test.ts @@ -234,6 +234,9 @@ describe('GraphHelpers', () => { lines: { lineWidth: 1, steps: false, fill: true }, shadowSize: 0, }, + selection: { + mode: 'x', + }, }); }); }); diff --git a/web/ui/react-app/src/pages/graph/GraphHelpers.ts b/web/ui/react-app/src/pages/graph/GraphHelpers.ts index 08f630ac4..55f64991a 100644 --- a/web/ui/react-app/src/pages/graph/GraphHelpers.ts +++ b/web/ui/react-app/src/pages/graph/GraphHelpers.ts @@ -147,6 +147,9 @@ export const getOptions = (stacked: boolean, useLocalTime: boolean): jquery.flot }, shadowSize: 0, }, + selection: { + mode: 'x', + }, }; }; diff --git a/web/ui/react-app/src/pages/graph/GraphTabContent.tsx b/web/ui/react-app/src/pages/graph/GraphTabContent.tsx index a29269cf5..66b1b8cf5 100644 --- a/web/ui/react-app/src/pages/graph/GraphTabContent.tsx +++ b/web/ui/react-app/src/pages/graph/GraphTabContent.tsx @@ -10,6 +10,7 @@ interface GraphTabContentProps { stacked: boolean; useLocalTime: boolean; showExemplars: boolean; + handleTimeRangeSelection: (startTime: number, endTime: number) => void; lastQueryParams: QueryParams | null; id: string; } @@ -21,6 +22,7 @@ export const GraphTabContent: FC = ({ useLocalTime, lastQueryParams, showExemplars, + handleTimeRangeSelection, id, }) => { if (!isPresent(data)) { @@ -41,6 +43,7 @@ export const GraphTabContent: FC = ({ stacked={stacked} useLocalTime={useLocalTime} showExemplars={showExemplars} + handleTimeRangeSelection={handleTimeRangeSelection} queryParams={lastQueryParams} id={id} /> diff --git a/web/ui/react-app/src/pages/graph/Panel.tsx b/web/ui/react-app/src/pages/graph/Panel.tsx index 3600edbdf..e58714f4c 100644 --- a/web/ui/react-app/src/pages/graph/Panel.tsx +++ b/web/ui/react-app/src/pages/graph/Panel.tsx @@ -255,6 +255,10 @@ class Panel extends Component { this.setOptions({ showExemplars: show }); }; + handleTimeRangeSelection = (startTime: number, endTime: number) => { + this.setOptions({ range: endTime - startTime, endTime: endTime }); + }; + render() { const { pastQueries, metricNames, options } = this.props; return ( @@ -356,6 +360,7 @@ class Panel extends Component { showExemplars={options.showExemplars} lastQueryParams={this.state.lastQueryParams} id={this.props.id} + handleTimeRangeSelection={this.handleTimeRangeSelection} /> )} diff --git a/web/ui/react-app/src/pages/graph/TimeInput.tsx b/web/ui/react-app/src/pages/graph/TimeInput.tsx index 2ab7e8a0c..4a0e4dd1e 100644 --- a/web/ui/react-app/src/pages/graph/TimeInput.tsx +++ b/web/ui/react-app/src/pages/graph/TimeInput.tsx @@ -79,7 +79,9 @@ class TimeInput extends Component { }); this.$time.on('change.datetimepicker', (e: any) => { - if (e.date) { + // The end time can also be set by dragging a section on the graph, + // and that value will have decimal places. + if (e.date && e.date.valueOf() !== Math.trunc(this.props.time?.valueOf()!)) { this.props.onChangeTime(e.date.valueOf()); } }); diff --git a/web/ui/react-app/src/types/index.d.ts b/web/ui/react-app/src/types/index.d.ts index 565069002..c809c1d90 100644 --- a/web/ui/react-app/src/types/index.d.ts +++ b/web/ui/react-app/src/types/index.d.ts @@ -43,6 +43,9 @@ declare namespace jquery.flot { series: { [K in keyof jquery.flot.seriesOptions]: jq.flot.seriesOptions[K] } & { stack: boolean; }; + selection: { + mode: string; + }; } } diff --git a/web/ui/react-app/src/vendor/flot/jquery.flot.selection.js b/web/ui/react-app/src/vendor/flot/jquery.flot.selection.js new file mode 100644 index 000000000..deb35abb1 --- /dev/null +++ b/web/ui/react-app/src/vendor/flot/jquery.flot.selection.js @@ -0,0 +1,412 @@ +/** + * + * THIS FILE WAS COPIED INTO PROMETHEUS FROM GRAFANA'S VENDORED FORK OF FLOT + * (LIVING AT https://github.com/grafana/grafana/tree/v7.5.8/public/vendor/flot). + * THE ORIGINAL FLOT CODE WAS LICENSED UNDER THE MIT LICENSE AS STATED BELOW. + * ADDITIONAL CHANGES HAVE BEEN CONTRIBUTED TO THE GRAFANA FORK UNDER AN + * APACHE 2 LICENSE, SEE https://github.com/grafana/grafana/blob/v7.5.8/LICENSE. + */ + +/* eslint-disable prefer-spread */ +/* eslint-disable no-loop-func */ +/* eslint-disable @typescript-eslint/no-this-alias */ +/* eslint-disable no-redeclare */ +/* eslint-disable no-useless-escape */ +/* eslint-disable prefer-const */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-use-before-define */ +/* eslint-disable eqeqeq */ +/* eslint-disable no-var */ +/* Flot plugin for selecting regions of a plot. + +Copyright (c) 2007-2013 IOLA and Ole Laursen. +Licensed under the MIT license. + +The plugin supports these options: + +selection: { + mode: null or "x" or "y" or "xy", + color: color, + shape: "round" or "miter" or "bevel", + minSize: number of pixels +} + +Selection support is enabled by setting the mode to one of "x", "y" or "xy". +In "x" mode, the user will only be able to specify the x range, similarly for +"y" mode. For "xy", the selection becomes a rectangle where both ranges can be +specified. "color" is color of the selection (if you need to change the color +later on, you can get to it with plot.getOptions().selection.color). "shape" +is the shape of the corners of the selection. + +"minSize" is the minimum size a selection can be in pixels. This value can +be customized to determine the smallest size a selection can be and still +have the selection rectangle be displayed. When customizing this value, the +fact that it refers to pixels, not axis units must be taken into account. +Thus, for example, if there is a bar graph in time mode with BarWidth set to 1 +minute, setting "minSize" to 1 will not make the minimum selection size 1 +minute, but rather 1 pixel. Note also that setting "minSize" to 0 will prevent +"plotunselected" events from being fired when the user clicks the mouse without +dragging. + +When selection support is enabled, a "plotselected" event will be emitted on +the DOM element you passed into the plot function. The event handler gets a +parameter with the ranges selected on the axes, like this: + + placeholder.bind( "plotselected", function( event, ranges ) { + alert("You selected " + ranges.xaxis.from + " to " + ranges.xaxis.to) + // similar for yaxis - with multiple axes, the extra ones are in + // x2axis, x3axis, ... + }); + +The "plotselected" event is only fired when the user has finished making the +selection. A "plotselecting" event is fired during the process with the same +parameters as the "plotselected" event, in case you want to know what's +happening while it's happening, + +A "plotunselected" event with no arguments is emitted when the user clicks the +mouse to remove the selection. As stated above, setting "minSize" to 0 will +destroy this behavior. + +The plugin allso adds the following methods to the plot object: + +- setSelection( ranges, preventEvent ) + + Set the selection rectangle. The passed in ranges is on the same form as + returned in the "plotselected" event. If the selection mode is "x", you + should put in either an xaxis range, if the mode is "y" you need to put in + an yaxis range and both xaxis and yaxis if the selection mode is "xy", like + this: + + setSelection({ xaxis: { from: 0, to: 10 }, yaxis: { from: 40, to: 60 } }); + + setSelection will trigger the "plotselected" event when called. If you don't + want that to happen, e.g. if you're inside a "plotselected" handler, pass + true as the second parameter. If you are using multiple axes, you can + specify the ranges on any of those, e.g. as x2axis/x3axis/... instead of + xaxis, the plugin picks the first one it sees. + +- clearSelection( preventEvent ) + + Clear the selection rectangle. Pass in true to avoid getting a + "plotunselected" event. + +- getSelection() + + Returns the current selection in the same format as the "plotselected" + event. If there's currently no selection, the function returns null. + +*/ + +(function($) { + function init(plot) { + var selection = { + first: { x: -1, y: -1 }, + second: { x: -1, y: -1 }, + show: false, + active: false, + }; + + // FIXME: The drag handling implemented here should be + // abstracted out, there's some similar code from a library in + // the navigation plugin, this should be massaged a bit to fit + // the Flot cases here better and reused. Doing this would + // make this plugin much slimmer. + var savedhandlers = {}; + + var mouseUpHandler = null; + + function onMouseMove(e) { + if (selection.active) { + updateSelection(e); + + plot.getPlaceholder().trigger('plotselecting', [getSelection()]); + } + } + + function onMouseDown(e) { + if (e.which != 1) + // only accept left-click + return; + + // cancel out any text selections + document.body.focus(); + + // prevent text selection and drag in old-school browsers + if (document.onselectstart !== undefined && savedhandlers.onselectstart == null) { + savedhandlers.onselectstart = document.onselectstart; + document.onselectstart = function() { + return false; + }; + } + if (document.ondrag !== undefined && savedhandlers.ondrag == null) { + savedhandlers.ondrag = document.ondrag; + document.ondrag = function() { + return false; + }; + } + + setSelectionPos(selection.first, e); + + selection.active = true; + + // this is a bit silly, but we have to use a closure to be + // able to whack the same handler again + mouseUpHandler = function(e) { + onMouseUp(e); + }; + + $(document).one('mouseup', mouseUpHandler); + } + + function onMouseUp(e) { + mouseUpHandler = null; + + // revert drag stuff for old-school browsers + if (document.onselectstart !== undefined) document.onselectstart = savedhandlers.onselectstart; + if (document.ondrag !== undefined) document.ondrag = savedhandlers.ondrag; + + // no more dragging + selection.active = false; + updateSelection(e); + + if (selectionIsSane()) triggerSelectedEvent(e); + else { + // this counts as a clear + plot.getPlaceholder().trigger('plotunselected', []); + plot.getPlaceholder().trigger('plotselecting', [null]); + } + + setTimeout(function() { + plot.isSelecting = false; + }, 10); + + return false; + } + + function getSelection() { + if (!selectionIsSane()) return null; + + if (!selection.show) return null; + + var r = {}, + c1 = selection.first, + c2 = selection.second; + var axes = plot.getAxes(); + // look if no axis is used + var noAxisInUse = true; + $.each(axes, function(name, axis) { + if (axis.used) { + //anyUsed = false; + } + }); + + $.each(axes, function(name, axis) { + if (axis.used || noAxisInUse) { + var p1 = axis.c2p(c1[axis.direction]), + p2 = axis.c2p(c2[axis.direction]); + r[name] = { from: Math.min(p1, p2), to: Math.max(p1, p2) }; + } + }); + return r; + } + + function triggerSelectedEvent(event) { + var r = getSelection(); + + // Add ctrlKey and metaKey to event + r.ctrlKey = event.ctrlKey; + r.metaKey = event.metaKey; + + plot.getPlaceholder().trigger('plotselected', [r]); + + // backwards-compat stuff, to be removed in future + if (r.xaxis && r.yaxis) + plot.getPlaceholder().trigger('selected', [{ x1: r.xaxis.from, y1: r.yaxis.from, x2: r.xaxis.to, y2: r.yaxis.to }]); + } + + function clamp(min, value, max) { + return value < min ? min : value > max ? max : value; + } + + function setSelectionPos(pos, e) { + var o = plot.getOptions(); + var offset = plot.getPlaceholder().offset(); + var plotOffset = plot.getPlotOffset(); + pos.x = clamp(0, e.pageX - offset.left - plotOffset.left, plot.width()); + pos.y = clamp(0, e.pageY - offset.top - plotOffset.top, plot.height()); + + if (o.selection.mode == 'y') pos.x = pos == selection.first ? 0 : plot.width(); + + if (o.selection.mode == 'x') pos.y = pos == selection.first ? 0 : plot.height(); + } + + function updateSelection(pos) { + if (pos.pageX == null) return; + + setSelectionPos(selection.second, pos); + if (selectionIsSane()) { + plot.isSelecting = true; + selection.show = true; + plot.triggerRedrawOverlay(); + } else clearSelection(true); + } + + function clearSelection(preventEvent) { + if (selection.show) { + selection.show = false; + plot.triggerRedrawOverlay(); + if (!preventEvent) plot.getPlaceholder().trigger('plotunselected', []); + } + } + + // function taken from markings support in Flot + function extractRange(ranges, coord) { + var axis, + from, + to, + key, + axes = plot.getAxes(); + + for (var k in axes) { + axis = axes[k]; + if (axis.direction == coord) { + key = coord + axis.n + 'axis'; + if (!ranges[key] && axis.n == 1) key = coord + 'axis'; // support x1axis as xaxis + if (ranges[key]) { + from = ranges[key].from; + to = ranges[key].to; + break; + } + } + } + + // backwards-compat stuff - to be removed in future + if (!ranges[key]) { + axis = coord == 'x' ? plot.getXAxes()[0] : plot.getYAxes()[0]; + from = ranges[coord + '1']; + to = ranges[coord + '2']; + } + + // auto-reverse as an added bonus + if (from != null && to != null && from > to) { + var tmp = from; + from = to; + to = tmp; + } + + return { from: from, to: to, axis: axis }; + } + + function setSelection(ranges, preventEvent) { + var range, + o = plot.getOptions(); + + if (o.selection.mode == 'y') { + selection.first.x = 0; + selection.second.x = plot.width(); + } else { + range = extractRange(ranges, 'x'); + + selection.first.x = range.axis.p2c(range.from); + selection.second.x = range.axis.p2c(range.to); + } + + if (o.selection.mode == 'x') { + selection.first.y = 0; + selection.second.y = plot.height(); + } else { + range = extractRange(ranges, 'y'); + + selection.first.y = range.axis.p2c(range.from); + selection.second.y = range.axis.p2c(range.to); + } + + selection.show = true; + plot.triggerRedrawOverlay(); + if (!preventEvent && selectionIsSane()) triggerSelectedEvent(); + } + + function selectionIsSane() { + var minSize = plot.getOptions().selection.minSize; + return ( + Math.abs(selection.second.x - selection.first.x) >= minSize && + Math.abs(selection.second.y - selection.first.y) >= minSize + ); + } + + plot.clearSelection = clearSelection; + plot.setSelection = setSelection; + plot.getSelection = getSelection; + + plot.hooks.bindEvents.push(function(plot, eventHolder) { + var o = plot.getOptions(); + if (o.selection.mode != null) { + eventHolder.mousemove(onMouseMove); + eventHolder.mousedown(onMouseDown); + } + }); + + plot.hooks.drawOverlay.push(function(plot, ctx) { + // draw selection + if (selection.show && selectionIsSane()) { + var plotOffset = plot.getPlotOffset(); + var o = plot.getOptions(); + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + var c = $.color.parse(o.selection.color); + + ctx.strokeStyle = c.scale('a', 0.8).toString(); + ctx.lineWidth = 1; + ctx.lineJoin = o.selection.shape; + ctx.fillStyle = c.scale('a', 0.4).toString(); + + var x = Math.min(selection.first.x, selection.second.x) + 0.5, + y = Math.min(selection.first.y, selection.second.y) + 0.5, + w = Math.abs(selection.second.x - selection.first.x) - 1, + h = Math.abs(selection.second.y - selection.first.y) - 1; + + ctx.fillRect(x, y, w, h); + ctx.strokeRect(x, y, w, h); + + ctx.restore(); + } + }); + + plot.hooks.shutdown.push(function(plot, eventHolder) { + eventHolder.unbind('mousemove', onMouseMove); + eventHolder.unbind('mousedown', onMouseDown); + + if (mouseUpHandler) { + $(document).unbind('mouseup', mouseUpHandler); + // grafana addition + // In L114 this plugin is overrinding document.onselectstart handler to prevent default or custom behaviour + // Then this patch is being restored during mouseup event. But, mouseup handler is unbound when this plugin is destroyed + // and the overridden onselectstart handler is not restored. The problematic behaviour surfaces when flot is re-rendered + // as a consequence of panel's model update. When i.e. options are applied via onBlur + // event on some input which results in flot re-render. The mouseup handler should be called to resture the original handlers + // but by the time the document mouseup event occurs, the event handler is no longer there, so onselectstart is permanently overridden. + // To fix that we are making sure that the overrides are reverted when this plugin is destroyed, the same way as they would + // via mouseup event handler (L138) + + if (document.onselectstart !== undefined) document.onselectstart = savedhandlers.onselectstart; + if (document.ondrag !== undefined) document.ondrag = savedhandlers.ondrag; + } + }); + } + + $.plot.plugins.push({ + init: init, + options: { + selection: { + mode: null, // one of null, "x", "y" or "xy" + color: '#e8cfac', + shape: 'round', // one of "round", "miter", or "bevel" + minSize: 5, // minimum number of pixels + }, + }, + name: 'selection', + version: '1.1', + }); +})(window.jQuery); diff --git a/web/ui/react-app/yarn.lock b/web/ui/react-app/yarn.lock index 2f5254662..2fba496ff 100644 --- a/web/ui/react-app/yarn.lock +++ b/web/ui/react-app/yarn.lock @@ -16,10 +16,10 @@ dependencies: "@babel/highlight" "^7.14.5" -"@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5", "@babel/compat-data@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.5.tgz#8ef4c18e58e801c5c95d3c1c0f2874a2680fadea" - integrity sha512-kixrYn4JwfAVPa0f2yfzc2AWti6WRRyO3XjWW5PJAvtE11qhSayrrcrEnee05KAtNaPC+EwehE8Qt1UedEVB8w== +"@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5", "@babel/compat-data@^7.14.7", "@babel/compat-data@^7.9.0": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.7.tgz#7b047d7a3a89a67d2258dc61f604f098f1bc7e08" + integrity sha512-nS6dZaISCXJ3+518CWiBfEr//gHyMO02uDxBkXTKZDN5POruCnOZ1N4YBRZDCabwF8nZMWBpRxIicmXtBs+fvw== "@babel/core@7.9.0": version "7.9.0" @@ -44,19 +44,19 @@ source-map "^0.5.0" "@babel/core@^7.1.0", "@babel/core@^7.4.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.5.tgz#d281f46a9905f07d1b3bf71ead54d9c7d89cb1e3" - integrity sha512-RN/AwP2DJmQTZSfiDaD+JQQ/J99KsIpOCfBE5pL+5jJSt7nI3nYGoAXZu+ffYSQ029NLs2DstZb+eR81uuARgg== + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.8.tgz#20cdf7c84b5d86d83fac8710a8bc605a7ba3f010" + integrity sha512-/AtaeEhT6ErpDhInbXmjHcUQXH0L0TEgscfcxk1qbOvLuKCa5aZT0SOOtDKFY96/CLROwbLSKyFor6idgNaU4Q== dependencies: "@babel/code-frame" "^7.14.5" - "@babel/generator" "^7.14.5" + "@babel/generator" "^7.14.8" "@babel/helper-compilation-targets" "^7.14.5" - "@babel/helper-module-transforms" "^7.14.5" - "@babel/helpers" "^7.14.5" - "@babel/parser" "^7.14.5" + "@babel/helper-module-transforms" "^7.14.8" + "@babel/helpers" "^7.14.8" + "@babel/parser" "^7.14.8" "@babel/template" "^7.14.5" - "@babel/traverse" "^7.14.5" - "@babel/types" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -64,12 +64,12 @@ semver "^6.3.0" source-map "^0.5.0" -"@babel/generator@^7.14.5", "@babel/generator@^7.4.0", "@babel/generator@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.5.tgz#848d7b9f031caca9d0cd0af01b063f226f52d785" - integrity sha512-y3rlP+/G25OIX3mYKKIOlQRcqj7YgrvHxOLbVmyLJ9bPmi5ttvUmpydVjcFjZphOktWuA7ovbx91ECloWTfjIA== +"@babel/generator@^7.14.8", "@babel/generator@^7.4.0", "@babel/generator@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.8.tgz#bf86fd6af96cf3b74395a8ca409515f89423e070" + integrity sha512-cYDUpvIzhBVnMzRoY1fkSEhK/HmwEVwlyULYgn/tMQYd6Obag3ylCjONle3gdErfXBW61SVTlR9QR7uWlgeIkg== dependencies: - "@babel/types" "^7.14.5" + "@babel/types" "^7.14.8" jsesc "^2.5.1" source-map "^0.5.0" @@ -98,14 +98,14 @@ browserslist "^4.16.6" semver "^6.3.0" -"@babel/helper-create-class-features-plugin@^7.14.5", "@babel/helper-create-class-features-plugin@^7.8.3": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.5.tgz#8842ec495516dd1ed8f6c572be92ba78b1e9beef" - integrity sha512-Uq9z2e7ZtcnDMirRqAGLRaLwJn+Lrh388v5ETrR3pALJnElVh2zqQmdbz4W2RUJYohAPh2mtyPUgyMHMzXMncQ== +"@babel/helper-create-class-features-plugin@^7.14.5", "@babel/helper-create-class-features-plugin@^7.14.6", "@babel/helper-create-class-features-plugin@^7.8.3": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.8.tgz#a6f8c3de208b1e5629424a9a63567f56501955fc" + integrity sha512-bpYvH8zJBWzeqi1o+co8qOrw+EXzQ/0c74gVmY205AWXy9nifHrOg77y+1zwxX5lXE7Icq4sPlSQ4O2kWBrteQ== dependencies: "@babel/helper-annotate-as-pure" "^7.14.5" "@babel/helper-function-name" "^7.14.5" - "@babel/helper-member-expression-to-functions" "^7.14.5" + "@babel/helper-member-expression-to-functions" "^7.14.7" "@babel/helper-optimise-call-expression" "^7.14.5" "@babel/helper-replace-supers" "^7.14.5" "@babel/helper-split-export-declaration" "^7.14.5" @@ -162,10 +162,10 @@ dependencies: "@babel/types" "^7.14.5" -"@babel/helper-member-expression-to-functions@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.5.tgz#d5c70e4ad13b402c95156c7a53568f504e2fb7b8" - integrity sha512-UxUeEYPrqH1Q/k0yRku1JE7dyfyehNwT6SVkMHvYvPDv4+uu627VXBckVj891BO8ruKBkiDoGnZf4qPDD8abDQ== +"@babel/helper-member-expression-to-functions@^7.14.5", "@babel/helper-member-expression-to-functions@^7.14.7": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.7.tgz#97e56244beb94211fe277bd818e3a329c66f7970" + integrity sha512-TMUt4xKxJn6ccjcOW7c4hlwyJArizskAhoSTOCkA0uZ+KghIaci0Qg9R043kUMWI9mtQfgny+NQ5QATnZ+paaA== dependencies: "@babel/types" "^7.14.5" @@ -176,19 +176,19 @@ dependencies: "@babel/types" "^7.14.5" -"@babel/helper-module-transforms@^7.14.5", "@babel/helper-module-transforms@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.5.tgz#7de42f10d789b423eb902ebd24031ca77cb1e10e" - integrity sha512-iXpX4KW8LVODuAieD7MzhNjmM6dzYY5tfRqT+R9HDXWl0jPn/djKmA+G9s/2C2T9zggw5tK1QNqZ70USfedOwA== +"@babel/helper-module-transforms@^7.14.5", "@babel/helper-module-transforms@^7.14.8", "@babel/helper-module-transforms@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.8.tgz#d4279f7e3fd5f4d5d342d833af36d4dd87d7dc49" + integrity sha512-RyE+NFOjXn5A9YU1dkpeBaduagTlZ0+fccnIcAGbv1KGUlReBj7utF7oEth8IdIBQPcux0DDgW5MFBH2xu9KcA== dependencies: "@babel/helper-module-imports" "^7.14.5" "@babel/helper-replace-supers" "^7.14.5" - "@babel/helper-simple-access" "^7.14.5" + "@babel/helper-simple-access" "^7.14.8" "@babel/helper-split-export-declaration" "^7.14.5" - "@babel/helper-validator-identifier" "^7.14.5" + "@babel/helper-validator-identifier" "^7.14.8" "@babel/template" "^7.14.5" - "@babel/traverse" "^7.14.5" - "@babel/types" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" "@babel/helper-optimise-call-expression@^7.14.5": version "7.14.5" @@ -221,12 +221,12 @@ "@babel/traverse" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/helper-simple-access@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.14.5.tgz#66ea85cf53ba0b4e588ba77fc813f53abcaa41c4" - integrity sha512-nfBN9xvmCt6nrMZjfhkl7i0oTV3yxR4/FztsbOASyTvVcoYd0TRHh7eMLdlEcCqobydC0LAF3LtC92Iwxo0wyw== +"@babel/helper-simple-access@^7.14.5", "@babel/helper-simple-access@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.14.8.tgz#82e1fec0644a7e775c74d305f212c39f8fe73924" + integrity sha512-TrFN4RHh9gnWEU+s7JloIho2T76GPwRHhdzOWLqTrMnlas8T9O7ec+oEDNsRXndOmru9ymH9DFrEOxpzPoSbdg== dependencies: - "@babel/types" "^7.14.5" + "@babel/types" "^7.14.8" "@babel/helper-skip-transparent-expression-wrappers@^7.14.5": version "7.14.5" @@ -242,10 +242,10 @@ dependencies: "@babel/types" "^7.14.5" -"@babel/helper-validator-identifier@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" - integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== +"@babel/helper-validator-identifier@^7.14.5", "@babel/helper-validator-identifier@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.8.tgz#32be33a756f29e278a0d644fa08a2c9e0f88a34c" + integrity sha512-ZGy6/XQjllhYQrNw/3zfWRwZCTVSiBLZ9DHVZxn9n2gip/7ab8mv2TWlKPIBk26RwedCBoWdjLmn+t9na2Gcow== "@babel/helper-validator-option@^7.14.5": version "7.14.5" @@ -262,14 +262,14 @@ "@babel/traverse" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/helpers@^7.14.5", "@babel/helpers@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.14.5.tgz#4870f8d9a6fdbbd65e5674a3558b4ff7fef0d9b2" - integrity sha512-xtcWOuN9VL6nApgVHtq3PPcQv5qFBJzoSZzJ/2c0QK/IP/gxVcoWSNQwFEGvmbQsuS9rhYqjILDGGXcTkA705Q== +"@babel/helpers@^7.14.8", "@babel/helpers@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.14.8.tgz#839f88f463025886cff7f85a35297007e2da1b77" + integrity sha512-ZRDmI56pnV+p1dH6d+UN6GINGz7Krps3+270qqI9UJ4wxYThfAIcI5i7j5vXC4FJ3Wap+S9qcebxeYiqn87DZw== dependencies: "@babel/template" "^7.14.5" - "@babel/traverse" "^7.14.5" - "@babel/types" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" "@babel/highlight@^7.14.5", "@babel/highlight@^7.8.3": version "7.14.5" @@ -280,10 +280,10 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.5", "@babel/parser@^7.4.3", "@babel/parser@^7.7.0", "@babel/parser@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.5.tgz#4cd2f346261061b2518873ffecdf1612cb032829" - integrity sha512-TM8C+xtH/9n1qzX+JNHi7AN2zHMTiPUtspO0ZdHflW8KaskkALhMmuMHb4bCmNdv9VAPzJX3/bXqkVLnAvsPfg== +"@babel/parser@^7.1.0", "@babel/parser@^7.14.5", "@babel/parser@^7.14.8", "@babel/parser@^7.4.3", "@babel/parser@^7.7.0", "@babel/parser@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.8.tgz#66fd41666b2d7b840bd5ace7f7416d5ac60208d4" + integrity sha512-syoCQFOoo/fzkWDeM0dLEZi5xqurb5vuyzwIMNZRNun+N/9A4cUZeQaE7dTrB8jGaKuJRBtEOajtnmw0I5hvvA== "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.14.5": version "7.14.5" @@ -294,10 +294,10 @@ "@babel/helper-skip-transparent-expression-wrappers" "^7.14.5" "@babel/plugin-proposal-optional-chaining" "^7.14.5" -"@babel/plugin-proposal-async-generator-functions@^7.14.5", "@babel/plugin-proposal-async-generator-functions@^7.8.3": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.14.5.tgz#4024990e3dd74181f4f426ea657769ff49a2df39" - integrity sha512-tbD/CG3l43FIXxmu4a7RBe4zH7MLJ+S/lFowPFO7HetS2hyOZ/0nnnznegDuzFzfkyQYTxqdTH/hKmuBngaDAA== +"@babel/plugin-proposal-async-generator-functions@^7.14.7", "@babel/plugin-proposal-async-generator-functions@^7.8.3": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.14.7.tgz#784a48c3d8ed073f65adcf30b57bcbf6c8119ace" + integrity sha512-RK8Wj7lXLY3bqei69/cc25gwS5puEc3dknoFPFbqfy3XxYQBQFvu4ioWpafMBAB+L9NyptQK4nMOa5Xz16og8Q== dependencies: "@babel/helper-plugin-utils" "^7.14.5" "@babel/helper-remap-async-to-generator" "^7.14.5" @@ -401,12 +401,12 @@ "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-numeric-separator" "^7.10.4" -"@babel/plugin-proposal-object-rest-spread@^7.14.5", "@babel/plugin-proposal-object-rest-spread@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.14.5.tgz#e581d5ccdfa187ea6ed73f56c6a21c1580b90fbf" - integrity sha512-VzMyY6PWNPPT3pxc5hi9LloKNr4SSrVCg7Yr6aZpW4Ym07r7KqSU/QXYwjXLVxqwSv0t/XSXkFoKBPUkZ8vb2A== +"@babel/plugin-proposal-object-rest-spread@^7.14.7", "@babel/plugin-proposal-object-rest-spread@^7.9.0": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.14.7.tgz#5920a2b3df7f7901df0205974c0641b13fd9d363" + integrity sha512-082hsZz+sVabfmDWo1Oct1u1AgbKbUAyVgmX4otIc7bdsRgHBXwTwb3DpDmD4Eyyx6DNiuz5UAATT655k+kL5g== dependencies: - "@babel/compat-data" "^7.14.5" + "@babel/compat-data" "^7.14.7" "@babel/helper-compilation-targets" "^7.14.5" "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-object-rest-spread" "^7.8.3" @@ -639,10 +639,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-destructuring@^7.14.5", "@babel/plugin-transform-destructuring@^7.8.3": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.14.5.tgz#d32ad19ff1a6da1e861dc62720d80d9776e3bf35" - integrity sha512-wU9tYisEbRMxqDezKUqC9GleLycCRoUsai9ddlsq54r8QRLaeEhc+d+9DqCG+kV9W2GgQjTZESPTpn5bAFMDww== +"@babel/plugin-transform-destructuring@^7.14.7", "@babel/plugin-transform-destructuring@^7.8.3": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.14.7.tgz#0ad58ed37e23e22084d109f185260835e5557576" + integrity sha512-0mDE99nK+kVh3xlc5vKwB6wnP9ecuSj+zQCa/n0voENtP/zymdT4HH6QEb65wjjcbqr1Jb/7z9Qp7TF5FtwYGw== dependencies: "@babel/helper-plugin-utils" "^7.14.5" @@ -744,10 +744,10 @@ "@babel/helper-module-transforms" "^7.14.5" "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-named-capturing-groups-regex@^7.14.5", "@babel/plugin-transform-named-capturing-groups-regex@^7.8.3": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.14.5.tgz#d537e8ee083ee6f6aa4f4eef9d2081d555746e4c" - integrity sha512-+Xe5+6MWFo311U8SchgeX5c1+lJM+eZDBZgD+tvXu9VVQPXwwVzeManMMjYX6xw2HczngfOSZjoFYKwdeB/Jvw== +"@babel/plugin-transform-named-capturing-groups-regex@^7.14.7", "@babel/plugin-transform-named-capturing-groups-regex@^7.8.3": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.14.7.tgz#60c06892acf9df231e256c24464bfecb0908fd4e" + integrity sha512-DTNOTaS7TkW97xsDMrp7nycUVh6sn/eq22VaxWfEdzuEbRsiaOU0pqU7DlyUGHVsbQbSghvjKRpEl+nUCKGQSg== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.14.5" @@ -872,10 +872,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-spread@^7.14.5", "@babel/plugin-transform-spread@^7.8.3": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.14.5.tgz#bd269fb4119754d2ce7f4cc39a96b4f71baae356" - integrity sha512-/3iqoQdiWergnShZYl0xACb4ADeYCJ7X/RgmwtXshn6cIvautRPAFzhd58frQlokLO6Jb4/3JXvmm6WNTPtiTw== +"@babel/plugin-transform-spread@^7.14.6", "@babel/plugin-transform-spread@^7.8.3": + version "7.14.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.14.6.tgz#6bd40e57fe7de94aa904851963b5616652f73144" + integrity sha512-Zr0x0YroFJku7n7+/HH3A2eIrGMjbmAIbJSVv0IZ+t3U2WUQUA64S/oeied2e+MaGSjmt4alzBCsK9E8gh+fag== dependencies: "@babel/helper-plugin-utils" "^7.14.5" "@babel/helper-skip-transparent-expression-wrappers" "^7.14.5" @@ -902,11 +902,11 @@ "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-transform-typescript@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.14.5.tgz#5b41b59072f765bd1ec1d0b694e08c7df0f6f8a0" - integrity sha512-cFD5PKp4b8/KkwQ7h71FdPXFvz1RgwTFF9akRZwFldb9G0AHf7CgoPx96c4Q/ZVjh6V81tqQwW5YiHws16OzPg== + version "7.14.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.14.6.tgz#6e9c2d98da2507ebe0a883b100cde3c7279df36c" + integrity sha512-XlTdBq7Awr4FYIzqhmYY80WN0V0azF74DMPyFqVHBvf81ZUgc4X7ZOpx6O8eLDK6iM5cCQzeyJw0ynTaefixRA== dependencies: - "@babel/helper-create-class-features-plugin" "^7.14.5" + "@babel/helper-create-class-features-plugin" "^7.14.6" "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-typescript" "^7.14.5" @@ -992,16 +992,16 @@ semver "^5.5.0" "@babel/preset-env@^7.4.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.14.5.tgz#c0c84e763661fd0e74292c3d511cb33b0c668997" - integrity sha512-ci6TsS0bjrdPpWGnQ+m4f+JSSzDKlckqKIJJt9UZ/+g7Zz9k0N8lYU8IeLg/01o2h8LyNZDMLGgRLDTxpudLsA== + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.14.8.tgz#254942f5ca80ccabcfbb2a9f524c74bca574005b" + integrity sha512-a9aOppDU93oArQ51H+B8M1vH+tayZbuBqzjOhntGetZVa+4tTu5jp+XTwqHGG2lxslqomPYVSjIxQkFwXzgnxg== dependencies: - "@babel/compat-data" "^7.14.5" + "@babel/compat-data" "^7.14.7" "@babel/helper-compilation-targets" "^7.14.5" "@babel/helper-plugin-utils" "^7.14.5" "@babel/helper-validator-option" "^7.14.5" "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.14.5" - "@babel/plugin-proposal-async-generator-functions" "^7.14.5" + "@babel/plugin-proposal-async-generator-functions" "^7.14.7" "@babel/plugin-proposal-class-properties" "^7.14.5" "@babel/plugin-proposal-class-static-block" "^7.14.5" "@babel/plugin-proposal-dynamic-import" "^7.14.5" @@ -1010,7 +1010,7 @@ "@babel/plugin-proposal-logical-assignment-operators" "^7.14.5" "@babel/plugin-proposal-nullish-coalescing-operator" "^7.14.5" "@babel/plugin-proposal-numeric-separator" "^7.14.5" - "@babel/plugin-proposal-object-rest-spread" "^7.14.5" + "@babel/plugin-proposal-object-rest-spread" "^7.14.7" "@babel/plugin-proposal-optional-catch-binding" "^7.14.5" "@babel/plugin-proposal-optional-chaining" "^7.14.5" "@babel/plugin-proposal-private-methods" "^7.14.5" @@ -1036,7 +1036,7 @@ "@babel/plugin-transform-block-scoping" "^7.14.5" "@babel/plugin-transform-classes" "^7.14.5" "@babel/plugin-transform-computed-properties" "^7.14.5" - "@babel/plugin-transform-destructuring" "^7.14.5" + "@babel/plugin-transform-destructuring" "^7.14.7" "@babel/plugin-transform-dotall-regex" "^7.14.5" "@babel/plugin-transform-duplicate-keys" "^7.14.5" "@babel/plugin-transform-exponentiation-operator" "^7.14.5" @@ -1048,7 +1048,7 @@ "@babel/plugin-transform-modules-commonjs" "^7.14.5" "@babel/plugin-transform-modules-systemjs" "^7.14.5" "@babel/plugin-transform-modules-umd" "^7.14.5" - "@babel/plugin-transform-named-capturing-groups-regex" "^7.14.5" + "@babel/plugin-transform-named-capturing-groups-regex" "^7.14.7" "@babel/plugin-transform-new-target" "^7.14.5" "@babel/plugin-transform-object-super" "^7.14.5" "@babel/plugin-transform-parameters" "^7.14.5" @@ -1056,18 +1056,18 @@ "@babel/plugin-transform-regenerator" "^7.14.5" "@babel/plugin-transform-reserved-words" "^7.14.5" "@babel/plugin-transform-shorthand-properties" "^7.14.5" - "@babel/plugin-transform-spread" "^7.14.5" + "@babel/plugin-transform-spread" "^7.14.6" "@babel/plugin-transform-sticky-regex" "^7.14.5" "@babel/plugin-transform-template-literals" "^7.14.5" "@babel/plugin-transform-typeof-symbol" "^7.14.5" "@babel/plugin-transform-unicode-escapes" "^7.14.5" "@babel/plugin-transform-unicode-regex" "^7.14.5" "@babel/preset-modules" "^0.1.4" - "@babel/types" "^7.14.5" + "@babel/types" "^7.14.8" babel-plugin-polyfill-corejs2 "^0.2.2" babel-plugin-polyfill-corejs3 "^0.2.2" babel-plugin-polyfill-regenerator "^0.2.2" - core-js-compat "^3.14.0" + core-js-compat "^3.15.0" semver "^6.3.0" "@babel/preset-modules@^0.1.3", "@babel/preset-modules@^0.1.4": @@ -1114,11 +1114,11 @@ "@babel/plugin-transform-typescript" "^7.9.0" "@babel/runtime-corejs3@^7.10.2", "@babel/runtime-corejs3@^7.12.1": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.14.5.tgz#0d9bf00d59c0b73185c462c323efffd0f4c37283" - integrity sha512-cBbwXj3F2xjnQJ0ERaFRLjxhUSBYsQPXJ7CERz/ecx6q6hzQ99eTflAPFC3ks4q/IG4CWupNVdflc4jlFBJVsg== + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.14.8.tgz#68539e0129f13eb1ed9a9aa273d3542b93c88384" + integrity sha512-4dMD5QRBkumn45oweR0SxoNtt15oz3BUBAQ8cIx7HJqZTtE8zjpM0My8aHJHVnyf4XfRg6DNzaE1080WLBiC1w== dependencies: - core-js-pure "^3.14.0" + core-js-pure "^3.15.0" regenerator-runtime "^0.13.4" "@babel/runtime@7.9.0": @@ -1129,9 +1129,9 @@ regenerator-runtime "^0.13.4" "@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.5", "@babel/runtime@^7.3.4", "@babel/runtime@^7.4.5", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.5.tgz#665450911c6031af38f81db530f387ec04cd9a98" - integrity sha512-121rumjddw9c3NCQ55KGkyE1h/nzWhU/owjhw0l4mQrkzz4x9SGS1X8gFLraHwX7td3Yo4QTL+qj0NcIzN87BA== + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.8.tgz#7119a56f421018852694290b9f9148097391b446" + integrity sha512-twj3L8Og5SaCRCErB4x4ajbvBIVV77CGeFglHpeg5WC5FF8TZzBWXtTJ4MqaD9QszLYTtr+IsaAL2rEUevb+eg== dependencies: regenerator-runtime "^0.13.4" @@ -1144,27 +1144,27 @@ "@babel/parser" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/traverse@^7.1.0", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.3", "@babel/traverse@^7.7.0", "@babel/traverse@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.5.tgz#c111b0f58afab4fea3d3385a406f692748c59870" - integrity sha512-G3BiS15vevepdmFqmUc9X+64y0viZYygubAMO8SvBmKARuF6CPSZtH4Ng9vi/lrWlZFGe3FWdXNy835akH8Glg== +"@babel/traverse@^7.1.0", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.14.8", "@babel/traverse@^7.4.3", "@babel/traverse@^7.7.0", "@babel/traverse@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.8.tgz#c0253f02677c5de1a8ff9df6b0aacbec7da1a8ce" + integrity sha512-kexHhzCljJcFNn1KYAQ6A5wxMRzq9ebYpEDV4+WdNyr3i7O44tanbDOR/xjiG2F3sllan+LgwK+7OMk0EmydHg== dependencies: "@babel/code-frame" "^7.14.5" - "@babel/generator" "^7.14.5" + "@babel/generator" "^7.14.8" "@babel/helper-function-name" "^7.14.5" "@babel/helper-hoist-variables" "^7.14.5" "@babel/helper-split-export-declaration" "^7.14.5" - "@babel/parser" "^7.14.5" - "@babel/types" "^7.14.5" + "@babel/parser" "^7.14.8" + "@babel/types" "^7.14.8" debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.14.5", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0", "@babel/types@^7.9.0": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.5.tgz#3bb997ba829a2104cedb20689c4a5b8121d383ff" - integrity sha512-M/NzBpEL95I5Hh4dwhin5JlE7EzO5PHMAuzjxss3tiOBD46KfQvVedN/3jEPZvdRvtsK2222XfdHogNIttFgcg== +"@babel/types@^7.0.0", "@babel/types@^7.14.5", "@babel/types@^7.14.8", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.0", "@babel/types@^7.9.0": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.8.tgz#38109de8fcadc06415fbd9b74df0065d4d41c728" + integrity sha512-iob4soQa7dZw8nodR/KlOQkPh9S4I8RwCxwRIFuiMRYjOzH/KJzdUfDgz6cGi5dDaclXF4P2PAhCdrBJNIg68Q== dependencies: - "@babel/helper-validator-identifier" "^7.14.5" + "@babel/helper-validator-identifier" "^7.14.8" to-fast-properties "^2.0.0" "@cnakazawa/watch@^1.0.3": @@ -1176,9 +1176,9 @@ minimist "^1.2.0" "@codemirror/autocomplete@^0.18.3": - version "0.18.7" - resolved "https://registry.yarnpkg.com/@codemirror/autocomplete/-/autocomplete-0.18.7.tgz#5d99cb8c4adf6abbc4b0cad9a14dcd2aae618916" - integrity sha512-4x6xH2jDuJzweB0Wdx3VQrFUvZF0V7RAHoomQOCq3NxUOZd/UXcSXrj3LXItFAxi6K4OlTLayFuI3z4k3UKUQQ== + version "0.18.8" + resolved "https://registry.yarnpkg.com/@codemirror/autocomplete/-/autocomplete-0.18.8.tgz#e82847071bd28029356055a498492659911c70a2" + integrity sha512-Va1Q763Vu/rVmIazru/ZnO2kkWVq6SlmMEjeD0qmxLAypyP6j/QNdpmaPDI1qb/+Mb9VFZBbac6a0aLTTi8qxQ== dependencies: "@codemirror/language" "^0.18.0" "@codemirror/state" "^0.18.0" @@ -1280,9 +1280,9 @@ "@codemirror/view" "^0.18.0" "@codemirror/rangeset@^0.18.0", "@codemirror/rangeset@^0.18.2": - version "0.18.3" - resolved "https://registry.yarnpkg.com/@codemirror/rangeset/-/rangeset-0.18.3.tgz#78d001f17385cf462647a60f8b15dab11990c2b1" - integrity sha512-p6bPVr6Cw0yh/QSelsg0RoNaG4btuzZo7YMT+WFwZsjbr7+X0dVpd2vqLAHIeDUfvOzrEI/dXXPKLpZZgYeU+g== + version "0.18.4" + resolved "https://registry.yarnpkg.com/@codemirror/rangeset/-/rangeset-0.18.4.tgz#84b615906236019107332a023116d2031f63b7bf" + integrity sha512-HHAebb85KoNrnzKG7Z3bOK3HirstPFFB5tu70QfSaUgSZeWnsSgcmGk033rMLGYV9GNTiibWKpqRPaCuyOUk7w== dependencies: "@codemirror/state" "^0.18.0" @@ -1305,10 +1305,10 @@ dependencies: "@codemirror/text" "^0.18.0" -"@codemirror/text@^0.18.0": - version "0.18.0" - resolved "https://registry.yarnpkg.com/@codemirror/text/-/text-0.18.0.tgz#a4a98862989ccef5545e730b269136d524c6a7c7" - integrity sha512-HMzHNIAbjCiCf3tEJMRg6ul01KPuXxQGNiHlHgAnqPguq/CX+L4Nvj5JlWQAI91Pupk18zhmM1c6eaazX4YeTg== +"@codemirror/text@^0.18.0", "@codemirror/text@^0.18.1": + version "0.18.1" + resolved "https://registry.yarnpkg.com/@codemirror/text/-/text-0.18.1.tgz#badacd628b8485932f8a42468bca1a106e210f93" + integrity sha512-vjXs6mi1F418kucTPlFvnCt9glKnjtYssdXb8mm1oaY/F5O+tgGVepm9Z8F7AKWCQvW8Bns1D3uLz/DOIEywIw== "@codemirror/tooltip@^0.18.4": version "0.18.4" @@ -1319,13 +1319,13 @@ "@codemirror/view" "^0.18.0" "@codemirror/view@^0.18.0", "@codemirror/view@^0.18.3": - version "0.18.17" - resolved "https://registry.yarnpkg.com/@codemirror/view/-/view-0.18.17.tgz#16723d36a2079168782ea1d993b11ea6fcc397b5" - integrity sha512-AneqrYFgQJFZY5CdIRvllaLWL7r966JZK64d05PrScYhhRT6y5iiq0VBO9nxX5Y2gbTXBdO1/eZdtJlhwa6hww== + version "0.18.19" + resolved "https://registry.yarnpkg.com/@codemirror/view/-/view-0.18.19.tgz#9d4f943a9e79ae00f6ad2ace0d6c759624faa86f" + integrity sha512-TmazVl8H3L+aYwlNb8xk6qADRb8KiYOO047pz51R4mGCg4Ja2siSjXktZgUvklsyWbUY7h9q+oAf4piH+mQZTw== dependencies: "@codemirror/rangeset" "^0.18.2" "@codemirror/state" "^0.18.0" - "@codemirror/text" "^0.18.0" + "@codemirror/text" "^0.18.1" style-mod "^4.0.0" w3c-keyname "^2.2.4" @@ -1581,6 +1581,11 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@nexucis/fuzzy@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@nexucis/fuzzy/-/fuzzy-0.2.2.tgz#60b2bd611e50a82170634027bd52751042622bea" + integrity sha512-XcBAj4bePw7rvQB86AOCnDCKAkm5JkVZh4JyVFlo4XXC/yuI4u2oTr4IQZLBsqrhI4ekZwVy3kwmO5kQ02NZzA== + "@nodelib/fs.stat@^1.1.2": version "1.1.3" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" @@ -1741,9 +1746,9 @@ integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== "@types/babel__core@^7.1.0": - version "7.1.14" - resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402" - integrity sha512-zGZJzzBUVDo/eV6KgbE0f0ZI7dInEYvo12Rb70uNQDshC3SkRMb67ja0GgRHZgAX3Za6rhaWlvbDO8rrGyAb1g== + version "7.1.15" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.15.tgz#2ccfb1ad55a02c83f8e0ad327cbc332f55eb1024" + integrity sha512-bxlMKPDbY8x5h6HBwVzEOk2C8fb6SLfYQ5Jw3uBYuYF1lfWk/kbLd81la82vrIkBb0l+JdmrZaDikPrNxpS/Ew== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" @@ -1752,31 +1757,31 @@ "@types/babel__traverse" "*" "@types/babel__generator@*": - version "7.6.2" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.2.tgz#f3d71178e187858f7c45e30380f8f1b7415a12d8" - integrity sha512-MdSJnBjl+bdwkLskZ3NGFp9YcXGx5ggLpQQPqtgakVhsWK0hTtNYhjpZLlWQTviGTvF8at+Bvli3jV7faPdgeQ== + version "7.6.3" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.3.tgz#f456b4b2ce79137f768aa130d2423d2f0ccfaba5" + integrity sha512-/GWCmzJWqV7diQW54smJZzWbSFf4QYtF71WCKhcx6Ru/tFyQIY2eiiITcCAeuPbNSvT9YCGkVMqqvSk2Z0mXiA== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.0" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.0.tgz#0c888dd70b3ee9eebb6e4f200e809da0076262be" - integrity sha512-NTPErx4/FiPCGScH7foPyr+/1Dkzkni+rHiYHHoTjvwou7AQzJkNeD60A9CXRy+ZEN2B1bggmkTMCDb+Mv5k+A== + version "7.4.1" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.1.tgz#3d1a48fd9d6c0edfd56f2ff578daed48f36c8969" + integrity sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" "@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": - version "7.11.1" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.11.1.tgz#654f6c4f67568e24c23b367e947098c6206fa639" - integrity sha512-Vs0hm0vPahPMYi9tDjtP66llufgO3ST16WXaSTtDGEl9cewAl3AibmxWw6TINOqHPT9z0uABKAYjT9jNSg4npw== + version "7.14.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.14.2.tgz#ffcd470bbb3f8bf30481678fb5502278ca833a43" + integrity sha512-K2waXdXBi2302XUdcHcR1jCeU0LL4TD9HRs/gk0N2Xvrht+G/BfJa4QObBQZfhMdxiCpV3COl5Nfq4uKTeTnJA== dependencies: "@babel/types" "^7.3.0" "@types/cheerio@*", "@types/cheerio@^0.22.22": - version "0.22.29" - resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.29.tgz#7115e9688bfc9e2f2730327c674b3d6a7e753e09" - integrity sha512-rNX1PsrDPxiNiyLnRKiW2NXHJFHqx0Fl3J2WsZq0MTBspa/FgwlqhXJE2crIcc+/2IglLHtSWw7g053oUR8fOg== + version "0.22.30" + resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.30.tgz#6c1ded70d20d890337f0f5144be2c5e9ce0936e6" + integrity sha512-t7ZVArWZlq3dFa9Yt33qFBQIK4CQd1Q3UJp0V+UhP6vgLWLM6Qug7vZuRSGXg45zXeB1Fm5X2vmBkEX58LV2Tw== dependencies: "@types/node" "*" @@ -1788,9 +1793,9 @@ "@types/enzyme" "*" "@types/enzyme@*", "@types/enzyme@^3.10.3": - version "3.10.8" - resolved "https://registry.yarnpkg.com/@types/enzyme/-/enzyme-3.10.8.tgz#ad7ac9d3af3de6fd0673773123fafbc63db50d42" - integrity sha512-vlOuzqsTHxog6PV79+tvOHFb6hq4QZKMq1lLD9MaWD1oec2lHTKndn76XOpSwCA0oFTaIbKVPrgM3k78Jjd16g== + version "3.10.9" + resolved "https://registry.yarnpkg.com/@types/enzyme/-/enzyme-3.10.9.tgz#b2d7c7429a37d994c156b6f361e83f271a60c8aa" + integrity sha512-dx5UvcWe2Vtye6S9Hw2rFB7Ul9uMXOAje2FAbXvVYieQDNle9qPAo7DfvFMSztZ9NFiD3dVZ4JsRYGTrSLynJg== dependencies: "@types/cheerio" "*" "@types/react" "*" @@ -1808,9 +1813,9 @@ "@types/jquery" "*" "@types/glob@^7.1.1": - version "7.1.3" - resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.3.tgz#e6ba80f36b7daad2c685acd9266382e68985c183" - integrity sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w== + version "7.1.4" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.4.tgz#ea59e21d2ee5c517914cb4bc8e4153b99e566672" + integrity sha512-w+LsMxKyYQm347Otw+IfBXOv9UWVjpHpCDdbBMt8Kz/xbvCYNjP+0qPh91Km3iKfSRLBB0P7fAMf0KHrPu+MyA== dependencies: "@types/minimatch" "*" "@types/node" "*" @@ -1843,34 +1848,29 @@ "@types/istanbul-lib-report" "*" "@types/jest@^26.0.10": - version "26.0.23" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.23.tgz#a1b7eab3c503b80451d019efb588ec63522ee4e7" - integrity sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA== + version "26.0.24" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.24.tgz#943d11976b16739185913a1936e0de0c4a7d595a" + integrity sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w== dependencies: jest-diff "^26.0.0" pretty-format "^26.0.0" "@types/jquery@*", "@types/jquery@^3.5.1": - version "3.5.5" - resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.5.tgz#2c63f47c9c8d96693d272f5453602afd8338c903" - integrity sha512-6RXU9Xzpc6vxNrS6FPPapN1SxSHgQ336WC6Jj/N8q30OiaBZ00l1GBgeP7usjVZPivSkGUfL1z/WW6TX989M+w== + version "3.5.6" + resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.6.tgz#97ac8e36dccd8ad8ed3f3f3b48933614d9fd8cf0" + integrity sha512-SmgCQRzGPId4MZQKDj9Hqc6kSXFNWZFHpELkyK8AQhf8Zr6HKfCzFv9ZC1Fv3FyQttJZOlap3qYb12h61iZAIg== dependencies: "@types/sizzle" "*" "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.5": - version "7.0.7" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" - integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== - -"@types/json5@^0.0.29": - version "0.0.29" - resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" - integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= + version "7.0.8" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.8.tgz#edf1bf1dbf4e04413ca8e5b17b3b7d7d54b59818" + integrity sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg== "@types/minimatch@*": - version "3.0.4" - resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.4.tgz#f0ec25dbf2f0e4b18647313ac031134ca5b24b21" - integrity sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA== + version "3.0.5" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" + integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== "@types/moment-timezone@^0.5.10": version "0.5.30" @@ -1880,14 +1880,14 @@ moment-timezone "*" "@types/node@*": - version "15.12.2" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.2.tgz#1f2b42c4be7156ff4a6f914b2fb03d05fa84e38d" - integrity sha512-zjQ69G564OCIWIOHSXyQEEDpdpGl+G348RAKY0XXy9Z5kU9Vzv1GMNnkar/ZJ8dzXB3COzD9Mo9NtRZ4xfgUww== + version "16.4.6" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.4.6.tgz#1734d119dfa8fede5606d35ae10f9fc9c84d889b" + integrity sha512-FKyawK3o5KL16AwbeFajen8G4K3mmqUrQsehn5wNKs8IzlKHE8TfnSmILXVMVziAEcnB23u1RCFU1NT6hSyr7Q== "@types/node@^12.11.1": - version "12.20.15" - resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.15.tgz#10ee6a6a3f971966fddfa3f6e89ef7a73ec622df" - integrity sha512-F6S4Chv4JicJmyrwlDkxUdGNSplsQdGwp1A0AJloEVDirWdZOAiRHhovDlsFkKUrquUXhz1imJhXHsf59auyAg== + version "12.20.17" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.17.tgz#ffd44c2801fc527a6fe6e86bc9b900261df1c87e" + integrity sha512-so8EHl4S6MmatPS0f9sE1ND94/ocbcEshW5OpyYthRqeRpiYyW2uXYTo/84kmfdfeNrDycARkvuiXl6nO40NGg== "@types/parse-json@^4.0.0": version "4.0.0" @@ -1895,33 +1895,33 @@ integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== "@types/prop-types@*": - version "15.7.3" - resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.3.tgz#2ab0d5da2e5815f94b0b9d4b95d1e5f243ab2ca7" - integrity sha512-KfRL3PuHmqQLOG+2tGpRO26Ctg+Cq1E01D2DMriKEATHgWLfeNDmq9e29Q9WIky0dQ3NPkd1mzYH8Lm936Z9qw== + version "15.7.4" + resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.4.tgz#fcf7205c25dff795ee79af1e30da2c9790808f11" + integrity sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ== "@types/q@^1.5.1": - version "1.5.4" - resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.4.tgz#15925414e0ad2cd765bfef58842f7e26a7accb24" - integrity sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug== + version "1.5.5" + resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.5.tgz#75a2a8e7d8ab4b230414505d92335d1dcb53a6df" + integrity sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ== "@types/reach__router@^1.2.6": - version "1.3.8" - resolved "https://registry.yarnpkg.com/@types/reach__router/-/reach__router-1.3.8.tgz#7b8607abf13704f918a9543257bcb7ec63028bfa" - integrity sha512-cjjT0FPdwuvhLWpCDt2WCh4sdBqNzJe3XhxXmRQGsY3IvT58M8sE4E7A0QaFYuJs3ar+McSJTiJxdYKWAXbBhw== + version "1.3.9" + resolved "https://registry.yarnpkg.com/@types/reach__router/-/reach__router-1.3.9.tgz#d3aaac0072665c81063cc6c557c18dadd642b226" + integrity sha512-N6rqQqTTAV/zKLfK3iq9Ww3wqCEhTZvsilhl0zI09zETdVq1QGmJH6+/xnj8AFUWIrle2Cqo+PGM/Ltr1vBb9w== dependencies: "@types/react" "*" "@types/react-copy-to-clipboard@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@types/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.0.tgz#38b035ca0c28334d3e0efaf3f319b81eea9690cd" - integrity sha512-faUg6Kx3Dfv0MBIcs+xzIptlRtjEVSaNjqyC14YAp4UwSiTHghnKtBOt9ERRTZZJfoJgnw10tomVaqG86GzdAw== + version "5.0.1" + resolved "https://registry.yarnpkg.com/@types/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.1.tgz#7d9c2c0af52e2e8106ebe2b9fde4f02859af7a3d" + integrity sha512-CDuRrJWEIdfKFC4vbwpXT3vk0O4gA/I/Kxu/1npUvGc2Yey5swPvsgO3JEnQkIUwdnYUbwUYDE/fTFQVgqr4oA== dependencies: "@types/react" "*" "@types/react-dom@^16.8.0": - version "16.9.13" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-16.9.13.tgz#5898f0ee68fe200685e6b61d3d7d8828692814d0" - integrity sha512-34Hr3XnmUSJbUVDxIw/e7dhQn2BJZhJmlAaPyPwfTQyuVS9mV/CeyghFcXyvkJXxI7notQJz8mF8FeCVvloJrA== + version "16.9.14" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-16.9.14.tgz#674b8f116645fe5266b40b525777fc6bb8eb3bcd" + integrity sha512-FIX2AVmPTGP30OUJ+0vadeIFJJ07Mh1m+U0rxfgyW34p3rTlXI+nlenvAxNn4BP36YyI9IJ/+UJ7Wu22N1pI7A== dependencies: "@types/react" "^16" @@ -1940,18 +1940,18 @@ "@types/react" "*" "@types/react@*": - version "17.0.11" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.11.tgz#67fcd0ddbf5a0b083a0f94e926c7d63f3b836451" - integrity sha512-yFRQbD+whVonItSk7ZzP/L+gPTJVBkL/7shLEF+i9GC/1cV3JmUxEQz6+9ylhUpWSDuqo1N9qEvqS6vTj4USUA== + version "17.0.15" + resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.15.tgz#c7533dc38025677e312606502df7656a6ea626d0" + integrity sha512-uTKHDK9STXFHLaKv6IMnwp52fm0hwU+N89w/p9grdUqcFA6WuqDyPhaWopbNyE1k/VhgzmHl8pu1L4wITtmlLw== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" csstype "^3.0.2" "@types/react@^16", "@types/react@^16.8.2": - version "16.14.8" - resolved "https://registry.yarnpkg.com/@types/react/-/react-16.14.8.tgz#4aee3ab004cb98451917c9b7ada3c7d7e52db3fe" - integrity sha512-QN0/Qhmx+l4moe7WJuTxNiTsjBwlBGHqKGvInSQCBdo7Qio0VtOqwsC0Wq7q3PbJlB0cR4Y4CVo1OOe6BOsOmA== + version "16.14.11" + resolved "https://registry.yarnpkg.com/@types/react/-/react-16.14.11.tgz#992a0cd4b66b9f27315042b5d96e976717368f04" + integrity sha512-Don0MtsZZ3fjwTJ2BsoqkyOy7e176KplEAKOpr/4XDdzinlyJBn9yfsKn5mcSgn4kh1B22+3tBnzBC1z63ybtQ== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" @@ -1972,9 +1972,9 @@ htmlparser2 "^4.1.0" "@types/scheduler@*": - version "0.16.1" - resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.1.tgz#18845205e86ff0038517aab7a18a62a6b9f71275" - integrity sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA== + version "0.16.2" + resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" + integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== "@types/sinon@^9.0.4": version "9.0.11" @@ -1984,9 +1984,9 @@ "@types/sinonjs__fake-timers" "*" "@types/sinonjs__fake-timers@*": - version "6.0.2" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz#3a84cf5ec3249439015e14049bd3161419bf9eae" - integrity sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg== + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08" + integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g== "@types/sizzle@*": version "2.3.3" @@ -2006,21 +2006,21 @@ "@types/react-test-renderer" "*" "@types/yargs-parser@*": - version "20.2.0" - resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9" - integrity sha512-37RSHht+gzzgYeobbG+KWryeAW8J33Nhr69cjTqSYymXVZEN9NbRYWoYlRtDhHKPVT1FyNKwaTPC1NynKZpzRA== + version "20.2.1" + resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.1.tgz#3b9ce2489919d9e4fea439b76916abc34b2df129" + integrity sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw== "@types/yargs@^13.0.0": - version "13.0.11" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-13.0.11.tgz#def2f0c93e4bdf2c61d7e34899b17e34be28d3b1" - integrity sha512-NRqD6T4gktUrDi1o1wLH3EKC1o2caCr7/wR87ODcbVITQF106OM3sFN92ysZ++wqelOd1CTzatnOBRDYYG6wGQ== + version "13.0.12" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-13.0.12.tgz#d895a88c703b78af0465a9de88aa92c61430b092" + integrity sha512-qCxJE1qgz2y0hA4pIxjBR+PelCH0U5CK1XJXFwCNqfmliatKp47UCXXE9Dyk1OXBDLvsCF57TqQEJaeLfDYEOQ== dependencies: "@types/yargs-parser" "*" "@types/yargs@^15.0.0": - version "15.0.13" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.13.tgz#34f7fec8b389d7f3c1fd08026a5763e072d3c6dc" - integrity sha512-kQ5JNTrbDv3Rp5X2n/iUu37IJBDU2gsZ5R/g1/KHOOEc5IKfUFjXT6DENPGduh08I/pamwtEq4oul7gUqKTQDQ== + version "15.0.14" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.14.tgz#26d821ddb89e70492160b66d10a0eb6df8f6fb06" + integrity sha512-yEJzHoxf6SyQGhBhIYGXQDSCkJjB6HohDShto7m8vaKg9Yp0Yn8+71J9eakh2bnPg6BfsH9PRMhiRTZnd4eXGQ== dependencies: "@types/yargs-parser" "*" @@ -2253,9 +2253,9 @@ acorn-globals@^6.0.0: acorn-walk "^7.1.1" acorn-jsx@^5.2.0: - version "5.3.1" - resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" - integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn-walk@^6.0.1: version "6.2.0" @@ -2283,9 +2283,9 @@ acorn@^7.1.1: integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== acorn@^8.2.4: - version "8.4.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.4.0.tgz#af53266e698d7cffa416714b503066a82221be60" - integrity sha512-ULr0LDaEqQrMFGyQ3bhJkLsbtrQ8QibAseGZeaSUiT/6zb9IvIkomWHJIvgvwad+hinRAgsI51JcWk2yvwyL+w== + version "8.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.4.1.tgz#56c36251fc7cabc7096adc18f05afe814321a28c" + integrity sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA== address@1.1.2, address@^1.0.1: version "1.1.2" @@ -2424,7 +2424,7 @@ anymatch@^2.0.0: micromatch "^3.1.4" normalize-path "^2.1.1" -anymatch@~3.1.1: +anymatch@~3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== @@ -2667,9 +2667,9 @@ aws4@^1.8.0: integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== axe-core@^4.0.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.2.2.tgz#0c987d82c8b82b4b9b7a945f1b5ef0d8fed586ed" - integrity sha512-OKRkKM4ojMEZRJ5UNJHmq9tht7cEnRnqKG6KyB/trYws00Xtkv12mHtlJ0SK7cmuNbrU8dPUova3ELTuilfBbw== + version "4.3.2" + resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.3.2.tgz#fcf8777b82c62cfc69c7e9f32c0d2226287680e7" + integrity sha512-5LMaDRWm8ZFPAEdzTYmgjjEdj1YnQcpfrVajO/sn/LhbpGp0Y0H64c2hLZI1gRMxfA+w1S71Uc/nHaOXgcCvGg== axobject-query@^2.0.2, axobject-query@^2.2.0: version "2.2.0" @@ -2776,12 +2776,12 @@ babel-plugin-polyfill-corejs2@^0.2.2: semver "^6.1.1" babel-plugin-polyfill-corejs3@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.2.2.tgz#7424a1682ee44baec817327710b1b094e5f8f7f5" - integrity sha512-l1Cf8PKk12eEk5QP/NQ6TH8A1pee6wWDJ96WjxrMXFLHLOBFzYM4moG80HFgduVhTqAFez4alnZKEhP/bYHg0A== + version "0.2.4" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.2.4.tgz#68cb81316b0e8d9d721a92e0009ec6ecd4cd2ca9" + integrity sha512-z3HnJE5TY/j4EFEa/qpQMSbcUJZ5JQi+3UFjXzn6pQCmIKc5Ug5j98SuYyH+m4xQnvKlMDIW4plLfgyVnd0IcQ== dependencies: "@babel/helper-define-polyfill-provider" "^0.2.2" - core-js-compat "^3.9.1" + core-js-compat "^3.14.0" babel-plugin-polyfill-regenerator@^0.2.2: version "0.2.2" @@ -3257,9 +3257,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001035, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001219: - version "1.0.30001237" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz#4b7783661515b8e7151fc6376cfd97f0e427b9e5" - integrity sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw== + version "1.0.30001248" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001248.tgz#26ab45e340f155ea5da2920dadb76a533cb8ebce" + integrity sha512-NwlQbJkxUFJ8nMErnGtT0QTM2TJ33xgz4KXJSMIrjXIbDVdaYueGyjOrLKRtJC+rTiWfi6j5cnZN1NBiSBJGNw== capture-exit@^2.0.0: version "2.0.0" @@ -3336,19 +3336,19 @@ cheerio@^1.0.0-rc.3: tslib "^2.2.0" "chokidar@>=3.0.0 <4.0.0", chokidar@^3.3.0, chokidar@^3.4.1: - version "3.5.1" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a" - integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw== + version "3.5.2" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75" + integrity sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ== dependencies: - anymatch "~3.1.1" + anymatch "~3.1.2" braces "~3.0.2" - glob-parent "~5.1.0" + glob-parent "~5.1.2" is-binary-path "~2.1.0" is-glob "~4.0.1" normalize-path "~3.0.0" - readdirp "~3.5.0" + readdirp "~3.6.0" optionalDependencies: - fsevents "~2.3.1" + fsevents "~2.3.2" chokidar@^2.1.8: version "2.1.8" @@ -3495,7 +3495,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.1: +color-convert@^1.9.0, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -3519,21 +3519,21 @@ color-name@^1.0.0, color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -color-string@^1.5.4: - version "1.5.5" - resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.5.tgz#65474a8f0e7439625f3d27a6a19d89fc45223014" - integrity sha512-jgIoum0OfQfq9Whcfc2z/VhCNcmQjWbey6qBX0vqt7YICflUmBCh9E9CiQD5GSJ+Uehixm3NUwHVhqUAWRivZg== +color-string@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.6.0.tgz#c3915f61fe267672cb7e1e064c9d692219f6c312" + integrity sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA== dependencies: color-name "^1.0.0" simple-swizzle "^0.2.2" color@^3.0.0: - version "3.1.3" - resolved "https://registry.yarnpkg.com/color/-/color-3.1.3.tgz#ca67fb4e7b97d611dcde39eceed422067d91596e" - integrity sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ== + version "3.2.1" + resolved "https://registry.yarnpkg.com/color/-/color-3.2.1.tgz#3544dc198caf4490c3ecc9a790b54fe9ff45e164" + integrity sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA== dependencies: - color-convert "^1.9.1" - color-string "^1.5.4" + color-convert "^1.9.3" + color-string "^1.6.0" colorette@^1.2.1, colorette@^1.2.2: version "1.2.2" @@ -3656,7 +3656,7 @@ content-type@~1.0.4: resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== -convert-source-map@1.7.0, convert-source-map@^1.4.0, convert-source-map@^1.7.0: +convert-source-map@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442" integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA== @@ -3668,6 +3668,13 @@ convert-source-map@^0.3.3: resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-0.3.5.tgz#f1d802950af7dd2631a1febe0596550c86ab3190" integrity sha1-8dgClQr33SYxof6+BZZVDIarMZA= +convert-source-map@^1.4.0, convert-source-map@^1.7.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.8.0.tgz#f3373c32d21b4d780dd8004514684fb791ca4369" + integrity sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA== + dependencies: + safe-buffer "~5.1.1" + cookie-signature@1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" @@ -3702,18 +3709,18 @@ copy-to-clipboard@^3: dependencies: toggle-selection "^1.0.6" -core-js-compat@^3.14.0, core-js-compat@^3.6.2, core-js-compat@^3.9.1: - version "3.14.0" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.14.0.tgz#b574dabf29184681d5b16357bd33d104df3d29a5" - integrity sha512-R4NS2eupxtiJU+VwgkF9WTpnSfZW4pogwKHd8bclWU2sp93Pr5S1uYJI84cMOubJRou7bcfL0vmwtLslWN5p3A== +core-js-compat@^3.14.0, core-js-compat@^3.15.0, core-js-compat@^3.6.2: + version "3.15.2" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.15.2.tgz#47272fbb479880de14b4e6081f71f3492f5bd3cb" + integrity sha512-Wp+BJVvwopjI+A1EFqm2dwUmWYXrvucmtIB2LgXn/Rb+gWPKYxtmb4GKHGKG/KGF1eK9jfjzT38DITbTOCX/SQ== dependencies: browserslist "^4.16.6" semver "7.0.0" -core-js-pure@^3.14.0: - version "3.14.0" - resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.14.0.tgz#72bcfacba74a65ffce04bf94ae91d966e80ee553" - integrity sha512-YVh+LN2FgNU0odThzm61BsdkwrbrchumFq3oztnE9vTKC4KS2fvnPmcx8t6jnqAyOTCTF4ZSiuK8Qhh7SNcL4g== +core-js-pure@^3.15.0: + version "3.15.2" + resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.15.2.tgz#c8e0874822705f3385d3197af9348f7c9ae2e3ce" + integrity sha512-D42L7RYh1J2grW8ttxoY1+17Y4wXZeKe7uyplAI3FkNQyI5OgBIAjUfFiTPfL1rs0qLpxaabITNbjKl1Sp82tA== core-js@^2.4.0: version "2.6.12" @@ -3721,9 +3728,9 @@ core-js@^2.4.0: integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== core-js@^3.5.0: - version "3.14.0" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.14.0.tgz#62322b98c71cc2018b027971a69419e2425c2a6c" - integrity sha512-3s+ed8er9ahK+zJpp9ZtuVcDoFzHNiZsPbNAAE4KXgrRHbjSqqNN6xGSXq6bq7TZIbKj4NLrLb6bJ5i+vSVjHA== + version "3.15.2" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.15.2.tgz#740660d2ff55ef34ce664d7e2455119c5bdd3d61" + integrity sha512-tKs41J7NJVuaya8DxIOCnl8QuPHx5/ZVbFo1oKgVl1qHFBBrDctzQGtuLjPpRdNTWmKPH6oEvgN/MUID+l485Q== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -4129,9 +4136,9 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9: ms "2.0.0" debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + version "4.3.2" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" + integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== dependencies: ms "2.1.2" @@ -4148,9 +4155,9 @@ decamelize@^1.2.0: integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= decimal.js@^10.2.1: - version "10.2.1" - resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.2.1.tgz#238ae7b0f0c793d3e3cea410108b35a2c01426a3" - integrity sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw== + version "10.3.1" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" + integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== decode-uri-component@^0.2.0: version "0.2.0" @@ -4504,9 +4511,9 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= electron-to-chromium@^1.3.378, electron-to-chromium@^1.3.723: - version "1.3.752" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.752.tgz#0728587f1b9b970ec9ffad932496429aef750d09" - integrity sha512-2Tg+7jSl3oPxgsBsWKh5H83QazTkmWG/cnNwJplmyZc7KcN61+I10oUgaXSVk/NwfvN3BdkKDR4FYuRBQQ2v0A== + version "1.3.790" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.790.tgz#5c569290929d92c8094fa08c79bc9393ca9e94e7" + integrity sha512-epMH/S2MkhBv+Y0+nHK8dC7bzmOaPwcmiYqt+VwxSUJLgPzkqZnGUEQ8eVhy5zGmgWm9tDDdXkHDzOEsVU979A== elliptic@^6.5.3: version "6.5.4" @@ -5601,7 +5608,7 @@ fsevents@^1.2.7: bindings "^1.5.0" nan "^2.12.1" -fsevents@~2.3.1: +fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== @@ -5631,11 +5638,6 @@ functions-have-names@^1.2.2: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.2.tgz#98d93991c39da9361f8e50b337c4f6e41f120e21" integrity sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA== -fuzzy@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/fuzzy/-/fuzzy-0.1.3.tgz#4c76ec2ff0ac1a36a9dccf9a00df8623078d4ed8" - integrity sha1-THbsL/CsGjap3M+aAN+GIweNTtg= - gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -5692,7 +5694,7 @@ glob-parent@^3.1.0: is-glob "^3.1.0" path-dirname "^1.0.0" -glob-parent@^5.0.0, glob-parent@~5.1.0: +glob-parent@^5.0.0, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== @@ -6420,9 +6422,9 @@ is-color-stop@^1.0.0: rgba-regex "^1.0.0" is-core-module@^2.2.0, is-core-module@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.4.0.tgz#8e9fc8e15027b011418026e98f0e6f4d86305cc1" - integrity sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A== + version "2.5.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.5.0.tgz#f754843617c70bfd29b7bd87327400cda5c18491" + integrity sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg== dependencies: has "^1.0.3" @@ -7335,7 +7337,7 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" -json5@^2.1.2: +json5@^2.1.2, json5@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== @@ -7829,17 +7831,17 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" -mime-db@1.48.0, "mime-db@>= 1.43.0 < 2": - version "1.48.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" - integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== +mime-db@1.49.0, "mime-db@>= 1.43.0 < 2": + version "1.49.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed" + integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA== mime-types@^2.1.12, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24: - version "2.1.31" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" - integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== + version "2.1.32" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5" + integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A== dependencies: - mime-db "1.48.0" + mime-db "1.49.0" mime@1.6.0: version "1.6.0" @@ -8276,9 +8278,9 @@ object-hash@^2.0.1: integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw== object-inspect@^1.10.3, object-inspect@^1.7.0, object-inspect@^1.9.0: - version "1.10.3" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.10.3.tgz#c2aa7d2d09f50c99375704f7a0adf24c5782d369" - integrity sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw== + version "1.11.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.11.0.tgz#9dceb146cedd4148a0d9e51ab88d34cf509922b1" + integrity sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg== object-is@^1.0.1, object-is@^1.0.2, object-is@^1.1.2: version "1.1.5" @@ -9498,9 +9500,9 @@ postcss@^7, postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.17, po supports-color "^6.1.0" postcss@^8.0.2: - version "8.3.4" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.4.tgz#41ece1c43f2f7c74dc7d90144047ce052757b822" - integrity sha512-/tZY0PXExXXnNhKv3TOvZAOUYRyuqcCbBm2c17YMDK0PlVII3K7/LKdt3ScHL+hhouddjUWi+1sKDf9xXW+8YA== + version "8.3.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.6.tgz#2730dd76a97969f37f53b9a6096197be311cc4ea" + integrity sha512-wG1cc/JhRgdqB6WHEuyLTedf3KIRuD0hG6ldkFEZNCjRxiC+3i6kkWUUbiJQayP28iwG35cEmAbe98585BYV0A== dependencies: colorette "^1.2.2" nanoid "^3.1.23" @@ -10058,10 +10060,10 @@ readdirp@^2.2.1: micromatch "^3.1.10" readable-stream "^2.0.2" -readdirp@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e" - integrity sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ== +readdirp@~3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" + integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" @@ -10102,9 +10104,9 @@ regenerator-runtime@^0.11.0: integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg== regenerator-runtime@^0.13.3, regenerator-runtime@^0.13.4: - version "0.13.7" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55" - integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew== + version "0.13.9" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz#8925742a98ffd90814988d7566ad30ca3b263b52" + integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA== regenerator-transform@^0.14.2: version "0.14.5" @@ -11488,12 +11490,11 @@ ts-pnp@^1.1.6: integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw== tsconfig-paths@^3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz#098547a6c4448807e8fcb8eae081064ee9a3c90b" - integrity sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw== + version "3.10.1" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz#79ae67a68c15289fdf5c51cb74f397522d795ed7" + integrity sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q== dependencies: - "@types/json5" "^0.0.29" - json5 "^1.0.1" + json5 "^2.2.0" minimist "^1.2.0" strip-bom "^3.0.0" @@ -11582,9 +11583,9 @@ typedarray@^0.0.6: integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= typescript@^3.3.3: - version "3.9.9" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.9.tgz#e69905c54bc0681d0518bd4d587cc6f2d0b1a674" - integrity sha512-kdMjTiekY+z/ubJCATUPlRDl39vXYiMV9iyeMuEuXZh2we6zz80uovNN2WlAxmmdE/Z/YQe+EbOEXB5RHEED3w== + version "3.9.10" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8" + integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== unbox-primitive@^1.0.1: version "1.0.1" @@ -11703,9 +11704,9 @@ url-loader@2.3.0: schema-utils "^2.5.0" url-parse@^1.4.3: - version "1.5.1" - resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.1.tgz#d5fa9890af8a5e1f274a2c98376510f6425f6e3b" - integrity sha512-HOfCOUJt7iSYzEx/UqgtwKRMC6EU91NFhsCHMv9oM03VJcVo2Qrp8T8kI9D7amFf1cu+/3CEhgb3rF9zL7k85Q== + version "1.5.3" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.3.tgz#71c1303d38fb6639ade183c2992c8cc0686df862" + integrity sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ== dependencies: querystringify "^2.1.1" requires-port "^1.0.0" @@ -12072,9 +12073,9 @@ whatwg-url@^7.0.0: webidl-conversions "^4.0.2" whatwg-url@^8.0.0, whatwg-url@^8.5.0: - version "8.6.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.6.0.tgz#27c0205a4902084b872aecb97cf0f2a7a3011f4c" - integrity sha512-os0KkeeqUOl7ccdDT1qqUcS4KH4tcBTSKK5Nl5WKb2lyxInIZ/CpjkqKa1Ss12mjfdcRX9mHmPPs7/SxG1Hbdw== + version "8.7.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" + integrity sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg== dependencies: lodash "^4.7.0" tr46 "^2.1.0" @@ -12309,9 +12310,9 @@ ws@^6.1.2, ws@^6.2.1: async-limiter "~1.0.0" ws@^7.4.5: - version "7.4.6" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" - integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== + version "7.5.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" + integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== xml-name-validator@^3.0.0: version "3.0.0"