From 139929a363092f6d24202eaf55d726f711ca9097 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sat, 4 Nov 2017 17:26:17 +0100 Subject: [PATCH 1/3] Update vendoring of prometheus/tsdb and prometheus/common --- .../prometheus/common/expfmt/text_parse.go | 4 ++ .../prometheus/common/promlog/flag/flag.go | 2 +- vendor/github.com/prometheus/tsdb/db.go | 35 +++++++++---- vendor/github.com/prometheus/tsdb/index.go | 13 +++-- vendor/vendor.json | 50 +++++++++++-------- 5 files changed, 66 insertions(+), 38 deletions(-) diff --git a/vendor/github.com/prometheus/common/expfmt/text_parse.go b/vendor/github.com/prometheus/common/expfmt/text_parse.go index ef9a15077..54bcfde29 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_parse.go +++ b/vendor/github.com/prometheus/common/expfmt/text_parse.go @@ -315,6 +315,10 @@ func (p *TextParser) startLabelValue() stateFn { if p.readTokenAsLabelValue(); p.err != nil { return nil } + if !model.LabelValue(p.currentToken.String()).IsValid() { + p.parseError(fmt.Sprintf("invalid label value %q", p.currentToken.String())) + return nil + } p.currentLabelPair.Value = proto.String(p.currentToken.String()) // Special treatment of summaries: // - Quantile labels are special, will result in dto.Quantile later. diff --git a/vendor/github.com/prometheus/common/promlog/flag/flag.go b/vendor/github.com/prometheus/common/promlog/flag/flag.go index 282dfd6eb..b9d361e43 100644 --- a/vendor/github.com/prometheus/common/promlog/flag/flag.go +++ b/vendor/github.com/prometheus/common/promlog/flag/flag.go @@ -26,7 +26,7 @@ const LevelFlagName = "log.level" const LevelFlagHelp = "Only log messages with the given severity or above. One of: [debug, info, warn, error]" // AddFlags adds the flags used by this package to the Kingpin application. -// To use the default Kingpin application, call AddFlags(kingpin.CommandLine, ..). +// To use the default Kingpin application, call AddFlags(kingpin.CommandLine) func AddFlags(a *kingpin.Application, logLevel *promlog.AllowedLevel) { a.Flag(LevelFlagName, LevelFlagHelp). Default("info").SetValue(logLevel) diff --git a/vendor/github.com/prometheus/tsdb/db.go b/vendor/github.com/prometheus/tsdb/db.go index 9d472e91a..3622a77f5 100644 --- a/vendor/github.com/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/tsdb/db.go @@ -378,15 +378,9 @@ func (db *DB) compact() (changes bool, err error) { return changes, errors.Wrapf(err, "compact %s", plan) } changes = true - - for _, pd := range plan { - if err := os.RemoveAll(pd); err != nil { - return changes, errors.Wrap(err, "delete compacted block") - } - } runtime.GC() - if err := db.reload(); err != nil { + if err := db.reload(plan...); err != nil { return changes, errors.Wrap(err, "reload blocks") } runtime.GC() @@ -440,7 +434,18 @@ func (db *DB) getBlock(id ulid.ULID) (*Block, bool) { return nil, false } -func (db *DB) reload() (err error) { +func stringsContain(set []string, elem string) bool { + for _, e := range set { + if elem == e { + return true + } + } + return false +} + +// reload on-disk blocks and trigger head truncation if new blocks appeared. It takes +// a list of block directories which should be deleted during reload. +func (db *DB) reload(deleteable ...string) (err error) { defer func() { if err != nil { db.metrics.reloadsFailed.Inc() @@ -462,6 +467,10 @@ func (db *DB) reload() (err error) { if err != nil { return errors.Wrapf(err, "read meta information %s", dir) } + // If the block is pending for deletion, don't add it to the new block set. + if stringsContain(deleteable, dir) { + continue + } b, ok := db.getBlock(meta.ULID) if !ok { @@ -487,8 +496,14 @@ func (db *DB) reload() (err error) { db.mtx.Unlock() for _, b := range oldBlocks { - if _, ok := exist[b.Meta().ULID]; !ok { - b.Close() + if _, ok := exist[b.Meta().ULID]; ok { + continue + } + if err := b.Close(); err != nil { + level.Warn(db.logger).Log("msg", "closing block failed", "err", err) + } + if err := os.RemoveAll(b.Dir()); err != nil { + level.Warn(db.logger).Log("msg", "deleting block failed", "err", err) } } diff --git a/vendor/github.com/prometheus/tsdb/index.go b/vendor/github.com/prometheus/tsdb/index.go index f976de695..258db74d2 100644 --- a/vendor/github.com/prometheus/tsdb/index.go +++ b/vendor/github.com/prometheus/tsdb/index.go @@ -35,6 +35,8 @@ const ( MagicIndex = 0xBAAAD700 indexFormatV1 = 1 + + size_unit = 4 ) const indexFilename = "index" @@ -203,12 +205,13 @@ func (w *indexWriter) write(bufs ...[]byte) error { return nil } -// addPadding adds zero byte padding until the file size is a multiple of n. -func (w *indexWriter) addPadding(n int) error { - p := n - (int(w.pos) % n) +// addPadding adds zero byte padding until the file size is a multiple size_unit. +func (w *indexWriter) addPadding() error { + p := w.pos % size_unit if p == 0 { return nil } + p = size_unit - p return errors.Wrap(w.write(make([]byte, p)), "add padding") } @@ -373,7 +376,7 @@ func (w *indexWriter) WriteLabelIndex(names []string, values []string) error { sort.Sort(valt) // Align beginning to 4 bytes for more efficient index list scans. - if err := w.addPadding(4); err != nil { + if err := w.addPadding(); err != nil { return err } @@ -446,7 +449,7 @@ func (w *indexWriter) WritePostings(name, value string, it Postings) error { } // Align beginning to 4 bytes for more efficient postings list scans. - if err := w.addPadding(4); err != nil { + if err := w.addPadding(); err != nil { return err } diff --git a/vendor/vendor.json b/vendor/vendor.json index 04a794917..0766c91a7 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -728,40 +728,46 @@ "revisionTime": "2015-02-12T10:17:44Z" }, { - "checksumSHA1": "Wtpzndm/+bdwwNU5PCTfb4oUhc8=", + "checksumSHA1": "xfnn0THnqNwjwimeTClsxahYrIo=", "path": "github.com/prometheus/common/expfmt", - "revision": "9e0844febd9e2856f839c9cb974fbd676d1755a8", - "revisionTime": "2017-04-18T15:52:10Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "GWlM3d2vPYyNATtTFgftS10/A9w=", "path": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg", - "revision": "9e0844febd9e2856f839c9cb974fbd676d1755a8", - "revisionTime": "2017-04-18T15:52:10Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "3VoqH7TFfzA6Ds0zFzIbKCUvBmw=", "path": "github.com/prometheus/common/model", - "revision": "0d0c3d572886e0f2323ed376557f9eb99b97d25b", - "revisionTime": "2017-06-16T14:41:04Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "Yseprf8kAFr/s7wztkQnrFuFN+8=", "path": "github.com/prometheus/common/promlog", - "revision": "2f17f4a9d485bf34b4bfaccc273805040e4f86c8", - "revisionTime": "2017-09-08T16:18:22Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" + }, + { + "checksumSHA1": "1H28FCxsaAIm6kvue+Wfdd8Lq6M=", + "path": "github.com/prometheus/common/promlog/flag", + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "9aDxDuzZt1l7FQJ9qpn2kPcF7NU=", "path": "github.com/prometheus/common/route", - "revision": "9e0844febd9e2856f839c9cb974fbd676d1755a8", - "revisionTime": "2017-04-18T15:52:10Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "91KYK0SpvkaMJJA2+BcxbVnyRO0=", "path": "github.com/prometheus/common/version", - "revision": "9e0844febd9e2856f839c9cb974fbd676d1755a8", - "revisionTime": "2017-04-18T15:52:10Z" + "revision": "e3fb1a1acd7605367a2b378bc2e2f893c05174b7", + "revisionTime": "2017-11-04T09:59:07Z" }, { "checksumSHA1": "W218eJZPXJG783fUr/z6IaAZyes=", @@ -770,28 +776,28 @@ "revisionTime": "2016-04-11T19:08:41Z" }, { - "checksumSHA1": "MZoz9kpR5PSUM9mJLh3c7nSrk9c=", + "checksumSHA1": "Bty/r75M8kM+GA80eMM5p0cLTi0=", "path": "github.com/prometheus/tsdb", - "revision": "b1df85781931b0ff48d09a364174016d16a4dc3e", - "revisionTime": "2017-11-01T17:11:22Z" + "revision": "706602daed1487f7849990678b4ece4599745905", + "revisionTime": "2017-11-04T07:45:56Z" }, { "checksumSHA1": "uy6ySJ6EZqof+yMD2wTkYob8BeU=", "path": "github.com/prometheus/tsdb/chunks", - "revision": "5d28c849c7ff3b43e2829a44a9aac16468e076ce", - "revisionTime": "2017-10-25T14:52:11Z" + "revision": "706602daed1487f7849990678b4ece4599745905", + "revisionTime": "2017-11-04T07:45:56Z" }, { "checksumSHA1": "7RhNAVcmDmLFqn9nWiudT0B76f8=", "path": "github.com/prometheus/tsdb/fileutil", - "revision": "5d28c849c7ff3b43e2829a44a9aac16468e076ce", - "revisionTime": "2017-10-25T14:52:11Z" + "revision": "706602daed1487f7849990678b4ece4599745905", + "revisionTime": "2017-11-04T07:45:56Z" }, { "checksumSHA1": "zhmlvc322RH1L3l9DaA9d/HVVWs=", "path": "github.com/prometheus/tsdb/labels", - "revision": "5d28c849c7ff3b43e2829a44a9aac16468e076ce", - "revisionTime": "2017-10-25T14:52:11Z" + "revision": "706602daed1487f7849990678b4ece4599745905", + "revisionTime": "2017-11-04T07:45:56Z" }, { "checksumSHA1": "5SYLEhADhdBVZAGPVHWggQl7H8k=", From 463faf5816d4ab1445ba2f2ecee08983b55ce640 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sat, 4 Nov 2017 17:40:11 +0100 Subject: [PATCH 2/3] Ignore staticcheck's SA1019 in discovery/kubernetes/kubernetes.go, too This showed up all of a sudden, even if checking out older commits. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 00c3bf248..5535427c6 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ ifdef DEBUG endif STATICCHECK_IGNORE = \ + github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:SA1019 \ github.com/prometheus/prometheus/discovery/kubernetes/node.go:SA1019 \ github.com/prometheus/prometheus/documentation/examples/remote_storage/remote_storage_adapter/main.go:SA1019 \ github.com/prometheus/prometheus/pkg/textparse/lex.l.go:SA4006 \ From 36a6ed4fe1487f8540014cc9def6741fdc0974d9 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Sat, 4 Nov 2017 17:30:58 +0100 Subject: [PATCH 3/3] Cut v2.0.0-rc.3 --- CHANGELOG.md | 32 +++++++++++++++++--------------- VERSION | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2553068..07cbfa91d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,24 @@ +## 2.0.0-rc.3 / 2017-11-04 + +_Changes compared to 2.0.0-rc.2._ + +* [ENHANCEMENT] Remove remote read from the query path if no remote storage + is configured. +* [ENHANCEMENT] Bump Consul HTTP client timeout to not match the Consul SD + watch timeout. +* [ENHANCEMENT] Bump up a too small max block duration to the min block + duration instead of returning an error. +* [BUGFIX] Avoid needless padding of 4 zero bytes in index files. +* [BUGFIX] Delete old blocks during reload (necessary on MS Windows). +* [BUGFIX] Fix regression of alert rules state loss on config reload. +* [BUGFIX] Serialize background WAL operations to avoid data races. + +This also contains bugfixes and remote-storage features from the 1.8 branch. + ## 1.8.2 / 2017-11-04 * [BUGFIX] EC2 service discovery: Do not crash if tags are empty. -## 2.0.0-rc.2 / 2017-10-25 - -* [ENHANCEMENT] handle WAL segments with corrupted header gracefully -* [ENHANCEMENT] stabilize memory usage during WAL replay -* [CHANGE] Prefix all storage metrics with `prometheus_` -* [BUGFIX] Correctly handle label removal in remote read -* [BUGFIX] Fix chunk misalignment causing out-of-order samples -* [BUGFIX] Fix connection leak in Consul SD -* [BUGFIX] Handle invalid chunk derefernces gracefully -* [BUGFIX] Prevent potential deadlock during failing querier construction - -Data written in previous pre-release versions may have been affected by the out-of-order -bug. Reading this data may reveal artefacts and incorrect data. -Starting with a clean storage directory is advised. The WAL directory may safely be kept. - ## 1.8.1 / 2017-10-19 * [BUGFIX] Correctly handle external labels on remote read endpoint diff --git a/VERSION b/VERSION index c8f3a156f..b87c4d410 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0-rc.2 +2.0.0-rc.3