From 8e4407c226aa3ede45220255ad1351366bff4372 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 4 Mar 2022 11:45:45 +0000 Subject: [PATCH 1/5] Update grafana/regexp package to reject very deeply nested regexps (#10397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a cherry-pick of https://github.com/golang/go/commit/ac071634c487eb6ac5422652de3c7c18fba7c522 with the following description: regexp/syntax: reject very deeply nested regexps in Parse The regexp code assumes it can recurse over the structure of a regexp safely. Go's growable stacks make that reasonable for all plausible regexps, but implausible ones can reach the “infinite recursion?” stack limit. This CL limits the depth of any parsed regexp to 1000. That is, the depth of the parse tree is required to be ≤ 1000. Regexps that require deeper parse trees will return ErrInternalError. Signed-off-by: Bryan Boreham --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3fa90c933..58e840f2d 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/pprof v0.0.0-20220218203455-0368bd9e19a7 github.com/gophercloud/gophercloud v0.24.0 - github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280 + github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/consul/api v1.12.0 github.com/hashicorp/go-hclog v0.12.2 // indirect diff --git a/go.sum b/go.sum index bd8081c9f..36af05221 100644 --- a/go.sum +++ b/go.sum @@ -682,8 +682,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280 h1:MOND6wXrwVXEzmL2bZ+Jcbgycwt1LD5q6NQbqz/Nlic= -github.com/grafana/regexp v0.0.0-20220202152315-e74e38789280/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 h1:uirlL/j72L93RhV4+mkWhjv0cov2I0MIgPOG9rMDr1k= +github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= From f620efee49446aafdebd25f27611ca97783e965a Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Tue, 8 Mar 2022 11:08:20 -0700 Subject: [PATCH 2/5] Cut 2.34.0-rc.1 Signed-off-by: Chris Marchbanks --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d5272a0..ef5b9ac1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.34.0-rc.1 / 2022-03-08 + +* [ENHANCEMENT] HTTP SD: Add a failure counter. #10372 +* [BUGFIX] Parser: Specify type in metadata parser errors. #10269 +* [BUGFIX] Scrape: Fix label limit changes not applying. #10370 + ## 2.34.0-rc.0 / 2022-02-24 * [CHANGE] UI: Classic UI removed. #10208 diff --git a/VERSION b/VERSION index 30b7b9fa0..62f91b470 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.34.0-rc.0 +2.34.0-rc.1 From b025390cb408965aef4c81edde9350ab99424e47 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Fri, 11 Mar 2022 13:26:59 -0300 Subject: [PATCH 3/5] Disable chunk write queue by default, allow user to configure the exact size (#10425) * Disable chunk write queue by default Signed-off-by: Mauro Stettler * update flag description Signed-off-by: Mauro Stettler --- cmd/prometheus/main.go | 5 +++ tsdb/chunks/head_chunks.go | 57 ++++++++++++++++++++++++--------- tsdb/chunks/head_chunks_test.go | 18 ++++++++++- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index d1563b1d5..6e8fad806 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -305,6 +305,9 @@ func main() { serverOnlyFlag(a, "storage.tsdb.wal-compression", "Compress the tsdb WAL."). Hidden().Default("true").BoolVar(&cfg.tsdb.WALCompression) + serverOnlyFlag(a, "storage.tsdb.head-chunks-write-queue-size", "Size of the queue through which head chunks are written to the disk to be m-mapped, 0 disables the queue completely. Experimental."). + Default("0").IntVar(&cfg.tsdb.HeadChunksWriteQueueSize) + agentOnlyFlag(a, "storage.agent.path", "Base path for metrics storage."). Default("data-agent/").StringVar(&cfg.agentStoragePath) @@ -1484,6 +1487,7 @@ type tsdbOptions struct { NoLockfile bool AllowOverlappingBlocks bool WALCompression bool + HeadChunksWriteQueueSize int StripeSize int MinBlockDuration model.Duration MaxBlockDuration model.Duration @@ -1501,6 +1505,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { NoLockfile: opts.NoLockfile, AllowOverlappingBlocks: opts.AllowOverlappingBlocks, WALCompression: opts.WALCompression, + HeadChunksWriteQueueSize: opts.HeadChunksWriteQueueSize, StripeSize: opts.StripeSize, MinBlockDuration: int64(time.Duration(opts.MinBlockDuration) / time.Millisecond), MaxBlockDuration: int64(time.Duration(opts.MaxBlockDuration) / time.Millisecond), diff --git a/tsdb/chunks/head_chunks.go b/tsdb/chunks/head_chunks.go index 94b20e129..307ecfc96 100644 --- a/tsdb/chunks/head_chunks.go +++ b/tsdb/chunks/head_chunks.go @@ -69,7 +69,8 @@ const ( // DefaultWriteBufferSize is the default write buffer size. DefaultWriteBufferSize = 4 * 1024 * 1024 // 4 MiB. // DefaultWriteQueueSize is the default size of the in-memory queue used before flushing chunks to the disk. - DefaultWriteQueueSize = 1000 + // A value of 0 completely disables this feature. + DefaultWriteQueueSize = 0 ) // ChunkDiskMapperRef represents the location of a head chunk on disk. @@ -249,7 +250,10 @@ func NewChunkDiskMapper(reg prometheus.Registerer, dir string, pool chunkenc.Poo crc32: newCRC32(), chunkBuffer: newChunkBuffer(), } - m.writeQueue = newChunkWriteQueue(reg, writeQueueSize, m.writeChunk) + + if writeQueueSize > 0 { + m.writeQueue = newChunkWriteQueue(reg, writeQueueSize, m.writeChunk) + } if m.pool == nil { m.pool = chunkenc.NewPool() @@ -375,18 +379,33 @@ func repairLastChunkFile(files map[int]string) (_ map[int]string, returnErr erro // WriteChunk writes the chunk to the disk. // The returned chunk ref is the reference from where the chunk encoding starts for the chunk. func (cdm *ChunkDiskMapper) WriteChunk(seriesRef HeadSeriesRef, mint, maxt int64, chk chunkenc.Chunk, callback func(err error)) (chkRef ChunkDiskMapperRef) { - var err error - defer func() { - if err != nil && callback != nil { - callback(err) - } - }() - - // cdm.evtlPosMtx must be held to serialize the calls to .getNextChunkRef() and .addJob(). + // cdm.evtlPosMtx must be held to serialize the calls to cdm.evtlPos.getNextChunkRef() and the writing of the chunk (either with or without queue). cdm.evtlPosMtx.Lock() defer cdm.evtlPosMtx.Unlock() - ref, cutFile := cdm.evtlPos.getNextChunkRef(chk) + + if cdm.writeQueue != nil { + return cdm.writeChunkViaQueue(ref, cutFile, seriesRef, mint, maxt, chk, callback) + } + + err := cdm.writeChunk(seriesRef, mint, maxt, chk, ref, cutFile) + if callback != nil { + callback(err) + } + + return ref +} + +func (cdm *ChunkDiskMapper) writeChunkViaQueue(ref ChunkDiskMapperRef, cutFile bool, seriesRef HeadSeriesRef, mint, maxt int64, chk chunkenc.Chunk, callback func(err error)) (chkRef ChunkDiskMapperRef) { + var err error + if callback != nil { + defer func() { + if err != nil { + callback(err) + } + }() + } + err = cdm.writeQueue.addJob(chunkWriteJob{ cutFile: cutFile, seriesRef: seriesRef, @@ -473,6 +492,10 @@ func (cdm *ChunkDiskMapper) CutNewFile() { } func (cdm *ChunkDiskMapper) IsQueueEmpty() bool { + if cdm.writeQueue == nil { + return true + } + return cdm.writeQueue.queueIsEmpty() } @@ -602,9 +625,11 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, ErrChunkDiskMapperClosed } - chunk := cdm.writeQueue.get(ref) - if chunk != nil { - return chunk, nil + if cdm.writeQueue != nil { + chunk := cdm.writeQueue.get(ref) + if chunk != nil { + return chunk, nil + } } sgmIndex, chkStart := ref.Unpack() @@ -962,7 +987,9 @@ func (cdm *ChunkDiskMapper) Close() error { cdm.evtlPosMtx.Lock() defer cdm.evtlPosMtx.Unlock() - cdm.writeQueue.stop() + if cdm.writeQueue != nil { + cdm.writeQueue.stop() + } // 'WriteChunk' locks writePathMtx first and then readPathMtx for cutting head chunk file. // The lock order should not be reversed here else it can cause deadlocks. diff --git a/tsdb/chunks/head_chunks_test.go b/tsdb/chunks/head_chunks_test.go index 9dca41d78..57c9c3401 100644 --- a/tsdb/chunks/head_chunks_test.go +++ b/tsdb/chunks/head_chunks_test.go @@ -27,6 +27,22 @@ import ( "github.com/prometheus/prometheus/tsdb/chunkenc" ) +var writeQueueSize int + +func TestMain(m *testing.M) { + // Run all tests with the chunk write queue disabled. + writeQueueSize = 0 + exitVal := m.Run() + if exitVal != 0 { + os.Exit(exitVal) + } + + // Re-run all tests with the chunk write queue size of 1e6. + writeQueueSize = 1000000 + exitVal = m.Run() + os.Exit(exitVal) +} + func TestChunkDiskMapper_WriteChunk_Chunk_IterateChunks(t *testing.T) { hrw := createChunkDiskMapper(t, "") defer func() { @@ -453,7 +469,7 @@ func createChunkDiskMapper(t *testing.T, dir string) *ChunkDiskMapper { dir = t.TempDir() } - hrw, err := NewChunkDiskMapper(nil, dir, chunkenc.NewPool(), DefaultWriteBufferSize, DefaultWriteQueueSize) + hrw, err := NewChunkDiskMapper(nil, dir, chunkenc.NewPool(), DefaultWriteBufferSize, writeQueueSize) require.NoError(t, err) require.False(t, hrw.fileMaxtSet) require.NoError(t, hrw.IterateAllChunks(func(_ HeadSeriesRef, _ ChunkDiskMapperRef, _, _ int64, _ uint16) error { return nil })) From 6c05a851edff5ab67ea0b3b9aa292f9ed827ecdb Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Thu, 10 Mar 2022 10:49:44 -0700 Subject: [PATCH 4/5] Cut 2.34.0-rc.2 Signed-off-by: Chris Marchbanks --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef5b9ac1b..e36064bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.34.0-rc.2 / 2022-03-11 + +* [ENHANCEMENT] TSDB: Disable the chunk write queue by default and allow configuration with the experimental flag `--storage.tsdb.head-chunks-write-queue-size`. #10425 + ## 2.34.0-rc.1 / 2022-03-08 * [ENHANCEMENT] HTTP SD: Add a failure counter. #10372 diff --git a/VERSION b/VERSION index 62f91b470..640981d5e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.34.0-rc.1 +2.34.0-rc.2 From 5b3792ca690cd10d84a4dfbe594a59179d0fed85 Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Mon, 14 Mar 2022 08:08:00 -0600 Subject: [PATCH 5/5] Cut 2.34.0 Signed-off-by: Chris Marchbanks --- CHANGELOG.md | 16 +++++----------- VERSION | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e36064bc7..1026a6767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,9 @@ -## 2.34.0-rc.2 / 2022-03-11 - -* [ENHANCEMENT] TSDB: Disable the chunk write queue by default and allow configuration with the experimental flag `--storage.tsdb.head-chunks-write-queue-size`. #10425 - -## 2.34.0-rc.1 / 2022-03-08 - -* [ENHANCEMENT] HTTP SD: Add a failure counter. #10372 -* [BUGFIX] Parser: Specify type in metadata parser errors. #10269 -* [BUGFIX] Scrape: Fix label limit changes not applying. #10370 - -## 2.34.0-rc.0 / 2022-02-24 +## 2.34.0 / 2022-03-15 * [CHANGE] UI: Classic UI removed. #10208 * [CHANGE] Tracing: Migrate from Jaeger to OpenTelemetry based tracing. #9724, #10203, #10276 +* [ENHANCEMENT] TSDB: Disable the chunk write queue by default and allow configuration with the experimental flag `--storage.tsdb.head-chunks-write-queue-size`. #10425 +* [ENHANCEMENT] HTTP SD: Add a failure counter. #10372 * [ENHANCEMENT] Azure SD: Set Prometheus User-Agent on requests. #10209 * [ENHANCEMENT] Uyuni SD: Reduce the number of logins to Uyuni. #10072 * [ENHANCEMENT] Scrape: Log when an invalid media type is encountered during a scrape. #10186 @@ -23,6 +15,8 @@ * [BUGFIX] PromQL: Properly return an error from histogram_quantile when metrics have the same labelset. #10140 * [BUGFIX] UI: Fix bug that sets the range input to the resolution. #10227 * [BUGFIX] TSDB: Fix a query panic when `memory-snapshot-on-shutdown` is enabled. #10348 +* [BUGFIX] Parser: Specify type in metadata parser errors. #10269 +* [BUGFIX] Scrape: Fix label limit changes not applying. #10370 ## 2.33.5 / 2022-03-08 diff --git a/VERSION b/VERSION index 640981d5e..dc1591835 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.34.0-rc.2 +2.34.0