From 7f0254da299c60f776ffa0c5b8d24d4419d88620 Mon Sep 17 00:00:00 2001
From: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
Date: Fri, 9 Aug 2024 22:50:30 +0530
Subject: [PATCH 01/18] chore: migrate stale check to use stale github action
Signed-off-by: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
---
.github/stale.yml | 56 -------------------------------------
.github/workflows/stale.yml | 32 +++++++++++++++++++++
2 files changed, 32 insertions(+), 56 deletions(-)
delete mode 100644 .github/stale.yml
create mode 100644 .github/workflows/stale.yml
diff --git a/.github/stale.yml b/.github/stale.yml
deleted file mode 100644
index 66a72af533..0000000000
--- a/.github/stale.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 60
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: false
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
- - keepalive
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking as stale
-staleLabel: stale
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: false
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-# Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-# closeComment: >
-# Your comment here.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-only: pulls
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-# daysUntilStale: 30
-# markComment: >
-# This pull request has been automatically marked as stale because it has not had
-# recent activity. It will be closed if no further activity occurs. Thank you
-# for your contributions.
-
-# issues:
-# exemptLabels:
-# - confirmed
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
new file mode 100644
index 0000000000..f817644489
--- /dev/null
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,32 @@
+name: Stale Check
+on:
+ workflow_dispatch: {}
+ schedule:
+ - cron: '0 0 * * *'
+permissions:
+ issues: write
+ pull-requests: write
+jobs:
+ stale:
+ # only run on repos in prometheus org, and skip this workflow in forks.
+ if: github.repository_owner == 'prometheus'
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@v9
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ # opt out of defaults to avoid marking issues as stale and closing them
+ # https://github.com/actions/stale#days-before-close
+ # https://github.com/actions/stale#days-before-stale
+ days-before-stale: -1
+ days-before-close: -1
+ # Setting it to empty string to skip comments.
+ # https://github.com/actions/stale#stale-pr-message
+ # https://github.com/actions/stale#stale-issue-message
+ stale-pr-message: ''
+ stale-issue-message: ''
+ operations-per-run: 30
+ # override days-before-stale, for only marking the pull requests as stale
+ days-before-pr-stale: 60
+ stale-pr-label: stale
+ exempt-pr-labels: keepalive
From 5dcaaee72a629388d7859cfe925ffe26c245cc76 Mon Sep 17 00:00:00 2001
From: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
Date: Fri, 9 Aug 2024 22:59:27 +0530
Subject: [PATCH 02/18] add stale check in list of files to sync
Signed-off-by: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
---
scripts/sync_repo_files.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/sync_repo_files.sh b/scripts/sync_repo_files.sh
index 6459fb1e7a..837811a56d 100755
--- a/scripts/sync_repo_files.sh
+++ b/scripts/sync_repo_files.sh
@@ -37,7 +37,7 @@ if [ -z "${GITHUB_TOKEN}" ]; then
fi
# List of files that should be synced.
-SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common SECURITY.md .yamllint scripts/golangci-lint.yml .github/workflows/scorecards.yml .github/workflows/container_description.yml"
+SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common SECURITY.md .yamllint scripts/golangci-lint.yml .github/workflows/scorecards.yml .github/workflows/container_description.yml .github/workflows/stale.yml"
# Go to the root of the repo
cd "$(git rev-parse --show-cdup)" || exit 1
From a3b36c12256de33dee5618fcc2de3ba53f24b8f6 Mon Sep 17 00:00:00 2001
From: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
Date: Fri, 9 Aug 2024 23:06:11 +0530
Subject: [PATCH 03/18] pin to SHA of github action
Signed-off-by: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
---
.github/workflows/stale.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index f817644489..0a57810a0b 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -12,7 +12,8 @@ jobs:
if: github.repository_owner == 'prometheus'
runs-on: ubuntu-latest
steps:
- - uses: actions/stale@v9
+ # pin to SHA of v9.0.0: https://github.com/actions/stale/releases/tag/v9.0.0
+ - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# opt out of defaults to avoid marking issues as stale and closing them
From 74c1af3fd57cdd25913ec5b59cd0492d7e2a9f84 Mon Sep 17 00:00:00 2001
From: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
Date: Mon, 12 Aug 2024 23:36:48 +0530
Subject: [PATCH 04/18] address review comments
Signed-off-by: Suraj Nath <9503187+electron0zero@users.noreply.github.com>
---
.github/workflows/stale.yml | 8 +++-----
scripts/sync_repo_files.sh | 2 +-
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 0a57810a0b..d71bcbc9d8 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -2,18 +2,16 @@ name: Stale Check
on:
workflow_dispatch: {}
schedule:
- - cron: '0 0 * * *'
+ - cron: '16 22 * * *'
permissions:
issues: write
pull-requests: write
jobs:
stale:
- # only run on repos in prometheus org, and skip this workflow in forks.
- if: github.repository_owner == 'prometheus'
+ if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
runs-on: ubuntu-latest
steps:
- # pin to SHA of v9.0.0: https://github.com/actions/stale/releases/tag/v9.0.0
- - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e
+ - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# opt out of defaults to avoid marking issues as stale and closing them
diff --git a/scripts/sync_repo_files.sh b/scripts/sync_repo_files.sh
index 837811a56d..1029336298 100755
--- a/scripts/sync_repo_files.sh
+++ b/scripts/sync_repo_files.sh
@@ -37,7 +37,7 @@ if [ -z "${GITHUB_TOKEN}" ]; then
fi
# List of files that should be synced.
-SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common SECURITY.md .yamllint scripts/golangci-lint.yml .github/workflows/scorecards.yml .github/workflows/container_description.yml .github/workflows/stale.yml"
+SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common SECURITY.md .yamllint scripts/golangci-lint.yml .github/workflows/scorecards.yml .github/workflows/container_description.yml .github/workflows/stale.yml"
# Go to the root of the repo
cd "$(git rev-parse --show-cdup)" || exit 1
From 5bd898863790bd94d9f762443aa21b84c3ac23b6 Mon Sep 17 00:00:00 2001
From: Arthur Silva Sens <arthursens2005@gmail.com>
Date: Thu, 29 Aug 2024 04:40:17 -0300
Subject: [PATCH 05/18] Simplify 'TestManagerCTZeroIngestion' (#14756)
Signed-off-by: Arthur Silva Sens <arthursens2005@gmail.com>
---
scrape/manager_test.go | 59 +++++++++++++++++++-----------------------
1 file changed, 26 insertions(+), 33 deletions(-)
diff --git a/scrape/manager_test.go b/scrape/manager_test.go
index a2a3bba6ff..f260167b5e 100644
--- a/scrape/manager_test.go
+++ b/scrape/manager_test.go
@@ -724,8 +724,6 @@ func TestManagerCTZeroIngestion(t *testing.T) {
name string
counterSample *dto.Counter
enableCTZeroIngestion bool
-
- expectedValues []float64
}{
{
name: "disabled with CT on counter",
@@ -734,7 +732,6 @@ func TestManagerCTZeroIngestion(t *testing.T) {
// Timestamp does not matter as long as it exists in this test.
CreatedTimestamp: timestamppb.Now(),
},
- expectedValues: []float64{1.0},
},
{
name: "enabled with CT on counter",
@@ -744,7 +741,6 @@ func TestManagerCTZeroIngestion(t *testing.T) {
CreatedTimestamp: timestamppb.Now(),
},
enableCTZeroIngestion: true,
- expectedValues: []float64{0.0, 1.0},
},
{
name: "enabled without CT on counter",
@@ -752,7 +748,6 @@ func TestManagerCTZeroIngestion(t *testing.T) {
Value: proto.Float64(1.0),
},
enableCTZeroIngestion: true,
- expectedValues: []float64{1.0},
},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -819,46 +814,44 @@ func TestManagerCTZeroIngestion(t *testing.T) {
})
scrapeManager.reload()
+ var got []float64
// Wait for one scrape.
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
require.NoError(t, runutil.Retry(100*time.Millisecond, ctx.Done(), func() error {
- if countFloatSamples(app, mName) != len(tc.expectedValues) {
- return fmt.Errorf("expected %v samples", tc.expectedValues)
+ app.mtx.Lock()
+ defer app.mtx.Unlock()
+
+ // Check if scrape happened and grab the relevant samples, they have to be there - or it's a bug
+ // and it's not worth waiting.
+ for _, f := range app.resultFloats {
+ if f.metric.Get(model.MetricNameLabel) == mName {
+ got = append(got, f.f)
+ }
}
- return nil
+ if len(app.resultFloats) > 0 {
+ return nil
+ }
+ return fmt.Errorf("expected some samples, got none")
}), "after 1 minute")
scrapeManager.Stop()
- require.Equal(t, tc.expectedValues, getResultFloats(app, mName))
+ // Check for zero samples, assuming we only injected always one sample.
+ // Did it contain CT to inject? If yes, was CT zero enabled?
+ if tc.counterSample.CreatedTimestamp.IsValid() && tc.enableCTZeroIngestion {
+ require.Len(t, got, 2)
+ require.Equal(t, 0.0, got[0])
+ require.Equal(t, tc.counterSample.GetValue(), got[1])
+ return
+ }
+
+ // Expect only one, valid sample.
+ require.Len(t, got, 1)
+ require.Equal(t, tc.counterSample.GetValue(), got[0])
})
}
}
-func countFloatSamples(a *collectResultAppender, expectedMetricName string) (count int) {
- a.mtx.Lock()
- defer a.mtx.Unlock()
-
- for _, f := range a.resultFloats {
- if f.metric.Get(model.MetricNameLabel) == expectedMetricName {
- count++
- }
- }
- return count
-}
-
-func getResultFloats(app *collectResultAppender, expectedMetricName string) (result []float64) {
- app.mtx.Lock()
- defer app.mtx.Unlock()
-
- for _, f := range app.resultFloats {
- if f.metric.Get(model.MetricNameLabel) == expectedMetricName {
- result = append(result, f.f)
- }
- }
- return result
-}
-
func TestUnregisterMetrics(t *testing.T) {
reg := prometheus.NewRegistry()
// Check that all metrics can be unregistered, allowing a second manager to be created.
From 838e49e7b8dfc05967bf4ddc95025550b80cd5cd Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Fri, 23 Aug 2024 16:57:43 +0100
Subject: [PATCH 06/18] [REFACTOR] TSDB: move chunkFromSeries from
headChunkReader to head
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
tsdb/head_read.go | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/tsdb/head_read.go b/tsdb/head_read.go
index 24d75f9c72..780eaa6690 100644
--- a/tsdb/head_read.go
+++ b/tsdb/head_read.go
@@ -376,12 +376,12 @@ func (h *headChunkReader) chunk(meta chunks.Meta, copyLastChunk bool) (chunkenc.
s.Lock()
defer s.Unlock()
- return h.chunkFromSeries(s, cid, copyLastChunk)
+ return h.head.chunkFromSeries(s, cid, h.mint, h.maxt, h.isoState, copyLastChunk)
}
// Call with s locked.
-func (h *headChunkReader) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID, copyLastChunk bool) (chunkenc.Chunk, int64, error) {
- c, headChunk, isOpen, err := s.chunk(cid, h.head.chunkDiskMapper, &h.head.memChunkPool)
+func (h *Head) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID, mint, maxt int64, isoState *isolationState, copyLastChunk bool) (chunkenc.Chunk, int64, error) {
+ c, headChunk, isOpen, err := s.chunk(cid, h.chunkDiskMapper, &h.memChunkPool)
if err != nil {
return nil, 0, err
}
@@ -390,12 +390,12 @@ func (h *headChunkReader) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID,
// Set this to nil so that Go GC can collect it after it has been used.
c.chunk = nil
c.prev = nil
- h.head.memChunkPool.Put(c)
+ h.memChunkPool.Put(c)
}
}()
// This means that the chunk is outside the specified range.
- if !c.OverlapsClosedInterval(h.mint, h.maxt) {
+ if !c.OverlapsClosedInterval(mint, maxt) {
return nil, 0, storage.ErrNotFound
}
@@ -407,7 +407,7 @@ func (h *headChunkReader) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID,
newB := make([]byte, len(b))
copy(newB, b) // TODO(codesome): Use bytes.Clone() when we upgrade to Go 1.20.
// TODO(codesome): Put back in the pool (non-trivial).
- chk, err = h.head.opts.ChunkPool.Get(s.headChunks.chunk.Encoding(), newB)
+ chk, err = h.opts.ChunkPool.Get(s.headChunks.chunk.Encoding(), newB)
if err != nil {
return nil, 0, err
}
@@ -417,7 +417,7 @@ func (h *headChunkReader) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID,
Chunk: chk,
s: s,
cid: cid,
- isoState: h.isoState,
+ isoState: isoState,
}, maxTime, nil
}
@@ -538,7 +538,7 @@ func (s *memSeries) mergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMapper,
case c.ref == 0: // This is an in-order head chunk.
_, cid := chunks.HeadChunkRef(c.meta.Ref).Unpack()
var err error
- iterable, _, err = hr.chunkFromSeries(s, cid, false)
+ iterable, _, err = hr.head.chunkFromSeries(s, cid, hr.mint, hr.maxt, hr.isoState, false)
if err != nil {
return nil, fmt.Errorf("invalid head chunk: %w", err)
}
From cde42f30e9c91a23458661ee33d47f828fa63b43 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Fri, 23 Aug 2024 17:10:02 +0100
Subject: [PATCH 07/18] TSDB: streamline reading of overlapping head chunks
`getOOOSeriesChunks` was already finding sets of overlapping chunks; we
store those in a `multiMeta` struct so that `ChunkOrIterable` can
reconstruct an `Iterable` easily and predictably.
We no longer need a `MergeOOO` flag to indicate that this Meta should
be merged with other ones; this is explicit in the `multiMeta` structure.
We also no longer need `chunkMetaAndChunkDiskMapperRef`.
Add `wrapOOOHeadChunk` to defeat `chunkenc.Pool` - chunks are reset
during compaction, but if we wrap them (like `safeHeadChunk` was doing
then this is skipped) .
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
tsdb/chunks/chunks.go | 3 -
tsdb/head_read.go | 101 ++++++--------------------
tsdb/ooo_head_read.go | 143 +++++++++++++++++++------------------
tsdb/ooo_head_read_test.go | 97 ++-----------------------
4 files changed, 98 insertions(+), 246 deletions(-)
diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go
index 69201c6db7..ec0f6d4036 100644
--- a/tsdb/chunks/chunks.go
+++ b/tsdb/chunks/chunks.go
@@ -133,9 +133,6 @@ type Meta struct {
// Time range the data covers.
// When MaxTime == math.MaxInt64 the chunk is still open and being appended to.
MinTime, MaxTime int64
-
- // Flag to indicate that this meta needs merge with OOO data.
- MergeOOO bool
}
// ChunkFromSamples requires all samples to have the same type.
diff --git a/tsdb/head_read.go b/tsdb/head_read.go
index 780eaa6690..d81ffbb6a0 100644
--- a/tsdb/head_read.go
+++ b/tsdb/head_read.go
@@ -366,7 +366,7 @@ func (h *headChunkReader) ChunkOrIterableWithCopy(meta chunks.Meta) (chunkenc.Ch
// If copyLastChunk is true, then it makes a copy of the head chunk if asked for it.
// Also returns max time of the chunk.
func (h *headChunkReader) chunk(meta chunks.Meta, copyLastChunk bool) (chunkenc.Chunk, int64, error) {
- sid, cid := chunks.HeadChunkRef(meta.Ref).Unpack()
+ sid, cid, isOOO := unpackHeadChunkRef(meta.Ref)
s := h.head.series.getByID(sid)
// This means that the series has been garbage collected.
@@ -376,11 +376,20 @@ func (h *headChunkReader) chunk(meta chunks.Meta, copyLastChunk bool) (chunkenc.
s.Lock()
defer s.Unlock()
- return h.head.chunkFromSeries(s, cid, h.mint, h.maxt, h.isoState, copyLastChunk)
+ return h.head.chunkFromSeries(s, cid, isOOO, h.mint, h.maxt, h.isoState, copyLastChunk)
+}
+
+// Dumb thing to defeat chunk pool.
+type wrapOOOHeadChunk struct {
+ chunkenc.Chunk
}
// Call with s locked.
-func (h *Head) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID, mint, maxt int64, isoState *isolationState, copyLastChunk bool) (chunkenc.Chunk, int64, error) {
+func (h *Head) chunkFromSeries(s *memSeries, cid chunks.HeadChunkID, isOOO bool, mint, maxt int64, isoState *isolationState, copyLastChunk bool) (chunkenc.Chunk, int64, error) {
+ if isOOO {
+ chk, maxTime, err := s.oooChunk(cid, h.chunkDiskMapper, &h.memChunkPool)
+ return wrapOOOHeadChunk{chk}, maxTime, err
+ }
c, headChunk, isOpen, err := s.chunk(cid, h.chunkDiskMapper, &h.memChunkPool)
if err != nil {
return nil, 0, err
@@ -481,85 +490,19 @@ func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDi
return elem, true, offset == 0, nil
}
-// mergedChunks return an iterable over all chunks that overlap the
-// time window [mint,maxt], plus meta.Chunk if populated.
-// If hr is non-nil then in-order chunks are included.
-// This function is not thread safe unless the caller holds a lock.
-// The caller must ensure that s.ooo is not nil.
-func (s *memSeries) mergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMapper, hr *headChunkReader, mint, maxt int64, maxMmapRef chunks.ChunkDiskMapperRef) (chunkenc.Iterable, error) {
- // We create a temporary slice of chunk metas to hold the information of all
- // possible chunks that may overlap with the requested chunk.
- tmpChks := make([]chunkMetaAndChunkDiskMapperRef, 0, len(s.ooo.oooMmappedChunks)+1)
+// oooChunk returns the chunk for the HeadChunkID by m-mapping it from the disk.
+// It never returns the head OOO chunk.
+func (s *memSeries) oooChunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDiskMapper, memChunkPool *sync.Pool) (chunk chunkenc.Chunk, maxTime int64, err error) {
+ // ix represents the index of chunk in the s.ooo.oooMmappedChunks slice. The chunk id's are
+ // incremented by 1 when new chunk is created, hence (id - firstOOOChunkID) gives the slice index.
+ ix := int(id) - int(s.ooo.firstOOOChunkID)
- for i, c := range s.ooo.oooMmappedChunks {
- if maxMmapRef != 0 && c.ref > maxMmapRef {
- break
- }
- if c.OverlapsClosedInterval(mint, maxt) {
- tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{
- meta: chunks.Meta{
- MinTime: c.minTime,
- MaxTime: c.maxTime,
- Ref: chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(i))),
- },
- ref: c.ref,
- })
- }
- }
- // Add in data copied from the head OOO chunk.
- if meta.Chunk != nil {
- tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{meta: meta})
+ if ix < 0 || ix >= len(s.ooo.oooMmappedChunks) {
+ return nil, 0, storage.ErrNotFound
}
- if hr != nil { // Include in-order chunks.
- metas := appendSeriesChunks(s, max(meta.MinTime, mint), min(meta.MaxTime, maxt), nil)
- for _, m := range metas {
- tmpChks = append(tmpChks, chunkMetaAndChunkDiskMapperRef{
- meta: m,
- ref: 0, // This tells the loop below it's an in-order head chunk.
- })
- }
- }
-
- // Next we want to sort all the collected chunks by min time so we can find
- // those that overlap and stop when we know the rest don't.
- slices.SortFunc(tmpChks, refLessByMinTimeAndMinRef)
-
- mc := &mergedOOOChunks{}
- absoluteMax := int64(math.MinInt64)
- for _, c := range tmpChks {
- if c.meta.Ref != meta.Ref && (len(mc.chunkIterables) == 0 || c.meta.MinTime > absoluteMax) {
- continue
- }
- var iterable chunkenc.Iterable
- switch {
- case c.meta.Chunk != nil:
- iterable = c.meta.Chunk
- case c.ref == 0: // This is an in-order head chunk.
- _, cid := chunks.HeadChunkRef(c.meta.Ref).Unpack()
- var err error
- iterable, _, err = hr.head.chunkFromSeries(s, cid, hr.mint, hr.maxt, hr.isoState, false)
- if err != nil {
- return nil, fmt.Errorf("invalid head chunk: %w", err)
- }
- default:
- chk, err := cdm.Chunk(c.ref)
- if err != nil {
- var cerr *chunks.CorruptionErr
- if errors.As(err, &cerr) {
- return nil, fmt.Errorf("invalid ooo mmapped chunk: %w", err)
- }
- return nil, err
- }
- iterable = chk
- }
- mc.chunkIterables = append(mc.chunkIterables, iterable)
- if c.meta.MaxTime > absoluteMax {
- absoluteMax = c.meta.MaxTime
- }
- }
-
- return mc, nil
+ chk, err := chunkDiskMapper.Chunk(s.ooo.oooMmappedChunks[ix].ref)
+ return chk, s.ooo.oooMmappedChunks[ix].maxTime, err
}
// safeHeadChunk makes sure that the chunk can be accessed without a race condition.
diff --git a/tsdb/ooo_head_read.go b/tsdb/ooo_head_read.go
index a50decd623..7b58ec566f 100644
--- a/tsdb/ooo_head_read.go
+++ b/tsdb/ooo_head_read.go
@@ -16,6 +16,7 @@ package tsdb
import (
"context"
"errors"
+ "fmt"
"math"
"slices"
@@ -91,11 +92,10 @@ func getOOOSeriesChunks(s *memSeries, mint, maxt int64, lastGarbageCollectedMmap
addChunk := func(minT, maxT int64, ref chunks.ChunkRef, chunk chunkenc.Chunk) {
tmpChks = append(tmpChks, chunks.Meta{
- MinTime: minT,
- MaxTime: maxT,
- Ref: ref,
- Chunk: chunk,
- MergeOOO: true,
+ MinTime: minT,
+ MaxTime: maxT,
+ Ref: ref,
+ Chunk: chunk,
})
}
@@ -140,34 +140,39 @@ func getOOOSeriesChunks(s *memSeries, mint, maxt int64, lastGarbageCollectedMmap
// those that overlap.
slices.SortFunc(tmpChks, lessByMinTimeAndMinRef)
- // Next we want to iterate the sorted collected chunks and only return the
- // chunks Meta the first chunk that overlaps with others.
+ // Next we want to iterate the sorted collected chunks and return composites for chunks that overlap with others.
// Example chunks of a series: 5:(100, 200) 6:(500, 600) 7:(150, 250) 8:(550, 650)
- // In the example 5 overlaps with 7 and 6 overlaps with 8 so we only want to
- // return chunk Metas for chunk 5 and chunk 6e
- *chks = append(*chks, tmpChks[0])
- maxTime := tmpChks[0].MaxTime // Tracks the maxTime of the previous "to be merged chunk".
+ // In the example 5 overlaps with 7 and 6 overlaps with 8 so we will return
+ // [5,7], [6,8].
+ toBeMerged := tmpChks[0]
for _, c := range tmpChks[1:] {
- switch {
- case c.MinTime > maxTime:
- *chks = append(*chks, c)
- maxTime = c.MaxTime
- case c.MaxTime > maxTime:
- maxTime = c.MaxTime
- (*chks)[len(*chks)-1].MaxTime = c.MaxTime
- fallthrough
- default:
- // If the head OOO chunk is part of an output chunk, copy the chunk pointer.
- if c.Chunk != nil {
- (*chks)[len(*chks)-1].Chunk = c.Chunk
+ if c.MinTime > toBeMerged.MaxTime {
+ // This chunk doesn't overlap. Send current toBeMerged to output and start a new one.
+ *chks = append(*chks, toBeMerged)
+ toBeMerged = c
+ } else {
+ // Merge this chunk with existing toBeMerged.
+ if mm, ok := toBeMerged.Chunk.(*multiMeta); ok {
+ mm.metas = append(mm.metas, c)
+ } else {
+ toBeMerged.Chunk = &multiMeta{metas: []chunks.Meta{toBeMerged, c}}
+ }
+ if toBeMerged.MaxTime < c.MaxTime {
+ toBeMerged.MaxTime = c.MaxTime
}
- (*chks)[len(*chks)-1].MergeOOO = (*chks)[len(*chks)-1].MergeOOO || c.MergeOOO
}
}
+ *chks = append(*chks, toBeMerged)
return nil
}
+// Fake Chunk object to pass a set of Metas inside Meta.Chunk.
+type multiMeta struct {
+ chunkenc.Chunk // We don't expect any of the methods to be called.
+ metas []chunks.Meta
+}
+
// LabelValues needs to be overridden from the headIndexReader implementation
// so we can return labels within either in-order range or ooo range.
func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) {
@@ -182,29 +187,6 @@ func (oh *HeadAndOOOIndexReader) LabelValues(ctx context.Context, name string, m
return labelValuesWithMatchers(ctx, oh, name, matchers...)
}
-type chunkMetaAndChunkDiskMapperRef struct {
- meta chunks.Meta
- ref chunks.ChunkDiskMapperRef
-}
-
-func refLessByMinTimeAndMinRef(a, b chunkMetaAndChunkDiskMapperRef) int {
- switch {
- case a.meta.MinTime < b.meta.MinTime:
- return -1
- case a.meta.MinTime > b.meta.MinTime:
- return 1
- }
-
- switch {
- case a.meta.Ref < b.meta.Ref:
- return -1
- case a.meta.Ref > b.meta.Ref:
- return 1
- default:
- return 0
- }
-}
-
func lessByMinTimeAndMinRef(a, b chunks.Meta) int {
switch {
case a.MinTime < b.MinTime:
@@ -243,36 +225,55 @@ func NewHeadAndOOOChunkReader(head *Head, mint, maxt int64, cr *headChunkReader,
}
func (cr *HeadAndOOOChunkReader) ChunkOrIterable(meta chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, error) {
- sid, _, _ := unpackHeadChunkRef(meta.Ref)
- if !meta.MergeOOO {
- return cr.cr.ChunkOrIterable(meta)
- }
-
- s := cr.head.series.getByID(sid)
- // This means that the series has been garbage collected.
- if s == nil {
- return nil, nil, storage.ErrNotFound
- }
-
- s.Lock()
- if s.ooo == nil { // Must have s.ooo non-nil to call mergedChunks().
- s.Unlock()
- return cr.cr.ChunkOrIterable(meta)
- }
- mc, err := s.mergedChunks(meta, cr.head.chunkDiskMapper, cr.cr, cr.mint, cr.maxt, cr.maxMmapRef)
- s.Unlock()
-
- return nil, mc, err
+ c, it, _, err := cr.chunkOrIterable(meta, false)
+ return c, it, err
}
// ChunkOrIterableWithCopy implements ChunkReaderWithCopy. The special Copy
// behaviour is only implemented for the in-order head chunk.
func (cr *HeadAndOOOChunkReader) ChunkOrIterableWithCopy(meta chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, int64, error) {
- if !meta.MergeOOO {
- return cr.cr.ChunkOrIterableWithCopy(meta)
+ return cr.chunkOrIterable(meta, true)
+}
+
+func (cr *HeadAndOOOChunkReader) chunkOrIterable(meta chunks.Meta, copyLastChunk bool) (chunkenc.Chunk, chunkenc.Iterable, int64, error) {
+ sid, cid, isOOO := unpackHeadChunkRef(meta.Ref)
+ s := cr.head.series.getByID(sid)
+ // This means that the series has been garbage collected.
+ if s == nil {
+ return nil, nil, 0, storage.ErrNotFound
}
- chk, iter, err := cr.ChunkOrIterable(meta)
- return chk, iter, 0, err
+ var isoState *isolationState
+ if cr.cr != nil {
+ isoState = cr.cr.isoState
+ }
+
+ s.Lock()
+ defer s.Unlock()
+
+ if meta.Chunk == nil {
+ c, maxt, err := cr.head.chunkFromSeries(s, cid, isOOO, meta.MinTime, meta.MaxTime, isoState, copyLastChunk)
+ return c, nil, maxt, err
+ }
+ mm, ok := meta.Chunk.(*multiMeta)
+ if !ok { // Complete chunk was supplied.
+ return meta.Chunk, nil, meta.MaxTime, nil
+ }
+ // We have a composite meta: construct a composite iterable.
+ mc := &mergedOOOChunks{}
+ for _, m := range mm.metas {
+ switch {
+ case m.Chunk != nil:
+ mc.chunkIterables = append(mc.chunkIterables, m.Chunk)
+ default:
+ _, cid, isOOO := unpackHeadChunkRef(m.Ref)
+ iterable, _, err := cr.head.chunkFromSeries(s, cid, isOOO, m.MinTime, m.MaxTime, isoState, copyLastChunk)
+ if err != nil {
+ return nil, nil, 0, fmt.Errorf("invalid head chunk: %w", err)
+ }
+ mc.chunkIterables = append(mc.chunkIterables, iterable)
+ }
+ }
+ return nil, mc, meta.MaxTime, nil
}
func (cr *HeadAndOOOChunkReader) Close() error {
diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go
index e565933f83..545bc7b6eb 100644
--- a/tsdb/ooo_head_read_test.go
+++ b/tsdb/ooo_head_read_test.go
@@ -308,10 +308,9 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
var expChunks []chunks.Meta
for _, e := range tc.expChunks {
meta := chunks.Meta{
- Chunk: chunkenc.Chunk(nil),
- MinTime: e.mint,
- MaxTime: e.maxt,
- MergeOOO: true, // Only OOO chunks are tested here, so we always request merge from OOO head.
+ Chunk: chunkenc.Chunk(nil),
+ MinTime: e.mint,
+ MaxTime: e.maxt,
}
// Ref to whatever Ref the chunk has, that we refer to by ID
@@ -485,7 +484,7 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
cr := NewHeadAndOOOChunkReader(db.head, 0, 1000, nil, nil, 0)
defer cr.Close()
c, iterable, err := cr.ChunkOrIterable(chunks.Meta{
- Ref: 0x1800000, Chunk: chunkenc.Chunk(nil), MinTime: 100, MaxTime: 300, MergeOOO: true,
+ Ref: 0x1800000, Chunk: chunkenc.Chunk(nil), MinTime: 100, MaxTime: 300,
})
require.Nil(t, iterable)
require.Equal(t, err, fmt.Errorf("not found"))
@@ -1030,94 +1029,6 @@ func testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
}
}
-// TestSortByMinTimeAndMinRef tests that the sort function for chunk metas does sort
-// by chunk meta MinTime and in case of same references by the lower reference.
-func TestSortByMinTimeAndMinRef(t *testing.T) {
- tests := []struct {
- name string
- input []chunkMetaAndChunkDiskMapperRef
- exp []chunkMetaAndChunkDiskMapperRef
- }{
- {
- name: "chunks are ordered by min time",
- input: []chunkMetaAndChunkDiskMapperRef{
- {
- meta: chunks.Meta{
- Ref: 0,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(0),
- },
- {
- meta: chunks.Meta{
- Ref: 1,
- MinTime: 1,
- },
- ref: chunks.ChunkDiskMapperRef(1),
- },
- },
- exp: []chunkMetaAndChunkDiskMapperRef{
- {
- meta: chunks.Meta{
- Ref: 0,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(0),
- },
- {
- meta: chunks.Meta{
- Ref: 1,
- MinTime: 1,
- },
- ref: chunks.ChunkDiskMapperRef(1),
- },
- },
- },
- {
- name: "if same mintime, lower reference goes first",
- input: []chunkMetaAndChunkDiskMapperRef{
- {
- meta: chunks.Meta{
- Ref: 10,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(0),
- },
- {
- meta: chunks.Meta{
- Ref: 5,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(1),
- },
- },
- exp: []chunkMetaAndChunkDiskMapperRef{
- {
- meta: chunks.Meta{
- Ref: 5,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(1),
- },
- {
- meta: chunks.Meta{
- Ref: 10,
- MinTime: 0,
- },
- ref: chunks.ChunkDiskMapperRef(0),
- },
- },
- },
- }
-
- for _, tc := range tests {
- t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
- slices.SortFunc(tc.input, refLessByMinTimeAndMinRef)
- require.Equal(t, tc.exp, tc.input)
- })
- }
-}
-
// TestSortMetaByMinTimeAndMinRef tests that the sort function for chunk metas does sort
// by chunk meta MinTime and in case of same references by the lower reference.
func TestSortMetaByMinTimeAndMinRef(t *testing.T) {
From 1f38ae7bca8471e90a6fa227cd4cd567e25952c4 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Sat, 24 Aug 2024 16:50:15 +0100
Subject: [PATCH 08/18] [TESTS] TSDB: fix up OOO tests for new Series behaviour
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
tsdb/ooo_head_read_test.go | 112 +++++++++++++++++++++++--------------
1 file changed, 69 insertions(+), 43 deletions(-)
diff --git a/tsdb/ooo_head_read_test.go b/tsdb/ooo_head_read_test.go
index 545bc7b6eb..40e37043b8 100644
--- a/tsdb/ooo_head_read_test.go
+++ b/tsdb/ooo_head_read_test.go
@@ -39,6 +39,11 @@ type chunkInterval struct {
maxt int64
}
+type expChunk struct {
+ c chunkInterval
+ m []chunkInterval
+}
+
// permutateChunkIntervals returns all possible orders of the given chunkIntervals.
func permutateChunkIntervals(in []chunkInterval, out [][]chunkInterval, left, right int) [][]chunkInterval {
if left == right {
@@ -65,7 +70,7 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
queryMinT int64
queryMaxT int64
inputChunkIntervals []chunkInterval
- expChunks []chunkInterval
+ expChunks []expChunk
}{
{
name: "Empty result and no error when head is empty",
@@ -107,8 +112,8 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// ts 0 100 150 200 250 300 350 400 450 500 550 600 650 700
// Query Interval [-----------------------------------------------------------]
// Chunk 0: [---------------------------------------]
- expChunks: []chunkInterval{
- {0, 150, 350},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 150, 350}},
},
},
{
@@ -121,8 +126,8 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// ts 0 100 150 200 250 300 350 400 450 500 550 600 650 700
// Query Interval: [---------------------------------------]
// Chunk 0: [-----------------------------------------------------------]
- expChunks: []chunkInterval{
- {0, 100, 400},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 400}},
},
},
{
@@ -142,9 +147,9 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 2: [-------------------]
// Chunk 3: [-------------------]
// Output Graphically [-----------------------------] [-----------------------------]
- expChunks: []chunkInterval{
- {0, 100, 250},
- {1, 500, 650},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 250}, m: []chunkInterval{{0, 100, 200}, {2, 150, 250}}},
+ {c: chunkInterval{1, 500, 650}, m: []chunkInterval{{1, 500, 600}, {3, 550, 650}}},
},
},
{
@@ -164,8 +169,8 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 2: [-------------------]
// Chunk 3: [------------------]
// Output Graphically [------------------------------------------------------------------------------]
- expChunks: []chunkInterval{
- {0, 100, 500},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 500}, m: []chunkInterval{{0, 100, 200}, {1, 200, 300}, {2, 300, 400}, {3, 400, 500}}},
},
},
{
@@ -185,11 +190,11 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 2: [------------------]
// Chunk 3: [------------------]
// Output Graphically [------------------][------------------][------------------][------------------]
- expChunks: []chunkInterval{
- {0, 100, 199},
- {1, 200, 299},
- {2, 300, 399},
- {3, 400, 499},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 199}},
+ {c: chunkInterval{1, 200, 299}},
+ {c: chunkInterval{2, 300, 399}},
+ {c: chunkInterval{3, 400, 499}},
},
},
{
@@ -209,8 +214,8 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 2: [------------------]
// Chunk 3: [------------------]
// Output Graphically [-----------------------------------------------]
- expChunks: []chunkInterval{
- {0, 100, 350},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 350}, m: []chunkInterval{{0, 100, 200}, {1, 150, 300}, {2, 250, 350}}},
},
},
{
@@ -228,8 +233,8 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 1: [-----------------------------]
// Chunk 2: [------------------------------]
// Output Graphically [-----------------------------------------------------------------------------------------]
- expChunks: []chunkInterval{
- {1, 0, 500},
+ expChunks: []expChunk{
+ {c: chunkInterval{1, 0, 500}, m: []chunkInterval{{1, 0, 200}, {2, 150, 300}, {0, 250, 500}}},
},
},
{
@@ -251,9 +256,9 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 3: [-------------------]
// Chunk 4: [---------------------------------------]
// Output Graphically [---------------------------------------] [------------------------------------------------]
- expChunks: []chunkInterval{
- {0, 100, 300},
- {4, 600, 850},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 300}, m: []chunkInterval{{0, 100, 300}, {2, 150, 250}}},
+ {c: chunkInterval{4, 600, 850}, m: []chunkInterval{{4, 600, 800}, {3, 650, 750}, {1, 770, 850}}},
},
},
{
@@ -271,10 +276,10 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
// Chunk 1: [----------]
// Chunk 2: [--------]
// Output Graphically [-------] [--------] [----------]
- expChunks: []chunkInterval{
- {0, 100, 150},
- {1, 300, 350},
- {2, 200, 250},
+ expChunks: []expChunk{
+ {c: chunkInterval{0, 100, 150}},
+ {c: chunkInterval{2, 200, 250}},
+ {c: chunkInterval{1, 300, 350}},
},
},
}
@@ -305,24 +310,38 @@ func TestOOOHeadIndexReader_Series(t *testing.T) {
s1.ooo = &memSeriesOOOFields{}
// define our expected chunks, by looking at the expected ChunkIntervals and setting...
+ // Ref to whatever Ref the chunk has, that we refer to by ID
+ findID := func(id int) chunks.ChunkRef {
+ for ref, c := range intervals {
+ if c.ID == id {
+ return chunks.ChunkRef(chunks.NewHeadChunkRef(chunks.HeadSeriesRef(s1ID), s1.oooHeadChunkID(ref)))
+ }
+ }
+ return 0
+ }
var expChunks []chunks.Meta
for _, e := range tc.expChunks {
- meta := chunks.Meta{
- Chunk: chunkenc.Chunk(nil),
- MinTime: e.mint,
- MaxTime: e.maxt,
- }
-
- // Ref to whatever Ref the chunk has, that we refer to by ID
- for ref, c := range intervals {
- if c.ID == e.ID {
- meta.Ref = chunks.ChunkRef(chunks.NewHeadChunkRef(chunks.HeadSeriesRef(s1ID), s1.oooHeadChunkID(ref)))
- break
+ var chunk chunkenc.Chunk
+ if len(e.m) > 0 {
+ mm := &multiMeta{}
+ for _, x := range e.m {
+ meta := chunks.Meta{
+ MinTime: x.mint,
+ MaxTime: x.maxt,
+ Ref: findID(x.ID),
+ }
+ mm.metas = append(mm.metas, meta)
}
+ chunk = mm
+ }
+ meta := chunks.Meta{
+ Chunk: chunk,
+ MinTime: e.c.mint,
+ MaxTime: e.c.maxt,
+ Ref: findID(e.c.ID),
}
expChunks = append(expChunks, meta)
}
- slices.SortFunc(expChunks, lessByMinTimeAndMinRef) // We always want the chunks to come back sorted by minTime asc.
if headChunk && len(intervals) > 0 {
// Put the last interval in the head chunk
@@ -497,6 +516,7 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
queryMaxT int64
firstInOrderSampleAt int64
inputSamples []testValue
+ expSingleChunks bool
expChunkError bool
expChunksSamples []chunks.SampleSlice
}{
@@ -509,7 +529,8 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
{Ts: minutes(30), V: 0},
{Ts: minutes(40), V: 0},
},
- expChunkError: false,
+ expChunkError: false,
+ expSingleChunks: true,
// ts (in minutes) 0 10 20 30 40 50 60 70 80 90 100
// Query Interval [------------------------------------------------------------------------------------------]
// Chunk 0: Current Head [--------] (With 2 samples)
@@ -689,7 +710,8 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
{Ts: minutes(40), V: 3},
{Ts: minutes(42), V: 3},
},
- expChunkError: false,
+ expChunkError: false,
+ expSingleChunks: true,
// ts (in minutes) 0 10 20 30 40 50 60 70 80 90 100
// Query Interval [------------------------------------------------------------------------------------------]
// Chunk 0 [-------]
@@ -844,9 +866,13 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
for i := 0; i < len(chks); i++ {
c, iterable, err := cr.ChunkOrIterable(chks[i])
require.NoError(t, err)
- require.Nil(t, c)
-
- it := iterable.Iterator(nil)
+ var it chunkenc.Iterator
+ if tc.expSingleChunks {
+ it = c.Iterator(nil)
+ } else {
+ require.Nil(t, c)
+ it = iterable.Iterator(nil)
+ }
resultSamples, err := storage.ExpandSamples(it, nil)
require.NoError(t, err)
requireEqualSamples(t, s1.String(), tc.expChunksSamples[i], resultSamples, true)
From f90c7a11d16da9244d6acd971513f409211a5748 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Thu, 29 Aug 2024 11:57:31 +0100
Subject: [PATCH 09/18] [REFACTOR] OTLP translator: simplify time conversion
We don't need multiple levels of abstraction to convert nanoseconds to
milliseconds.
We do benefit from tests, however.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
.../prometheusremotewrite/helper.go | 3 +--
.../prometheusremotewrite/helper_test.go | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/storage/remote/otlptranslator/prometheusremotewrite/helper.go
index f2d7ecd4e3..67cf28119d 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/helper.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/helper.go
@@ -24,7 +24,6 @@ import (
"slices"
"sort"
"strconv"
- "time"
"unicode/utf8"
"github.com/cespare/xxhash/v2"
@@ -594,5 +593,5 @@ func addResourceTargetInfo(resource pcommon.Resource, settings Settings, timesta
// convertTimeStamp converts OTLP timestamp in ns to timestamp in ms
func convertTimeStamp(timestamp pcommon.Timestamp) int64 {
- return timestamp.AsTime().UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond))
+ return int64(timestamp) / 1_000_000
}
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
index c4dd781ae6..729d99b8a8 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
@@ -14,6 +14,7 @@ package prometheusremotewrite
import (
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/collector/pdata/pcommon"
@@ -159,3 +160,21 @@ func TestCreateAttributes(t *testing.T) {
})
}
}
+
+func Test_convertTimeStamp(t *testing.T) {
+ tests := []struct {
+ name string
+ arg pcommon.Timestamp
+ want int64
+ }{
+ {"zero", 0, 0},
+ {"1ms", 1_000_000, 1},
+ {"1s", pcommon.Timestamp(time.Unix(1, 0).UnixNano()), 1000},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := convertTimeStamp(tt.arg)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
From e9e3d64b7c9df354d46c73da464b56a7c4cb7233 Mon Sep 17 00:00:00 2001
From: Jorge Creixell <jcreixell@gmail.com>
Date: Thu, 29 Aug 2024 15:50:39 +0200
Subject: [PATCH 10/18] PromQL engine: Delay deletion of __name__ label to the
end of the query evaluation (#14477)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
PromQL engine: Delay deletion of __name__ label to the end of the query evaluation
- This change allows optionally preserving the `__name__` label via the `label_replace` and `label_join` functions, and helps prevent the dreaded "vector cannot contain metrics with the same labelset" error.
- The implementation extends the `Series` and `Sample` structs with a boolean flag indicating whether the `__name__` label should be deleted at the end of the query evaluation.
- The `label_replace` and `label_join` functions can still access the value of the `__name__` label, even if it has been previously marked for deletion. If `__name__` is used as target label, it won't be dropped at the end of the query evaluation.
- Fixes https://github.com/prometheus/prometheus/issues/11397
- See https://github.com/jcreixell/prometheus/pull/2 for previous discussion, including the decision to create this PR and benchmark it before considering other alternatives (like refactoring `labels.Labels`).
- See https://github.com/jcreixell/prometheus/pull/1 for an alternative implementation using a special label instead of boolean flags.
- Note: a feature flag `promql-delayed-name-removal` has been added as it changes the behavior of some "weird" queries (see https://github.com/prometheus/prometheus/issues/11397#issuecomment-1451998792)
Example (this always fails, as `__name__` is being dropped by `count_over_time`):
```
count_over_time({__name__!=""}[1m])
=> Error executing query: vector cannot contain metrics with the same labelset
```
Before:
```
label_replace(count_over_time({__name__!=""}[1m]), "__name__", "count_$1", "__name__", "(.+)")
=> Error executing query: vector cannot contain metrics with the same labelset
```
After:
```
label_replace(count_over_time({__name__!=""}[1m]), "__name__", "count_$1", "__name__", "(.+)")
=>
count_go_gc_cycles_automatic_gc_cycles_total{instance="localhost:9090", job="prometheus"} 1
count_go_gc_cycles_forced_gc_cycles_total{instance="localhost:9090", job="prometheus"} 1
...
```
Signed-off-by: Jorge Creixell <jcreixell@gmail.com>
---------
Signed-off-by: Jorge Creixell <jcreixell@gmail.com>
Signed-off-by: Björn Rabenstein <github@rabenste.in>
---
cmd/prometheus/main.go | 14 +-
docs/command-line/prometheus.md | 2 +-
docs/feature_flags.md | 8 ++
promql/engine.go | 124 +++++++++++++-----
promql/engine_test.go | 29 ++--
promql/functions.go | 120 +++++++++++++----
promql/promqltest/test.go | 2 +
.../testdata/name_label_dropping.test | 84 ++++++++++++
promql/value.go | 6 +
9 files changed, 309 insertions(+), 80 deletions(-)
create mode 100644 promql/promqltest/testdata/name_label_dropping.test
diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index c402fbcb80..65ffd7de5a 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -169,6 +169,8 @@ type flagConfig struct {
corsRegexString string
promlogConfig promlog.Config
+
+ promqlEnableDelayedNameRemoval bool
}
// setFeatureListOptions sets the corresponding options from the featureList.
@@ -238,6 +240,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
case "delayed-compaction":
c.tsdb.EnableDelayedCompaction = true
level.Info(logger).Log("msg", "Experimental delayed compaction is enabled.")
+ case "promql-delayed-name-removal":
+ c.promqlEnableDelayedNameRemoval = true
+ level.Info(logger).Log("msg", "Experimental PromQL delayed name removal enabled.")
case "utf8-names":
model.NameValidationScheme = model.UTF8Validation
level.Info(logger).Log("msg", "Experimental UTF-8 support enabled")
@@ -487,7 +492,7 @@ func main() {
a.Flag("scrape.name-escaping-scheme", `Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots".`).Default(scrape.DefaultNameEscapingScheme.String()).StringVar(&cfg.nameEscapingScheme)
- a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+ a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, auto-gomaxprocs, auto-gomemlimit, concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, expand-external-labels, extra-scrape-metrics, memory-snapshot-on-shutdown, native-histograms, new-service-discovery-manager, no-default-scrape-port, otlp-write-receiver, promql-experimental-functions, promql-delayed-name-removal, promql-per-step-stats, remote-write-receiver (DEPRECATED), utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
Default("").StringsVar(&cfg.featureList)
promlogflag.AddFlags(a, &cfg.promlogConfig)
@@ -799,9 +804,10 @@ func main() {
NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
// EnableAtModifier and EnableNegativeOffset have to be
// always on for regular PromQL as of Prometheus v2.33.
- EnableAtModifier: true,
- EnableNegativeOffset: true,
- EnablePerStepStats: cfg.enablePerStepStats,
+ EnableAtModifier: true,
+ EnableNegativeOffset: true,
+ EnablePerStepStats: cfg.enablePerStepStats,
+ EnableDelayedNameRemoval: cfg.promqlEnableDelayedNameRemoval,
}
queryEngine = promql.NewEngine(opts)
diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md
index f65c260c40..7d9e5a3c80 100644
--- a/docs/command-line/prometheus.md
+++ b/docs/command-line/prometheus.md
@@ -57,7 +57,7 @@ The Prometheus monitoring server
| <code class="text-nowrap">--query.max-concurrency</code> | Maximum number of queries executed concurrently. Use with server mode only. | `20` |
| <code class="text-nowrap">--query.max-samples</code> | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` |
| <code class="text-nowrap">--scrape.name-escaping-scheme</code> | Method for escaping legacy invalid names when sending to Prometheus that does not support UTF-8. Can be one of "values", "underscores", or "dots". | `values` |
-| <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable. Valid options: agent, auto-gomemlimit, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | |
+| <code class="text-nowrap">--enable-feature</code> <code class="text-nowrap">...<code class="text-nowrap"> | Comma separated feature names to enable. Valid options: agent, auto-gomaxprocs, auto-gomemlimit, concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, expand-external-labels, extra-scrape-metrics, memory-snapshot-on-shutdown, native-histograms, new-service-discovery-manager, no-default-scrape-port, otlp-write-receiver, promql-experimental-functions, promql-delayed-name-removal, promql-per-step-stats, remote-write-receiver (DEPRECATED), utf8-names. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | |
| <code class="text-nowrap">--log.level</code> | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` |
| <code class="text-nowrap">--log.format</code> | Output format of log messages. One of: [logfmt, json] | `logfmt` |
diff --git a/docs/feature_flags.md b/docs/feature_flags.md
index 0a908bb91d..7b07a04d0e 100644
--- a/docs/feature_flags.md
+++ b/docs/feature_flags.md
@@ -250,6 +250,14 @@ Note that during this delay, the Head continues its usual operations, which incl
Despite the delay in compaction, the blocks produced are time-aligned in the same manner as they would be if the delay was not in place.
+## Delay __name__ label removal for PromQL engine
+
+`--enable-feature=promql-delayed-name-removal`
+
+When enabled, Prometheus will change the way in which the `__name__` label is removed from PromQL query results (for functions and expressions for which this is necessary). Specifically, it will delay the removal to the last step of the query evaluation, instead of every time an expression or function creating derived metrics is evaluated.
+
+This allows optionally preserving the `__name__` label via the `label_replace` and `label_join` functions, and helps prevent the "vector cannot contain metrics with the same labelset" error, which can happen when applying a regex-matcher to the `__name__` label.
+
## UTF-8 Name Support
`--enable-feature=utf8-names`
diff --git a/promql/engine.go b/promql/engine.go
index 60c8e06394..b54ce2d6dc 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -313,6 +313,11 @@ type EngineOpts struct {
// EnablePerStepStats if true allows for per-step stats to be computed on request. Disabled otherwise.
EnablePerStepStats bool
+
+ // EnableDelayedNameRemoval delays the removal of the __name__ label to the last step of the query evaluation.
+ // This is useful in certain scenarios where the __name__ label must be preserved or where applying a
+ // regex-matcher to the __name__ label may otherwise lead to duplicate labelset errors.
+ EnableDelayedNameRemoval bool
}
// Engine handles the lifetime of queries from beginning to end.
@@ -330,6 +335,7 @@ type Engine struct {
enableAtModifier bool
enableNegativeOffset bool
enablePerStepStats bool
+ enableDelayedNameRemoval bool
}
// NewEngine returns a new engine.
@@ -420,6 +426,7 @@ func NewEngine(opts EngineOpts) *Engine {
enableAtModifier: opts.EnableAtModifier,
enableNegativeOffset: opts.EnableNegativeOffset,
enablePerStepStats: opts.EnablePerStepStats,
+ enableDelayedNameRemoval: opts.EnableDelayedNameRemoval,
}
}
@@ -712,6 +719,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
lookbackDelta: s.LookbackDelta,
samplesStats: query.sampleStats,
noStepSubqueryIntervalFn: ng.noStepSubqueryIntervalFn,
+ enableDelayedNameRemoval: ng.enableDelayedNameRemoval,
}
query.sampleStats.InitStepTracking(start, start, 1)
@@ -743,9 +751,9 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
// Point might have a different timestamp, force it to the evaluation
// timestamp as that is when we ran the evaluation.
if len(s.Histograms) > 0 {
- vector[i] = Sample{Metric: s.Metric, H: s.Histograms[0].H, T: start}
+ vector[i] = Sample{Metric: s.Metric, H: s.Histograms[0].H, T: start, DropName: s.DropName}
} else {
- vector[i] = Sample{Metric: s.Metric, F: s.Floats[0].F, T: start}
+ vector[i] = Sample{Metric: s.Metric, F: s.Floats[0].F, T: start, DropName: s.DropName}
}
}
return vector, warnings, nil
@@ -770,6 +778,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
lookbackDelta: s.LookbackDelta,
samplesStats: query.sampleStats,
noStepSubqueryIntervalFn: ng.noStepSubqueryIntervalFn,
+ enableDelayedNameRemoval: ng.enableDelayedNameRemoval,
}
query.sampleStats.InitStepTracking(evaluator.startTimestamp, evaluator.endTimestamp, evaluator.interval)
val, warnings, err := evaluator.Eval(s.Expr)
@@ -1032,6 +1041,7 @@ type evaluator struct {
lookbackDelta time.Duration
samplesStats *stats.QuerySamples
noStepSubqueryIntervalFn func(rangeMillis int64) int64
+ enableDelayedNameRemoval bool
}
// errorf causes a panic with the input formatted into an error.
@@ -1073,6 +1083,9 @@ func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws annotations.Anno
defer ev.recover(expr, &ws, &err)
v, ws = ev.eval(expr)
+ if ev.enableDelayedNameRemoval {
+ ev.cleanupMetricLabels(v)
+ }
return v, ws, nil
}
@@ -1101,6 +1114,9 @@ type EvalNodeHelper struct {
rightSigs map[string]Sample
matchedSigs map[string]map[uint64]struct{}
resultMetric map[string]labels.Labels
+
+ // Additional options for the evaluation.
+ enableDelayedNameRemoval bool
}
func (enh *EvalNodeHelper) resetBuilder(lbls labels.Labels) {
@@ -1150,7 +1166,7 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper)
biggestLen = len(matrixes[i])
}
}
- enh := &EvalNodeHelper{Out: make(Vector, 0, biggestLen)}
+ enh := &EvalNodeHelper{Out: make(Vector, 0, biggestLen), enableDelayedNameRemoval: ev.enableDelayedNameRemoval}
type seriesAndTimestamp struct {
Series
ts int64
@@ -1196,12 +1212,12 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper)
for si, series := range matrixes[i] {
switch {
case len(series.Floats) > 0 && series.Floats[0].T == ts:
- vectors[i] = append(vectors[i], Sample{Metric: series.Metric, F: series.Floats[0].F, T: ts})
+ vectors[i] = append(vectors[i], Sample{Metric: series.Metric, F: series.Floats[0].F, T: ts, DropName: series.DropName})
// Move input vectors forward so we don't have to re-scan the same
// past points at the next step.
matrixes[i][si].Floats = series.Floats[1:]
case len(series.Histograms) > 0 && series.Histograms[0].T == ts:
- vectors[i] = append(vectors[i], Sample{Metric: series.Metric, H: series.Histograms[0].H, T: ts})
+ vectors[i] = append(vectors[i], Sample{Metric: series.Metric, H: series.Histograms[0].H, T: ts, DropName: series.DropName})
matrixes[i][si].Histograms = series.Histograms[1:]
default:
continue
@@ -1240,15 +1256,15 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper)
// If this could be an instant query, shortcut so as not to change sort order.
if ev.endTimestamp == ev.startTimestamp {
- if result.ContainsSameLabelset() {
+ if !ev.enableDelayedNameRemoval && result.ContainsSameLabelset() {
ev.errorf("vector cannot contain metrics with the same labelset")
}
mat := make(Matrix, len(result))
for i, s := range result {
if s.H == nil {
- mat[i] = Series{Metric: s.Metric, Floats: []FPoint{{T: ts, F: s.F}}}
+ mat[i] = Series{Metric: s.Metric, Floats: []FPoint{{T: ts, F: s.F}}, DropName: s.DropName}
} else {
- mat[i] = Series{Metric: s.Metric, Histograms: []HPoint{{T: ts, H: s.H}}}
+ mat[i] = Series{Metric: s.Metric, Histograms: []HPoint{{T: ts, H: s.H}}, DropName: s.DropName}
}
}
ev.currentSamples = originalNumSamples + mat.TotalSamples()
@@ -1266,7 +1282,7 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper)
}
ss.ts = ts
} else {
- ss = seriesAndTimestamp{Series{Metric: sample.Metric}, ts}
+ ss = seriesAndTimestamp{Series{Metric: sample.Metric, DropName: sample.DropName}, ts}
}
addToSeries(&ss.Series, enh.Ts, sample.F, sample.H, numSteps)
seriess[h] = ss
@@ -1302,7 +1318,7 @@ func (ev *evaluator) rangeEvalAgg(aggExpr *parser.AggregateExpr, sortedGrouping
var warnings annotations.Annotations
- enh := &EvalNodeHelper{}
+ enh := &EvalNodeHelper{enableDelayedNameRemoval: ev.enableDelayedNameRemoval}
tempNumSamples := ev.currentSamples
// Create a mapping from input series to output groups.
@@ -1611,10 +1627,17 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
var prevSS *Series
inMatrix := make(Matrix, 1)
inArgs[matrixArgIndex] = inMatrix
- enh := &EvalNodeHelper{Out: make(Vector, 0, 1)}
+ enh := &EvalNodeHelper{Out: make(Vector, 0, 1), enableDelayedNameRemoval: ev.enableDelayedNameRemoval}
// Process all the calls for one time series at a time.
it := storage.NewBuffer(selRange)
var chkIter chunkenc.Iterator
+
+ // The last_over_time function acts like offset; thus, it
+ // should keep the metric name. For all the other range
+ // vector functions, the only change needed is to drop the
+ // metric name in the output.
+ dropName := e.Func.Name != "last_over_time"
+
for i, s := range selVS.Series {
if err := contextDone(ev.ctx, "expression evaluation"); err != nil {
ev.error(err)
@@ -1629,15 +1652,12 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
chkIter = s.Iterator(chkIter)
it.Reset(chkIter)
metric := selVS.Series[i].Labels()
- // The last_over_time function acts like offset; thus, it
- // should keep the metric name. For all the other range
- // vector functions, the only change needed is to drop the
- // metric name in the output.
- if e.Func.Name != "last_over_time" {
+ if !ev.enableDelayedNameRemoval && dropName {
metric = metric.DropMetricName()
}
ss := Series{
- Metric: metric,
+ Metric: metric,
+ DropName: dropName,
}
inMatrix[0].Metric = selVS.Series[i].Labels()
for ts, step := ev.startTimestamp, -1; ts <= ev.endTimestamp; ts += ev.interval {
@@ -1752,16 +1772,16 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
return Matrix{
Series{
- Metric: createLabelsForAbsentFunction(e.Args[0]),
- Floats: newp,
+ Metric: createLabelsForAbsentFunction(e.Args[0]),
+ Floats: newp,
+ DropName: dropName,
},
}, warnings
}
- if mat.ContainsSameLabelset() {
+ if !ev.enableDelayedNameRemoval && mat.ContainsSameLabelset() {
ev.errorf("vector cannot contain metrics with the same labelset")
}
-
return mat, warnings
case *parser.ParenExpr:
@@ -1772,12 +1792,15 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
mat := val.(Matrix)
if e.Op == parser.SUB {
for i := range mat {
- mat[i].Metric = mat[i].Metric.DropMetricName()
+ if !ev.enableDelayedNameRemoval {
+ mat[i].Metric = mat[i].Metric.DropMetricName()
+ }
+ mat[i].DropName = true
for j := range mat[i].Floats {
mat[i].Floats[j].F = -mat[i].Floats[j].F
}
}
- if mat.ContainsSameLabelset() {
+ if !ev.enableDelayedNameRemoval && mat.ContainsSameLabelset() {
ev.errorf("vector cannot contain metrics with the same labelset")
}
}
@@ -1913,6 +1936,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
lookbackDelta: ev.lookbackDelta,
samplesStats: ev.samplesStats.NewChild(),
noStepSubqueryIntervalFn: ev.noStepSubqueryIntervalFn,
+ enableDelayedNameRemoval: ev.enableDelayedNameRemoval,
}
if e.Step != 0 {
@@ -1957,6 +1981,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio
lookbackDelta: ev.lookbackDelta,
samplesStats: ev.samplesStats.NewChild(),
noStepSubqueryIntervalFn: ev.noStepSubqueryIntervalFn,
+ enableDelayedNameRemoval: ev.enableDelayedNameRemoval,
}
res, ws := newEv.eval(e.Expr)
ev.currentSamples = newEv.currentSamples
@@ -2553,7 +2578,7 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
continue
}
metric := resultMetric(ls.Metric, rs.Metric, op, matching, enh)
- if returnBool {
+ if !ev.enableDelayedNameRemoval && returnBool {
metric = metric.DropMetricName()
}
insertedSigs, exists := matchedSigs[sig]
@@ -2578,9 +2603,10 @@ func (ev *evaluator) VectorBinop(op parser.ItemType, lhs, rhs Vector, matching *
}
enh.Out = append(enh.Out, Sample{
- Metric: metric,
- F: floatValue,
- H: histogramValue,
+ Metric: metric,
+ F: floatValue,
+ H: histogramValue,
+ DropName: returnBool,
})
}
return enh.Out, lastErr
@@ -2680,7 +2706,10 @@ func (ev *evaluator) VectorscalarBinop(op parser.ItemType, lhs Vector, rhs Scala
lhsSample.F = float
lhsSample.H = histogram
if shouldDropMetricName(op) || returnBool {
- lhsSample.Metric = lhsSample.Metric.DropMetricName()
+ if !ev.enableDelayedNameRemoval {
+ lhsSample.Metric = lhsSample.Metric.DropMetricName()
+ }
+ lhsSample.DropName = true
}
enh.Out = append(enh.Out, lhsSample)
}
@@ -3019,6 +3048,7 @@ func (ev *evaluator) aggregation(e *parser.AggregateExpr, q float64, inputMatrix
ss := &outputMatrix[ri]
addToSeries(ss, enh.Ts, aggr.floatValue, aggr.histogramValue, numSteps)
+ ss.DropName = inputMatrix[ri].DropName
}
return annos
@@ -3045,7 +3075,7 @@ seriesLoop:
if !ok {
continue
}
- s = Sample{Metric: inputMatrix[si].Metric, F: f}
+ s = Sample{Metric: inputMatrix[si].Metric, F: f, DropName: inputMatrix[si].DropName}
group := &groups[seriesToResult[si]]
// Initialize this group if it's the first time we've seen it.
@@ -3129,16 +3159,16 @@ seriesLoop:
mat = make(Matrix, 0, len(groups))
}
- add := func(lbls labels.Labels, f float64) {
+ add := func(lbls labels.Labels, f float64, dropName bool) {
// If this could be an instant query, add directly to the matrix so the result is in consistent order.
if ev.endTimestamp == ev.startTimestamp {
- mat = append(mat, Series{Metric: lbls, Floats: []FPoint{{T: enh.Ts, F: f}}})
+ mat = append(mat, Series{Metric: lbls, Floats: []FPoint{{T: enh.Ts, F: f}}, DropName: dropName})
} else {
// Otherwise the results are added into seriess elements.
hash := lbls.Hash()
ss, ok := seriess[hash]
if !ok {
- ss = Series{Metric: lbls}
+ ss = Series{Metric: lbls, DropName: dropName}
}
addToSeries(&ss, enh.Ts, f, nil, numSteps)
seriess[hash] = ss
@@ -3155,7 +3185,7 @@ seriesLoop:
sort.Sort(sort.Reverse(aggr.heap))
}
for _, v := range aggr.heap {
- add(v.Metric, v.F)
+ add(v.Metric, v.F, v.DropName)
}
case parser.BOTTOMK:
@@ -3164,12 +3194,12 @@ seriesLoop:
sort.Sort(sort.Reverse((*vectorByReverseValueHeap)(&aggr.heap)))
}
for _, v := range aggr.heap {
- add(v.Metric, v.F)
+ add(v.Metric, v.F, v.DropName)
}
case parser.LIMITK, parser.LIMIT_RATIO:
for _, v := range aggr.heap {
- add(v.Metric, v.F)
+ add(v.Metric, v.F, v.DropName)
}
}
}
@@ -3221,6 +3251,30 @@ func (ev *evaluator) aggregationCountValues(e *parser.AggregateExpr, grouping []
return enh.Out, nil
}
+func (ev *evaluator) cleanupMetricLabels(v parser.Value) {
+ if v.Type() == parser.ValueTypeMatrix {
+ mat := v.(Matrix)
+ for i := range mat {
+ if mat[i].DropName {
+ mat[i].Metric = mat[i].Metric.DropMetricName()
+ }
+ }
+ if mat.ContainsSameLabelset() {
+ ev.errorf("vector cannot contain metrics with the same labelset")
+ }
+ } else if v.Type() == parser.ValueTypeVector {
+ vec := v.(Vector)
+ for i := range vec {
+ if vec[i].DropName {
+ vec[i].Metric = vec[i].Metric.DropMetricName()
+ }
+ }
+ if vec.ContainsSameLabelset() {
+ ev.errorf("vector cannot contain metrics with the same labelset")
+ }
+ }
+}
+
func addToSeries(ss *Series, ts int64, f float64, h *histogram.FloatHistogram, numSteps int) {
if h == nil {
if ss.Floats == nil {
diff --git a/promql/engine_test.go b/promql/engine_test.go
index 37cba975b3..3e159782a3 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -1714,7 +1714,8 @@ load 1ms
{F: 3600, T: 6 * 60 * 1000},
{F: 3600, T: 7 * 60 * 1000},
},
- Metric: labels.EmptyLabels(),
+ Metric: labels.EmptyLabels(),
+ DropName: true,
},
},
},
@@ -1930,20 +1931,24 @@ func TestSubquerySelector(t *testing.T) {
nil,
promql.Matrix{
promql.Series{
- Floats: []promql.FPoint{{F: 3, T: 7985000}, {F: 3, T: 7990000}, {F: 3, T: 7995000}, {F: 3, T: 8000000}},
- Metric: labels.FromStrings("job", "api-server", "instance", "0", "group", "canary"),
+ Floats: []promql.FPoint{{F: 3, T: 7985000}, {F: 3, T: 7990000}, {F: 3, T: 7995000}, {F: 3, T: 8000000}},
+ Metric: labels.FromStrings("job", "api-server", "instance", "0", "group", "canary"),
+ DropName: true,
},
promql.Series{
- Floats: []promql.FPoint{{F: 4, T: 7985000}, {F: 4, T: 7990000}, {F: 4, T: 7995000}, {F: 4, T: 8000000}},
- Metric: labels.FromStrings("job", "api-server", "instance", "1", "group", "canary"),
+ Floats: []promql.FPoint{{F: 4, T: 7985000}, {F: 4, T: 7990000}, {F: 4, T: 7995000}, {F: 4, T: 8000000}},
+ Metric: labels.FromStrings("job", "api-server", "instance", "1", "group", "canary"),
+ DropName: true,
},
promql.Series{
- Floats: []promql.FPoint{{F: 1, T: 7985000}, {F: 1, T: 7990000}, {F: 1, T: 7995000}, {F: 1, T: 8000000}},
- Metric: labels.FromStrings("job", "api-server", "instance", "0", "group", "production"),
+ Floats: []promql.FPoint{{F: 1, T: 7985000}, {F: 1, T: 7990000}, {F: 1, T: 7995000}, {F: 1, T: 8000000}},
+ Metric: labels.FromStrings("job", "api-server", "instance", "0", "group", "production"),
+ DropName: true,
},
promql.Series{
- Floats: []promql.FPoint{{F: 2, T: 7985000}, {F: 2, T: 7990000}, {F: 2, T: 7995000}, {F: 2, T: 8000000}},
- Metric: labels.FromStrings("job", "api-server", "instance", "1", "group", "production"),
+ Floats: []promql.FPoint{{F: 2, T: 7985000}, {F: 2, T: 7990000}, {F: 2, T: 7995000}, {F: 2, T: 8000000}},
+ Metric: labels.FromStrings("job", "api-server", "instance", "1", "group", "production"),
+ DropName: true,
},
},
nil,
@@ -3470,11 +3475,11 @@ func TestNativeHistogram_MulDivOperator(t *testing.T) {
// histogram * scalar.
queryString := fmt.Sprintf(`%s * %f`, seriesName, c.scalar)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels()}})
+ queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels(), DropName: true}})
// scalar * histogram.
queryString = fmt.Sprintf(`%f * %s`, c.scalar, seriesName)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels()}})
+ queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels(), DropName: true}})
// histogram * float.
queryString = fmt.Sprintf(`%s * %s`, seriesName, floatSeriesName)
@@ -3486,7 +3491,7 @@ func TestNativeHistogram_MulDivOperator(t *testing.T) {
// histogram / scalar.
queryString = fmt.Sprintf(`%s / %f`, seriesName, c.scalar)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedDiv, Metric: labels.EmptyLabels()}})
+ queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedDiv, Metric: labels.EmptyLabels(), DropName: true}})
// histogram / float.
queryString = fmt.Sprintf(`%s / %s`, seriesName, floatSeriesName)
diff --git a/promql/functions.go b/promql/functions.go
index 61a2dd0765..9fa7fbe190 100644
--- a/promql/functions.go
+++ b/promql/functions.go
@@ -483,9 +483,13 @@ func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper
return enh.Out, nil
}
for _, el := range vec {
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: math.Max(minVal, math.Min(maxVal, el.F)),
+ Metric: el.Metric,
+ F: math.Max(minVal, math.Min(maxVal, el.F)),
+ DropName: true,
})
}
return enh.Out, nil
@@ -496,9 +500,13 @@ func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
vec := vals[0].(Vector)
maxVal := vals[1].(Vector)[0].F
for _, el := range vec {
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: math.Min(maxVal, el.F),
+ Metric: el.Metric,
+ F: math.Min(maxVal, el.F),
+ DropName: true,
})
}
return enh.Out, nil
@@ -509,9 +517,13 @@ func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel
vec := vals[0].(Vector)
minVal := vals[1].(Vector)[0].F
for _, el := range vec {
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: math.Max(minVal, el.F),
+ Metric: el.Metric,
+ F: math.Max(minVal, el.F),
+ DropName: true,
})
}
return enh.Out, nil
@@ -532,8 +544,9 @@ func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper
for _, el := range vec {
f := math.Floor(el.F*toNearestInverse+0.5) / toNearestInverse
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: f,
+ Metric: el.Metric,
+ F: f,
+ DropName: true,
})
}
return enh.Out, nil
@@ -882,9 +895,13 @@ func funcPresentOverTime(vals []parser.Value, args parser.Expressions, enh *Eval
func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float64) Vector {
for _, el := range vals[0].(Vector) {
if el.H == nil { // Process only float samples.
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: f(el.F),
+ Metric: el.Metric,
+ F: f(el.F),
+ DropName: true,
})
}
}
@@ -1028,9 +1045,13 @@ func funcSgn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper)
func funcTimestamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) {
vec := vals[0].(Vector)
for _, el := range vec {
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: float64(el.T) / 1000,
+ Metric: el.Metric,
+ F: float64(el.T) / 1000,
+ DropName: true,
})
}
return enh.Out, nil
@@ -1137,9 +1158,13 @@ func funcHistogramCount(vals []parser.Value, args parser.Expressions, enh *EvalN
if sample.H == nil {
continue
}
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: sample.H.Count,
+ Metric: sample.Metric,
+ F: sample.H.Count,
+ DropName: true,
})
}
return enh.Out, nil
@@ -1154,9 +1179,13 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod
if sample.H == nil {
continue
}
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: sample.H.Sum,
+ Metric: sample.Metric,
+ F: sample.H.Sum,
+ DropName: true,
})
}
return enh.Out, nil
@@ -1171,9 +1200,13 @@ func funcHistogramAvg(vals []parser.Value, args parser.Expressions, enh *EvalNod
if sample.H == nil {
continue
}
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: sample.H.Sum / sample.H.Count,
+ Metric: sample.Metric,
+ F: sample.H.Sum / sample.H.Count,
+ DropName: true,
})
}
return enh.Out, nil
@@ -1210,9 +1243,13 @@ func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *Eval
}
variance += cVariance
variance /= sample.H.Count
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: math.Sqrt(variance),
+ Metric: sample.Metric,
+ F: math.Sqrt(variance),
+ DropName: true,
})
}
return enh.Out, nil
@@ -1249,9 +1286,13 @@ func funcHistogramStdVar(vals []parser.Value, args parser.Expressions, enh *Eval
}
variance += cVariance
variance /= sample.H.Count
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: variance,
+ Metric: sample.Metric,
+ F: variance,
+ DropName: true,
})
}
return enh.Out, nil
@@ -1268,9 +1309,13 @@ func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *Ev
if sample.H == nil {
continue
}
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: histogramFraction(lower, upper, sample.H),
+ Metric: sample.Metric,
+ F: histogramFraction(lower, upper, sample.H),
+ DropName: true,
})
}
return enh.Out, nil
@@ -1338,9 +1383,13 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev
continue
}
+ if !enh.enableDelayedNameRemoval {
+ sample.Metric = sample.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: sample.Metric.DropMetricName(),
- F: histogramQuantile(q, sample.H),
+ Metric: sample.Metric,
+ F: histogramQuantile(q, sample.H),
+ DropName: true,
})
}
@@ -1442,6 +1491,11 @@ func (ev *evaluator) evalLabelReplace(args parser.Expressions) (parser.Value, an
lb.Reset(el.Metric)
lb.Set(dst, string(res))
matrix[i].Metric = lb.Labels()
+ if dst == model.MetricNameLabel {
+ matrix[i].DropName = false
+ } else {
+ matrix[i].DropName = el.DropName
+ }
}
}
if matrix.ContainsSameLabelset() {
@@ -1496,6 +1550,12 @@ func (ev *evaluator) evalLabelJoin(args parser.Expressions) (parser.Value, annot
lb.Reset(el.Metric)
lb.Set(dst, strval)
matrix[i].Metric = lb.Labels()
+
+ if dst == model.MetricNameLabel {
+ matrix[i].DropName = false
+ } else {
+ matrix[i].DropName = el.DropName
+ }
}
return matrix, ws
@@ -1518,9 +1578,13 @@ func dateWrapper(vals []parser.Value, enh *EvalNodeHelper, f func(time.Time) flo
for _, el := range vals[0].(Vector) {
t := time.Unix(int64(el.F), 0).UTC()
+ if !enh.enableDelayedNameRemoval {
+ el.Metric = el.Metric.DropMetricName()
+ }
enh.Out = append(enh.Out, Sample{
- Metric: el.Metric.DropMetricName(),
- F: f(t),
+ Metric: el.Metric,
+ F: f(t),
+ DropName: true,
})
}
return enh.Out
diff --git a/promql/promqltest/test.go b/promql/promqltest/test.go
index 4d12dead9b..f43d7a5741 100644
--- a/promql/promqltest/test.go
+++ b/promql/promqltest/test.go
@@ -90,6 +90,7 @@ func NewTestEngine(enablePerStepStats bool, lookbackDelta time.Duration, maxSamp
EnableNegativeOffset: true,
EnablePerStepStats: enablePerStepStats,
LookbackDelta: lookbackDelta,
+ EnableDelayedNameRemoval: true,
})
}
@@ -1362,6 +1363,7 @@ func (ll *LazyLoader) clear() error {
NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(ll.SubqueryInterval) },
EnableAtModifier: ll.opts.EnableAtModifier,
EnableNegativeOffset: ll.opts.EnableNegativeOffset,
+ EnableDelayedNameRemoval: true,
}
ll.queryEngine = promql.NewEngine(opts)
diff --git a/promql/promqltest/testdata/name_label_dropping.test b/promql/promqltest/testdata/name_label_dropping.test
new file mode 100644
index 0000000000..1f1dac3602
--- /dev/null
+++ b/promql/promqltest/testdata/name_label_dropping.test
@@ -0,0 +1,84 @@
+# Test for __name__ label drop.
+load 5m
+ metric{env="1"} 0 60 120
+ another_metric{env="1"} 60 120 180
+
+# Does not drop __name__ for vector selector
+eval instant at 15m metric{env="1"}
+ metric{env="1"} 120
+
+# Drops __name__ for unary operators
+eval instant at 15m -metric
+ {env="1"} -120
+
+# Drops __name__ for binary operators
+eval instant at 15m metric + another_metric
+ {env="1"} 300
+
+# Does not drop __name__ for binary comparison operators
+eval instant at 15m metric <= another_metric
+ metric{env="1"} 120
+
+# Drops __name__ for binary comparison operators with "bool" modifier
+eval instant at 15m metric <= bool another_metric
+ {env="1"} 1
+
+# Drops __name__ for vector-scalar operations
+eval instant at 15m metric * 2
+ {env="1"} 240
+
+# Drops __name__ for instant-vector functions
+eval instant at 15m clamp(metric, 0, 100)
+ {env="1"} 100
+
+# Drops __name__ for range-vector functions
+eval instant at 15m rate(metric{env="1"}[10m])
+ {env="1"} 0.2
+
+# Does not drop __name__ for last_over_time function
+eval instant at 15m last_over_time(metric{env="1"}[10m])
+ metric{env="1"} 120
+
+# Drops name for other _over_time functions
+eval instant at 15m max_over_time(metric{env="1"}[10m])
+ {env="1"} 120
+
+# Allows relabeling (to-be-dropped) __name__ via label_replace
+eval instant at 15m label_replace(rate({env="1"}[10m]), "my_name", "rate_$1", "__name__", "(.+)")
+ {my_name="rate_metric", env="1"} 0.2
+ {my_name="rate_another_metric", env="1"} 0.2
+
+# Allows preserving __name__ via label_replace
+eval instant at 15m label_replace(rate({env="1"}[10m]), "__name__", "rate_$1", "__name__", "(.+)")
+ rate_metric{env="1"} 0.2
+ rate_another_metric{env="1"} 0.2
+
+# Allows relabeling (to-be-dropped) __name__ via label_join
+eval instant at 15m label_join(rate({env="1"}[10m]), "my_name", "_", "__name__")
+ {my_name="metric", env="1"} 0.2
+ {my_name="another_metric", env="1"} 0.2
+
+# Allows preserving __name__ via label_join
+eval instant at 15m label_join(rate({env="1"}[10m]), "__name__", "_", "__name__", "env")
+ metric_1{env="1"} 0.2
+ another_metric_1{env="1"} 0.2
+
+# Does not drop metric names fro aggregation operators
+eval instant at 15m sum by (__name__, env) (metric{env="1"})
+ metric{env="1"} 120
+
+# Aggregation operators by __name__ lead to duplicate labelset errors (aggregation is partitioned by not yet removed __name__ label)
+# This is an accidental side effect of delayed __name__ label dropping
+eval_fail instant at 15m sum by (__name__) (rate({env="1"}[10m]))
+
+# Aggregation operators aggregate metrics with same labelset and to-be-dropped names
+# This is an accidental side effect of delayed __name__ label dropping
+eval instant at 15m sum(rate({env="1"}[10m])) by (env)
+ {env="1"} 0.4
+
+# Aggregationk operators propagate __name__ label dropping information
+eval instant at 15m topk(10, sum by (__name__, env) (metric{env="1"}))
+ metric{env="1"} 120
+
+eval instant at 15m topk(10, sum by (__name__, env) (rate(metric{env="1"}[10m])))
+ {env="1"} 0.2
diff --git a/promql/value.go b/promql/value.go
index f129137d80..f25dbcd780 100644
--- a/promql/value.go
+++ b/promql/value.go
@@ -68,6 +68,9 @@ type Series struct {
Metric labels.Labels `json:"metric"`
Floats []FPoint `json:"values,omitempty"`
Histograms []HPoint `json:"histograms,omitempty"`
+ // DropName is used to indicate whether the __name__ label should be dropped
+ // as part of the query evaluation.
+ DropName bool `json:"-"`
}
func (s Series) String() string {
@@ -194,6 +197,9 @@ type Sample struct {
H *histogram.FloatHistogram
Metric labels.Labels
+ // DropName is used to indicate whether the __name__ label should be dropped
+ // as part of the query evaluation.
+ DropName bool
}
func (s Sample) String() string {
From 8c7bf39d9671f717df8286d94869485a562f1ac9 Mon Sep 17 00:00:00 2001
From: Neeraj Gartia <80708727+NeerajGartia21@users.noreply.github.com>
Date: Thu, 29 Aug 2024 20:12:35 +0530
Subject: [PATCH 11/18] Moves TestNativeHistogram_MulDivOperator to promql
testing framework (#14688)
PromQL: add test for mul and div operator
Also, remove the converted test from the engine_test.go file.
This also includes an extension of the test framework to allow NaN/Inf in histogram buckets.
---------
Signed-off-by: Neeraj Gartia <neerajgartia211002@gmail.com>
---
promql/engine_test.go | 166 ------------------
promql/parser/lex.go | 10 ++
promql/parser/lex_test.go | 23 +++
.../testdata/native_histograms.test | 46 +++++
4 files changed, 79 insertions(+), 166 deletions(-)
diff --git a/promql/engine_test.go b/promql/engine_test.go
index 3e159782a3..923d1264d6 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -17,7 +17,6 @@ import (
"context"
"errors"
"fmt"
- "math"
"os"
"sort"
"strconv"
@@ -3337,171 +3336,6 @@ func TestNativeHistogram_SubOperator(t *testing.T) {
}
}
-func TestNativeHistogram_MulDivOperator(t *testing.T) {
- // TODO(codesome): Integrate histograms into the PromQL testing framework
- // and write more tests there.
- originalHistogram := histogram.Histogram{
- Schema: 0,
- Count: 21,
- Sum: 33,
- ZeroThreshold: 0.001,
- ZeroCount: 3,
- PositiveSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- PositiveBuckets: []int64{3, 0, 0},
- NegativeSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- NegativeBuckets: []int64{3, 0, 0},
- }
-
- cases := []struct {
- scalar float64
- histogram histogram.Histogram
- expectedMul histogram.FloatHistogram
- expectedDiv histogram.FloatHistogram
- }{
- {
- scalar: 3,
- histogram: originalHistogram,
- expectedMul: histogram.FloatHistogram{
- Schema: 0,
- Count: 63,
- Sum: 99,
- ZeroThreshold: 0.001,
- ZeroCount: 9,
- PositiveSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- PositiveBuckets: []float64{9, 9, 9},
- NegativeSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- NegativeBuckets: []float64{9, 9, 9},
- },
- expectedDiv: histogram.FloatHistogram{
- Schema: 0,
- Count: 7,
- Sum: 11,
- ZeroThreshold: 0.001,
- ZeroCount: 1,
- PositiveSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- PositiveBuckets: []float64{1, 1, 1},
- NegativeSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- NegativeBuckets: []float64{1, 1, 1},
- },
- },
- {
- scalar: 0,
- histogram: originalHistogram,
- expectedMul: histogram.FloatHistogram{
- Schema: 0,
- Count: 0,
- Sum: 0,
- ZeroThreshold: 0.001,
- ZeroCount: 0,
- PositiveSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- PositiveBuckets: []float64{0, 0, 0},
- NegativeSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- NegativeBuckets: []float64{0, 0, 0},
- },
- expectedDiv: histogram.FloatHistogram{
- Schema: 0,
- Count: math.Inf(1),
- Sum: math.Inf(1),
- ZeroThreshold: 0.001,
- ZeroCount: math.Inf(1),
- PositiveSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- PositiveBuckets: []float64{math.Inf(1), math.Inf(1), math.Inf(1)},
- NegativeSpans: []histogram.Span{
- {Offset: 0, Length: 3},
- },
- NegativeBuckets: []float64{math.Inf(1), math.Inf(1), math.Inf(1)},
- },
- },
- }
-
- idx0 := int64(0)
- for _, c := range cases {
- for _, floatHisto := range []bool{true, false} {
- t.Run(fmt.Sprintf("floatHistogram=%t %d", floatHisto, idx0), func(t *testing.T) {
- storage := teststorage.New(t)
- t.Cleanup(func() { storage.Close() })
-
- seriesName := "sparse_histogram_series"
- floatSeriesName := "float_series"
-
- engine := newTestEngine()
-
- ts := idx0 * int64(10*time.Minute/time.Millisecond)
- app := storage.Appender(context.Background())
- h := c.histogram
- lbls := labels.FromStrings("__name__", seriesName)
- // Since we mutate h later, we need to create a copy here.
- var err error
- if floatHisto {
- _, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat(nil))
- } else {
- _, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil)
- }
- require.NoError(t, err)
- _, err = app.Append(0, labels.FromStrings("__name__", floatSeriesName), ts, c.scalar)
- require.NoError(t, err)
- require.NoError(t, app.Commit())
-
- queryAndCheck := func(queryString string, exp promql.Vector) {
- qry, err := engine.NewInstantQuery(context.Background(), storage, nil, queryString, timestamp.Time(ts))
- require.NoError(t, err)
-
- res := qry.Exec(context.Background())
- require.NoError(t, res.Err)
-
- vector, err := res.Vector()
- require.NoError(t, err)
-
- testutil.RequireEqual(t, exp, vector)
- }
-
- // histogram * scalar.
- queryString := fmt.Sprintf(`%s * %f`, seriesName, c.scalar)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels(), DropName: true}})
-
- // scalar * histogram.
- queryString = fmt.Sprintf(`%f * %s`, c.scalar, seriesName)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels(), DropName: true}})
-
- // histogram * float.
- queryString = fmt.Sprintf(`%s * %s`, seriesName, floatSeriesName)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels()}})
-
- // float * histogram.
- queryString = fmt.Sprintf(`%s * %s`, floatSeriesName, seriesName)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedMul, Metric: labels.EmptyLabels()}})
-
- // histogram / scalar.
- queryString = fmt.Sprintf(`%s / %f`, seriesName, c.scalar)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedDiv, Metric: labels.EmptyLabels(), DropName: true}})
-
- // histogram / float.
- queryString = fmt.Sprintf(`%s / %s`, seriesName, floatSeriesName)
- queryAndCheck(queryString, []promql.Sample{{T: ts, H: &c.expectedDiv, Metric: labels.EmptyLabels()}})
- })
- idx0++
- }
- }
-}
-
func TestQueryLookbackDelta(t *testing.T) {
var (
load = `load 5m
diff --git a/promql/parser/lex.go b/promql/parser/lex.go
index 0cefa30c8f..d031e83307 100644
--- a/promql/parser/lex.go
+++ b/promql/parser/lex.go
@@ -617,6 +617,16 @@ func lexBuckets(l *Lexer) stateFn {
l.bracketOpen = false
l.emit(RIGHT_BRACKET)
return lexHistogram
+ case isAlpha(r):
+ // Current word is Inf or NaN.
+ word := l.input[l.start:l.pos]
+ if desc, ok := key[strings.ToLower(word)]; ok {
+ if desc == NUMBER {
+ l.emit(desc)
+ return lexStatements
+ }
+ }
+ return lexBuckets
default:
return l.errorf("invalid character in buckets description: %q", r)
}
diff --git a/promql/parser/lex_test.go b/promql/parser/lex_test.go
index ac9aa27625..c5475a8b94 100644
--- a/promql/parser/lex_test.go
+++ b/promql/parser/lex_test.go
@@ -639,6 +639,29 @@ var tests = []struct {
},
seriesDesc: true,
},
+ {
+ input: `{} {{buckets: [Inf NaN] schema:1}}`,
+ expected: []Item{
+ {LEFT_BRACE, 0, `{`},
+ {RIGHT_BRACE, 1, `}`},
+ {SPACE, 2, ` `},
+ {OPEN_HIST, 3, `{{`},
+ {BUCKETS_DESC, 5, `buckets`},
+ {COLON, 12, `:`},
+ {SPACE, 13, ` `},
+ {LEFT_BRACKET, 14, `[`},
+ {NUMBER, 15, `Inf`},
+ {SPACE, 18, ` `},
+ {NUMBER, 19, `NaN`},
+ {RIGHT_BRACKET, 22, `]`},
+ {SPACE, 23, ` `},
+ {SCHEMA_DESC, 24, `schema`},
+ {COLON, 30, `:`},
+ {NUMBER, 31, `1`},
+ {CLOSE_HIST, 32, `}}`},
+ },
+ seriesDesc: true,
+ },
{ // Series with sum as -Inf and count as NaN.
input: `{} {{buckets: [5 10 7] sum:Inf count:NaN}}`,
expected: []Item{
diff --git a/promql/promqltest/testdata/native_histograms.test b/promql/promqltest/testdata/native_histograms.test
index 110fd69e7e..71e102dcee 100644
--- a/promql/promqltest/testdata/native_histograms.test
+++ b/promql/promqltest/testdata/native_histograms.test
@@ -718,6 +718,52 @@ eval instant at 10m histogram_fraction(-Inf, +Inf, histogram_fraction_4)
eval instant at 10m histogram_sum(scalar(histogram_fraction(-Inf, +Inf, sum(histogram_fraction_4))) * histogram_fraction_4)
{} 100
+# Apply multiplication and division operator to histogram.
+load 10m
+ histogram_mul_div {{schema:0 count:21 sum:33 z_bucket:3 z_bucket_w:0.001 buckets:[3 3 3] n_buckets:[3 3 3]}}x1
+ float_series_3 3+0x1
+ float_series_0 0+0x1
+
+eval instant at 10m histogram_mul_div*3
+ {} {{schema:0 count:63 sum:99 z_bucket:9 z_bucket_w:0.001 buckets:[9 9 9] n_buckets:[9 9 9]}}
+
+eval instant at 10m 3*histogram_mul_div
+ {} {{schema:0 count:63 sum:99 z_bucket:9 z_bucket_w:0.001 buckets:[9 9 9] n_buckets:[9 9 9]}}
+
+eval instant at 10m histogram_mul_div*float_series_3
+ {} {{schema:0 count:63 sum:99 z_bucket:9 z_bucket_w:0.001 buckets:[9 9 9] n_buckets:[9 9 9]}}
+
+eval instant at 10m float_series_3*histogram_mul_div
+ {} {{schema:0 count:63 sum:99 z_bucket:9 z_bucket_w:0.001 buckets:[9 9 9] n_buckets:[9 9 9]}}
+
+eval instant at 10m histogram_mul_div/3
+ {} {{schema:0 count:7 sum:11 z_bucket:1 z_bucket_w:0.001 buckets:[1 1 1] n_buckets:[1 1 1]}}
+
+eval instant at 10m histogram_mul_div/float_series_3
+ {} {{schema:0 count:7 sum:11 z_bucket:1 z_bucket_w:0.001 buckets:[1 1 1] n_buckets:[1 1 1]}}
+
+eval instant at 10m histogram_mul_div*0
+ {} {{schema:0 count:0 sum:0 z_bucket:0 z_bucket_w:0.001 buckets:[0 0 0] n_buckets:[0 0 0]}}
+
+eval instant at 10m 0*histogram_mul_div
+ {} {{schema:0 count:0 sum:0 z_bucket:0 z_bucket_w:0.001 buckets:[0 0 0] n_buckets:[0 0 0]}}
+
+eval instant at 10m histogram_mul_div*float_series_0
+ {} {{schema:0 count:0 sum:0 z_bucket:0 z_bucket_w:0.001 buckets:[0 0 0] n_buckets:[0 0 0]}}
+
+eval instant at 10m float_series_0*histogram_mul_div
+ {} {{schema:0 count:0 sum:0 z_bucket:0 z_bucket_w:0.001 buckets:[0 0 0] n_buckets:[0 0 0]}}
+
+# TODO: (NeerajGartia21) remove all the histogram buckets in case of division with zero. See: https://github.com/prometheus/prometheus/issues/13934
+eval instant at 10m histogram_mul_div/0
+ {} {{schema:0 count:Inf sum:Inf z_bucket:Inf z_bucket_w:0.001 buckets:[Inf Inf Inf] n_buckets:[Inf Inf Inf]}}
+
+eval instant at 10m histogram_mul_div/float_series_0
+ {} {{schema:0 count:Inf sum:Inf z_bucket:Inf z_bucket_w:0.001 buckets:[Inf Inf Inf] n_buckets:[Inf Inf Inf]}}
+
+eval instant at 10m histogram_mul_div*0/0
+ {} {{schema:0 count:NaN sum:NaN z_bucket:NaN z_bucket_w:0.001 buckets:[NaN NaN NaN] n_buckets:[NaN NaN NaN]}}
+
clear
# Counter reset only noticeable in a single bucket.
From b0677263c3d40efcb86ff21afe9b29d860660a45 Mon Sep 17 00:00:00 2001
From: Kevin Rawal <84058124+kevinrawal@users.noreply.github.com>
Date: Thu, 29 Aug 2024 23:06:31 +0530
Subject: [PATCH 12/18] Update basics.md
fix broken URL for feature flags native histograms in querying prometheus documentation
Signed-off-by: Kevin Rawal <84058124+kevinrawal@users.noreply.github.com>
---
docs/querying/basics.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/querying/basics.md b/docs/querying/basics.md
index 304c9f07d4..81ffb4e0f3 100644
--- a/docs/querying/basics.md
+++ b/docs/querying/basics.md
@@ -41,7 +41,7 @@ vector is the only type which can be graphed.
_Notes about the experimental native histograms:_
* Ingesting native histograms has to be enabled via a [feature
- flag](../../feature_flags.md#native-histograms).
+ flag](../feature_flags.md#native-histograms).
* Once native histograms have been ingested into the TSDB (and even after
disabling the feature flag again), both instant vectors and range vectors may
now contain samples that aren't simple floating point numbers (float samples)
From a77f5007f9debcc1fc5b7133ac422e2b164a8c99 Mon Sep 17 00:00:00 2001
From: Callum Styan <callumstyan@gmail.com>
Date: Fri, 30 Aug 2024 00:14:20 -0700
Subject: [PATCH 13/18] fix bug with metadata for rw2 (#14766)
Signed-off-by: Callum Styan <callumstyan@gmail.com>
---
tsdb/wlog/watcher.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tsdb/wlog/watcher.go b/tsdb/wlog/watcher.go
index 86e6f9c81d..ac5041e87b 100644
--- a/tsdb/wlog/watcher.go
+++ b/tsdb/wlog/watcher.go
@@ -603,7 +603,7 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error {
}
case record.Metadata:
- if !w.sendMetadata || !tail {
+ if !w.sendMetadata {
break
}
meta, err := dec.Metadata(rec, metadata[:0])
From d550c4a0b97ff1ad7cd427f562d7eadde894d417 Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Fri, 30 Aug 2024 10:13:00 +0100
Subject: [PATCH 14/18] [COMMENT] Improve comment on almost.Equal; add tests
At first it wasn't obvious to me why the first check would give a
different result to the second.
Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
---
util/almost/almost.go | 6 ++---
util/almost/almost_test.go | 50 ++++++++++++++++++++++++++++++++++++++
2 files changed, 53 insertions(+), 3 deletions(-)
create mode 100644 util/almost/almost_test.go
diff --git a/util/almost/almost.go b/util/almost/almost.go
index a404626586..5f866b89b3 100644
--- a/util/almost/almost.go
+++ b/util/almost/almost.go
@@ -22,10 +22,10 @@ import (
var minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
// Equal returns true if a and b differ by less than their sum
-// multiplied by epsilon.
+// multiplied by epsilon, or if both are StaleNaN, or if both are any other NaN.
func Equal(a, b, epsilon float64) bool {
- // StaleNaN is a special value that is used as staleness maker, so
- // the two values are equal when both are exactly equals to stale NaN.
+ // StaleNaN is a special value that is used as staleness maker, and
+ // we don't want it to compare equal to any other NaN.
if value.IsStaleNaN(a) || value.IsStaleNaN(b) {
return value.IsStaleNaN(a) && value.IsStaleNaN(b)
}
diff --git a/util/almost/almost_test.go b/util/almost/almost_test.go
new file mode 100644
index 0000000000..fba37f13f6
--- /dev/null
+++ b/util/almost/almost_test.go
@@ -0,0 +1,50 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package almost
+
+import (
+ "fmt"
+ "math"
+ "testing"
+
+ "github.com/prometheus/prometheus/model/value"
+)
+
+func TestEqual(t *testing.T) {
+ staleNaN := math.Float64frombits(value.StaleNaN)
+ tests := []struct {
+ a float64
+ b float64
+ epsilon float64
+ want bool
+ }{
+ {0.0, 0.0, 0.0, true},
+ {0.0, 0.1, 0.0, false},
+ {1.0, 1.1, 0.1, true},
+ {-1.0, -1.1, 0.1, true},
+ {math.MaxFloat64, math.MaxFloat64 / 10, 0.1, false},
+ {1.0, math.NaN(), 0.1, false},
+ {math.NaN(), math.NaN(), 0.1, true},
+ {math.NaN(), staleNaN, 0.1, false},
+ {staleNaN, math.NaN(), 0.1, false},
+ {staleNaN, staleNaN, 0.1, true},
+ }
+ for _, tt := range tests {
+ t.Run(fmt.Sprintf("%v,%v,%v", tt.a, tt.b, tt.epsilon), func(t *testing.T) {
+ if got := Equal(tt.a, tt.b, tt.epsilon); got != tt.want {
+ t.Errorf("Equal(%v,%v,%v) = %v, want %v", tt.a, tt.b, tt.epsilon, got, tt.want)
+ }
+ })
+ }
+}
From bc6c2c5d3502926852f33df0b0a8da1cb3c7bd00 Mon Sep 17 00:00:00 2001
From: Arve Knudsen <arve.knudsen@gmail.com>
Date: Fri, 30 Aug 2024 11:30:57 +0200
Subject: [PATCH 15/18] OTLP Receiver: Add tests (#14764)
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
---
.../prometheusremotewrite/helper_test.go | 225 +++++
.../prometheusremotewrite/histograms_test.go | 771 ++++++++++++++++++
.../number_data_points_test.go | 258 ++++++
.../prometheusremotewrite/testutil_test.go | 55 ++
4 files changed, 1309 insertions(+)
create mode 100644 storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go
create mode 100644 storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go
create mode 100644 storage/remote/otlptranslator/prometheusremotewrite/testutil_test.go
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
index 729d99b8a8..e02ebbf5de 100644
--- a/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
+++ b/storage/remote/otlptranslator/prometheusremotewrite/helper_test.go
@@ -10,6 +10,10 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
+// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/debbf30360b8d3a0ded8db09c4419d2a9c99b94a/pkg/translator/prometheusremotewrite/helper_test.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
+
package prometheusremotewrite
import (
@@ -18,6 +22,9 @@ import (
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+
+ "github.com/prometheus/common/model"
"github.com/prometheus/prometheus/prompb"
)
@@ -178,3 +185,221 @@ func Test_convertTimeStamp(t *testing.T) {
})
}
}
+
+func TestPrometheusConverter_AddSummaryDataPoints(t *testing.T) {
+ ts := pcommon.Timestamp(time.Now().UnixNano())
+ tests := []struct {
+ name string
+ metric func() pmetric.Metric
+ want func() map[uint64]*prompb.TimeSeries
+ }{
+ {
+ name: "summary with start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_summary")
+ metric.SetEmptySummary()
+
+ dp := metric.Summary().DataPoints().AppendEmpty()
+ dp.SetTimestamp(ts)
+ dp.SetStartTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_summary" + countStr},
+ }
+ createdLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_summary" + createdSuffix},
+ }
+ sumLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_summary" + sumStr},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(sumLabels): {
+ Labels: sumLabels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(createdLabels): {
+ Labels: createdLabels,
+ Samples: []prompb.Sample{
+ {Value: float64(convertTimeStamp(ts)), Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "summary without start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_summary")
+ metric.SetEmptySummary()
+
+ dp := metric.Summary().DataPoints().AppendEmpty()
+ dp.SetTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_summary" + countStr},
+ }
+ sumLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_summary" + sumStr},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(sumLabels): {
+ Labels: sumLabels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ metric := tt.metric()
+ converter := NewPrometheusConverter()
+
+ converter.addSummaryDataPoints(
+ metric.Summary().DataPoints(),
+ pcommon.NewResource(),
+ Settings{
+ ExportCreatedMetric: true,
+ },
+ metric.Name(),
+ )
+
+ assert.Equal(t, tt.want(), converter.unique)
+ assert.Empty(t, converter.conflicts)
+ })
+ }
+}
+
+func TestPrometheusConverter_AddHistogramDataPoints(t *testing.T) {
+ ts := pcommon.Timestamp(time.Now().UnixNano())
+ tests := []struct {
+ name string
+ metric func() pmetric.Metric
+ want func() map[uint64]*prompb.TimeSeries
+ }{
+ {
+ name: "histogram with start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_hist")
+ metric.SetEmptyHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+
+ pt := metric.Histogram().DataPoints().AppendEmpty()
+ pt.SetTimestamp(ts)
+ pt.SetStartTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist" + countStr},
+ }
+ createdLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist" + createdSuffix},
+ }
+ infLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist_bucket"},
+ {Name: model.BucketLabel, Value: "+Inf"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(infLabels): {
+ Labels: infLabels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(createdLabels): {
+ Labels: createdLabels,
+ Samples: []prompb.Sample{
+ {Value: float64(convertTimeStamp(ts)), Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "histogram without start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_hist")
+ metric.SetEmptyHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+
+ pt := metric.Histogram().DataPoints().AppendEmpty()
+ pt.SetTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist" + countStr},
+ }
+ infLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist_bucket"},
+ {Name: model.BucketLabel, Value: "+Inf"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(infLabels): {
+ Labels: infLabels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ metric := tt.metric()
+ converter := NewPrometheusConverter()
+
+ converter.addHistogramDataPoints(
+ metric.Histogram().DataPoints(),
+ pcommon.NewResource(),
+ Settings{
+ ExportCreatedMetric: true,
+ },
+ metric.Name(),
+ )
+
+ assert.Equal(t, tt.want(), converter.unique)
+ assert.Empty(t, converter.conflicts)
+ })
+ }
+}
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go b/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go
new file mode 100644
index 0000000000..cd1c858ac1
--- /dev/null
+++ b/storage/remote/otlptranslator/prometheusremotewrite/histograms_test.go
@@ -0,0 +1,771 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/247a9f996e09a83cdc25addf70c05e42b8b30186/pkg/translator/prometheusremotewrite/histograms_test.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
+
+package prometheusremotewrite
+
+import (
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/prometheus/common/model"
+ "github.com/prometheus/prometheus/prompb"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+
+ prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
+)
+
+type expectedBucketLayout struct {
+ wantSpans []prompb.BucketSpan
+ wantDeltas []int64
+}
+
+func TestConvertBucketsLayout(t *testing.T) {
+ tests := []struct {
+ name string
+ buckets func() pmetric.ExponentialHistogramDataPointBuckets
+ wantLayout map[int32]expectedBucketLayout
+ }{
+ {
+ name: "zero offset",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(0)
+ b.BucketCounts().FromRaw([]uint64{4, 3, 2, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 1,
+ Length: 4,
+ },
+ },
+ wantDeltas: []int64{4, -1, -1, -1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 1,
+ Length: 2,
+ },
+ },
+ // 4+3, 2+1 = 7, 3 =delta= 7, -4
+ wantDeltas: []int64{7, -4},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 1,
+ Length: 1,
+ },
+ },
+ // 4+3+2+1 = 10 =delta= 10
+ wantDeltas: []int64{10},
+ },
+ },
+ },
+ {
+ name: "offset 1",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(1)
+ b.BucketCounts().FromRaw([]uint64{4, 3, 2, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 2,
+ Length: 4,
+ },
+ },
+ wantDeltas: []int64{4, -1, -1, -1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 1,
+ Length: 3,
+ },
+ },
+ wantDeltas: []int64{4, 1, -4}, // 0+4, 3+2, 1+0 = 4, 5, 1
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 1,
+ Length: 2,
+ },
+ },
+ wantDeltas: []int64{9, -8}, // 0+4+3+2, 1+0+0+0 = 9, 1
+ },
+ },
+ },
+ {
+ name: "positive offset",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(4)
+ b.BucketCounts().FromRaw([]uint64{4, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 5,
+ Length: 4,
+ },
+ {
+ Offset: 12,
+ Length: 1,
+ },
+ },
+ wantDeltas: []int64{4, -2, -2, 2, -1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 3,
+ Length: 2,
+ },
+ {
+ Offset: 6,
+ Length: 1,
+ },
+ },
+ // Downscale:
+ // 4+2, 0+2, 0+0, 0+0, 0+0, 0+0, 0+0, 0+0, 1+0 = 6, 2, 0, 0, 0, 0, 0, 0, 1
+ wantDeltas: []int64{6, -4, -1},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 2,
+ Length: 1,
+ },
+ {
+ Offset: 3,
+ Length: 1,
+ },
+ },
+ // Downscale:
+ // 4+2+0+2, 0+0+0+0, 0+0+0+0, 0+0+0+0, 1+0+0+0 = 8, 0, 0, 0, 1
+ // Check from sclaing from previous: 6+2, 0+0, 0+0, 0+0, 1+0 = 8, 0, 0, 0, 1
+ wantDeltas: []int64{8, -7},
+ },
+ },
+ },
+ {
+ name: "scaledown merges spans",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(4)
+ b.BucketCounts().FromRaw([]uint64{4, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 5,
+ Length: 4,
+ },
+ {
+ Offset: 8,
+ Length: 1,
+ },
+ },
+ wantDeltas: []int64{4, -2, -2, 2, -1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 3,
+ Length: 2,
+ },
+ {
+ Offset: 4,
+ Length: 1,
+ },
+ },
+ // Downscale:
+ // 4+2, 0+2, 0+0, 0+0, 0+0, 0+0, 1+0 = 6, 2, 0, 0, 0, 0, 1
+ wantDeltas: []int64{6, -4, -1},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 2,
+ Length: 4,
+ },
+ },
+ // Downscale:
+ // 4+2+0+2, 0+0+0+0, 0+0+0+0, 1+0+0+0 = 8, 0, 0, 1
+ // Check from sclaing from previous: 6+2, 0+0, 0+0, 1+0 = 8, 0, 0, 1
+ wantDeltas: []int64{8, -8, 0, 1},
+ },
+ },
+ },
+ {
+ name: "negative offset",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(-2)
+ b.BucketCounts().FromRaw([]uint64{3, 1, 0, 0, 0, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: -1,
+ Length: 2,
+ },
+ {
+ Offset: 3,
+ Length: 1,
+ },
+ },
+ wantDeltas: []int64{3, -2, 0},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 3,
+ },
+ },
+ // Downscale:
+ // 3+1, 0+0, 0+1 = 4, 0, 1
+ wantDeltas: []int64{4, -4, 1},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 2,
+ },
+ },
+ // Downscale:
+ // 0+0+3+1, 0+0+0+0 = 4, 1
+ wantDeltas: []int64{4, -3},
+ },
+ },
+ },
+ {
+ name: "buckets with gaps of size 1",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(-2)
+ b.BucketCounts().FromRaw([]uint64{3, 1, 0, 1, 0, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: -1,
+ Length: 6,
+ },
+ },
+ wantDeltas: []int64{3, -2, -1, 1, -1, 1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 3,
+ },
+ },
+ // Downscale:
+ // 3+1, 0+1, 0+1 = 4, 1, 1
+ wantDeltas: []int64{4, -3, 0},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 2,
+ },
+ },
+ // Downscale:
+ // 0+0+3+1, 0+1+0+1 = 4, 2
+ wantDeltas: []int64{4, -2},
+ },
+ },
+ },
+ {
+ name: "buckets with gaps of size 2",
+ buckets: func() pmetric.ExponentialHistogramDataPointBuckets {
+ b := pmetric.NewExponentialHistogramDataPointBuckets()
+ b.SetOffset(-2)
+ b.BucketCounts().FromRaw([]uint64{3, 0, 0, 1, 0, 0, 1})
+ return b
+ },
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: -1,
+ Length: 7,
+ },
+ },
+ wantDeltas: []int64{3, -3, 0, 1, -1, 0, 1},
+ },
+ 1: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 4,
+ },
+ },
+ // Downscale:
+ // 3+0, 0+1, 0+0, 0+1 = 3, 1, 0, 1
+ wantDeltas: []int64{3, -2, -1, 1},
+ },
+ 2: {
+ wantSpans: []prompb.BucketSpan{
+ {
+ Offset: 0,
+ Length: 3,
+ },
+ },
+ // Downscale:
+ // 0+0+3+0, 0+1+0+0, 1+0+0+0 = 3, 1, 1
+ wantDeltas: []int64{3, -2, 0},
+ },
+ },
+ },
+ {
+ name: "zero buckets",
+ buckets: pmetric.NewExponentialHistogramDataPointBuckets,
+ wantLayout: map[int32]expectedBucketLayout{
+ 0: {
+ wantSpans: nil,
+ wantDeltas: nil,
+ },
+ 1: {
+ wantSpans: nil,
+ wantDeltas: nil,
+ },
+ 2: {
+ wantSpans: nil,
+ wantDeltas: nil,
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ for scaleDown, wantLayout := range tt.wantLayout {
+ t.Run(fmt.Sprintf("%s-scaleby-%d", tt.name, scaleDown), func(t *testing.T) {
+ gotSpans, gotDeltas := convertBucketsLayout(tt.buckets(), scaleDown)
+ assert.Equal(t, wantLayout.wantSpans, gotSpans)
+ assert.Equal(t, wantLayout.wantDeltas, gotDeltas)
+ })
+ }
+ }
+}
+
+func BenchmarkConvertBucketLayout(b *testing.B) {
+ scenarios := []struct {
+ gap int
+ }{
+ {gap: 0},
+ {gap: 1},
+ {gap: 2},
+ {gap: 3},
+ }
+
+ for _, scenario := range scenarios {
+ buckets := pmetric.NewExponentialHistogramDataPointBuckets()
+ buckets.SetOffset(0)
+ for i := 0; i < 1000; i++ {
+ if i%(scenario.gap+1) == 0 {
+ buckets.BucketCounts().Append(10)
+ } else {
+ buckets.BucketCounts().Append(0)
+ }
+ }
+ b.Run(fmt.Sprintf("gap %d", scenario.gap), func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ convertBucketsLayout(buckets, 0)
+ }
+ })
+ }
+}
+
+func TestExponentialToNativeHistogram(t *testing.T) {
+ tests := []struct {
+ name string
+ exponentialHist func() pmetric.ExponentialHistogramDataPoint
+ wantNativeHist func() prompb.Histogram
+ wantErrMessage string
+ }{
+ {
+ name: "convert exp. to native histogram",
+ exponentialHist: func() pmetric.ExponentialHistogramDataPoint {
+ pt := pmetric.NewExponentialHistogramDataPoint()
+ pt.SetStartTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(100)))
+ pt.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(500)))
+ pt.SetCount(4)
+ pt.SetSum(10.1)
+ pt.SetScale(1)
+ pt.SetZeroCount(1)
+
+ pt.Positive().BucketCounts().FromRaw([]uint64{1, 1})
+ pt.Positive().SetOffset(1)
+
+ pt.Negative().BucketCounts().FromRaw([]uint64{1, 1})
+ pt.Negative().SetOffset(1)
+
+ return pt
+ },
+ wantNativeHist: func() prompb.Histogram {
+ return prompb.Histogram{
+ Count: &prompb.Histogram_CountInt{CountInt: 4},
+ Sum: 10.1,
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 1},
+ NegativeSpans: []prompb.BucketSpan{{Offset: 2, Length: 2}},
+ NegativeDeltas: []int64{1, 0},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 2, Length: 2}},
+ PositiveDeltas: []int64{1, 0},
+ Timestamp: 500,
+ }
+ },
+ },
+ {
+ name: "convert exp. to native histogram with no sum",
+ exponentialHist: func() pmetric.ExponentialHistogramDataPoint {
+ pt := pmetric.NewExponentialHistogramDataPoint()
+ pt.SetStartTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(100)))
+ pt.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(500)))
+
+ pt.SetCount(4)
+ pt.SetScale(1)
+ pt.SetZeroCount(1)
+
+ pt.Positive().BucketCounts().FromRaw([]uint64{1, 1})
+ pt.Positive().SetOffset(1)
+
+ pt.Negative().BucketCounts().FromRaw([]uint64{1, 1})
+ pt.Negative().SetOffset(1)
+
+ return pt
+ },
+ wantNativeHist: func() prompb.Histogram {
+ return prompb.Histogram{
+ Count: &prompb.Histogram_CountInt{CountInt: 4},
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 1},
+ NegativeSpans: []prompb.BucketSpan{{Offset: 2, Length: 2}},
+ NegativeDeltas: []int64{1, 0},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 2, Length: 2}},
+ PositiveDeltas: []int64{1, 0},
+ Timestamp: 500,
+ }
+ },
+ },
+ {
+ name: "invalid negative scale",
+ exponentialHist: func() pmetric.ExponentialHistogramDataPoint {
+ pt := pmetric.NewExponentialHistogramDataPoint()
+ pt.SetScale(-10)
+ return pt
+ },
+ wantErrMessage: "cannot convert exponential to native histogram." +
+ " Scale must be >= -4, was -10",
+ },
+ {
+ name: "no downscaling at scale 8",
+ exponentialHist: func() pmetric.ExponentialHistogramDataPoint {
+ pt := pmetric.NewExponentialHistogramDataPoint()
+ pt.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(500)))
+ pt.SetCount(6)
+ pt.SetSum(10.1)
+ pt.SetScale(8)
+ pt.SetZeroCount(1)
+
+ pt.Positive().BucketCounts().FromRaw([]uint64{1, 1, 1})
+ pt.Positive().SetOffset(1)
+
+ pt.Negative().BucketCounts().FromRaw([]uint64{1, 1, 1})
+ pt.Negative().SetOffset(2)
+ return pt
+ },
+ wantNativeHist: func() prompb.Histogram {
+ return prompb.Histogram{
+ Count: &prompb.Histogram_CountInt{CountInt: 6},
+ Sum: 10.1,
+ Schema: 8,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 1},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 2, Length: 3}},
+ PositiveDeltas: []int64{1, 0, 0}, // 1, 1, 1
+ NegativeSpans: []prompb.BucketSpan{{Offset: 3, Length: 3}},
+ NegativeDeltas: []int64{1, 0, 0}, // 1, 1, 1
+ Timestamp: 500,
+ }
+ },
+ },
+ {
+ name: "downsample if scale is more than 8",
+ exponentialHist: func() pmetric.ExponentialHistogramDataPoint {
+ pt := pmetric.NewExponentialHistogramDataPoint()
+ pt.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(500)))
+ pt.SetCount(6)
+ pt.SetSum(10.1)
+ pt.SetScale(9)
+ pt.SetZeroCount(1)
+
+ pt.Positive().BucketCounts().FromRaw([]uint64{1, 1, 1})
+ pt.Positive().SetOffset(1)
+
+ pt.Negative().BucketCounts().FromRaw([]uint64{1, 1, 1})
+ pt.Negative().SetOffset(2)
+ return pt
+ },
+ wantNativeHist: func() prompb.Histogram {
+ return prompb.Histogram{
+ Count: &prompb.Histogram_CountInt{CountInt: 6},
+ Sum: 10.1,
+ Schema: 8,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 1},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 1, Length: 2}},
+ PositiveDeltas: []int64{1, 1}, // 0+1, 1+1 = 1, 2
+ NegativeSpans: []prompb.BucketSpan{{Offset: 2, Length: 2}},
+ NegativeDeltas: []int64{2, -1}, // 1+1, 1+0 = 2, 1
+ Timestamp: 500,
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ validateExponentialHistogramCount(t, tt.exponentialHist()) // Sanity check.
+ got, annots, err := exponentialToNativeHistogram(tt.exponentialHist())
+ if tt.wantErrMessage != "" {
+ assert.ErrorContains(t, err, tt.wantErrMessage)
+ return
+ }
+
+ require.NoError(t, err)
+ require.Empty(t, annots)
+ assert.Equal(t, tt.wantNativeHist(), got)
+ validateNativeHistogramCount(t, got)
+ })
+ }
+}
+
+func validateExponentialHistogramCount(t *testing.T, h pmetric.ExponentialHistogramDataPoint) {
+ actualCount := uint64(0)
+ for _, bucket := range h.Positive().BucketCounts().AsRaw() {
+ actualCount += bucket
+ }
+ for _, bucket := range h.Negative().BucketCounts().AsRaw() {
+ actualCount += bucket
+ }
+ require.Equal(t, h.Count(), actualCount, "exponential histogram count mismatch")
+}
+
+func validateNativeHistogramCount(t *testing.T, h prompb.Histogram) {
+ require.NotNil(t, h.Count)
+ require.IsType(t, &prompb.Histogram_CountInt{}, h.Count)
+ want := h.Count.(*prompb.Histogram_CountInt).CountInt
+ var (
+ actualCount uint64
+ prevBucket int64
+ )
+ for _, delta := range h.PositiveDeltas {
+ prevBucket += delta
+ actualCount += uint64(prevBucket)
+ }
+ prevBucket = 0
+ for _, delta := range h.NegativeDeltas {
+ prevBucket += delta
+ actualCount += uint64(prevBucket)
+ }
+ assert.Equal(t, want, actualCount, "native histogram count mismatch")
+}
+
+func TestPrometheusConverter_addExponentialHistogramDataPoints(t *testing.T) {
+ tests := []struct {
+ name string
+ metric func() pmetric.Metric
+ wantSeries func() map[uint64]*prompb.TimeSeries
+ }{
+ {
+ name: "histogram data points with same labels",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_hist")
+ metric.SetEmptyExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+
+ pt := metric.ExponentialHistogram().DataPoints().AppendEmpty()
+ pt.SetCount(7)
+ pt.SetScale(1)
+ pt.Positive().SetOffset(-1)
+ pt.Positive().BucketCounts().FromRaw([]uint64{4, 2})
+ pt.Exemplars().AppendEmpty().SetDoubleValue(1)
+ pt.Attributes().PutStr("attr", "test_attr")
+
+ pt = metric.ExponentialHistogram().DataPoints().AppendEmpty()
+ pt.SetCount(4)
+ pt.SetScale(1)
+ pt.Positive().SetOffset(-1)
+ pt.Positive().BucketCounts().FromRaw([]uint64{4, 2, 1})
+ pt.Exemplars().AppendEmpty().SetDoubleValue(2)
+ pt.Attributes().PutStr("attr", "test_attr")
+
+ return metric
+ },
+ wantSeries: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist"},
+ {Name: "attr", Value: "test_attr"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Histograms: []prompb.Histogram{
+ {
+ Count: &prompb.Histogram_CountInt{CountInt: 7},
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 2}},
+ PositiveDeltas: []int64{4, -2},
+ },
+ {
+ Count: &prompb.Histogram_CountInt{CountInt: 4},
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 3}},
+ PositiveDeltas: []int64{4, -2, -1},
+ },
+ },
+ Exemplars: []prompb.Exemplar{
+ {Value: 1},
+ {Value: 2},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "histogram data points with different labels",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_hist")
+ metric.SetEmptyExponentialHistogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+
+ pt := metric.ExponentialHistogram().DataPoints().AppendEmpty()
+ pt.SetCount(7)
+ pt.SetScale(1)
+ pt.Positive().SetOffset(-1)
+ pt.Positive().BucketCounts().FromRaw([]uint64{4, 2})
+ pt.Exemplars().AppendEmpty().SetDoubleValue(1)
+ pt.Attributes().PutStr("attr", "test_attr")
+
+ pt = metric.ExponentialHistogram().DataPoints().AppendEmpty()
+ pt.SetCount(4)
+ pt.SetScale(1)
+ pt.Negative().SetOffset(-1)
+ pt.Negative().BucketCounts().FromRaw([]uint64{4, 2, 1})
+ pt.Exemplars().AppendEmpty().SetDoubleValue(2)
+ pt.Attributes().PutStr("attr", "test_attr_two")
+
+ return metric
+ },
+ wantSeries: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist"},
+ {Name: "attr", Value: "test_attr"},
+ }
+ labelsAnother := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_hist"},
+ {Name: "attr", Value: "test_attr_two"},
+ }
+
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Histograms: []prompb.Histogram{
+ {
+ Count: &prompb.Histogram_CountInt{CountInt: 7},
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
+ PositiveSpans: []prompb.BucketSpan{{Offset: 0, Length: 2}},
+ PositiveDeltas: []int64{4, -2},
+ },
+ },
+ Exemplars: []prompb.Exemplar{
+ {Value: 1},
+ },
+ },
+ timeSeriesSignature(labelsAnother): {
+ Labels: labelsAnother,
+ Histograms: []prompb.Histogram{
+ {
+ Count: &prompb.Histogram_CountInt{CountInt: 4},
+ Schema: 1,
+ ZeroThreshold: defaultZeroThreshold,
+ ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0},
+ NegativeSpans: []prompb.BucketSpan{{Offset: 0, Length: 3}},
+ NegativeDeltas: []int64{4, -2, -1},
+ },
+ },
+ Exemplars: []prompb.Exemplar{
+ {Value: 2},
+ },
+ },
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ metric := tt.metric()
+
+ converter := NewPrometheusConverter()
+ annots, err := converter.addExponentialHistogramDataPoints(
+ metric.ExponentialHistogram().DataPoints(),
+ pcommon.NewResource(),
+ Settings{
+ ExportCreatedMetric: true,
+ },
+ prometheustranslator.BuildCompliantName(metric, "", true),
+ )
+ require.NoError(t, err)
+ require.Empty(t, annots)
+
+ assert.Equal(t, tt.wantSeries(), converter.unique)
+ assert.Empty(t, converter.conflicts)
+ })
+ }
+}
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go
new file mode 100644
index 0000000000..41afc8c4c3
--- /dev/null
+++ b/storage/remote/otlptranslator/prometheusremotewrite/number_data_points_test.go
@@ -0,0 +1,258 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/247a9f996e09a83cdc25addf70c05e42b8b30186/pkg/translator/prometheusremotewrite/number_data_points_test.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
+
+package prometheusremotewrite
+
+import (
+ "testing"
+ "time"
+
+ "github.com/prometheus/common/model"
+ "github.com/prometheus/prometheus/prompb"
+ "github.com/stretchr/testify/assert"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+)
+
+func TestPrometheusConverter_addGaugeNumberDataPoints(t *testing.T) {
+ ts := uint64(time.Now().UnixNano())
+ tests := []struct {
+ name string
+ metric func() pmetric.Metric
+ want func() map[uint64]*prompb.TimeSeries
+ }{
+ {
+ name: "gauge",
+ metric: func() pmetric.Metric {
+ return getIntGaugeMetric(
+ "test",
+ pcommon.NewMap(),
+ 1, ts,
+ )
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {
+ Value: 1,
+ Timestamp: convertTimeStamp(pcommon.Timestamp(ts)),
+ }},
+ },
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ metric := tt.metric()
+ converter := NewPrometheusConverter()
+
+ converter.addGaugeNumberDataPoints(
+ metric.Gauge().DataPoints(),
+ pcommon.NewResource(),
+ Settings{
+ ExportCreatedMetric: true,
+ },
+ metric.Name(),
+ )
+
+ assert.Equal(t, tt.want(), converter.unique)
+ assert.Empty(t, converter.conflicts)
+ })
+ }
+}
+
+func TestPrometheusConverter_addSumNumberDataPoints(t *testing.T) {
+ ts := pcommon.Timestamp(time.Now().UnixNano())
+ tests := []struct {
+ name string
+ metric func() pmetric.Metric
+ want func() map[uint64]*prompb.TimeSeries
+ }{
+ {
+ name: "sum",
+ metric: func() pmetric.Metric {
+ return getIntSumMetric(
+ "test",
+ pcommon.NewMap(),
+ 1,
+ uint64(ts.AsTime().UnixNano()),
+ )
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {
+ Value: 1,
+ Timestamp: convertTimeStamp(ts),
+ }},
+ },
+ }
+ },
+ },
+ {
+ name: "sum with exemplars",
+ metric: func() pmetric.Metric {
+ m := getIntSumMetric(
+ "test",
+ pcommon.NewMap(),
+ 1,
+ uint64(ts.AsTime().UnixNano()),
+ )
+ m.Sum().DataPoints().At(0).Exemplars().AppendEmpty().SetDoubleValue(2)
+ return m
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{{
+ Value: 1,
+ Timestamp: convertTimeStamp(ts),
+ }},
+ Exemplars: []prompb.Exemplar{
+ {Value: 2},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "monotonic cumulative sum with start timestamp",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_sum")
+ metric.SetEmptySum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+ metric.SetEmptySum().SetIsMonotonic(true)
+
+ dp := metric.Sum().DataPoints().AppendEmpty()
+ dp.SetDoubleValue(1)
+ dp.SetTimestamp(ts)
+ dp.SetStartTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_sum"},
+ }
+ createdLabels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_sum" + createdSuffix},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 1, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ timeSeriesSignature(createdLabels): {
+ Labels: createdLabels,
+ Samples: []prompb.Sample{
+ {Value: float64(convertTimeStamp(ts)), Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "monotonic cumulative sum with no start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_sum")
+ metric.SetEmptySum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+ metric.SetEmptySum().SetIsMonotonic(true)
+
+ dp := metric.Sum().DataPoints().AppendEmpty()
+ dp.SetTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_sum"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ {
+ name: "non-monotonic cumulative sum with start time",
+ metric: func() pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName("test_sum")
+ metric.SetEmptySum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+ metric.SetEmptySum().SetIsMonotonic(false)
+
+ dp := metric.Sum().DataPoints().AppendEmpty()
+ dp.SetTimestamp(ts)
+
+ return metric
+ },
+ want: func() map[uint64]*prompb.TimeSeries {
+ labels := []prompb.Label{
+ {Name: model.MetricNameLabel, Value: "test_sum"},
+ }
+ return map[uint64]*prompb.TimeSeries{
+ timeSeriesSignature(labels): {
+ Labels: labels,
+ Samples: []prompb.Sample{
+ {Value: 0, Timestamp: convertTimeStamp(ts)},
+ },
+ },
+ }
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ metric := tt.metric()
+ converter := NewPrometheusConverter()
+
+ converter.addSumNumberDataPoints(
+ metric.Sum().DataPoints(),
+ pcommon.NewResource(),
+ metric,
+ Settings{
+ ExportCreatedMetric: true,
+ },
+ metric.Name(),
+ )
+
+ assert.Equal(t, tt.want(), converter.unique)
+ assert.Empty(t, converter.conflicts)
+ })
+ }
+}
diff --git a/storage/remote/otlptranslator/prometheusremotewrite/testutil_test.go b/storage/remote/otlptranslator/prometheusremotewrite/testutil_test.go
new file mode 100644
index 0000000000..187127fcb2
--- /dev/null
+++ b/storage/remote/otlptranslator/prometheusremotewrite/testutil_test.go
@@ -0,0 +1,55 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// Provenance-includes-location: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/247a9f996e09a83cdc25addf70c05e42b8b30186/pkg/translator/prometheusremotewrite/testutil_test.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Copyright The OpenTelemetry Authors.
+
+package prometheusremotewrite
+
+import (
+ "strings"
+
+ "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+)
+
+func getIntGaugeMetric(name string, attributes pcommon.Map, value int64, ts uint64) pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName(name)
+ dp := metric.SetEmptyGauge().DataPoints().AppendEmpty()
+ if strings.HasPrefix(name, "staleNaN") {
+ dp.SetFlags(pmetric.DefaultDataPointFlags.WithNoRecordedValue(true))
+ }
+ dp.SetIntValue(value)
+ attributes.CopyTo(dp.Attributes())
+
+ dp.SetStartTimestamp(pcommon.Timestamp(0))
+ dp.SetTimestamp(pcommon.Timestamp(ts))
+ return metric
+}
+
+func getIntSumMetric(name string, attributes pcommon.Map, value int64, ts uint64) pmetric.Metric {
+ metric := pmetric.NewMetric()
+ metric.SetName(name)
+ metric.SetEmptySum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+ dp := metric.Sum().DataPoints().AppendEmpty()
+ if strings.HasPrefix(name, "staleNaN") {
+ dp.SetFlags(pmetric.DefaultDataPointFlags.WithNoRecordedValue(true))
+ }
+ dp.SetIntValue(value)
+ attributes.CopyTo(dp.Attributes())
+
+ dp.SetStartTimestamp(pcommon.Timestamp(0))
+ dp.SetTimestamp(pcommon.Timestamp(ts))
+ return metric
+}
From d23d196db58cbf5ee67d64932aeb14b6038d16c9 Mon Sep 17 00:00:00 2001
From: machine424 <ayoubmrini424@gmail.com>
Date: Fri, 30 Aug 2024 13:37:25 +0200
Subject: [PATCH 16/18] fix(discovery): prevent the manager from storing stale
targetGroups
Signed-off-by: machine424 <ayoubmrini424@gmail.com>
---
discovery/manager.go | 10 +++++++++-
discovery/manager_test.go | 18 ++++++++----------
scrape/manager_test.go | 2 +-
3 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/discovery/manager.go b/discovery/manager.go
index b7ebdb7e0b..cefa90a866 100644
--- a/discovery/manager.go
+++ b/discovery/manager.go
@@ -394,8 +394,16 @@ func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) {
m.targets[poolKey] = make(map[string]*targetgroup.Group)
}
for _, tg := range tgs {
- if tg != nil { // Some Discoverers send nil target group so need to check for it to avoid panics.
+ // Some Discoverers send nil target group so need to check for it to avoid panics.
+ if tg == nil {
+ continue
+ }
+ if len(tg.Targets) > 0 {
m.targets[poolKey][tg.Source] = tg
+ } else {
+ // The target group is empty, drop the corresponding entry to avoid leaks.
+ // In case the group yielded targets before, allGroups() will take care of making consumers drop them.
+ delete(m.targets[poolKey], tg.Source)
}
}
}
diff --git a/discovery/manager_test.go b/discovery/manager_test.go
index 707c3931d7..831cefe514 100644
--- a/discovery/manager_test.go
+++ b/discovery/manager_test.go
@@ -1051,8 +1051,8 @@ func TestDiscovererConfigs(t *testing.T) {
}
// TestTargetSetRecreatesEmptyStaticConfigs ensures that reloading a config file after
-// removing all targets from the static_configs sends an update with empty targetGroups.
-// This is required to signal the receiver that this target set has no current targets.
+// removing all targets from the static_configs cleans the corresponding targetGroups entries to avoid leaks and sends an empty update.
+// The update is required to signal the consumers that the previous targets should be dropped.
func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -1085,16 +1085,14 @@ func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
discoveryManager.ApplyConfig(c)
syncedTargets = <-discoveryManager.SyncCh()
+ require.Len(t, discoveryManager.targets, 1)
p = pk("static", "prometheus", 1)
targetGroups, ok := discoveryManager.targets[p]
- require.True(t, ok, "'%v' should be present in target groups", p)
- group, ok := targetGroups[""]
- require.True(t, ok, "missing '' key in target groups %v", targetGroups)
-
- require.Empty(t, group.Targets, "Invalid number of targets.")
- require.Len(t, syncedTargets, 1)
- require.Len(t, syncedTargets["prometheus"], 1)
- require.Nil(t, syncedTargets["prometheus"][0].Labels)
+ require.True(t, ok, "'%v' should be present in targets", p)
+ // Otherwise the targetGroups will leak, see https://github.com/prometheus/prometheus/issues/12436.
+ require.Empty(t, targetGroups, 0, "'%v' should no longer have any associated target groups", p)
+ require.Len(t, syncedTargets, 1, "an update with no targetGroups should still be sent.")
+ require.Empty(t, syncedTargets["prometheus"], 0)
}
func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
diff --git a/scrape/manager_test.go b/scrape/manager_test.go
index f260167b5e..c71691c95d 100644
--- a/scrape/manager_test.go
+++ b/scrape/manager_test.go
@@ -1178,7 +1178,7 @@ scrape_configs:
)
}
-// TestOnlyStaleTargetsAreDropped makes sure that when a job has multiple providers, when aone of them should no,
+// TestOnlyStaleTargetsAreDropped makes sure that when a job has multiple providers, when one of them should no
// longer discover targets, only the stale targets of that provier are dropped.
func TestOnlyStaleTargetsAreDropped(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
From fae66d0bba182f6e3745a8ff04237d59fb9a575d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Sep 2024 23:49:48 +0000
Subject: [PATCH 17/18] Bump github.com/hashicorp/consul/api from 1.29.2 to
1.29.4
Bumps [github.com/hashicorp/consul/api](https://github.com/hashicorp/consul) from 1.29.2 to 1.29.4.
- [Release notes](https://github.com/hashicorp/consul/releases)
- [Changelog](https://github.com/hashicorp/consul/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hashicorp/consul/compare/api/v1.29.2...api/v1.29.4)
---
updated-dependencies:
- dependency-name: github.com/hashicorp/consul/api
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index af327c64ad..2f5524dd4e 100644
--- a/go.mod
+++ b/go.mod
@@ -36,7 +36,7 @@ require (
github.com/gophercloud/gophercloud v1.14.0
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
github.com/grpc-ecosystem/grpc-gateway v1.16.0
- github.com/hashicorp/consul/api v1.29.2
+ github.com/hashicorp/consul/api v1.29.4
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3
github.com/hetznercloud/hcloud-go/v2 v2.12.0
github.com/ionos-cloud/sdk-go/v6 v6.2.0
diff --git a/go.sum b/go.sum
index 933ef94201..20f88c1023 100644
--- a/go.sum
+++ b/go.sum
@@ -353,8 +353,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
-github.com/hashicorp/consul/api v1.29.2 h1:aYyRn8EdE2mSfG14S1+L9Qkjtz8RzmaWh6AcNGRNwPw=
-github.com/hashicorp/consul/api v1.29.2/go.mod h1:0YObcaLNDSbtlgzIRtmRXI1ZkeuK0trCBxwZQ4MYnIk=
+github.com/hashicorp/consul/api v1.29.4 h1:P6slzxDLBOxUSj3fWo2o65VuKtbtOXFi7TSSgtXutuE=
+github.com/hashicorp/consul/api v1.29.4/go.mod h1:HUlfw+l2Zy68ceJavv2zAyArl2fqhGWnMycyt56sBgg=
github.com/hashicorp/consul/proto-public v0.6.2 h1:+DA/3g/IiKlJZb88NBn0ZgXrxJp2NlvCZdEyl+qxvL0=
github.com/hashicorp/consul/proto-public v0.6.2/go.mod h1:cXXbOg74KBNGajC+o8RlA502Esf0R9prcoJgiOX/2Tg=
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
From 02d1cabd0a182dd6c98fd522cdfd1aee885ffa04 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Sep 2024 23:50:13 +0000
Subject: [PATCH 18/18] Bump github.com/hetznercloud/hcloud-go/v2 from 2.12.0
to 2.13.1
Bumps [github.com/hetznercloud/hcloud-go/v2](https://github.com/hetznercloud/hcloud-go) from 2.12.0 to 2.13.1.
- [Release notes](https://github.com/hetznercloud/hcloud-go/releases)
- [Changelog](https://github.com/hetznercloud/hcloud-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hetznercloud/hcloud-go/compare/v2.12.0...v2.13.1)
---
updated-dependencies:
- dependency-name: github.com/hetznercloud/hcloud-go/v2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index af327c64ad..86e17fec3b 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway v1.16.0
github.com/hashicorp/consul/api v1.29.2
github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3
- github.com/hetznercloud/hcloud-go/v2 v2.12.0
+ github.com/hetznercloud/hcloud-go/v2 v2.13.1
github.com/ionos-cloud/sdk-go/v6 v6.2.0
github.com/json-iterator/go v1.1.12
github.com/klauspost/compress v1.17.9
diff --git a/go.sum b/go.sum
index 933ef94201..1195580280 100644
--- a/go.sum
+++ b/go.sum
@@ -414,8 +414,8 @@ github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtx
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/hetznercloud/hcloud-go/v2 v2.12.0 h1:nOgfNTo0gyXZJJdM8mo/XH5MO/e80wAEpldRzdWayhY=
-github.com/hetznercloud/hcloud-go/v2 v2.12.0/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0=
+github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXfGmtD1a/qckQ=
+github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=