diff --git a/tsdb/head.go b/tsdb/head.go index b2e897b5a..6917ad80d 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -274,9 +274,9 @@ type headMetrics struct { chunksCreated prometheus.Counter chunksRemoved prometheus.Counter gcDuration prometheus.Summary - samplesAppended prometheus.Counter - outOfBoundSamples prometheus.Counter - outOfOrderSamples prometheus.Counter + samplesAppended *prometheus.CounterVec + outOfBoundSamples *prometheus.CounterVec + outOfOrderSamples *prometheus.CounterVec walTruncateDuration prometheus.Summary walCorruptionsTotal prometheus.Counter walTotalReplayDuration prometheus.Gauge @@ -288,12 +288,13 @@ type headMetrics struct { checkpointCreationTotal prometheus.Counter mmapChunkCorruptionTotal prometheus.Counter snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. - - // Sparse histogram metrics for experiments. - // TODO: remove these in the final version. - histogramSamplesTotal prometheus.Counter } +const ( + sampleMetricTypeFloat = "float" + sampleMetricTypeHistogram = "histogram" +) + func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { m := &headMetrics{ activeAppenders: prometheus.NewGauge(prometheus.GaugeOpts{ @@ -346,18 +347,18 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_data_replay_duration_seconds", Help: "Time taken to replay the data on disk.", }), - samplesAppended: prometheus.NewCounter(prometheus.CounterOpts{ + samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "prometheus_tsdb_head_samples_appended_total", Help: "Total number of appended samples.", - }), - outOfBoundSamples: prometheus.NewCounter(prometheus.CounterOpts{ + }, []string{"type"}), + outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "prometheus_tsdb_out_of_bound_samples_total", Help: "Total number of out of bound samples ingestion failed attempts.", - }), - outOfOrderSamples: prometheus.NewCounter(prometheus.CounterOpts{ + }, []string{"type"}), + outOfOrderSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "prometheus_tsdb_out_of_order_samples_total", Help: "Total number of out of order samples ingestion failed attempts.", - }), + }, []string{"type"}), headTruncateFail: prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_head_truncations_failed_total", Help: "Total number of head truncations that failed.", @@ -390,10 +391,6 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { Name: "prometheus_tsdb_snapshot_replay_error_total", Help: "Total number snapshot replays that failed.", }), - histogramSamplesTotal: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "prometheus_tsdb_histogram_samples_total", - Help: "Total number of histograms samples added.", - }), } if r != nil { @@ -421,7 +418,6 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { m.checkpointCreationTotal, m.mmapChunkCorruptionTotal, m.snapshotReplayErrorTotal, - m.histogramSamplesTotal, // Metrics bound to functions and not needed in tests // can be created and registered on the spot. prometheus.NewGaugeFunc(prometheus.GaugeOpts{ diff --git a/tsdb/head_append.go b/tsdb/head_append.go index 037b03407..6565be52d 100644 --- a/tsdb/head_append.go +++ b/tsdb/head_append.go @@ -297,7 +297,7 @@ type headAppender struct { func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { if t < a.minValidTime { - a.head.metrics.outOfBoundSamples.Inc() + a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc() return 0, storage.ErrOutOfBounds } @@ -335,7 +335,7 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64 if err := s.appendable(t, v); err != nil { s.Unlock() if err == storage.ErrOutOfOrderSample { - a.head.metrics.outOfOrderSamples.Inc() + a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() } return 0, err } @@ -444,7 +444,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels } if t < a.minValidTime { - a.head.metrics.outOfBoundSamples.Inc() + a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() return 0, storage.ErrOutOfBounds } @@ -483,7 +483,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels if err := s.appendableHistogram(t, h); err != nil { s.Unlock() if err == storage.ErrOutOfOrderSample { - a.head.metrics.outOfOrderSamples.Inc() + a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() } return 0, err } @@ -728,7 +728,7 @@ func (a *headAppender) Commit() (err error) { if !ok { total-- - a.head.metrics.outOfOrderSamples.Inc() + a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc() } if chunkCreated { a.head.metrics.chunks.Inc() @@ -736,7 +736,7 @@ func (a *headAppender) Commit() (err error) { } } - total += len(a.histograms) // TODO: different metric? + histogramsTotal := len(a.histograms) for i, s := range a.histograms { series = a.histogramSeries[i] series.Lock() @@ -745,11 +745,9 @@ func (a *headAppender) Commit() (err error) { series.pendingCommit = false series.Unlock() - if ok { - a.head.metrics.histogramSamplesTotal.Inc() - } else { - total-- - a.head.metrics.outOfOrderSamples.Inc() + if !ok { + histogramsTotal-- + a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc() } if chunkCreated { a.head.metrics.chunks.Inc() @@ -764,7 +762,8 @@ func (a *headAppender) Commit() (err error) { series.Unlock() } - a.head.metrics.samplesAppended.Add(float64(total)) + a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(total)) + a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsTotal)) a.head.updateMinMaxTime(a.mint, a.maxt) return nil diff --git a/tsdb/head_test.go b/tsdb/head_test.go index e747885b9..16d667f17 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -2131,19 +2131,19 @@ func TestOutOfOrderSamplesMetric(t *testing.T) { require.NoError(t, app.Commit()) // Test out of order metric. - require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) app = db.Appender(ctx) _, err = app.Append(0, labels.FromStrings("a", "b"), 2, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) _, err = app.Append(0, labels.FromStrings("a", "b"), 3, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) _, err = app.Append(0, labels.FromStrings("a", "b"), 4, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) require.NoError(t, app.Commit()) // Compact Head to test out of bound metric. @@ -2159,11 +2159,11 @@ func TestOutOfOrderSamplesMetric(t *testing.T) { app = db.Appender(ctx) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-2, 99) require.Equal(t, storage.ErrOutOfBounds, err) - require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples)) + require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat))) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-1, 99) require.Equal(t, storage.ErrOutOfBounds, err) - require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples)) + require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat))) require.NoError(t, app.Commit()) // Some more valid samples for out of order. @@ -2178,15 +2178,15 @@ func TestOutOfOrderSamplesMetric(t *testing.T) { app = db.Appender(ctx) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+2, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+3, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+4, 99) require.Equal(t, storage.ErrOutOfOrderSample, err) - require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) + require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))) require.NoError(t, app.Commit()) } @@ -3267,7 +3267,7 @@ func TestHistogramMetrics(t *testing.T) { } } - require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.histogramSamplesTotal)) + require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) require.NoError(t, head.Close()) w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false) @@ -3276,7 +3276,7 @@ func TestHistogramMetrics(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.histogramSamplesTotal)) // Counter reset. + require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) // Counter reset. } func TestHistogramStaleSample(t *testing.T) {