From f70b52d0b6d91bfb19fc45ac5ef7872d2ba78064 Mon Sep 17 00:00:00 2001 From: Jeremy Meulemans Date: Thu, 16 Feb 2017 17:55:42 -0600 Subject: [PATCH 1/3] Adding gauge for number of open head chunks. Fixes #1710 --- storage/local/storage.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/storage/local/storage.go b/storage/local/storage.go index 25dc30b0a..81e1fd730 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -180,6 +180,7 @@ type MemorySeriesStorage struct { persistErrors prometheus.Counter queuedChunksToPersist prometheus.Counter numSeries prometheus.Gauge + numHeadChunks prometheus.Gauge dirtySeries prometheus.Gauge seriesOps *prometheus.CounterVec ingestedSamplesCount prometheus.Counter @@ -254,6 +255,12 @@ func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) *MemorySeriesStorage Name: "memory_series", Help: "The current number of series in memory.", }), + numHeadChunks: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "num_head_chunks_open", + Help: "The current number of open head chunks.", + }), dirtySeries: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, @@ -372,6 +379,16 @@ func (s *MemorySeriesStorage) Start() (err error) { log.Info("Loading series map and head chunks...") s.fpToSeries, s.numChunksToPersist, err = p.loadSeriesMapAndHeads() + openHeadCount := 0 + for fp := range s.fpToSeries.fpIter() { + if series, ok := s.fpToSeries.get(fp); ok { + if !series.headChunkClosed { + openHeadCount += 1 + } + } + } + s.numHeadChunks.Set(float64(openHeadCount)) + if err != nil { return err } @@ -977,6 +994,9 @@ func (s *MemorySeriesStorage) getOrCreateSeries(fp model.Fingerprint, m model.Me } s.fpToSeries.put(fp, series) s.numSeries.Inc() + if !series.headChunkClosed { + s.numHeadChunks.Inc() + } } return series, nil } @@ -1383,6 +1403,7 @@ func (s *MemorySeriesStorage) maintainMemorySeries( } if closed { s.incNumChunksToPersist(1) + s.numHeadChunks.Dec() } seriesWasDirty := series.dirty @@ -1783,6 +1804,7 @@ func (s *MemorySeriesStorage) Describe(ch chan<- *prometheus.Desc) { ch <- maxChunksToPersistDesc ch <- numChunksToPersistDesc ch <- s.numSeries.Desc() + ch <- s.numHeadChunks.Desc() ch <- s.dirtySeries.Desc() s.seriesOps.Describe(ch) ch <- s.ingestedSamplesCount.Desc() @@ -1812,6 +1834,7 @@ func (s *MemorySeriesStorage) Collect(ch chan<- prometheus.Metric) { float64(s.getNumChunksToPersist()), ) ch <- s.numSeries + ch <- s.numHeadChunks ch <- s.dirtySeries s.seriesOps.Collect(ch) ch <- s.ingestedSamplesCount From 074050b8c053e65f383a9d59541191379bef1d42 Mon Sep 17 00:00:00 2001 From: Jeremy Meulemans Date: Thu, 16 Feb 2017 18:04:28 -0600 Subject: [PATCH 2/3] Updating for failed codeclimate check. --- storage/local/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/local/storage.go b/storage/local/storage.go index 81e1fd730..3945509ec 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -383,7 +383,7 @@ func (s *MemorySeriesStorage) Start() (err error) { for fp := range s.fpToSeries.fpIter() { if series, ok := s.fpToSeries.get(fp); ok { if !series.headChunkClosed { - openHeadCount += 1 + openHeadCount++ } } } From 025c828976b0ba92bb55a88dfee5a80572e72a78 Mon Sep 17 00:00:00 2001 From: Jeremy Meulemans Date: Fri, 17 Feb 2017 07:10:13 -0600 Subject: [PATCH 3/3] Changed to open_head_chunks to address review. Now incrementing numHeadChunks directly. --- storage/local/storage.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/local/storage.go b/storage/local/storage.go index 3945509ec..fa9001702 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -258,7 +258,7 @@ func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) *MemorySeriesStorage numHeadChunks: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "num_head_chunks_open", + Name: "open_head_chunks", Help: "The current number of open head chunks.", }), dirtySeries: prometheus.NewGauge(prometheus.GaugeOpts{ @@ -379,15 +379,13 @@ func (s *MemorySeriesStorage) Start() (err error) { log.Info("Loading series map and head chunks...") s.fpToSeries, s.numChunksToPersist, err = p.loadSeriesMapAndHeads() - openHeadCount := 0 for fp := range s.fpToSeries.fpIter() { if series, ok := s.fpToSeries.get(fp); ok { if !series.headChunkClosed { - openHeadCount++ + s.numHeadChunks.Inc() } } } - s.numHeadChunks.Set(float64(openHeadCount)) if err != nil { return err