diff --git a/collectors/monitors.go b/collectors/monitors.go index 9f44981..4c521e0 100644 --- a/collectors/monitors.go +++ b/collectors/monitors.go @@ -271,6 +271,14 @@ func (m *MonitorCollector) collect() error { return err } + // Reset daemon specifc metrics; daemons can leave the cluster + m.TotalKBs.Reset() + m.UsedKBs.Reset() + m.AvailKBs.Reset() + m.PercentAvail.Reset() + m.Latency.Reset() + m.ClockSkew.Reset() + for _, healthService := range stats.Health.Health.HealthServices { for _, monstat := range healthService.Mons { kbTotal, err := monstat.KBTotal.Float64() diff --git a/collectors/osd.go b/collectors/osd.go index 876f252..f3bb60b 100644 --- a/collectors/osd.go +++ b/collectors/osd.go @@ -700,6 +700,22 @@ func (o *OSDCollector) Describe(ch chan<- *prometheus.Desc) { // Collect sends all the collected metrics to the provided prometheus channel. // It requires the caller to handle synchronization. func (o *OSDCollector) Collect(ch chan<- prometheus.Metric) { + + // Reset daemon specifc metrics; daemons can leave the cluster + o.CrushWeight.Reset() + o.Depth.Reset() + o.Reweight.Reset() + o.Bytes.Reset() + o.UsedBytes.Reset() + o.AvailBytes.Reset() + o.Utilization.Reset() + o.Variance.Reset() + o.Pgs.Reset() + o.CommitLatency.Reset() + o.ApplyLatency.Reset() + o.OSDIn.Reset() + o.OSDUp.Reset() + if err := o.collectOSDPerf(); err != nil { log.Println("failed collecting osd perf stats:", err) } diff --git a/collectors/pool_usage.go b/collectors/pool_usage.go index 9924ac6..3ae1ccb 100644 --- a/collectors/pool_usage.go +++ b/collectors/pool_usage.go @@ -210,6 +210,17 @@ func (p *PoolUsageCollector) collect() error { return err } + // Reset pool specfic metrics, pools can be removed + p.UsedBytes.Reset() + p.RawUsedBytes.Reset() + p.MaxAvail.Reset() + p.Objects.Reset() + p.DirtyObjects.Reset() + p.ReadIO.Reset() + p.ReadBytes.Reset() + p.WriteIO.Reset() + p.WriteBytes.Reset() + for _, pool := range stats.Pools { p.UsedBytes.WithLabelValues(pool.Name).Set(pool.Stats.BytesUsed) p.RawUsedBytes.WithLabelValues(pool.Name).Set(pool.Stats.RawBytesUsed)