collectors: Reset metric vectors; pools and daemons can vanish

Signed-off-by: Jan Fajerski <jfajerski@suse.com>

collectors/monitors.go

@@ -271,6 +271,14 @@ func (m *MonitorCollector) collect() error {
 		return err
 	}
 
+        // Reset daemon specifc metrics; daemons can leave the cluster
+        m.TotalKBs.Reset()
+        m.UsedKBs.Reset()
+        m.AvailKBs.Reset()
+        m.PercentAvail.Reset()
+        m.Latency.Reset()
+        m.ClockSkew.Reset()
+
 	for _, healthService := range stats.Health.Health.HealthServices {
 		for _, monstat := range healthService.Mons {
 			kbTotal, err := monstat.KBTotal.Float64()

collectors/osd.go

@@ -700,6 +700,22 @@ func (o *OSDCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect sends all the collected metrics to the provided prometheus channel.
 // It requires the caller to handle synchronization.
 func (o *OSDCollector) Collect(ch chan<- prometheus.Metric) {
+
+        // Reset daemon specifc metrics; daemons can leave the cluster
+        o.CrushWeight.Reset()
+        o.Depth.Reset()
+        o.Reweight.Reset()
+        o.Bytes.Reset()
+        o.UsedBytes.Reset()
+        o.AvailBytes.Reset()
+        o.Utilization.Reset()
+        o.Variance.Reset()
+        o.Pgs.Reset()
+        o.CommitLatency.Reset()
+        o.ApplyLatency.Reset()
+        o.OSDIn.Reset()
+        o.OSDUp.Reset()
+
 	if err := o.collectOSDPerf(); err != nil {
 		log.Println("failed collecting osd perf stats:", err)
 	}

collectors/pool_usage.go

@@ -210,6 +210,17 @@ func (p *PoolUsageCollector) collect() error {
 		return err
 	}
 
+        // Reset pool specfic metrics, pools can be removed
+        p.UsedBytes.Reset()
+        p.RawUsedBytes.Reset()
+        p.MaxAvail.Reset()
+        p.Objects.Reset()
+        p.DirtyObjects.Reset()
+        p.ReadIO.Reset()
+        p.ReadBytes.Reset()
+        p.WriteIO.Reset()
+        p.WriteBytes.Reset()
+
 	for _, pool := range stats.Pools {
 		p.UsedBytes.WithLabelValues(pool.Name).Set(pool.Stats.BytesUsed)
 		p.RawUsedBytes.WithLabelValues(pool.Name).Set(pool.Stats.RawBytesUsed)