[ENHANCEMENT] Alerts: remove metrics for removed Alertmanagers (#13909)

* [ENHANCEMENT] Alerts: remove metrics for removed Alertmanagers

So they don't continue to report stale values.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
This commit is contained in:
Bryan Boreham 2024-09-26 15:32:18 +01:00 committed by GitHub
parent 751100b3d0
commit 5710ddf24f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -770,6 +770,7 @@ func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) {
s.mtx.Lock()
defer s.mtx.Unlock()
previousAms := s.ams
// Set new Alertmanagers and deduplicate them along their unique URL.
s.ams = []alertmanager{}
s.droppedAms = []alertmanager{}
@ -789,6 +790,17 @@ func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) {
seen[us] = struct{}{}
s.ams = append(s.ams, am)
}
// Now remove counters for any removed Alertmanagers.
for _, am := range previousAms {
us := am.url().String()
if _, ok := seen[us]; ok {
continue
}
s.metrics.latency.DeleteLabelValues(us)
s.metrics.sent.DeleteLabelValues(us)
s.metrics.errors.DeleteLabelValues(us)
seen[us] = struct{}{}
}
}
func postPath(pre string, v config.AlertmanagerAPIVersion) string {