Add debug logs for muted alerts (#3558)

This commit adds debug logs to MuteStage that logs when an alert
is muted. This can help operators root cause missing notifications
when alerts are silenced by mistake or purpose but then forgotten
about.

Signed-off-by: George Robinson <george.robinson@grafana.com>
This commit is contained in:
George Robinson 2023-10-24 18:00:38 +01:00 committed by GitHub
parent 280a742b76
commit d2501323d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -518,16 +518,24 @@ func NewMuteStage(m types.Muter) *MuteStage {
}
// Exec implements the Stage interface.
func (n *MuteStage) Exec(ctx context.Context, _ log.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {
var filtered []*types.Alert
func (n *MuteStage) Exec(ctx context.Context, logger log.Logger, alerts ...*types.Alert) (context.Context, []*types.Alert, error) {
var (
filtered []*types.Alert
muted []*types.Alert
)
for _, a := range alerts {
// TODO(fabxc): increment total alerts counter.
// Do not send the alert if muted.
if !n.muter.Mutes(a.Labels) {
if n.muter.Mutes(a.Labels) {
muted = append(muted, a)
} else {
filtered = append(filtered, a)
}
// TODO(fabxc): increment muted alerts counter if muted.
}
if len(muted) > 0 {
level.Debug(logger).Log("msg", "Notifications will not be sent for muted alerts", "alerts", fmt.Sprintf("%v", muted))
}
return ctx, filtered, nil
}