Fix race condition causing 1st alert to not be immediately delivered when group_wait is 0s
Signed-off-by: Marco Pracucci <marco@pracucci.com>
This commit is contained in:
parent
1f3796c5cc
commit
72ef6e04e1
|
@ -290,30 +290,36 @@ func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) {
|
||||||
d.aggrGroups[route] = group
|
d.aggrGroups[route] = group
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the group does not exist, create it.
|
|
||||||
ag, ok := group[fp]
|
ag, ok := group[fp]
|
||||||
if !ok {
|
if ok {
|
||||||
ag = newAggrGroup(d.ctx, groupLabels, route, d.timeout, d.logger)
|
ag.insert(alert)
|
||||||
group[fp] = ag
|
return
|
||||||
d.metrics.aggrGroups.Inc()
|
|
||||||
|
|
||||||
go ag.run(func(ctx context.Context, alerts ...*types.Alert) bool {
|
|
||||||
_, _, err := d.stage.Exec(ctx, d.logger, alerts...)
|
|
||||||
if err != nil {
|
|
||||||
lvl := level.Error(d.logger)
|
|
||||||
if ctx.Err() == context.Canceled {
|
|
||||||
// It is expected for the context to be canceled on
|
|
||||||
// configuration reload or shutdown. In this case, the
|
|
||||||
// message should only be logged at the debug level.
|
|
||||||
lvl = level.Debug(d.logger)
|
|
||||||
}
|
|
||||||
lvl.Log("msg", "Notify for alerts failed", "num_alerts", len(alerts), "err", err)
|
|
||||||
}
|
|
||||||
return err == nil
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the group does not exist, create it.
|
||||||
|
ag = newAggrGroup(d.ctx, groupLabels, route, d.timeout, d.logger)
|
||||||
|
group[fp] = ag
|
||||||
|
d.metrics.aggrGroups.Inc()
|
||||||
|
|
||||||
|
// Insert the 1st alert in the group before starting the group's run()
|
||||||
|
// function, to make sure that when the run() will be executed the 1st
|
||||||
|
// alert is already there.
|
||||||
ag.insert(alert)
|
ag.insert(alert)
|
||||||
|
|
||||||
|
go ag.run(func(ctx context.Context, alerts ...*types.Alert) bool {
|
||||||
|
_, _, err := d.stage.Exec(ctx, d.logger, alerts...)
|
||||||
|
if err != nil {
|
||||||
|
lvl := level.Error(d.logger)
|
||||||
|
if ctx.Err() == context.Canceled {
|
||||||
|
// It is expected for the context to be canceled on
|
||||||
|
// configuration reload or shutdown. In this case, the
|
||||||
|
// message should only be logged at the debug level.
|
||||||
|
lvl = level.Debug(d.logger)
|
||||||
|
}
|
||||||
|
lvl.Log("msg", "Notify for alerts failed", "num_alerts", len(alerts), "err", err)
|
||||||
|
}
|
||||||
|
return err == nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func getGroupLabels(alert *types.Alert, route *Route) model.LabelSet {
|
func getGroupLabels(alert *types.Alert, route *Route) model.LabelSet {
|
||||||
|
|
Loading…
Reference in New Issue