Instrument received alerts
This instruments the number of received alerts and makes handling of invalid alerts more graceful.
This commit is contained in:
parent
1eb64b4263
commit
b2fb6c8ab0
49
api.go
49
api.go
|
@ -32,6 +32,25 @@ import (
|
||||||
"github.com/prometheus/alertmanager/version"
|
"github.com/prometheus/alertmanager/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
numReceivedAlerts = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Namespace: "alertmanager",
|
||||||
|
Name: "alerts_received_total",
|
||||||
|
Help: "The total number of received alerts.",
|
||||||
|
}, []string{"status"})
|
||||||
|
|
||||||
|
numInvalidAlerts = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Namespace: "alertmanager",
|
||||||
|
Name: "invalid_alerts_total",
|
||||||
|
Help: "The total number of received alerts that were invalid.",
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
prometheus.Register(numReceivedAlerts)
|
||||||
|
prometheus.Register(numInvalidAlerts)
|
||||||
|
}
|
||||||
|
|
||||||
// API provides registration of handlers for API routes.
|
// API provides registration of handlers for API routes.
|
||||||
type API struct {
|
type API struct {
|
||||||
alerts provider.Alerts
|
alerts provider.Alerts
|
||||||
|
@ -208,30 +227,48 @@ func (api *API) insertAlerts(w http.ResponseWriter, r *http.Request, alerts ...*
|
||||||
for _, alert := range alerts {
|
for _, alert := range alerts {
|
||||||
alert.UpdatedAt = now
|
alert.UpdatedAt = now
|
||||||
|
|
||||||
|
// Ensure StartsAt is set.
|
||||||
if alert.StartsAt.IsZero() {
|
if alert.StartsAt.IsZero() {
|
||||||
alert.StartsAt = now
|
alert.StartsAt = now
|
||||||
}
|
}
|
||||||
|
// If no end time is defined, set a timeout after which an alert
|
||||||
|
// is marked resolved if it is not updated.
|
||||||
if alert.EndsAt.IsZero() {
|
if alert.EndsAt.IsZero() {
|
||||||
alert.Timeout = true
|
alert.Timeout = true
|
||||||
alert.EndsAt = alert.StartsAt.Add(api.resolveTimeout)
|
alert.EndsAt = alert.StartsAt.Add(api.resolveTimeout)
|
||||||
|
|
||||||
|
numReceivedAlerts.WithLabelValues("firing").Inc()
|
||||||
|
} else {
|
||||||
|
numReceivedAlerts.WithLabelValues("resolved").Inc()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only validate after we've ensured that StartsAt is set.
|
// Make a best effort to insert all alerts that are valid.
|
||||||
|
var (
|
||||||
|
validAlerts = make([]*types.Alert, 0, len(alerts))
|
||||||
|
validationErrs = &types.MultiError{}
|
||||||
|
)
|
||||||
for _, a := range alerts {
|
for _, a := range alerts {
|
||||||
if err := a.Validate(); err != nil {
|
if err := a.Validate(); err != nil {
|
||||||
|
validationErrs.Add(err)
|
||||||
|
numInvalidAlerts.Inc()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
validAlerts = append(validAlerts, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := api.alerts.Put(validAlerts...); err != nil {
|
||||||
respondError(w, apiError{
|
respondError(w, apiError{
|
||||||
typ: errorBadData,
|
typ: errorInternal,
|
||||||
err: err,
|
err: err,
|
||||||
}, nil)
|
}, nil)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if err := api.alerts.Put(alerts...); err != nil {
|
if validationErrs.Len() > 0 {
|
||||||
respondError(w, apiError{
|
respondError(w, apiError{
|
||||||
typ: errorInternal,
|
typ: errorBadData,
|
||||||
err: err,
|
err: validationErrs,
|
||||||
}, nil)
|
}, nil)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue