From b2fb6c8ab0ea4d996fc4763de3868aecb478bca3 Mon Sep 17 00:00:00 2001
From: Fabian Reinartz <fab.reinartz@gmail.com>
Date: Sat, 9 Jan 2016 13:29:02 +0100
Subject: [PATCH] Instrument received alerts

This instruments the number of received alerts and makes
handling of invalid alerts more graceful.
---
 api.go | 51 ++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 44 insertions(+), 7 deletions(-)

diff --git a/api.go b/api.go
index 5935a61a..45bbfbe3 100644
--- a/api.go
+++ b/api.go
@@ -32,6 +32,25 @@ import (
 	"github.com/prometheus/alertmanager/version"
 )
 
+var (
+	numReceivedAlerts = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "alertmanager",
+		Name:      "alerts_received_total",
+		Help:      "The total number of received alerts.",
+	}, []string{"status"})
+
+	numInvalidAlerts = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "alertmanager",
+		Name:      "invalid_alerts_total",
+		Help:      "The total number of received alerts that were invalid.",
+	})
+)
+
+func init() {
+	prometheus.Register(numReceivedAlerts)
+	prometheus.Register(numInvalidAlerts)
+}
+
 // API provides registration of handlers for API routes.
 type API struct {
 	alerts         provider.Alerts
@@ -208,27 +227,37 @@ func (api *API) insertAlerts(w http.ResponseWriter, r *http.Request, alerts ...*
 	for _, alert := range alerts {
 		alert.UpdatedAt = now
 
+		// Ensure StartsAt is set.
 		if alert.StartsAt.IsZero() {
 			alert.StartsAt = now
 		}
+		// If no end time is defined, set a timeout after which an alert
+		// is marked resolved if it is not updated.
 		if alert.EndsAt.IsZero() {
 			alert.Timeout = true
 			alert.EndsAt = alert.StartsAt.Add(api.resolveTimeout)
+
+			numReceivedAlerts.WithLabelValues("firing").Inc()
+		} else {
+			numReceivedAlerts.WithLabelValues("resolved").Inc()
 		}
 	}
 
-	// Only validate after we've ensured that StartsAt is set.
+	// Make a best effort to insert all alerts that are valid.
+	var (
+		validAlerts    = make([]*types.Alert, 0, len(alerts))
+		validationErrs = &types.MultiError{}
+	)
 	for _, a := range alerts {
 		if err := a.Validate(); err != nil {
-			respondError(w, apiError{
-				typ: errorBadData,
-				err: err,
-			}, nil)
-			return
+			validationErrs.Add(err)
+			numInvalidAlerts.Inc()
+			continue
 		}
+		validAlerts = append(validAlerts, a)
 	}
 
-	if err := api.alerts.Put(alerts...); err != nil {
+	if err := api.alerts.Put(validAlerts...); err != nil {
 		respondError(w, apiError{
 			typ: errorInternal,
 			err: err,
@@ -236,6 +265,14 @@ func (api *API) insertAlerts(w http.ResponseWriter, r *http.Request, alerts ...*
 		return
 	}
 
+	if validationErrs.Len() > 0 {
+		respondError(w, apiError{
+			typ: errorBadData,
+			err: validationErrs,
+		}, nil)
+		return
+	}
+
 	respond(w, nil)
 }