diff --git a/README.md b/README.md index c8ab647c..8e946b8f 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ following aspects: * aggregating alerts by labelset * handling notification repeats * sending alert notifications via external services (currently email, +generic web hook, [PagerDuty](http://www.pagerduty.com/), [HipChat](http://www.hipchat.com/), [Slack](http://www.slack.com/), diff --git a/config/config.proto b/config/config.proto index 5e97cc77..80d6d2fc 100644 --- a/config/config.proto +++ b/config/config.proto @@ -58,7 +58,7 @@ message HipChatConfig { // Configuration for notification via Slack. message SlackConfig { - // Slack webhook url, (https://api.slack.com/incoming-webhooks). + // Slack webhook URL, (https://api.slack.com/incoming-webhooks). optional string webhook_url = 1; // Slack channel override, (like #other-channel or @username). optional string channel = 2; @@ -70,6 +70,7 @@ message SlackConfig { optional bool send_resolved = 5 [default = false]; } +// Configuration for notification via Flowdock. message FlowdockConfig { // Flowdock flow API token. optional string api_token = 1; @@ -81,6 +82,14 @@ message FlowdockConfig { optional bool send_resolved = 4 [default = false]; } +// Configuration for notification via generic webhook. +message WebhookConfig { + // URL to send POST request to. + optional string url = 1; + // Notify when resolved. + optional bool send_resolved = 2 [default = false]; +} + // Notification configuration definition. message NotificationConfig { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -97,6 +106,8 @@ message NotificationConfig { repeated SlackConfig slack_config = 6; // Zero or more Flowdock notification configurations. repeated FlowdockConfig flowdock_config = 7; + // Zero or more generic web hook notification configurations. + repeated WebhookConfig webhook_config = 8; } // A regex-based label filter used in aggregations. diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index d4289ccd..8cf8733e 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -15,6 +15,7 @@ It has these top-level messages: HipChatConfig SlackConfig FlowdockConfig + WebhookConfig NotificationConfig Filter AggregationRule @@ -186,7 +187,7 @@ func (m *HipChatConfig) GetSendResolved() bool { // Configuration for notification via Slack. type SlackConfig struct { - // Slack webhook url, (https://api.slack.com/incoming-webhooks). + // Slack webhook URL, (https://api.slack.com/incoming-webhooks). WebhookUrl *string `protobuf:"bytes,1,opt,name=webhook_url" json:"webhook_url,omitempty"` // Slack channel override, (like #other-channel or @username). Channel *string `protobuf:"bytes,2,opt,name=channel" json:"channel,omitempty"` @@ -242,6 +243,7 @@ func (m *SlackConfig) GetSendResolved() bool { return Default_SlackConfig_SendResolved } +// Configuration for notification via Flowdock. type FlowdockConfig struct { // Flowdock flow API token. ApiToken *string `protobuf:"bytes,1,opt,name=api_token" json:"api_token,omitempty"` @@ -288,6 +290,35 @@ func (m *FlowdockConfig) GetSendResolved() bool { return Default_FlowdockConfig_SendResolved } +// Configuration for notification via generic webhook. +type WebhookConfig struct { + // URL to send POST request to. + Url *string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,2,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *WebhookConfig) Reset() { *m = WebhookConfig{} } +func (m *WebhookConfig) String() string { return proto.CompactTextString(m) } +func (*WebhookConfig) ProtoMessage() {} + +const Default_WebhookConfig_SendResolved bool = false + +func (m *WebhookConfig) GetUrl() string { + if m != nil && m.Url != nil { + return *m.Url + } + return "" +} + +func (m *WebhookConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_WebhookConfig_SendResolved +} + // Notification configuration definition. type NotificationConfig struct { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -303,8 +334,10 @@ type NotificationConfig struct { // Zero or more slack notification configurations. SlackConfig []*SlackConfig `protobuf:"bytes,6,rep,name=slack_config" json:"slack_config,omitempty"` // Zero or more Flowdock notification configurations. - FlowdockConfig []*FlowdockConfig `protobuf:"bytes,7,rep,name=flowdock_config" json:"flowdock_config,omitempty"` - XXX_unrecognized []byte `json:"-"` + FlowdockConfig []*FlowdockConfig `protobuf:"bytes,7,rep,name=flowdock_config" json:"flowdock_config,omitempty"` + // Zero or more generic web hook notification configurations. + WebhookConfig []*WebhookConfig `protobuf:"bytes,8,rep,name=webhook_config" json:"webhook_config,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *NotificationConfig) Reset() { *m = NotificationConfig{} } @@ -360,6 +393,13 @@ func (m *NotificationConfig) GetFlowdockConfig() []*FlowdockConfig { return nil } +func (m *NotificationConfig) GetWebhookConfig() []*WebhookConfig { + if m != nil { + return m.WebhookConfig + } + return nil +} + // A regex-based label filter used in aggregations. type Filter struct { // The regex matching the label name. diff --git a/main.go b/main.go index c3ec0654..106dc7e0 100644 --- a/main.go +++ b/main.go @@ -57,7 +57,7 @@ func main() { } saveSilencesTicker := time.NewTicker(10 * time.Second) go func() { - for _ = range saveSilencesTicker.C { + for range saveSilencesTicker.C { if err := silencer.SaveToFile(*silencesFile); err != nil { log.Error("Error saving silences to file: ", err) } diff --git a/manager/alert.go b/manager/alert.go index 73182acd..a3f4dd81 100644 --- a/manager/alert.go +++ b/manager/alert.go @@ -33,14 +33,14 @@ type Alerts []*Alert // Alert models an action triggered by Prometheus. type Alert struct { // Short summary of alert. - Summary string + Summary string `json:"summary"` // Long description of alert. - Description string + Description string `json:"description"` // Label value pairs for purpose of aggregation, matching, and disposition // dispatching. This must minimally include an "alertname" label. - Labels AlertLabelSet + Labels AlertLabelSet `json:"labels"` // Extra key/value information which is not used for aggregation. - Payload AlertPayload + Payload AlertPayload `json:"payload"` } func (a *Alert) Name() string { diff --git a/manager/manager.go b/manager/manager.go index 178f6194..294a46b7 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -403,7 +403,7 @@ func (s *memoryAlertManager) runIteration() { // Run the memoryAlertManager's main dispatcher loop. func (s *memoryAlertManager) Run() { iterationTicker := time.NewTicker(time.Second) - for _ = range iterationTicker.C { + for range iterationTicker.C { s.checkSanity() s.runIteration() } diff --git a/manager/notifier.go b/manager/notifier.go index ce6b0453..33e16b80 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -374,6 +374,40 @@ func newFlowdockMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) return msg } +type webhookMessage struct { + Version string `json:"version"` + Status string `json:"status"` + Alerts []Alert `json:"alert"` +} + +func (n *notifier) sendWebhookNotification(op notificationOp, config *pb.WebhookConfig, a *Alert) error { + status := "" + switch op { + case notificationOpTrigger: + status = "firing" + case notificationOpResolve: + status = "resolved" + } + + msg := &webhookMessage{ + Version: "1", + Status: status, + Alerts: []Alert{*a}, + } + jsonMessage, err := json.Marshal(msg) + if err != nil { + return err + } + httpResponse, err := postJSON(jsonMessage, config.GetUrl()) + if err != nil { + return err + } + if err := processResponse(httpResponse, "Webhook", a); err != nil { + return err + } + return nil +} + func postJSON(jsonMessage []byte, url string) (*http.Response, error) { timeout := time.Duration(5 * time.Second) client := http.Client{ @@ -582,6 +616,14 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No log.Errorln("Error sending Flowdock notification:", err) } } + for _, whConfig := range config.WebhookConfig { + if op == notificationOpResolve && !whConfig.GetSendResolved() { + continue + } + if err := n.sendWebhookNotification(op, whConfig, a); err != nil { + log.Errorln("Error sending Webhook notification:", err) + } + } } func (n *notifier) Dispatch() { diff --git a/manager/notifier_test.go b/manager/notifier_test.go index e73f6505..e538b56a 100644 --- a/manager/notifier_test.go +++ b/manager/notifier_test.go @@ -16,10 +16,17 @@ package manager import ( "bytes" "crypto/tls" + "encoding/json" "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" "os" + "reflect" "testing" "time" + + pb "github.com/prometheus/alertmanager/config/generated" ) func TestWriteEmailBody(t *testing.T) { @@ -169,3 +176,48 @@ func TestGetSMTPAuth(t *testing.T) { t.Errorf("PLAIN auth with bad host-port: expected error but got %T, %v", auth, cfg) } } + +func TestSendWebhookNotification(t *testing.T) { + var body []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + body, err = ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("error reading webhook notification: %s", err) + } + })) + defer ts.Close() + + config := &pb.WebhookConfig{ + Url: &ts.URL, + } + alert := &Alert{ + Summary: "Testsummary", + Description: "Test alert description, something went wrong here.", + Labels: AlertLabelSet{ + "alertname": "TestAlert", + }, + Payload: AlertPayload{ + "payload_label1": "payload_value1", + }, + } + n := ¬ifier{} + err := n.sendWebhookNotification(notificationOpTrigger, config, alert) + if err != nil { + t.Errorf("error sending webhook notification: %s", err) + } + + var msg webhookMessage + err = json.Unmarshal(body, &msg) + if err != nil { + t.Errorf("error unmarshalling webhook notification: %s", err) + } + expected := webhookMessage{ + Version: "1", + Status: "firing", + Alerts: []Alert{*alert}, + } + if !reflect.DeepEqual(msg, expected) { + t.Errorf("incorrect webhook notification: Expected: %s Actual: %s", expected, msg) + } +} diff --git a/web/api/api.go b/web/api/api.go index ab3e1164..8fb70592 100644 --- a/web/api/api.go +++ b/web/api/api.go @@ -25,19 +25,19 @@ import ( ) type AlertManagerService struct { - Manager manager.AlertManager - Silencer *manager.Silencer + Manager manager.AlertManager + Silencer *manager.Silencer PathPrefix string } func (s AlertManagerService) Handler() http.Handler { r := httprouter.New() - r.POST(s.PathPrefix + "api/alerts", s.addAlerts) - r.GET(s.PathPrefix + "api/silences", s.silenceSummary) - r.POST(s.PathPrefix + "api/silences", s.addSilence) - r.GET(s.PathPrefix + "api/silences/:id", s.getSilence) - r.POST(s.PathPrefix + "api/silences/:id", s.updateSilence) - r.DELETE(s.PathPrefix + "api/silences/:id", s.deleteSilence) + r.POST(s.PathPrefix+"api/alerts", s.addAlerts) + r.GET(s.PathPrefix+"api/silences", s.silenceSummary) + r.POST(s.PathPrefix+"api/silences", s.addSilence) + r.GET(s.PathPrefix+"api/silences/:id", s.getSilence) + r.POST(s.PathPrefix+"api/silences/:id", s.updateSilence) + r.DELETE(s.PathPrefix+"api/silences/:id", s.deleteSilence) return r } diff --git a/web/silences.go b/web/silences.go index 7387e509..facfce8a 100644 --- a/web/silences.go +++ b/web/silences.go @@ -24,7 +24,7 @@ type SilenceStatus struct { } type SilencesHandler struct { - Silencer *manager.Silencer + Silencer *manager.Silencer PathPrefix string }