From a0c156e3e65c5588176d6d4e1418d96b105abb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Lindstr=C3=B6m?= Date: Mon, 16 Feb 2015 20:11:00 +0000 Subject: [PATCH] add pushover as notification method This adds https://pushover.net/ as a way of notification. --- README.md | 4 +- config/config.go | 8 ++ config/config.proto | 10 ++ config/fixtures/sample.conf.input | 4 + config/generated/config.pb.go | 148 ++++++++++++++++++++++++++---- manager/notifier.go | 26 ++++++ 6 files changed, 183 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index aceebc5c..351dede1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ following aspects: * inhibiting alerts based on alert dependencies * aggregating alerts by labelset * handling notification repeats -* sending alert notifications via external services (currently email or [PagerDuty](http://www.pagerduty.com/)) +* sending alert notifications via external services (currently email, +[PagerDuty](http://www.pagerduty.com/) or +[Pushover](https://www.pushover.net/)) See [config/fixtures/sample.conf.input](config/fixtures/sample.conf.input) for an example config. The full configuration schema including a documentation for diff --git a/config/config.go b/config/config.go index 94129ece..cd3e1cc0 100644 --- a/config/config.go +++ b/config/config.go @@ -56,6 +56,14 @@ func (c Config) Validate() error { return fmt.Errorf("Missing email address in email notification config: %s", proto.MarshalTextString(ec)) } } + for _, ec := range nc.PushoverConfig { + if ec.Token == nil { + return fmt.Errorf("Missing token in Pushover notification config: %s", proto.MarshalTextString(ec)) + } + if ec.UserKey == nil { + return fmt.Errorf("Missing user key in Pushover notification config: %s", proto.MarshalTextString(ec)) + } + } if _, ok := ncNames[nc.GetName()]; ok { return fmt.Errorf("Notification config name not unique: %s", nc.GetName()) diff --git a/config/config.proto b/config/config.proto index 7eb8e7a5..3661b32e 100644 --- a/config/config.proto +++ b/config/config.proto @@ -26,6 +26,14 @@ message EmailConfig { optional string email = 1; } +// Configuration for notification via pushover.net. +message PushoverConfig { + // Pushover token + optional string token = 1; + // Pushover user_key + optional string user_key = 2; +} + // Notification configuration definition. message NotificationConfig { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -34,6 +42,8 @@ message NotificationConfig { repeated PagerDutyConfig pagerduty_config = 2; // Zero or more email notification configurations. repeated EmailConfig email_config = 3; + // Zero or more pushover notification configurations. + repeated PushoverConfig pushover_config = 4; } // A regex-based label filter used in aggregations. diff --git a/config/fixtures/sample.conf.input b/config/fixtures/sample.conf.input index fcf5119d..d343b233 100644 --- a/config/fixtures/sample.conf.input +++ b/config/fixtures/sample.conf.input @@ -6,6 +6,10 @@ notification_config { email_config { email: "test@testservice.org" } + pushover_config { + token: "mypushovertoken" + user_key: "mypushoverkey" + } } aggregation_rule { diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index adcce7b3..cd2ed8ef 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -13,7 +13,10 @@ var _ = proto.Marshal var _ = &json.SyntaxError{} var _ = math.Inf +// Configuration for notification via PagerDuty. type PagerDutyConfig struct { + // PagerDuty service key, see: + // http://developer.pagerduty.com/documentation/integration/events ServiceKey *string `protobuf:"bytes,1,opt,name=service_key" json:"service_key,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -29,7 +32,9 @@ func (m *PagerDutyConfig) GetServiceKey() string { return "" } +// Configuration for notification via mail. type EmailConfig struct { + // Email address to notify. Email *string `protobuf:"bytes,1,opt,name=email" json:"email,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -45,11 +50,44 @@ func (m *EmailConfig) GetEmail() string { return "" } +// Configuration for notification via pushover.net. +type PushoverConfig struct { + // Pushover token + Token *string `protobuf:"bytes,1,opt,name=token" json:"token,omitempty"` + // Pushover user_key + UserKey *string `protobuf:"bytes,2,opt,name=user_key" json:"user_key,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PushoverConfig) Reset() { *m = PushoverConfig{} } +func (m *PushoverConfig) String() string { return proto.CompactTextString(m) } +func (*PushoverConfig) ProtoMessage() {} + +func (m *PushoverConfig) GetToken() string { + if m != nil && m.Token != nil { + return *m.Token + } + return "" +} + +func (m *PushoverConfig) GetUserKey() string { + if m != nil && m.UserKey != nil { + return *m.UserKey + } + return "" +} + +// Notification configuration definition. type NotificationConfig struct { - Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - PagerdutyConfig []*PagerDutyConfig `protobuf:"bytes,2,rep,name=pagerduty_config" json:"pagerduty_config,omitempty"` - EmailConfig []*EmailConfig `protobuf:"bytes,3,rep,name=email_config" json:"email_config,omitempty"` - XXX_unrecognized []byte `json:"-"` + // Name of this NotificationConfig. Referenced from AggregationRule. + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Zero or more PagerDuty notification configurations. + PagerdutyConfig []*PagerDutyConfig `protobuf:"bytes,2,rep,name=pagerduty_config" json:"pagerduty_config,omitempty"` + // Zero or more email notification configurations. + EmailConfig []*EmailConfig `protobuf:"bytes,3,rep,name=email_config" json:"email_config,omitempty"` + // Zero or more pushover notification configurations. + PushoverConfig []*PushoverConfig `protobuf:"bytes,4,rep,name=pushover_config" json:"pushover_config,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *NotificationConfig) Reset() { *m = NotificationConfig{} } @@ -77,8 +115,18 @@ func (m *NotificationConfig) GetEmailConfig() []*EmailConfig { return nil } +func (m *NotificationConfig) GetPushoverConfig() []*PushoverConfig { + if m != nil { + return m.PushoverConfig + } + return nil +} + +// A regex-based label filter used in aggregations. type Filter struct { - NameRe *string `protobuf:"bytes,1,opt,name=name_re" json:"name_re,omitempty"` + // The regex matching the label name. + NameRe *string `protobuf:"bytes,1,opt,name=name_re" json:"name_re,omitempty"` + // The regex matching the label value. ValueRe *string `protobuf:"bytes,2,opt,name=value_re" json:"value_re,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -101,11 +149,16 @@ func (m *Filter) GetValueRe() string { return "" } +// Grouping and notification setting definitions for alerts. type AggregationRule struct { - Filter []*Filter `protobuf:"bytes,1,rep,name=filter" json:"filter,omitempty"` - RepeatRateSeconds *int32 `protobuf:"varint,2,opt,name=repeat_rate_seconds,def=7200" json:"repeat_rate_seconds,omitempty"` - NotificationConfigName *string `protobuf:"bytes,3,opt,name=notification_config_name" json:"notification_config_name,omitempty"` - XXX_unrecognized []byte `json:"-"` + // Filters that define which alerts are matched by this AggregationRule. + Filter []*Filter `protobuf:"bytes,1,rep,name=filter" json:"filter,omitempty"` + // How many seconds to wait before resending a notification for a specific alert. + RepeatRateSeconds *int32 `protobuf:"varint,2,opt,name=repeat_rate_seconds,def=7200" json:"repeat_rate_seconds,omitempty"` + // Notification configuration to use for this AggregationRule, referenced by + // their name. + NotificationConfigName *string `protobuf:"bytes,3,opt,name=notification_config_name" json:"notification_config_name,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *AggregationRule) Reset() { *m = AggregationRule{} } @@ -135,11 +188,70 @@ func (m *AggregationRule) GetNotificationConfigName() string { return "" } +// An InhibitRule specifies that a class of (source) alerts should inhibit +// notifications for another class of (target) alerts if all specified matching +// labels are equal between the two alerts. This may be used to inhibit alerts +// from sending notifications if their meaning is logically a subset of a +// higher-level alert. +// +// For example, if an entire job is down, there is little sense in sending a +// notification for every single instance of said job being down. This could be +// expressed as the following inhibit rule: +// +// inhibit_rule { +// # Select all source alerts that are candidates for being inhibitors. All +// # supplied source filters have to match in order to select a source alert. +// source_filter: { +// name_re: "alertname" +// value_re: "JobDown" +// } +// source_filter: { +// name_re: "service" +// value_re: "api" +// } +// +// # Select all target alerts that are candidates for being inhibited. All +// # supplied target filters have to match in order to select a target alert. +// target_filter: { +// name_re: "alertname" +// value_re: "InstanceDown" +// } +// target_filter: { +// name_re: "service" +// value_re: "api" +// } +// +// # A target alert only actually inhibits a source alert if they match on +// # these labels. I.e. the alerts needs to fire for the same job in the same +// # zone for the inhibit to take effect between them. +// match_on: "job" +// match_on: "zone" +// } +// +// In this example, when JobDown is firing for +// +// JobDown{zone="aa",job="test",service="api"} +// +// ...it would inhibit an InstanceDown alert for +// +// InstanceDown{zone="aa",job="test",instance="1",service="api"} +// +// However, an InstanceDown alert for another zone: +// +// {zone="ab",job="test",instance="1",service="api"} +// +// ...would still fire. type InhibitRule struct { - SourceFilter []*Filter `protobuf:"bytes,1,rep,name=source_filter" json:"source_filter,omitempty"` - TargetFilter []*Filter `protobuf:"bytes,2,rep,name=target_filter" json:"target_filter,omitempty"` - MatchOn []string `protobuf:"bytes,3,rep,name=match_on" json:"match_on,omitempty"` - XXX_unrecognized []byte `json:"-"` + // The set of Filters which define the group of source alerts (which inhibit + // the target alerts). + SourceFilter []*Filter `protobuf:"bytes,1,rep,name=source_filter" json:"source_filter,omitempty"` + // The set of Filters which define the group of target alerts (which are + // inhibited by the source alerts). + TargetFilter []*Filter `protobuf:"bytes,2,rep,name=target_filter" json:"target_filter,omitempty"` + // A set of label names whose label values need to be identical in source and + // target alerts in order for the inhibition to take effect. + MatchOn []string `protobuf:"bytes,3,rep,name=match_on" json:"match_on,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *InhibitRule) Reset() { *m = InhibitRule{} } @@ -167,11 +279,15 @@ func (m *InhibitRule) GetMatchOn() []string { return nil } +// Global alert manager configuration. type AlertManagerConfig struct { - AggregationRule []*AggregationRule `protobuf:"bytes,1,rep,name=aggregation_rule" json:"aggregation_rule,omitempty"` + // Aggregation rule definitions. + AggregationRule []*AggregationRule `protobuf:"bytes,1,rep,name=aggregation_rule" json:"aggregation_rule,omitempty"` + // Notification configuration definitions. NotificationConfig []*NotificationConfig `protobuf:"bytes,2,rep,name=notification_config" json:"notification_config,omitempty"` - InhibitRule []*InhibitRule `protobuf:"bytes,3,rep,name=inhibit_rule" json:"inhibit_rule,omitempty"` - XXX_unrecognized []byte `json:"-"` + // List of alert inhibition rules. + InhibitRule []*InhibitRule `protobuf:"bytes,3,rep,name=inhibit_rule" json:"inhibit_rule,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *AlertManagerConfig) Reset() { *m = AlertManagerConfig{} } diff --git a/manager/notifier.go b/manager/notifier.go index 1b79ab35..5e0c1d2c 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -26,6 +26,7 @@ import ( "text/template" "github.com/golang/glog" + "github.com/thorduri/pushover" pb "github.com/prometheus/alertmanager/config/generated" ) @@ -190,6 +191,26 @@ func (n *notifier) sendEmailNotification(email string, a *Alert) error { return writeEmailBody(wc, a) } +func (n *notifier) sendPushoverNotification(token, userKey string, a *Alert) error { + po, err := pushover.NewPushover(token, userKey) + if err != nil { + return err + } + + // Validate credentials + err = po.Validate() + if err != nil { + return err + } + + // Send pushover message + _, _, err = po.Push(&pushover.Message{ + Title: a.Summary, + Message: a.Description, + }) + return err +} + func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) { for _, pdConfig := range config.PagerdutyConfig { if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), a); err != nil { @@ -205,6 +226,11 @@ func (n *notifier) handleNotification(a *Alert, config *pb.NotificationConfig) { glog.Error("Error sending email notification: ", err) } } + for _, poConfig := range config.PushoverConfig { + if err := n.sendPushoverNotification(poConfig.GetToken(), poConfig.GetUserKey(), a); err != nil { + glog.Error("Error sending Pushover notification: ", err) + } + } } func (n *notifier) Dispatch() {