From bfcded79e8a816ab22a36b393b15972c3227dc32 Mon Sep 17 00:00:00 2001 From: Tomas Karasek Date: Mon, 18 May 2015 19:07:56 +0300 Subject: [PATCH 1/5] Added flowdock notifier --- config/config.go | 8 ++++ config/config.proto | 13 +++++++ config/generated/config.pb.go | 60 +++++++++++++++++++++++++++- manager/notifier.go | 73 +++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 5ccf68eb..025af917 100644 --- a/config/config.go +++ b/config/config.go @@ -77,6 +77,14 @@ func (c Config) Validate() error { return fmt.Errorf("Missing webhook URL in Slack config: %s", proto.MarshalTextString(sc)) } } + for _, fc := range nc.FlowdockConfig { + if fc.ApiToken == nil { + return fmt.Errorf("Missing API token in Flowdock config: %s", proto.MarshalTextString(fc)) + } + if fc.FromAddress == nil { + return fmt.Errorf("Missing from_address Flowdock config: %s", proto.MarshalTextString(fc)) + } + } if _, ok := ncNames[nc.GetName()]; ok { return fmt.Errorf("Notification config name not unique: %s", nc.GetName()) diff --git a/config/config.proto b/config/config.proto index 1948e5fe..e60ae3ca 100644 --- a/config/config.proto +++ b/config/config.proto @@ -70,6 +70,17 @@ message SlackConfig { optional bool send_resolved = 5 [default = false]; } +message FlowdockConfig { + // Flowdock flow API token + optional string api_token = 2; + // Flowdock from_address + optional string from_address = 3; + // Flowdock flow tags + repeated string tags = 4; + // Color of message when triggered. + optional bool send_resolved = 7 [default = false]; +} + // Notification configuration definition. message NotificationConfig { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -84,6 +95,8 @@ message NotificationConfig { repeated HipChatConfig hipchat_config = 5; // Zero or more slack notification configurations. repeated SlackConfig slack_config = 6; + // Zero or more Flowdock notification configurations. + repeated FlowdockConfig flowdock_config = 7; } // A regex-based label filter used in aggregations. diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index 2536c09d..32406f91 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -14,6 +14,7 @@ It has these top-level messages: PushoverConfig HipChatConfig SlackConfig + FlowdockConfig NotificationConfig Filter AggregationRule @@ -241,6 +242,52 @@ func (m *SlackConfig) GetSendResolved() bool { return Default_SlackConfig_SendResolved } +type FlowdockConfig struct { + // Flowdock flow API token + ApiToken *string `protobuf:"bytes,2,opt,name=api_token" json:"api_token,omitempty"` + // Flowdock from_address + FromAddress *string `protobuf:"bytes,3,opt,name=from_address" json:"from_address,omitempty"` + // Flowdock flow tags + Tags []string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"` + // Color of message when triggered. + SendResolved *bool `protobuf:"varint,7,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *FlowdockConfig) Reset() { *m = FlowdockConfig{} } +func (m *FlowdockConfig) String() string { return proto.CompactTextString(m) } +func (*FlowdockConfig) ProtoMessage() {} + +const Default_FlowdockConfig_SendResolved bool = false + +func (m *FlowdockConfig) GetApiToken() string { + if m != nil && m.ApiToken != nil { + return *m.ApiToken + } + return "" +} + +func (m *FlowdockConfig) GetFromAddress() string { + if m != nil && m.FromAddress != nil { + return *m.FromAddress + } + return "" +} + +func (m *FlowdockConfig) GetTags() []string { + if m != nil { + return m.Tags + } + return nil +} + +func (m *FlowdockConfig) GetSendResolved() bool { + if m != nil && m.SendResolved != nil { + return *m.SendResolved + } + return Default_FlowdockConfig_SendResolved +} + // Notification configuration definition. type NotificationConfig struct { // Name of this NotificationConfig. Referenced from AggregationRule. @@ -254,8 +301,10 @@ type NotificationConfig struct { // Zero or more hipchat notification configurations. HipchatConfig []*HipChatConfig `protobuf:"bytes,5,rep,name=hipchat_config" json:"hipchat_config,omitempty"` // Zero or more slack notification configurations. - SlackConfig []*SlackConfig `protobuf:"bytes,6,rep,name=slack_config" json:"slack_config,omitempty"` - XXX_unrecognized []byte `json:"-"` + SlackConfig []*SlackConfig `protobuf:"bytes,6,rep,name=slack_config" json:"slack_config,omitempty"` + // Zero or more Flowdock notification configurations. + FlowdockConfig []*FlowdockConfig `protobuf:"bytes,7,rep,name=flowdock_config" json:"flowdock_config,omitempty"` + XXX_unrecognized []byte `json:"-"` } func (m *NotificationConfig) Reset() { *m = NotificationConfig{} } @@ -304,6 +353,13 @@ func (m *NotificationConfig) GetSlackConfig() []*SlackConfig { return nil } +func (m *NotificationConfig) GetFlowdockConfig() []*FlowdockConfig { + if m != nil { + return m.FlowdockConfig + } + return nil +} + // A regex-based label filter used in aggregations. type Filter struct { // The regex matching the label name. diff --git a/manager/notifier.go b/manager/notifier.go index 0879ce9d..071374fc 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -65,6 +65,7 @@ var ( smtpSmartHost = flag.String("notification.smtp.smarthost", "", "Address of the smarthost to send all email notifications to.") smtpSender = flag.String("notification.smtp.sender", "alertmanager@example.org", "Sender email address to use in email notifications.") hipchatURL = flag.String("notification.hipchat.url", "https://api.hipchat.com/v2", "HipChat API V2 URL.") + flowdockURL = flag.String("notification.flowdock.url", "https://api.flowdock.com/v1/messages/team_inbox", "Flowdock API V1 URL.") ) type notificationOp int @@ -325,6 +326,59 @@ func (n *notifier) sendSlackNotification(op notificationOp, config *pb.SlackConf return nil } +// https://www.flowdock.com/api/team-inbox +// compulsory fields in Flowdock JSON: source, from_address, subject, content +// Content-type: application/json +// POST to https://api.flowdock.com/v1/messages/team_inbox/:flow_api_token +type FlowdockMessage struct { + Source string `json:"source"` + FromAddress string `json:"from_address"` + Subject string `json:"subject"` + Content string `json:"content"` + Format string `json:"format"` + Link string `json:"link"` + Tags []string `json:"tags,omitempty"` +} + +func jsonize(msg interface{}) []byte { + buf, err := json.Marshal(msg) + if err != nil { + glog.Errorln("Error compiling JSON:", err) + } + return buf +} +func getFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) *FlowdockMessage { + status := "" + switch op { + case notificationOpTrigger: + status = "firing" + case notificationOpResolve: + status = "resolved" + } + msg := &FlowdockMessage{ + Source: "Prometheus", + FromAddress: config.GetFromAddress(), + Subject: html.EscapeString(a.Summary), + Format: "html", + Content: fmt.Sprintf("*%s %s*: %s (<%s|view>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), + Link: a.Payload["GeneratorURL"], + Tags: config.GetTags(), + } + return msg +} + +func postJSONtoURL(jsonMessage []byte, url string) *http.Response { + timeout := time.Duration(5 * time.Second) + client := http.Client{ + Timeout: timeout, + } + response, err := client.Post(url, contentTypeJSON, bytes.NewBuffer(jsonMessage)) + if err != nil { + glog.Errorln("Error while sending Flowdock notification:", err) + } + return response +} + func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error { return writeEmailBodyWithTime(w, from, to, status, a, time.Now()) } @@ -452,6 +506,16 @@ func (n *notifier) sendPushoverNotification(token string, op notificationOp, use return err } +func processResponse(r *http.Response, targetName string, a *Alert) { + defer r.Body.Close() + + respBuf, err := ioutil.ReadAll(r.Body) + if err != nil { + glog.Errorln("Error reading HTTP response:", err) + } + glog.Infof("Sent %s notification for alert %v. Response: HTTP %d: %s", targetName, a.Fingerprint(), r.StatusCode, respBuf) +} + func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { for _, pdConfig := range config.PagerdutyConfig { if err := n.sendPagerDutyNotification(pdConfig.GetServiceKey(), op, a); err != nil { @@ -494,6 +558,15 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No glog.Errorln("Error sending Slack notification:", err) } } + for _, fdConfig := range config.FlowdockConfig { + if op == notificationOpResolve && !fdConfig.GetSendResolved() { + continue + } + flowdockMessage := getFlowdockNotificationMessage(op, fdConfig, a) + url := *flowdockURL + "/" + fdConfig.GetApiToken() + httpResponse := postJSONtoURL(jsonize(flowdockMessage), url) + processResponse(httpResponse, "Flowdock", a) + } } func (n *notifier) Dispatch() { From ffd54a3ee642a07080e58f53a5ea2e439c4c5d3d Mon Sep 17 00:00:00 2001 From: Tomas Karasek Date: Mon, 18 May 2015 19:17:19 +0300 Subject: [PATCH 2/5] added flowdock block to sample config --- config/fixtures/sample.conf.input | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/fixtures/sample.conf.input b/config/fixtures/sample.conf.input index 58edb0b9..7b1c0248 100644 --- a/config/fixtures/sample.conf.input +++ b/config/fixtures/sample.conf.input @@ -19,6 +19,10 @@ notification_config { webhook_url: "webhookurl" send_resolved: true } + flowdock_config { + api_token: "4c7234902348234902384234234cdb59" + from_address: "aliaswithgravatar@somehwere.com" + } } aggregation_rule { From 48cdc777ce3c57f96b14f01bd359684621f317b4 Mon Sep 17 00:00:00 2001 From: Tomas Karasek Date: Tue, 19 May 2015 11:37:22 +0300 Subject: [PATCH 3/5] improve Flowdock PR based on julius' comments --- README.md | 5 +++-- config/config.proto | 10 +++++----- config/fixtures/sample.conf.input | 1 + config/generated/config.pb.go | 14 +++++++------- manager/notifier.go | 21 +++++++++++---------- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8d298d65..c941163f 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,9 @@ following aspects: * sending alert notifications via external services (currently email, [PagerDuty](http://www.pagerduty.com/), [HipChat](http://www.hipchat.com/), -[Slack](http://www.slack.com/), or -[Pushover](https://www.pushover.net/)) +[Slack](http://www.slack.com/), +[Pushover](https://www.pushover.net/)), or +[Flowdock](https://www.flowdock.com/)) See [config/fixtures/sample.conf.input](config/fixtures/sample.conf.input) for an example config. The full configuration schema including a documentation for diff --git a/config/config.proto b/config/config.proto index e60ae3ca..d614791a 100644 --- a/config/config.proto +++ b/config/config.proto @@ -72,13 +72,13 @@ message SlackConfig { message FlowdockConfig { // Flowdock flow API token - optional string api_token = 2; + optional string api_token = 1; // Flowdock from_address - optional string from_address = 3; + optional string from_address = 2; // Flowdock flow tags - repeated string tags = 4; - // Color of message when triggered. - optional bool send_resolved = 7 [default = false]; + repeated string tag = 3; + // Notify when resolved. + optional bool send_resolved = 4 [default = false]; } // Notification configuration definition. diff --git a/config/fixtures/sample.conf.input b/config/fixtures/sample.conf.input index 7b1c0248..a4a312c9 100644 --- a/config/fixtures/sample.conf.input +++ b/config/fixtures/sample.conf.input @@ -22,6 +22,7 @@ notification_config { flowdock_config { api_token: "4c7234902348234902384234234cdb59" from_address: "aliaswithgravatar@somehwere.com" + tag: "monitoring" } } diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index 32406f91..b5793db4 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -244,13 +244,13 @@ func (m *SlackConfig) GetSendResolved() bool { type FlowdockConfig struct { // Flowdock flow API token - ApiToken *string `protobuf:"bytes,2,opt,name=api_token" json:"api_token,omitempty"` + ApiToken *string `protobuf:"bytes,1,opt,name=api_token" json:"api_token,omitempty"` // Flowdock from_address - FromAddress *string `protobuf:"bytes,3,opt,name=from_address" json:"from_address,omitempty"` + FromAddress *string `protobuf:"bytes,2,opt,name=from_address" json:"from_address,omitempty"` // Flowdock flow tags - Tags []string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"` - // Color of message when triggered. - SendResolved *bool `protobuf:"varint,7,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` + Tag []string `protobuf:"bytes,3,rep,name=tag" json:"tag,omitempty"` + // Notify when resolved. + SendResolved *bool `protobuf:"varint,4,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -274,9 +274,9 @@ func (m *FlowdockConfig) GetFromAddress() string { return "" } -func (m *FlowdockConfig) GetTags() []string { +func (m *FlowdockConfig) GetTag() []string { if m != nil { - return m.Tags + return m.Tag } return nil } diff --git a/manager/notifier.go b/manager/notifier.go index 071374fc..f55571ce 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -330,7 +330,7 @@ func (n *notifier) sendSlackNotification(op notificationOp, config *pb.SlackConf // compulsory fields in Flowdock JSON: source, from_address, subject, content // Content-type: application/json // POST to https://api.flowdock.com/v1/messages/team_inbox/:flow_api_token -type FlowdockMessage struct { +type flowdockMessage struct { Source string `json:"source"` FromAddress string `json:"from_address"` Subject string `json:"subject"` @@ -340,14 +340,14 @@ type FlowdockMessage struct { Tags []string `json:"tags,omitempty"` } -func jsonize(msg interface{}) []byte { +func marshallJSON(msg interface{}) []byte { buf, err := json.Marshal(msg) if err != nil { - glog.Errorln("Error compiling JSON:", err) + glog.Errorln("Error marshalling JSON:", err) } return buf } -func getFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) *FlowdockMessage { +func newFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) *flowdockMessage { status := "" switch op { case notificationOpTrigger: @@ -355,26 +355,27 @@ func getFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig case notificationOpResolve: status = "resolved" } - msg := &FlowdockMessage{ + + msg := &flowdockMessage{ Source: "Prometheus", FromAddress: config.GetFromAddress(), Subject: html.EscapeString(a.Summary), Format: "html", Content: fmt.Sprintf("*%s %s*: %s (<%s|view>)", html.EscapeString(a.Labels["alertname"]), status, html.EscapeString(a.Summary), a.Payload["GeneratorURL"]), Link: a.Payload["GeneratorURL"], - Tags: config.GetTags(), + Tags: append(config.GetTag(), status), } return msg } -func postJSONtoURL(jsonMessage []byte, url string) *http.Response { +func postJSON(jsonMessage []byte, url string) *http.Response { timeout := time.Duration(5 * time.Second) client := http.Client{ Timeout: timeout, } response, err := client.Post(url, contentTypeJSON, bytes.NewBuffer(jsonMessage)) if err != nil { - glog.Errorln("Error while sending Flowdock notification:", err) + glog.Errorln("Error while posting JSON:", err) } return response } @@ -562,9 +563,9 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No if op == notificationOpResolve && !fdConfig.GetSendResolved() { continue } - flowdockMessage := getFlowdockNotificationMessage(op, fdConfig, a) + flowdockMessage := newFlowdockNotificationMessage(op, fdConfig, a) url := *flowdockURL + "/" + fdConfig.GetApiToken() - httpResponse := postJSONtoURL(jsonize(flowdockMessage), url) + httpResponse := postJSON(marshallJSON(flowdockMessage), url) processResponse(httpResponse, "Flowdock", a) } } From 9632bf24f82863b9b7fc25e19f957bd50ed62252 Mon Sep 17 00:00:00 2001 From: Tomas Karasek Date: Tue, 19 May 2015 11:38:28 +0300 Subject: [PATCH 4/5] more readable HTTP response logging --- manager/notifier.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/manager/notifier.go b/manager/notifier.go index f55571ce..abf74491 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -508,13 +508,17 @@ func (n *notifier) sendPushoverNotification(token string, op notificationOp, use } func processResponse(r *http.Response, targetName string, a *Alert) { - defer r.Body.Close() - - respBuf, err := ioutil.ReadAll(r.Body) - if err != nil { - glog.Errorln("Error reading HTTP response:", err) + spec := fmt.Sprintf("%s notification for alert %v", targetName, a.Fingerprint()) + if r == nil { + glog.Errorln("No HTTP response for", spec) + } else { + defer r.Body.Close() + respBuf, err := ioutil.ReadAll(r.Body) + if err != nil { + glog.Errorln("Error reading HTTP response for", spec, err) + } + glog.Infof("Sent %s. Response: HTTP %d: %s", spec, r.StatusCode, respBuf) } - glog.Infof("Sent %s notification for alert %v. Response: HTTP %d: %s", targetName, a.Fingerprint(), r.StatusCode, respBuf) } func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { From 5b1c59f0cc1806bd0f4202c2e32928e89339f06b Mon Sep 17 00:00:00 2001 From: Tomas Karasek Date: Tue, 19 May 2015 14:31:06 +0300 Subject: [PATCH 5/5] rewrote to better fit in exisitng code --- config/config.proto | 6 ++-- config/generated/config.pb.go | 6 ++-- manager/notifier.go | 57 ++++++++++++++++++----------------- 3 files changed, 35 insertions(+), 34 deletions(-) diff --git a/config/config.proto b/config/config.proto index d614791a..5e97cc77 100644 --- a/config/config.proto +++ b/config/config.proto @@ -71,11 +71,11 @@ message SlackConfig { } message FlowdockConfig { - // Flowdock flow API token + // Flowdock flow API token. optional string api_token = 1; - // Flowdock from_address + // Flowdock from_address. optional string from_address = 2; - // Flowdock flow tags + // Flowdock flow tags. repeated string tag = 3; // Notify when resolved. optional bool send_resolved = 4 [default = false]; diff --git a/config/generated/config.pb.go b/config/generated/config.pb.go index b5793db4..d4289ccd 100644 --- a/config/generated/config.pb.go +++ b/config/generated/config.pb.go @@ -243,11 +243,11 @@ func (m *SlackConfig) GetSendResolved() bool { } type FlowdockConfig struct { - // Flowdock flow API token + // Flowdock flow API token. ApiToken *string `protobuf:"bytes,1,opt,name=api_token" json:"api_token,omitempty"` - // Flowdock from_address + // Flowdock from_address. FromAddress *string `protobuf:"bytes,2,opt,name=from_address" json:"from_address,omitempty"` - // Flowdock flow tags + // Flowdock flow tags. Tag []string `protobuf:"bytes,3,rep,name=tag" json:"tag,omitempty"` // Notify when resolved. SendResolved *bool `protobuf:"varint,4,opt,name=send_resolved,def=0" json:"send_resolved,omitempty"` diff --git a/manager/notifier.go b/manager/notifier.go index abf74491..78ec65a0 100644 --- a/manager/notifier.go +++ b/manager/notifier.go @@ -326,10 +326,6 @@ func (n *notifier) sendSlackNotification(op notificationOp, config *pb.SlackConf return nil } -// https://www.flowdock.com/api/team-inbox -// compulsory fields in Flowdock JSON: source, from_address, subject, content -// Content-type: application/json -// POST to https://api.flowdock.com/v1/messages/team_inbox/:flow_api_token type flowdockMessage struct { Source string `json:"source"` FromAddress string `json:"from_address"` @@ -340,14 +336,24 @@ type flowdockMessage struct { Tags []string `json:"tags,omitempty"` } -func marshallJSON(msg interface{}) []byte { - buf, err := json.Marshal(msg) +func (n *notifier) sendFlowdockNotification(op notificationOp, config *pb.FlowdockConfig, a *Alert) error { + flowdockMessage := newFlowdockMessage(op, config, a) + url := strings.TrimRight(*flowdockURL, "/") + "/" + config.GetApiToken() + jsonMessage, err := json.Marshal(flowdockMessage) if err != nil { - glog.Errorln("Error marshalling JSON:", err) + return err } - return buf + httpResponse, err := postJSON(jsonMessage, url) + if err != nil { + return err + } + if err := processResponse(httpResponse, "Flowdock", a); err != nil { + return err + } + return nil } -func newFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) *flowdockMessage { + +func newFlowdockMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert) *flowdockMessage { status := "" switch op { case notificationOpTrigger: @@ -368,16 +374,12 @@ func newFlowdockNotificationMessage(op notificationOp, config *pb.FlowdockConfig return msg } -func postJSON(jsonMessage []byte, url string) *http.Response { +func postJSON(jsonMessage []byte, url string) (*http.Response, error) { timeout := time.Duration(5 * time.Second) client := http.Client{ Timeout: timeout, } - response, err := client.Post(url, contentTypeJSON, bytes.NewBuffer(jsonMessage)) - if err != nil { - glog.Errorln("Error while posting JSON:", err) - } - return response + return client.Post(url, contentTypeJSON, bytes.NewBuffer(jsonMessage)) } func writeEmailBody(w io.Writer, from, to, status string, a *Alert) error { @@ -507,18 +509,18 @@ func (n *notifier) sendPushoverNotification(token string, op notificationOp, use return err } -func processResponse(r *http.Response, targetName string, a *Alert) { +func processResponse(r *http.Response, targetName string, a *Alert) error { spec := fmt.Sprintf("%s notification for alert %v", targetName, a.Fingerprint()) if r == nil { - glog.Errorln("No HTTP response for", spec) - } else { - defer r.Body.Close() - respBuf, err := ioutil.ReadAll(r.Body) - if err != nil { - glog.Errorln("Error reading HTTP response for", spec, err) - } - glog.Infof("Sent %s. Response: HTTP %d: %s", spec, r.StatusCode, respBuf) + return fmt.Errorf("No HTTP response for %s", spec) } + defer r.Body.Close() + respBuf, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + glog.Infof("Sent %s. Response: HTTP %d: %s", spec, r.StatusCode, respBuf) + return nil } func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.NotificationConfig) { @@ -567,10 +569,9 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No if op == notificationOpResolve && !fdConfig.GetSendResolved() { continue } - flowdockMessage := newFlowdockNotificationMessage(op, fdConfig, a) - url := *flowdockURL + "/" + fdConfig.GetApiToken() - httpResponse := postJSON(marshallJSON(flowdockMessage), url) - processResponse(httpResponse, "Flowdock", a) + if err := n.sendFlowdockNotification(op, fdConfig, a); err != nil { + glog.Errorln("Error sending Flowdock notification:", err) + } } }