2015-10-11 15:24:49 +00:00
|
|
|
// Copyright 2015 Prometheus Team
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2015-09-25 12:38:22 +00:00
|
|
|
package types
|
|
|
|
|
2015-09-25 16:14:46 +00:00
|
|
|
import (
|
|
|
|
"fmt"
|
2015-09-27 17:50:24 +00:00
|
|
|
"hash/fnv"
|
2015-11-20 14:10:38 +00:00
|
|
|
"regexp"
|
2015-10-11 10:40:43 +00:00
|
|
|
"strings"
|
2015-11-09 13:34:57 +00:00
|
|
|
"sync"
|
2015-09-25 16:14:46 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/prometheus/common/model"
|
|
|
|
)
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// Marker helps to mark alerts as silenced and/or inhibited.
|
|
|
|
// All methods are goroutine-safe.
|
2015-11-09 13:34:57 +00:00
|
|
|
type Marker interface {
|
|
|
|
SetInhibited(alert model.Fingerprint, b bool)
|
|
|
|
SetSilenced(alert model.Fingerprint, sil ...uint64)
|
|
|
|
|
|
|
|
Silenced(alert model.Fingerprint) (uint64, bool)
|
|
|
|
Inhibited(alert model.Fingerprint) bool
|
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// NewMarker returns an instance of a Marker implementation.
|
2015-11-09 13:34:57 +00:00
|
|
|
func NewMarker() Marker {
|
|
|
|
return &memMarker{
|
|
|
|
inhibited: map[model.Fingerprint]struct{}{},
|
|
|
|
silenced: map[model.Fingerprint]uint64{},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type memMarker struct {
|
|
|
|
inhibited map[model.Fingerprint]struct{}
|
|
|
|
silenced map[model.Fingerprint]uint64
|
|
|
|
|
|
|
|
mtx sync.RWMutex
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *memMarker) Inhibited(alert model.Fingerprint) bool {
|
|
|
|
m.mtx.RLock()
|
|
|
|
defer m.mtx.RUnlock()
|
|
|
|
|
|
|
|
_, ok := m.inhibited[alert]
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *memMarker) Silenced(alert model.Fingerprint) (uint64, bool) {
|
|
|
|
m.mtx.RLock()
|
|
|
|
defer m.mtx.RUnlock()
|
|
|
|
|
|
|
|
sid, ok := m.silenced[alert]
|
|
|
|
return sid, ok
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *memMarker) SetInhibited(alert model.Fingerprint, b bool) {
|
|
|
|
m.mtx.Lock()
|
|
|
|
defer m.mtx.Unlock()
|
|
|
|
|
|
|
|
if !b {
|
|
|
|
delete(m.inhibited, alert)
|
|
|
|
} else {
|
|
|
|
m.inhibited[alert] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *memMarker) SetSilenced(alert model.Fingerprint, sil ...uint64) {
|
|
|
|
m.mtx.Lock()
|
|
|
|
defer m.mtx.Unlock()
|
|
|
|
|
|
|
|
if len(sil) == 0 {
|
|
|
|
delete(m.silenced, alert)
|
|
|
|
} else {
|
|
|
|
m.silenced[alert] = sil[0]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// MultiError contains multiple errors and implements the error interface. Its
|
|
|
|
// zero value is ready to use. All its methods are goroutine safe.
|
|
|
|
type MultiError struct {
|
|
|
|
mtx sync.Mutex
|
|
|
|
errors []error
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add adds an error to the MultiError.
|
|
|
|
func (e *MultiError) Add(err error) {
|
|
|
|
e.mtx.Lock()
|
|
|
|
defer e.mtx.Unlock()
|
|
|
|
|
|
|
|
e.errors = append(e.errors, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Len returns the number of errors added to the MultiError.
|
|
|
|
func (e *MultiError) Len() int {
|
|
|
|
e.mtx.Lock()
|
|
|
|
defer e.mtx.Unlock()
|
|
|
|
|
|
|
|
return len(e.errors)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Errors returns the errors added to the MuliError. The returned slice is a
|
|
|
|
// copy of the internal slice of errors.
|
|
|
|
func (e *MultiError) Errors() []error {
|
|
|
|
e.mtx.Lock()
|
|
|
|
defer e.mtx.Unlock()
|
|
|
|
|
|
|
|
return append(make([]error, 0, len(e.errors)), e.errors...)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e *MultiError) Error() string {
|
|
|
|
e.mtx.Lock()
|
|
|
|
defer e.mtx.Unlock()
|
2015-10-11 10:40:43 +00:00
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
es := make([]string, 0, len(e.errors))
|
|
|
|
for _, err := range e.errors {
|
2015-10-11 10:40:43 +00:00
|
|
|
es = append(es, err.Error())
|
|
|
|
}
|
|
|
|
return strings.Join(es, "; ")
|
|
|
|
}
|
|
|
|
|
2015-10-01 12:53:49 +00:00
|
|
|
// Alert wraps a model.Alert with additional information relevant
|
2015-10-01 13:46:10 +00:00
|
|
|
// to internal of the Alertmanager.
|
|
|
|
// The type is never exposed to external communication and the
|
|
|
|
// embedded alert has to be sanitized beforehand.
|
2015-09-25 12:38:22 +00:00
|
|
|
type Alert struct {
|
2015-10-01 12:53:49 +00:00
|
|
|
model.Alert
|
2015-09-25 12:38:22 +00:00
|
|
|
|
|
|
|
// The authoritative timestamp.
|
2015-12-03 16:27:36 +00:00
|
|
|
UpdatedAt time.Time `json:"-"`
|
|
|
|
Timeout bool `json:"-"`
|
|
|
|
WasSilenced bool `json:"-"`
|
|
|
|
WasInhibited bool `json:"-"`
|
2015-09-30 12:53:05 +00:00
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// AlertSlice is a sortable slice of Alerts.
|
2015-10-27 17:24:09 +00:00
|
|
|
type AlertSlice []*Alert
|
|
|
|
|
|
|
|
func (as AlertSlice) Less(i, j int) bool { return as[i].UpdatedAt.Before(as[j].UpdatedAt) }
|
|
|
|
func (as AlertSlice) Swap(i, j int) { as[i], as[j] = as[j], as[i] }
|
|
|
|
func (as AlertSlice) Len() int { return len(as) }
|
|
|
|
|
2015-10-01 13:46:10 +00:00
|
|
|
// Alerts turns a sequence of internal alerts into a list of
|
|
|
|
// exposable model.Alert structures.
|
|
|
|
func Alerts(alerts ...*Alert) model.Alerts {
|
2015-11-20 14:10:38 +00:00
|
|
|
res := make(model.Alerts, 0, len(alerts))
|
2015-10-01 13:46:10 +00:00
|
|
|
for _, a := range alerts {
|
|
|
|
v := a.Alert
|
|
|
|
// If the end timestamp was set as the expected value in case
|
2015-10-11 10:33:54 +00:00
|
|
|
// of a timeout but is not reached yet, do not expose it.
|
|
|
|
if a.Timeout && !a.Resolved() {
|
2015-10-01 13:46:10 +00:00
|
|
|
v.EndsAt = time.Time{}
|
|
|
|
}
|
|
|
|
res = append(res, &v)
|
2015-09-30 12:53:05 +00:00
|
|
|
}
|
2015-10-01 13:46:10 +00:00
|
|
|
return res
|
2015-09-25 12:38:22 +00:00
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// Merge merges the timespan of two alerts based and overwrites annotations
|
|
|
|
// based on the authoritative timestamp. A new alert is returned, the labels
|
|
|
|
// are assumed to be equal.
|
2015-10-02 14:52:04 +00:00
|
|
|
func (a *Alert) Merge(o *Alert) *Alert {
|
|
|
|
// Let o always be the younger alert.
|
2015-10-15 14:21:36 +00:00
|
|
|
if o.UpdatedAt.Before(a.UpdatedAt) {
|
2015-10-02 14:52:04 +00:00
|
|
|
return o.Merge(a)
|
|
|
|
}
|
|
|
|
|
|
|
|
res := *o
|
|
|
|
|
|
|
|
// Always pick the earliest starting time.
|
|
|
|
if a.StartsAt.Before(o.StartsAt) {
|
|
|
|
res.StartsAt = a.StartsAt
|
|
|
|
}
|
|
|
|
|
2016-02-03 13:11:59 +00:00
|
|
|
// A non-timeout resolved timestamp always rules.
|
2015-10-02 14:52:04 +00:00
|
|
|
// The latest explicit resolved timestamp wins.
|
|
|
|
if a.EndsAt.After(o.EndsAt) && !a.Timeout {
|
|
|
|
res.EndsAt = a.EndsAt
|
|
|
|
}
|
|
|
|
|
|
|
|
return &res
|
|
|
|
}
|
2015-09-25 12:38:22 +00:00
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// A Muter determines whether a given label set is muted.
|
2015-09-28 12:13:01 +00:00
|
|
|
type Muter interface {
|
2015-09-25 16:14:46 +00:00
|
|
|
Mutes(model.LabelSet) bool
|
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// A MuteFunc is a function that implements the Muter interface.
|
2015-09-28 12:13:01 +00:00
|
|
|
type MuteFunc func(model.LabelSet) bool
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// Mutes implements the Muter interface.
|
2015-09-28 12:13:01 +00:00
|
|
|
func (f MuteFunc) Mutes(lset model.LabelSet) bool { return f(lset) }
|
|
|
|
|
2015-09-25 16:14:46 +00:00
|
|
|
// A Silence determines whether a given label set is muted
|
|
|
|
// at the current time.
|
|
|
|
type Silence struct {
|
2015-10-01 15:50:15 +00:00
|
|
|
model.Silence
|
2015-09-25 16:14:46 +00:00
|
|
|
|
2015-09-27 12:07:04 +00:00
|
|
|
// A set of matchers determining if an alert is affected
|
|
|
|
// by the silence.
|
2015-10-01 15:50:15 +00:00
|
|
|
Matchers Matchers `json:"-"`
|
2015-09-25 16:14:46 +00:00
|
|
|
|
|
|
|
// timeFunc provides the time against which to evaluate
|
|
|
|
// the silence.
|
|
|
|
timeFunc func() time.Time
|
|
|
|
}
|
|
|
|
|
2015-11-06 09:09:39 +00:00
|
|
|
// NewSilence creates a new internal Silence from a public silence object.
|
2015-10-01 15:50:15 +00:00
|
|
|
func NewSilence(s *model.Silence) *Silence {
|
|
|
|
sil := &Silence{
|
|
|
|
Silence: *s,
|
|
|
|
timeFunc: time.Now,
|
2015-09-27 12:25:56 +00:00
|
|
|
}
|
2015-10-01 15:50:15 +00:00
|
|
|
for _, m := range s.Matchers {
|
2015-09-27 12:25:56 +00:00
|
|
|
if !m.IsRegex {
|
|
|
|
sil.Matchers = append(sil.Matchers, NewMatcher(m.Name, m.Value))
|
|
|
|
continue
|
|
|
|
}
|
2015-11-20 14:10:38 +00:00
|
|
|
rem := NewRegexMatcher(m.Name, regexp.MustCompile("^(?:"+m.Value+")$"))
|
2015-09-27 12:25:56 +00:00
|
|
|
sil.Matchers = append(sil.Matchers, rem)
|
|
|
|
}
|
2015-10-01 15:50:15 +00:00
|
|
|
return sil
|
2015-09-27 12:25:56 +00:00
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// Mutes implements the Muter interface.
|
2015-10-01 15:50:15 +00:00
|
|
|
func (sil *Silence) Mutes(lset model.LabelSet) bool {
|
|
|
|
t := sil.timeFunc()
|
2015-09-27 12:25:56 +00:00
|
|
|
|
2015-10-01 15:50:15 +00:00
|
|
|
if t.Before(sil.StartsAt) || t.After(sil.EndsAt) {
|
|
|
|
return false
|
2015-09-27 12:25:56 +00:00
|
|
|
}
|
2015-10-01 15:50:15 +00:00
|
|
|
|
2015-10-01 20:16:44 +00:00
|
|
|
b := sil.Matchers.Match(lset)
|
|
|
|
|
|
|
|
return b
|
2015-09-27 12:25:56 +00:00
|
|
|
}
|
2015-09-27 17:50:24 +00:00
|
|
|
|
2015-11-06 09:09:39 +00:00
|
|
|
// NotifyInfo holds information about the last successful notification
|
2015-11-10 12:47:04 +00:00
|
|
|
// of an alert to a receiver.
|
2015-11-06 09:09:39 +00:00
|
|
|
type NotifyInfo struct {
|
2015-09-27 17:50:24 +00:00
|
|
|
Alert model.Fingerprint
|
2015-11-10 12:47:04 +00:00
|
|
|
Receiver string
|
2015-09-27 17:50:24 +00:00
|
|
|
Resolved bool
|
|
|
|
Timestamp time.Time
|
|
|
|
}
|
|
|
|
|
2015-11-06 09:09:39 +00:00
|
|
|
func (n *NotifyInfo) String() string {
|
2015-11-10 12:47:04 +00:00
|
|
|
return fmt.Sprintf("<Notify:%q@%s to=%v res=%v>", n.Alert, n.Timestamp, n.Receiver, n.Resolved)
|
2015-09-28 20:02:41 +00:00
|
|
|
}
|
|
|
|
|
2015-11-20 14:10:38 +00:00
|
|
|
// Fingerprint returns a quasi-unique fingerprint for the NotifyInfo.
|
2015-11-06 09:09:39 +00:00
|
|
|
func (n *NotifyInfo) Fingerprint() model.Fingerprint {
|
2015-09-27 17:50:24 +00:00
|
|
|
h := fnv.New64a()
|
2015-11-10 12:47:04 +00:00
|
|
|
h.Write([]byte(n.Receiver))
|
2015-09-27 17:50:24 +00:00
|
|
|
|
|
|
|
fp := model.Fingerprint(h.Sum64())
|
|
|
|
|
|
|
|
return fp ^ n.Alert
|
|
|
|
}
|