From 335a34486efcecb9e88e86bf1673eafc842d574f Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Mon, 17 Dec 2018 20:16:28 +0100 Subject: [PATCH 1/4] Add external labels to template expansion This affects the expansion of templates in alert labels and annotations and console templates. Signed-off-by: Sylvain Rabot --- cmd/prometheus/main.go | 6 +++++ config/config.go | 5 ++++ docs/configuration/alerting_rules.md | 2 +- pkg/rulefmt/rulefmt.go | 2 +- rules/alerting.go | 24 ++++++++++++++++---- template/template.go | 28 ++++++++++++++++++----- web/web.go | 34 ++++++++++++++++++++++------ 7 files changed, 81 insertions(+), 20 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index e002cff31..27484e899 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -748,6 +748,12 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval)) + + // Register conf as current config. + config.CurrentConfigMutex.Lock() + config.CurrentConfig = conf + config.CurrentConfigMutex.Unlock() + level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename) return nil } diff --git a/config/config.go b/config/config.go index f4d8d470d..7acd54ad4 100644 --- a/config/config.go +++ b/config/config.go @@ -20,6 +20,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "github.com/pkg/errors" @@ -68,6 +69,10 @@ func LoadFile(filename string) (*Config, error) { // The defaults applied before parsing the respective config sections. var ( + // CurrentConfig is a pointer to the current configuration. + CurrentConfig *Config + CurrentConfigMutex sync.RWMutex + // DefaultConfig is the default top-level configuration. DefaultConfig = Config{ GlobalConfig: DefaultGlobalConfig, diff --git a/docs/configuration/alerting_rules.md b/docs/configuration/alerting_rules.md index cc674a09b..e8110a1a8 100644 --- a/docs/configuration/alerting_rules.md +++ b/docs/configuration/alerting_rules.md @@ -43,7 +43,7 @@ The `annotations` clause specifies a set of informational labels that can be use #### Templating Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles). -The `$labels` variable holds the label key/value pairs of an alert instance +The `$labels` & `$externalLabels` variables hold the label key/value pairs of an alert instance and `$value` holds the evaluated value of an alert instance. # To insert a firing element's label values: diff --git a/pkg/rulefmt/rulefmt.go b/pkg/rulefmt/rulefmt.go index 60faee085..19a2d1836 100644 --- a/pkg/rulefmt/rulefmt.go +++ b/pkg/rulefmt/rulefmt.go @@ -155,7 +155,7 @@ func testTemplateParsing(rl *Rule) (errs []error) { } // Trying to parse templates. - tmplData := template.AlertTemplateData(make(map[string]string), 0) + tmplData := template.AlertTemplateData(make(map[string]string), make(map[string]string), 0) defs := "{{$labels := .Labels}}{{$value := .Value}}" parseTest := func(text string) error { tmpl := template.NewTemplateExpander( diff --git a/rules/alerting.go b/rules/alerting.go index 7292d287b..c215fc014 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -28,6 +28,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -79,8 +80,9 @@ func (s AlertState) String() string { type Alert struct { State AlertState - Labels labels.Labels - Annotations labels.Labels + Labels labels.Labels + ExternalLabels labels.Labels + Annotations labels.Labels // The value at the last evaluation of the alerting expression. Value float64 @@ -300,15 +302,27 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, var vec promql.Vector for _, smpl := range res { // Provide the alert information to the template. - l := make(map[string]string, len(smpl.Metric)) + l := make(map[string]string) for _, lbl := range smpl.Metric { l[lbl.Name] = lbl.Value } - tmplData := template.AlertTemplateData(l, smpl.V) + // Add external labels. + el := make(map[string]string) + if config.CurrentConfig != nil { + config.CurrentConfigMutex.RLock() + for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { + el[string(eln)] = string(elv) + } + config.CurrentConfigMutex.RUnlock() + } + + tmplData := template.AlertTemplateData(l, el, smpl.V) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}{{$value := .Value}}" + defs := "{{$labels := .Labels}}" + defs += "{{$externalLabels := .ExternalLabels}}" + defs += "{{$value := .Value}}" expand := func(text string) string { tmpl := template.NewTemplateExpander( diff --git a/template/template.go b/template/template.go index ef6212780..88b0e841d 100644 --- a/template/template.go +++ b/template/template.go @@ -28,9 +28,10 @@ import ( text_template "text/template" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) @@ -133,6 +134,19 @@ func NewTemplateExpander( "label": func(label string, s *sample) string { return s.Labels[label] }, + "externalLabel": func(label string) string { + if config.CurrentConfig == nil { + return "" + } + config.CurrentConfigMutex.RLock() + defer config.CurrentConfigMutex.RUnlock() + for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { + if label == string(eln) { + return string(elv) + } + } + return "" + }, "value": func(s *sample) float64 { return s.Value }, @@ -261,13 +275,15 @@ func NewTemplateExpander( } // AlertTemplateData returns the interface to be used in expanding the template. -func AlertTemplateData(labels map[string]string, value float64) interface{} { +func AlertTemplateData(labels map[string]string, externalLabels map[string]string, value float64) interface{} { return struct { - Labels map[string]string - Value float64 + Labels map[string]string + ExternalLabels map[string]string + Value float64 }{ - Labels: labels, - Value: value, + Labels: labels, + ExternalLabels: externalLabels, + Value: value, } } diff --git a/web/web.go b/web/web.go index 6a23e0254..4762bb6dd 100644 --- a/web/web.go +++ b/web/web.go @@ -541,19 +541,39 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { for k, v := range rawParams { params[k] = v[0] } + + // External labels + el := make(map[string]string) + if config.CurrentConfig != nil { + config.CurrentConfigMutex.RLock() + for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { + el[string(eln)] = string(elv) + } + config.CurrentConfigMutex.RUnlock() + } + + // Inject some convenience variables that are easier to remember for users + // who are not used to Go's templating system. + defs := "{{$rawParams := .RawParams }}" + defs += "{{$params := .Params}}" + defs += "{{$path := .Path}}" + defs += "{{$externalLabels := .ExternalLabels}}" + data := struct { - RawParams url.Values - Params map[string]string - Path string + RawParams url.Values + Params map[string]string + Path string + ExternalLabels map[string]string }{ - RawParams: rawParams, - Params: params, - Path: strings.TrimLeft(name, "/"), + RawParams: rawParams, + Params: params, + Path: strings.TrimLeft(name, "/"), + ExternalLabels: el, } tmpl := template.NewTemplateExpander( h.context, - string(text), + defs+string(text), "__console_"+name, data, h.now(), From a92ef68dd80a318b4a29722af085bd9dfaaf6acf Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Mon, 15 Apr 2019 19:06:25 +0200 Subject: [PATCH 2/4] Fix staticcheck errors Not sure why they only show up now. Signed-off-by: Bjoern Rabenstein --- cmd/promtool/http.go | 8 -------- discovery/openstack/mock_test.go | 6 +++--- promql/parse.go | 18 ++++++++---------- web/federate_test.go | 1 - 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/cmd/promtool/http.go b/cmd/promtool/http.go index 43656ad67..e269a23ed 100644 --- a/cmd/promtool/http.go +++ b/cmd/promtool/http.go @@ -14,8 +14,6 @@ package main import ( - "context" - "net/http" "time" "github.com/pkg/errors" @@ -42,12 +40,6 @@ func newPrometheusHTTPClient(serverURL string) (*prometheusHTTPClient, error) { }, nil } -func (c *prometheusHTTPClient) do(req *http.Request) (*http.Response, []byte, error) { - ctx, cancel := context.WithTimeout(context.Background(), c.requestTimeout) - defer cancel() - return c.httpClient.Do(ctx, req) -} - func (c *prometheusHTTPClient) urlJoin(path string) string { return c.httpClient.URL(path, nil).String() } diff --git a/discovery/openstack/mock_test.go b/discovery/openstack/mock_test.go index 67b14f6f6..13c25dfbb 100644 --- a/discovery/openstack/mock_test.go +++ b/discovery/openstack/mock_test.go @@ -247,7 +247,7 @@ func (m *SDMock) HandleHypervisorListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, hypervisorListBody) + fmt.Fprint(w, hypervisorListBody) }) } @@ -544,7 +544,7 @@ func (m *SDMock) HandleServerListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, serverListBody) + fmt.Fprint(w, serverListBody) }) } @@ -583,6 +583,6 @@ func (m *SDMock) HandleFloatingIPListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, listOutput) + fmt.Fprint(w, listOutput) }) } diff --git a/promql/parse.go b/promql/parse.go index 2a055fbbb..4bb4a11c6 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -326,17 +326,15 @@ var errUnexpected = errors.New("unexpected error") // recover is the handler that turns panics into returns from the top level of Parse. func (p *parser) recover(errp *error) { e := recover() - if e != nil { - if _, ok := e.(runtime.Error); ok { - // Print the stack trace but do not inhibit the running application. - buf := make([]byte, 64<<10) - buf = buf[:runtime.Stack(buf, false)] + if _, ok := e.(runtime.Error); ok { + // Print the stack trace but do not inhibit the running application. + buf := make([]byte, 64<<10) + buf = buf[:runtime.Stack(buf, false)] - fmt.Fprintf(os.Stderr, "parser panic: %v\n%s", e, buf) - *errp = errUnexpected - } else { - *errp = e.(error) - } + fmt.Fprintf(os.Stderr, "parser panic: %v\n%s", e, buf) + *errp = errUnexpected + } else if e != nil { + *errp = e.(error) } p.lex.close() } diff --git a/web/federate_test.go b/web/federate_test.go index 03c4763eb..1476bec3f 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -28,7 +28,6 @@ import ( var scenarios = map[string]struct { params string - accept string externalLabels labels.Labels code int body string From 38d518c0fe8abeeb8dcdb0472510cece3dabfd0e Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Mon, 15 Apr 2019 18:52:58 +0200 Subject: [PATCH 3/4] Rework #5009 after comments Signed-off-by: Bjoern Rabenstein --- cmd/prometheus/main.go | 13 +++-- cmd/promtool/unittest.go | 3 +- config/config.go | 5 -- docs/configuration/alerting_rules.md | 8 +-- docs/configuration/template_reference.md | 10 ++-- pkg/rulefmt/rulefmt.go | 11 ++-- rules/alerting.go | 65 +++++++++++++----------- rules/alerting_test.go | 4 +- rules/manager.go | 9 ++-- rules/manager_test.go | 24 ++++----- template/template.go | 14 ----- web/api/v1/api_test.go | 2 + web/web.go | 28 +++++----- 13 files changed, 98 insertions(+), 98 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 27484e899..108888d65 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -443,7 +443,11 @@ func main() { } files = append(files, fs...) } - return ruleManager.Update(time.Duration(cfg.GlobalConfig.EvaluationInterval), files) + return ruleManager.Update( + time.Duration(cfg.GlobalConfig.EvaluationInterval), + files, + cfg.GlobalConfig.ExternalLabels, + ) }, } @@ -747,13 +751,8 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config if failed { return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } + promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval)) - - // Register conf as current config. - config.CurrentConfigMutex.Lock() - config.CurrentConfig = conf - config.CurrentConfigMutex.Unlock() - level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename) return nil } diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 7a67aed64..c73895b93 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -163,7 +163,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou Logger: &dummyLogger{}, } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(tg.Interval, ruleFiles...) + // TODO(beorn7): Provide a way to pass in external labels. + groupsMap, ers := m.LoadGroups(tg.Interval, nil, ruleFiles...) if ers != nil { return ers } diff --git a/config/config.go b/config/config.go index 7acd54ad4..f4d8d470d 100644 --- a/config/config.go +++ b/config/config.go @@ -20,7 +20,6 @@ import ( "path/filepath" "regexp" "strings" - "sync" "time" "github.com/pkg/errors" @@ -69,10 +68,6 @@ func LoadFile(filename string) (*Config, error) { // The defaults applied before parsing the respective config sections. var ( - // CurrentConfig is a pointer to the current configuration. - CurrentConfig *Config - CurrentConfigMutex sync.RWMutex - // DefaultConfig is the default top-level configuration. DefaultConfig = Config{ GlobalConfig: DefaultGlobalConfig, diff --git a/docs/configuration/alerting_rules.md b/docs/configuration/alerting_rules.md index e8110a1a8..1bc435beb 100644 --- a/docs/configuration/alerting_rules.md +++ b/docs/configuration/alerting_rules.md @@ -42,9 +42,11 @@ The `annotations` clause specifies a set of informational labels that can be use #### Templating -Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles). -The `$labels` & `$externalLabels` variables hold the label key/value pairs of an alert instance -and `$value` holds the evaluated value of an alert instance. +Label and annotation values can be templated using [console +templates](https://prometheus.io/docs/visualization/consoles). The `$labels` +variable holds the label key/value pairs of an alert instance. The configured +external labels can be accessed via the `$externalLabels` variable. The +`$value` variable holds the evaluated value of an alert instance. # To insert a firing element's label values: {{ $labels. }} diff --git a/docs/configuration/template_reference.md b/docs/configuration/template_reference.md index 2e10e4247..8791d68b3 100644 --- a/docs/configuration/template_reference.md +++ b/docs/configuration/template_reference.md @@ -89,8 +89,10 @@ parameterize templates, and have a few other differences. ### Alert field templates -`.Value` and `.Labels` contain the alert value and labels. They are also exposed -as the `$value` and `$labels` variables for convenience. +`.Value`, `.Labels`, and `ExternalLabels` contain the alert value, the alert +labels, and the globally configured external labels, respectively. They are +also exposed as the `$value`, `$labels`, and `$externalLabels` variables for +convenience. ### Console templates @@ -104,7 +106,9 @@ auto-escaping. To bypass the auto-escaping use the `safe*` functions., URL parameters are available as a map in `.Params`. To access multiple URL parameters by the same name, `.RawParams` is a map of the list values for each parameter. The URL path is available in `.Path`, excluding the `/consoles/` -prefix. +prefix. The globally configured external labels are available as +`.ExternalLabels`. There are also convenience variables for all four: +`$rawParams`, `$params`, `$path`, and `$externalLabels`. Consoles also have access to all the templates defined with `{{define "templateName"}}...{{end}}` found in `*.lib` files in the directory pointed to diff --git a/pkg/rulefmt/rulefmt.go b/pkg/rulefmt/rulefmt.go index 19a2d1836..b89470d38 100644 --- a/pkg/rulefmt/rulefmt.go +++ b/pkg/rulefmt/rulefmt.go @@ -16,6 +16,7 @@ package rulefmt import ( "context" "io/ioutil" + "strings" "time" "github.com/pkg/errors" @@ -155,12 +156,16 @@ func testTemplateParsing(rl *Rule) (errs []error) { } // Trying to parse templates. - tmplData := template.AlertTemplateData(make(map[string]string), make(map[string]string), 0) - defs := "{{$labels := .Labels}}{{$value := .Value}}" + tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0) + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } parseTest := func(text string) error { tmpl := template.NewTemplateExpander( context.TODO(), - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+rl.Alert, tmplData, model.Time(timestamp.FromTime(time.Now())), diff --git a/rules/alerting.go b/rules/alerting.go index c215fc014..b66d6db88 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "net/url" + "strings" "sync" "time" @@ -28,7 +29,6 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -80,9 +80,8 @@ func (s AlertState) String() string { type Alert struct { State AlertState - Labels labels.Labels - ExternalLabels labels.Labels - Annotations labels.Labels + Labels labels.Labels + Annotations labels.Labels // The value at the last evaluation of the alerting expression. Value float64 @@ -121,6 +120,8 @@ type AlertingRule struct { labels labels.Labels // Non-identifying key/value pairs. annotations labels.Labels + // External labels from the global config. + externalLabels map[string]string // true if old state has been restored. We start persisting samples for ALERT_FOR_STATE // only after the restoration. restored bool @@ -142,17 +143,27 @@ type AlertingRule struct { } // NewAlertingRule constructs a new AlertingRule. -func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns labels.Labels, restored bool, logger log.Logger) *AlertingRule { +func NewAlertingRule( + name string, vec promql.Expr, hold time.Duration, + labels, annotations, externalLabels labels.Labels, + restored bool, logger log.Logger, +) *AlertingRule { + el := make(map[string]string, len(externalLabels)) + for _, lbl := range externalLabels { + el[lbl.Name] = lbl.Value + } + return &AlertingRule{ - name: name, - vector: vec, - holdDuration: hold, - labels: lbls, - annotations: anns, - health: HealthUnknown, - active: map[uint64]*Alert{}, - logger: logger, - restored: restored, + name: name, + vector: vec, + holdDuration: hold, + labels: labels, + annotations: annotations, + externalLabels: el, + health: HealthUnknown, + active: map[uint64]*Alert{}, + logger: logger, + restored: restored, } } @@ -302,32 +313,24 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, var vec promql.Vector for _, smpl := range res { // Provide the alert information to the template. - l := make(map[string]string) + l := make(map[string]string, len(smpl.Metric)) for _, lbl := range smpl.Metric { l[lbl.Name] = lbl.Value } - // Add external labels. - el := make(map[string]string) - if config.CurrentConfig != nil { - config.CurrentConfigMutex.RLock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - el[string(eln)] = string(elv) - } - config.CurrentConfigMutex.RUnlock() - } - - tmplData := template.AlertTemplateData(l, el, smpl.V) + tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}" - defs += "{{$externalLabels := .ExternalLabels}}" - defs += "{{$value := .Value}}" + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } expand := func(text string) string { tmpl := template.NewTemplateExpander( ctx, - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+r.Name(), tmplData, model.Time(timestamp.FromTime(ts)), @@ -463,7 +466,7 @@ func (r *AlertingRule) ForEachActiveAlert(f func(*Alert)) { } func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) { - alerts := make([]*Alert, 0) + alerts := []*Alert{} r.ForEachActiveAlert(func(alert *Alert) { if alert.needsSending(ts, resendDelay) { alert.LastSentAt = ts diff --git a/rules/alerting_test.go b/rules/alerting_test.go index 884da30bf..a862e0923 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -26,7 +26,7 @@ import ( func TestAlertingRuleHTMLSnippet(t *testing.T) { expr, err := promql.ParseExpr(`foo{html="BOLD"}`) testutil.Ok(t, err) - rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), false, nil) + rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), nil, false, nil) const want = `alert: testrule expr: foo{html="<b>BOLD<b>"} @@ -62,7 +62,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { // If an alert is going back and forth between two label values it will never fire. // Instead, you should write two alerts with constant labels. labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"), - nil, true, nil, + nil, nil, true, nil, ) results := []promql.Vector{ diff --git a/rules/manager.go b/rules/manager.go index 96766b2ab..77130489f 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -751,11 +751,11 @@ func (m *Manager) Stop() { // Update the rule manager's state as the config requires. If // loading the new rules failed the old rule set is restored. -func (m *Manager) Update(interval time.Duration, files []string) error { +func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error { m.mtx.Lock() defer m.mtx.Unlock() - groups, errs := m.LoadGroups(interval, files...) + groups, errs := m.LoadGroups(interval, externalLabels, files...) if errs != nil { for _, e := range errs { level.Error(m.logger).Log("msg", "loading groups failed", "err", e) @@ -803,7 +803,9 @@ func (m *Manager) Update(interval time.Duration, files []string) error { } // LoadGroups reads groups from a list of files. -func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[string]*Group, []error) { +func (m *Manager) LoadGroups( + interval time.Duration, externalLabels labels.Labels, filenames ...string, +) (map[string]*Group, []error) { groups := make(map[string]*Group) shouldRestore := !m.restored @@ -834,6 +836,7 @@ func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[s time.Duration(r.For), labels.FromMap(r.Labels), labels.FromMap(r.Annotations), + externalLabels, m.restored, log.With(m.logger, "alert", r.Alert), )) diff --git a/rules/manager_test.go b/rules/manager_test.go index 6dbff9c38..6298c0866 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -51,7 +51,7 @@ func TestAlertingRule(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -192,7 +192,7 @@ func TestForStateAddSamples(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -366,7 +366,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, true, nil, + nil, nil, true, nil, ) group := NewGroup("default", "", time.Second, []Rule{rule}, true, opts) @@ -426,7 +426,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, false, nil, + nil, nil, false, nil, ) newGroup := NewGroup("default", "", time.Second, []Rule{newRule}, true, opts) @@ -581,12 +581,12 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) { func TestCopyState(t *testing.T) { oldGroup := &Group{ rules: []Rule{ - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), NewRecordingRule("rule2", nil, nil), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), }, seriesInPreviousEval: []map[string]labels.Labels{ {"a": nil}, @@ -604,10 +604,10 @@ func TestCopyState(t *testing.T) { NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, true, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), NewRecordingRule("rule4", nil, nil), }, seriesInPreviousEval: make([]map[string]labels.Labels, 8), @@ -654,7 +654,7 @@ func TestUpdate(t *testing.T) { ruleManager.Run() defer ruleManager.Stop() - err := ruleManager.Update(10*time.Second, files) + err := ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) testutil.Assert(t, len(ruleManager.groups) > 0, "expected non-empty rule groups") for _, g := range ruleManager.groups { @@ -663,7 +663,7 @@ func TestUpdate(t *testing.T) { } } - err = ruleManager.Update(10*time.Second, files) + err = ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) for _, g := range ruleManager.groups { for _, actual := range g.seriesInPreviousEval { @@ -699,7 +699,7 @@ func TestNotify(t *testing.T) { expr, err := promql.ParseExpr("a > 1") testutil.Ok(t, err) - rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, true, log.NewNopLogger()) + rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger()) group := NewGroup("alert", "", time.Second, []Rule{rule}, true, opts) app, _ := storage.Appender() diff --git a/template/template.go b/template/template.go index 88b0e841d..de6ee5453 100644 --- a/template/template.go +++ b/template/template.go @@ -31,7 +31,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) @@ -134,19 +133,6 @@ func NewTemplateExpander( "label": func(label string, s *sample) string { return s.Labels[label] }, - "externalLabel": func(label string) string { - if config.CurrentConfig == nil { - return "" - } - config.CurrentConfigMutex.RLock() - defer config.CurrentConfigMutex.RUnlock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - if label == string(eln) { - return string(elv) - } - } - return "" - }, "value": func(s *sample) float64 { return s.Value }, diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 5082c1172..b2b8939de 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -142,6 +142,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) @@ -151,6 +152,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) diff --git a/web/web.go b/web/web.go index 4762bb6dd..77a059725 100644 --- a/web/web.go +++ b/web/web.go @@ -542,22 +542,22 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { params[k] = v[0] } - // External labels - el := make(map[string]string) - if config.CurrentConfig != nil { - config.CurrentConfigMutex.RLock() - for eln, elv := range (*config.CurrentConfig).GlobalConfig.ExternalLabels { - el[string(eln)] = string(elv) - } - config.CurrentConfigMutex.RUnlock() + externalLabels := map[string]string{} + h.mtx.RLock() + els := h.config.GlobalConfig.ExternalLabels + h.mtx.RUnlock() + for _, el := range els { + externalLabels[el.Name] = el.Value } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$rawParams := .RawParams }}" - defs += "{{$params := .Params}}" - defs += "{{$path := .Path}}" - defs += "{{$externalLabels := .ExternalLabels}}" + defs := []string{ + "{{$rawParams := .RawParams }}", + "{{$params := .Params}}", + "{{$path := .Path}}", + "{{$externalLabels := .ExternalLabels}}", + } data := struct { RawParams url.Values @@ -568,12 +568,12 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { RawParams: rawParams, Params: params, Path: strings.TrimLeft(name, "/"), - ExternalLabels: el, + ExternalLabels: externalLabels, } tmpl := template.NewTemplateExpander( h.context, - defs+string(text), + strings.Join(append(defs, string(text)), ""), "__console_"+name, data, h.now(), From 76102d570ce9d427d4b9a944a1c0ec45d2daae08 Mon Sep 17 00:00:00 2001 From: Bjoern Rabenstein Date: Sat, 20 Apr 2019 02:19:06 +0200 Subject: [PATCH 4/4] Add test for external labels in label template Signed-off-by: Bjoern Rabenstein --- rules/alerting_test.go | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/rules/alerting_test.go b/rules/alerting_test.go index a862e0923..8d611c3e4 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/go-kit/kit/log" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" @@ -142,3 +143,96 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { testutil.Equals(t, result, filteredRes) } } + +func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) { + suite, err := promql.NewTest(t, ` + load 1m + http_requests{job="app-server", instance="0"} 75 85 70 70 + `) + testutil.Ok(t, err) + defer suite.Close() + + err = suite.Run() + testutil.Ok(t, err) + + expr, err := promql.ParseExpr(`http_requests < 100`) + testutil.Ok(t, err) + + ruleWithoutExternalLabels := NewAlertingRule( + "ExternalLabelDoesNotExist", + expr, + time.Minute, + labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), + nil, + nil, + true, log.NewNopLogger(), + ) + ruleWithExternalLabels := NewAlertingRule( + "ExternalLabelExists", + expr, + time.Minute, + labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), + nil, + labels.FromStrings("foo", "bar", "dings", "bums"), + true, log.NewNopLogger(), + ) + result := promql.Vector{ + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalLabelDoesNotExist", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "There are 0 external Labels, of which foo is .", + ), + Point: promql.Point{V: 1}, + }, + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalLabelExists", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "There are 2 external Labels, of which foo is bar.", + ), + Point: promql.Point{V: 1}, + }, + } + + evalTime := time.Unix(0, 0) + result[0].Point.T = timestamp.FromTime(evalTime) + result[1].Point.T = timestamp.FromTime(evalTime) + + var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples. + res, err := ruleWithoutExternalLabels.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + testutil.Ok(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + testutil.Equals(t, smplName, "ALERTS_FOR_STATE") + } + } + + res, err = ruleWithExternalLabels.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + testutil.Ok(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + testutil.Equals(t, smplName, "ALERTS_FOR_STATE") + } + } + + testutil.Equals(t, result, filteredRes) +}