From b2cb637f97a7e049c1f057bc56f6981211b8b509 Mon Sep 17 00:00:00 2001 From: Silas Snider Date: Tue, 1 Sep 2015 10:18:39 -0700 Subject: [PATCH] Add instrumentation around configuration reloads. This commit enables automation (and alerting) around attempts to update prometheus server configuration automatically. --- cmd/prometheus/main.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 93b8e3b13..f2c518678 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -46,6 +46,19 @@ func main() { os.Exit(Main()) } +var ( + configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "prometheus", + Name: "config_last_reload_successful", + Help: "Whether the last configuration reload attempt was successful.", + }) + configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "prometheus", + Name: "config_last_reload_success_timestamp_seconds", + Help: "Timestamp of the last successful configuration reload.", + }) +) + // Main manages the startup and shutdown lifecycle of the entire Prometheus server. func Main() int { if err := parse(os.Args[1:]); err != nil { @@ -134,6 +147,8 @@ func Main() int { // The storage has to be fully initialized before registering. prometheus.MustRegister(memStorage) prometheus.MustRegister(notificationHandler) + prometheus.MustRegister(configSuccess) + prometheus.MustRegister(configSuccessTime) go ruleManager.Run() defer ruleManager.Stop() @@ -172,8 +187,16 @@ type Reloadable interface { ApplyConfig(*config.Config) bool } -func reloadConfig(filename string, rls ...Reloadable) bool { +func reloadConfig(filename string, rls ...Reloadable) (success bool) { log.Infof("Loading configuration file %s", filename) + defer func() { + if success { + configSuccess.Set(1) + configSuccessTime.Set(float64(time.Now().Unix())) + } else { + configSuccess.Set(0) + } + }() conf, err := config.LoadFile(filename) if err != nil { @@ -181,7 +204,7 @@ func reloadConfig(filename string, rls ...Reloadable) bool { log.Errorf("Note: The configuration format has changed with version 0.14. Please see the documentation (http://prometheus.io/docs/operating/configuration/) and the provided configuration migration tool (https://github.com/prometheus/migrate).") return false } - success := true + success = true for _, rl := range rls { success = success && rl.ApplyConfig(conf)