From 44e5f220c07c88e5e25650f278d3632b9df22cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Mon, 23 May 2022 15:00:59 +0100 Subject: [PATCH] Move prometheus_ready metric to web package (#10729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This moves prometheus_ready to the web package and links it with the ready variable that decides if HTTP requests should return 200 or 503. This is a follow up change from #10682 Signed-off-by: Łukasz Mierzwa --- cmd/prometheus/main.go | 9 ++------- web/web.go | 22 +++++++++++++++++----- web/web_test.go | 22 +++++++++++++++++----- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index ccc41a566..463b01ea8 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -86,10 +86,6 @@ var ( Name: "prometheus_config_last_reload_success_timestamp_seconds", Help: "Timestamp of the last successful configuration reload.", }) - readyStatus = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "prometheus_ready", - Help: "Whether Prometheus startup was fully completed and the server is ready for normal operation.", - }) defaultRetentionString = "15d" defaultRetentionDuration model.Duration @@ -756,7 +752,6 @@ func main() { prometheus.MustRegister(configSuccess) prometheus.MustRegister(configSuccessTime) - prometheus.MustRegister(readyStatus) // Start all components while we wait for TSDB to open but only load // initial config and mark ourselves as ready after it completed. @@ -812,6 +807,7 @@ func main() { }, func(err error) { close(cancel) + webHandler.SetReady(false) }, ) } @@ -949,9 +945,8 @@ func main() { reloadReady.Close() - webHandler.Ready() + webHandler.SetReady(true) level.Info(logger).Log("msg", "Server is ready to receive web requests.") - readyStatus.Set(1) <-cancel return nil }, diff --git a/web/web.go b/web/web.go index 132582872..959e6771c 100644 --- a/web/web.go +++ b/web/web.go @@ -109,6 +109,7 @@ type metrics struct { requestCounter *prometheus.CounterVec requestDuration *prometheus.HistogramVec responseSize *prometheus.HistogramVec + readyStatus prometheus.Gauge } func newMetrics(r prometheus.Registerer) *metrics { @@ -136,10 +137,14 @@ func newMetrics(r prometheus.Registerer) *metrics { }, []string{"handler"}, ), + readyStatus: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_ready", + Help: "Whether Prometheus startup was fully completed and the server is ready for normal operation.", + }), } if r != nil { - r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize) + r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize, m.readyStatus) registerFederationMetrics(r) } return m @@ -301,7 +306,7 @@ func New(logger log.Logger, o *Options) *Handler { now: model.Now, } - h.ready.Store(0) + h.SetReady(false) factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager } factoryAr := func(_ context.Context) api_v1.AlertmanagerRetriever { return h.notifier } @@ -503,9 +508,16 @@ func serveDebug(w http.ResponseWriter, req *http.Request) { } } -// Ready sets Handler to be ready. -func (h *Handler) Ready() { - h.ready.Store(1) +// SetReady sets the ready status of our web Handler +func (h *Handler) SetReady(v bool) { + if v { + h.ready.Store(1) + h.metrics.readyStatus.Set(1) + return + } + + h.ready.Store(0) + h.metrics.readyStatus.Set(0) } // Verifies whether the server is ready or not. diff --git a/web/web_test.go b/web/web_test.go index 2ea21a9ec..102b30a15 100644 --- a/web/web_test.go +++ b/web/web_test.go @@ -146,7 +146,7 @@ func TestReadyAndHealthy(t *testing.T) { cleanupTestResponse(t, resp) // Set to ready. - webHandler.Ready() + webHandler.SetReady(true) for _, u := range []string{ baseURL + "/-/healthy", @@ -245,7 +245,7 @@ func TestRoutePrefix(t *testing.T) { cleanupTestResponse(t, resp) // Set to ready. - webHandler.Ready() + webHandler.SetReady(true) resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy") require.NoError(t, err) @@ -292,7 +292,7 @@ func TestDebugHandler(t *testing.T) { }, } handler := New(nil, opts) - handler.Ready() + handler.SetReady(true) w := httptest.NewRecorder() @@ -329,16 +329,28 @@ func TestHTTPMetrics(t *testing.T) { code := getReady() require.Equal(t, http.StatusServiceUnavailable, code) + ready := handler.metrics.readyStatus + require.Equal(t, 0, int(prom_testutil.ToFloat64(ready))) counter := handler.metrics.requestCounter require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) - handler.Ready() + handler.SetReady(true) for range [2]int{} { code = getReady() require.Equal(t, http.StatusOK, code) } + require.Equal(t, 1, int(prom_testutil.ToFloat64(ready))) require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK))))) require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) + + handler.SetReady(false) + for range [2]int{} { + code = getReady() + require.Equal(t, http.StatusServiceUnavailable, code) + } + require.Equal(t, 0, int(prom_testutil.ToFloat64(ready))) + require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK))))) + require.Equal(t, 3, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) } func TestShutdownWithStaleConnection(t *testing.T) { @@ -510,7 +522,7 @@ func TestAgentAPIEndPoints(t *testing.T) { opts.Flags = map[string]string{} webHandler := New(nil, opts) - webHandler.Ready() + webHandler.SetReady(true) webHandler.config = &config.Config{} webHandler.notifier = ¬ifier.Manager{} l, err := webHandler.Listener()