RepoMirrors
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

web: fix flaky TestHTTPMetrics() (#5695) 

Signed-off-by: Simon Pasquier <spasquie@redhat.com>
This commit is contained in:
Simon Pasquier 2019-06-24 15:48:15 +02:00 committed by GitHub
parent ed5671d57f
commit be67b8d460
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 86 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/prometheus/main.go
View File

@ -123,6 +123,10 @@ func main() {
notifier: notifier.Options{
Registerer: prometheus.DefaultRegisterer,
},
web: web.Options{
Registerer: prometheus.DefaultRegisterer,
Gatherer: prometheus.DefaultGatherer,
},
promlogConfig: promlog.Config{},
}

129
web/web.go
View File

@ -89,40 +89,71 @@ func withStackTracer(h http.Handler, l log.Logger) http.Handler {
})
}
var (
requestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_http_requests_total",
Help: "Counter of HTTP requests.",
},
[]string{"handler", "code"},
)
requestDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "prometheus_http_request_duration_seconds",
Help: "Histogram of latencies for HTTP requests.",
Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
},
[]string{"handler"},
)
responseSize = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "prometheus_http_response_size_bytes",
Help: "Histogram of response size for HTTP requests.",
Buckets: prometheus.ExponentialBuckets(100, 10, 8),
},
[]string{"handler"},
)
)
type metrics struct {
requestCounter *prometheus.CounterVec
requestDuration *prometheus.HistogramVec
responseSize *prometheus.HistogramVec
}
func init() {
prometheus.MustRegister(requestCounter, requestDuration, responseSize)
func newMetrics(r prometheus.Registerer) *metrics {
m := &metrics{
requestCounter: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "prometheus_http_requests_total",
Help: "Counter of HTTP requests.",
},
[]string{"handler", "code"},
),
requestDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "prometheus_http_request_duration_seconds",
Help: "Histogram of latencies for HTTP requests.",
Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
},
[]string{"handler"},
),
responseSize: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "prometheus_http_response_size_bytes",
Help: "Histogram of response size for HTTP requests.",
Buckets: prometheus.ExponentialBuckets(100, 10, 8),
},
[]string{"handler"},
),
}
if r != nil {
r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize)
}
return m
}
func (m *metrics) instrumentHandlerWithPrefix(prefix string) func(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return func(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return m.instrumentHandler(prefix+handlerName, handler)
}
}
func (m *metrics) instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return promhttp.InstrumentHandlerCounter(
m.requestCounter.MustCurryWith(prometheus.Labels{"handler": handlerName}),
promhttp.InstrumentHandlerDuration(
m.requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}),
promhttp.InstrumentHandlerResponseSize(
m.responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}),
handler,
),
),
)
}
// Handler serves various HTTP endpoints of the Prometheus server
type Handler struct {
logger log.Logger
gatherer prometheus.Gatherer
metrics *metrics
scrapeManager *scrape.Manager
ruleManager *rules.Manager
queryEngine *promql.Engine
@ -197,41 +228,31 @@ type Options struct {
PageTitle string
RemoteReadSampleLimit int
RemoteReadConcurrencyLimit int
}
func instrumentHandlerWithPrefix(prefix string) func(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return func(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return instrumentHandler(prefix+handlerName, handler)
}
}
func instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc {
return promhttp.InstrumentHandlerCounter(
requestCounter.MustCurryWith(prometheus.Labels{"handler": handlerName}),
promhttp.InstrumentHandlerDuration(
requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}),
promhttp.InstrumentHandlerResponseSize(
responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}),
handler,
),
),
)
Gatherer prometheus.Gatherer
Registerer prometheus.Registerer
}
// New initializes a new web Handler.
func New(logger log.Logger, o *Options) *Handler {
router := route.New().WithInstrumentation(instrumentHandler)
cwd, err := os.Getwd()
if err != nil {
cwd = "<error retrieving current working directory>"
}
if logger == nil {
logger = log.NewNopLogger()
}
m := newMetrics(o.Registerer)
router := route.New().WithInstrumentation(m.instrumentHandler)
cwd, err := os.Getwd()
if err != nil {
cwd = "<error retrieving current working directory>"
}
h := &Handler{
logger: logger,
logger: logger,
gatherer: o.Gatherer,
metrics: m,
router: router,
quitCh: make(chan struct{}),
reloadCh: make(chan chan error),
@ -463,7 +484,7 @@ func (h *Handler) Run(ctx context.Context) error {
mux := http.NewServeMux()
mux.Handle("/", h.router)
av1 := route.New().WithInstrumentation(instrumentHandlerWithPrefix("/api/v1"))
av1 := route.New().WithInstrumentation(h.metrics.instrumentHandlerWithPrefix("/api/v1"))
h.apiV1.Register(av1)
apiPath := "/api"
if h.options.RoutePrefix != "/" {
@ -649,7 +670,7 @@ func (h *Handler) status(w http.ResponseWriter, r *http.Request) {
status.StorageRetention = status.StorageRetention + h.options.TSDBCfg.MaxBytes.String()
}
metrics, err := prometheus.DefaultGatherer.Gather()
metrics, err := h.gatherer.Gather()
if err != nil {
http.Error(w, fmt.Sprintf("error gathering runtime status: %s", err), http.StatusInternalServerError)
return

11
web/web_test.go
View File

@ -26,6 +26,7 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/prometheus/config"
@ -119,7 +120,8 @@ func TestReadyAndHealthy(t *testing.T) {
Host: "localhost:9090",
Path: "/",
},
Version: &PrometheusVersion{},
Version: &PrometheusVersion{},
Gatherer: prometheus.DefaultGatherer,
}
opts.Flags = map[string]string{}
@ -423,13 +425,14 @@ func TestHTTPMetrics(t *testing.T) {
code := getReady()
testutil.Equals(t, http.StatusServiceUnavailable, code)
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
counter := handler.metrics.requestCounter
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
handler.Ready()
for range [2]int{} {
code = getReady()
testutil.Equals(t, http.StatusOK, code)
}
testutil.Equals(t, 2, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(requestCounter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
testutil.Equals(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
}