// Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package web import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "net" "net/http" "net/http/pprof" "net/url" "os" "path" "path/filepath" "sort" "strings" "sync" "sync/atomic" "time" pprof_runtime "runtime/pprof" template_text "text/template" "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" "github.com/prometheus/common/model" "github.com/prometheus/common/route" "golang.org/x/net/context" "golang.org/x/net/netutil" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/retrieval" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage/local" "github.com/prometheus/prometheus/template" "github.com/prometheus/prometheus/util/httputil" api_v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/prometheus/prometheus/web/ui" ) var localhostRepresentations = []string{"127.0.0.1", "localhost"} // Handler serves various HTTP endpoints of the Prometheus server type Handler struct { targetManager *retrieval.TargetManager ruleManager *rules.Manager queryEngine *promql.Engine context context.Context storage local.Storage notifier *notifier.Notifier apiV1 *api_v1.API router *route.Router listenErrCh chan error quitCh chan struct{} reloadCh chan chan error options *Options config *config.Config versionInfo *PrometheusVersion birth time.Time cwd string flagsMap map[string]string mtx sync.RWMutex now func() model.Time ready uint32 // ready is uint32 rather than boolean to be able to use atomic functions. } // ApplyConfig updates the config field of the Handler struct func (h *Handler) ApplyConfig(conf *config.Config) error { h.mtx.Lock() defer h.mtx.Unlock() h.config = conf return nil } // PrometheusVersion contains build information about Prometheus. type PrometheusVersion struct { Version string `json:"version"` Revision string `json:"revision"` Branch string `json:"branch"` BuildUser string `json:"buildUser"` BuildDate string `json:"buildDate"` GoVersion string `json:"goVersion"` } // Options for the web Handler. type Options struct { Context context.Context Storage local.Storage QueryEngine *promql.Engine TargetManager *retrieval.TargetManager RuleManager *rules.Manager Notifier *notifier.Notifier Version *PrometheusVersion Flags map[string]string ListenAddress string ReadTimeout time.Duration MaxConnections int ExternalURL *url.URL RoutePrefix string MetricsPath string UseLocalAssets bool UserAssetsPath string ConsoleTemplatesPath string ConsoleLibrariesPath string EnableQuit bool } // New initializes a new web Handler. func New(o *Options) *Handler { router := route.New() cwd, err := os.Getwd() if err != nil { cwd = "" } h := &Handler{ router: router, listenErrCh: make(chan error), quitCh: make(chan struct{}), reloadCh: make(chan chan error), options: o, versionInfo: o.Version, birth: time.Now(), cwd: cwd, flagsMap: o.Flags, context: o.Context, targetManager: o.TargetManager, ruleManager: o.RuleManager, queryEngine: o.QueryEngine, storage: o.Storage, notifier: o.Notifier, now: model.Now, ready: 0, } h.apiV1 = api_v1.NewAPI( o.QueryEngine, o.Storage, o.TargetManager, o.Notifier, func() config.Config { h.mtx.RLock() defer h.mtx.RUnlock() return *h.config }, ) if o.RoutePrefix != "/" { // If the prefix is missing for the root path, prepend it. router.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, o.RoutePrefix, http.StatusFound) }) router = router.WithPrefix(o.RoutePrefix) } instrh := prometheus.InstrumentHandler instrf := prometheus.InstrumentHandlerFunc readyf := h.testReady router.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, path.Join(o.ExternalURL.Path, "/graph"), http.StatusFound) }) router.Get("/alerts", readyf(instrf("alerts", h.alerts))) router.Get("/graph", readyf(instrf("graph", h.graph))) router.Get("/status", readyf(instrf("status", h.status))) router.Get("/flags", readyf(instrf("flags", h.flags))) router.Get("/config", readyf(instrf("config", h.serveConfig))) router.Get("/rules", readyf(instrf("rules", h.rules))) router.Get("/targets", readyf(instrf("targets", h.targets))) router.Get("/version", readyf(instrf("version", h.version))) router.Get("/heap", readyf(instrf("heap", dumpHeap))) router.Get(o.MetricsPath, readyf(prometheus.Handler().ServeHTTP)) router.Get("/federate", readyf(instrh("federate", httputil.CompressionHandler{ Handler: http.HandlerFunc(h.federation), }))) h.apiV1.Register(router.WithPrefix("/api/v1")) router.Get("/consoles/*filepath", readyf(instrf("consoles", h.consoles))) router.Get("/static/*filepath", readyf(instrf("static", serveStaticAsset))) if o.UserAssetsPath != "" { router.Get("/user/*filepath", readyf(instrf("user", route.FileServe(o.UserAssetsPath)))) } if o.EnableQuit { router.Post("/-/quit", readyf(h.quit)) } router.Post("/-/reload", readyf(h.reload)) router.Get("/-/reload", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusMethodNotAllowed) fmt.Fprintf(w, "This endpoint requires a POST request.\n") }) router.Get("/debug/*subpath", readyf(serveDebug)) router.Post("/debug/*subpath", readyf(serveDebug)) router.Get("/-/healthy", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "Prometheus is Healthy.\n") }) router.Get("/-/ready", readyf(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "Prometheus is Ready.\n") })) return h } func serveDebug(w http.ResponseWriter, req *http.Request) { ctx := req.Context() subpath := route.Param(ctx, "subpath") // Based off paths from init() in golang.org/src/net/http/pprof/pprof.go if subpath == "/pprof/" { pprof.Index(w, req) } else if subpath == "/pprof/cmdline" { pprof.Cmdline(w, req) } else if subpath == "/pprof/profile" { pprof.Profile(w, req) } else if subpath == "/pprof/symbol" { pprof.Symbol(w, req) } else if subpath == "/pprof/trace" { pprof.Trace(w, req) } else { http.NotFound(w, req) } } func serveStaticAsset(w http.ResponseWriter, req *http.Request) { fp := route.Param(req.Context(), "filepath") fp = filepath.Join("web/ui/static", fp) info, err := ui.AssetInfo(fp) if err != nil { log.With("file", fp).Warn("Could not get file info: ", err) w.WriteHeader(http.StatusNotFound) return } file, err := ui.Asset(fp) if err != nil { if err != io.EOF { log.With("file", fp).Warn("Could not get file: ", err) } w.WriteHeader(http.StatusNotFound) return } http.ServeContent(w, req, info.Name(), info.ModTime(), bytes.NewReader(file)) } // Ready sets Handler to be ready. func (h *Handler) Ready() { atomic.StoreUint32(&h.ready, 1) } // Verifies whether the server is ready or not. func (h *Handler) isReady() bool { ready := atomic.LoadUint32(&h.ready) if ready == 0 { return false } return true } // Checks if server is ready, calls f if it is, returns 503 if it is not. func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if h.isReady() { f(w, r) } else { w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprintf(w, "Service Unavailable") } } } // ListenError returns the receive-only channel that signals errors while starting the web server. func (h *Handler) ListenError() <-chan error { return h.listenErrCh } // Quit returns the receive-only quit channel. func (h *Handler) Quit() <-chan struct{} { return h.quitCh } // Reload returns the receive-only channel that signals configuration reload requests. func (h *Handler) Reload() <-chan chan error { return h.reloadCh } // Run serves the HTTP endpoints. func (h *Handler) Run() { log.Infof("Listening on %s", h.options.ListenAddress) operationName := nethttp.OperationNameFunc(func(r *http.Request) string { return fmt.Sprintf("%s %s", r.Method, r.URL.Path) }) server := &http.Server{ Addr: h.options.ListenAddress, Handler: nethttp.Middleware(opentracing.GlobalTracer(), h.router, operationName), ErrorLog: log.NewErrorLogger(), ReadTimeout: h.options.ReadTimeout, } listener, err := net.Listen("tcp", h.options.ListenAddress) if err != nil { h.listenErrCh <- err } else { limitedListener := netutil.LimitListener(listener, h.options.MaxConnections) h.listenErrCh <- server.Serve(limitedListener) } } func (h *Handler) alerts(w http.ResponseWriter, r *http.Request) { alerts := h.ruleManager.AlertingRules() alertsSorter := byAlertStateAndNameSorter{alerts: alerts} sort.Sort(alertsSorter) alertStatus := AlertStatus{ AlertingRules: alertsSorter.alerts, AlertStateToRowClass: map[rules.AlertState]string{ rules.StateInactive: "success", rules.StatePending: "warning", rules.StateFiring: "danger", }, } h.executeTemplate(w, "alerts.html", alertStatus) } func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { ctx := r.Context() name := route.Param(ctx, "filepath") file, err := http.Dir(h.options.ConsoleTemplatesPath).Open(name) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return } text, err := ioutil.ReadAll(file) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Provide URL parameters as a map for easy use. Advanced users may have need for // parameters beyond the first, so provide RawParams. rawParams, err := url.ParseQuery(r.URL.RawQuery) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } params := map[string]string{} for k, v := range rawParams { params[k] = v[0] } data := struct { RawParams url.Values Params map[string]string Path string }{ RawParams: rawParams, Params: params, Path: strings.TrimLeft(name, "/"), } tmpl := template.NewTemplateExpander(h.context, string(text), "__console_"+name, data, h.now(), h.queryEngine, h.options.ExternalURL) filenames, err := filepath.Glob(h.options.ConsoleLibrariesPath + "/*.lib") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } result, err := tmpl.ExpandHTML(filenames) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } io.WriteString(w, result) } func (h *Handler) graph(w http.ResponseWriter, r *http.Request) { h.executeTemplate(w, "graph.html", nil) } func (h *Handler) status(w http.ResponseWriter, r *http.Request) { h.executeTemplate(w, "status.html", struct { Birth time.Time CWD string Version *PrometheusVersion Alertmanagers []*url.URL }{ Birth: h.birth, CWD: h.cwd, Version: h.versionInfo, Alertmanagers: h.notifier.Alertmanagers(), }) } func (h *Handler) flags(w http.ResponseWriter, r *http.Request) { h.executeTemplate(w, "flags.html", h.flagsMap) } func (h *Handler) serveConfig(w http.ResponseWriter, r *http.Request) { h.mtx.RLock() defer h.mtx.RUnlock() h.executeTemplate(w, "config.html", h.config.String()) } func (h *Handler) rules(w http.ResponseWriter, r *http.Request) { h.executeTemplate(w, "rules.html", h.ruleManager) } func (h *Handler) targets(w http.ResponseWriter, r *http.Request) { // Bucket targets by job label tps := map[string][]*retrieval.Target{} for _, t := range h.targetManager.Targets() { job := string(t.Labels()[model.JobLabel]) tps[job] = append(tps[job], t) } for _, targets := range tps { sort.Slice(targets, func(i, j int) bool { return targets[i].Labels()[model.InstanceLabel] < targets[j].Labels()[model.InstanceLabel] }) } h.executeTemplate(w, "targets.html", struct { TargetPools map[string][]*retrieval.Target }{ TargetPools: tps, }) } func (h *Handler) version(w http.ResponseWriter, r *http.Request) { dec := json.NewEncoder(w) if err := dec.Encode(h.versionInfo); err != nil { http.Error(w, fmt.Sprintf("error encoding JSON: %s", err), http.StatusInternalServerError) } } func (h *Handler) quit(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Requesting termination... Goodbye!") close(h.quitCh) } func (h *Handler) reload(w http.ResponseWriter, r *http.Request) { rc := make(chan error) h.reloadCh <- rc if err := <-rc; err != nil { http.Error(w, fmt.Sprintf("failed to reload config: %s", err), http.StatusInternalServerError) } } func (h *Handler) consolesPath() string { if _, err := os.Stat(h.options.ConsoleTemplatesPath + "/index.html"); !os.IsNotExist(err) { return h.options.ExternalURL.Path + "/consoles/index.html" } if h.options.UserAssetsPath != "" { if _, err := os.Stat(h.options.UserAssetsPath + "/index.html"); !os.IsNotExist(err) { return h.options.ExternalURL.Path + "/user/index.html" } } return "" } func tmplFuncs(consolesPath string, opts *Options) template_text.FuncMap { return template_text.FuncMap{ "since": func(t time.Time) time.Duration { return time.Since(t) / time.Millisecond * time.Millisecond }, "consolesPath": func() string { return consolesPath }, "pathPrefix": func() string { return opts.ExternalURL.Path }, "buildVersion": func() string { return opts.Version.Revision }, "stripLabels": func(lset model.LabelSet, labels ...model.LabelName) model.LabelSet { for _, ln := range labels { delete(lset, ln) } return lset }, "globalURL": func(u *url.URL) *url.URL { host, port, err := net.SplitHostPort(u.Host) if err != nil { return u } for _, lhr := range localhostRepresentations { if host == lhr { _, ownPort, err := net.SplitHostPort(opts.ListenAddress) if err != nil { return u } if port == ownPort { // Only in the case where the target is on localhost and its port is // the same as the one we're listening on, we know for sure that // we're monitoring our own process and that we need to change the // scheme, hostname, and port to the externally reachable ones as // well. We shouldn't need to touch the path at all, since if a // path prefix is defined, the path under which we scrape ourselves // should already contain the prefix. u.Scheme = opts.ExternalURL.Scheme u.Host = opts.ExternalURL.Host } else { // Otherwise, we only know that localhost is not reachable // externally, so we replace only the hostname by the one in the // external URL. It could be the wrong hostname for the service on // this port, but it's still the best possible guess. host, _, err := net.SplitHostPort(opts.ExternalURL.Host) if err != nil { return u } u.Host = host + ":" + port } break } } return u }, "numHealthy": func(pool []*retrieval.Target) int { alive := len(pool) for _, p := range pool { if p.Health() != retrieval.HealthGood { alive-- } } return alive }, "healthToClass": func(th retrieval.TargetHealth) string { switch th { case retrieval.HealthUnknown: return "warning" case retrieval.HealthGood: return "success" default: return "danger" } }, "alertStateToClass": func(as rules.AlertState) string { switch as { case rules.StateInactive: return "success" case rules.StatePending: return "warning" case rules.StateFiring: return "danger" default: panic("unknown alert state") } }, } } func (h *Handler) getTemplate(name string) (string, error) { baseTmpl, err := ui.Asset("web/ui/templates/_base.html") if err != nil { return "", fmt.Errorf("error reading base template: %s", err) } pageTmpl, err := ui.Asset(filepath.Join("web/ui/templates", name)) if err != nil { return "", fmt.Errorf("error reading page template %s: %s", name, err) } return string(baseTmpl) + string(pageTmpl), nil } func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) { text, err := h.getTemplate(name) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } tmpl := template.NewTemplateExpander(h.context, text, name, data, h.now(), h.queryEngine, h.options.ExternalURL) tmpl.Funcs(tmplFuncs(h.consolesPath(), h.options)) result, err := tmpl.ExpandHTML(nil) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } io.WriteString(w, result) } func dumpHeap(w http.ResponseWriter, r *http.Request) { target := fmt.Sprintf("/tmp/%d.heap", time.Now().Unix()) f, err := os.Create(target) if err != nil { log.Error("Could not dump heap: ", err) } fmt.Fprintf(w, "Writing to %s...", target) defer f.Close() pprof_runtime.WriteHeapProfile(f) fmt.Fprintf(w, "Done") } // AlertStatus bundles alerting rules and the mapping of alert states to row classes. type AlertStatus struct { AlertingRules []*rules.AlertingRule AlertStateToRowClass map[rules.AlertState]string } type byAlertStateAndNameSorter struct { alerts []*rules.AlertingRule } func (s byAlertStateAndNameSorter) Len() int { return len(s.alerts) } func (s byAlertStateAndNameSorter) Less(i, j int) bool { return s.alerts[i].State() > s.alerts[j].State() || (s.alerts[i].State() == s.alerts[j].State() && s.alerts[i].Name() < s.alerts[j].Name()) } func (s byAlertStateAndNameSorter) Swap(i, j int) { s.alerts[i], s.alerts[j] = s.alerts[j], s.alerts[i] }