mirror of
https://github.com/prometheus/prometheus
synced 2024-12-25 07:52:28 +00:00
d4b9fe801f
When appending to the head and a chunk is full it is flushed to the disk and m-mapped (memory mapped) to free up memory Prom startup now happens in these stages - Iterate the m-maped chunks from disk and keep a map of series reference to its slice of mmapped chunks. - Iterate the WAL as usual. Whenever we create a new series, look for it's mmapped chunks in the map created before and add it to that series. If a head chunk is corrupted the currpted one and all chunks after that are deleted and the data after the corruption is recovered from the existing WAL which means that a corruption in m-mapped files results in NO data loss. [Mmaped chunks format](https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/head_chunks.md) - main difference is that the chunk for mmaping now also includes series reference because there is no index for mapping series to chunks. [The block chunks](https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/chunks.md) are accessed from the index which includes the offsets for the chunks in the chunks file - example - chunks of series ID have offsets 200, 500 etc in the chunk files. In case of mmaped chunks, the offsets are stored in memory and accessed from that. During WAL replay, these offsets are restored by iterating all m-mapped chunks as stated above by matching the series id present in the chunk header and offset of that chunk in that file. **Prombench results** _WAL Replay_ 1h Wal reply time 30% less wal reply time - 4m31 vs 3m36 2h Wal reply time 20% less wal reply time - 8m16 vs 7m _Memory During WAL Replay_ High Churn: 10-15% less RAM - 32gb vs 28gb 20% less RAM after compaction 34gb vs 27gb No Churn: 20-30% less RAM - 23gb vs 18gb 40% less RAM after compaction 32.5gb vs 20gb Screenshots are in [this comment](https://github.com/prometheus/prometheus/pull/6679#issuecomment-621678932) Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>
462 lines
12 KiB
Go
462 lines
12 KiB
Go
// Copyright 2016 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package web
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/prometheus/prometheus/config"
|
|
"github.com/prometheus/prometheus/notifier"
|
|
"github.com/prometheus/prometheus/rules"
|
|
"github.com/prometheus/prometheus/scrape"
|
|
"github.com/prometheus/prometheus/tsdb"
|
|
"github.com/prometheus/prometheus/util/testutil"
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
// On linux with a global proxy the tests will fail as the go client(http,grpc) tries to connect through the proxy.
|
|
os.Setenv("no_proxy", "localhost,127.0.0.1,0.0.0.0,:")
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
func TestGlobalURL(t *testing.T) {
|
|
opts := &Options{
|
|
ListenAddress: ":9090",
|
|
ExternalURL: &url.URL{
|
|
Scheme: "https",
|
|
Host: "externalhost:80",
|
|
Path: "/path/prefix",
|
|
},
|
|
}
|
|
|
|
tests := []struct {
|
|
inURL string
|
|
outURL string
|
|
}{
|
|
{
|
|
// Nothing should change if the input URL is not on localhost, even if the port is our listening port.
|
|
inURL: "http://somehost:9090/metrics",
|
|
outURL: "http://somehost:9090/metrics",
|
|
},
|
|
{
|
|
// Port and host should change if target is on localhost and port is our listening port.
|
|
inURL: "http://localhost:9090/metrics",
|
|
outURL: "https://externalhost:80/metrics",
|
|
},
|
|
{
|
|
// Only the host should change if the port is not our listening port, but the host is localhost.
|
|
inURL: "http://localhost:8000/metrics",
|
|
outURL: "http://externalhost:8000/metrics",
|
|
},
|
|
{
|
|
// Alternative localhost representations should also work.
|
|
inURL: "http://127.0.0.1:9090/metrics",
|
|
outURL: "https://externalhost:80/metrics",
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
inURL, err := url.Parse(test.inURL)
|
|
|
|
testutil.Ok(t, err)
|
|
|
|
globalURL := tmplFuncs("", opts)["globalURL"].(func(u *url.URL) *url.URL)
|
|
outURL := globalURL(inURL)
|
|
|
|
testutil.Equals(t, test.outURL, outURL.String())
|
|
}
|
|
}
|
|
|
|
type dbAdapter struct {
|
|
*tsdb.DB
|
|
}
|
|
|
|
func (a *dbAdapter) Stats(statsByLabelName string) (*tsdb.Stats, error) {
|
|
return a.Head().Stats(statsByLabelName), nil
|
|
}
|
|
|
|
func TestReadyAndHealthy(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dbDir, err := ioutil.TempDir("", "tsdb-ready")
|
|
testutil.Ok(t, err)
|
|
defer testutil.Ok(t, os.RemoveAll(dbDir))
|
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil)
|
|
testutil.Ok(t, err)
|
|
|
|
opts := &Options{
|
|
ListenAddress: ":9090",
|
|
ReadTimeout: 30 * time.Second,
|
|
MaxConnections: 512,
|
|
Context: nil,
|
|
Storage: nil,
|
|
LocalStorage: &dbAdapter{db},
|
|
TSDBDir: dbDir,
|
|
QueryEngine: nil,
|
|
ScrapeManager: &scrape.Manager{},
|
|
RuleManager: &rules.Manager{},
|
|
Notifier: nil,
|
|
RoutePrefix: "/",
|
|
EnableAdminAPI: true,
|
|
ExternalURL: &url.URL{
|
|
Scheme: "http",
|
|
Host: "localhost:9090",
|
|
Path: "/",
|
|
},
|
|
Version: &PrometheusVersion{},
|
|
Gatherer: prometheus.DefaultGatherer,
|
|
}
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
webHandler := New(nil, opts)
|
|
|
|
webHandler.config = &config.Config{}
|
|
webHandler.notifier = ¬ifier.Manager{}
|
|
|
|
go func() {
|
|
err := webHandler.Run(context.Background())
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
}
|
|
}()
|
|
|
|
// TODO(bwplotka): Those tests create tons of new connection and memory that is never cleaned.
|
|
// Close and exhaust all response bodies.
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
// to be up before starting tests.
|
|
time.Sleep(5 * time.Second)
|
|
|
|
resp, err := http.Get("http://localhost:9090/-/healthy")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/-/ready")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/version")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/graph")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/snapshot", "", strings.NewReader(""))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/graph")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/alerts")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/flags")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/rules")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/service-discovery")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/targets")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/config")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/status")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
// Set to ready.
|
|
webHandler.Ready()
|
|
|
|
resp, err = http.Get("http://localhost:9090/-/healthy")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/-/ready")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/version")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/graph")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/snapshot", "", strings.NewReader(""))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/alerts")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/flags")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/rules")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/service-discovery")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/targets")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/config")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9090/status")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
}
|
|
|
|
func TestRoutePrefix(t *testing.T) {
|
|
t.Parallel()
|
|
dbDir, err := ioutil.TempDir("", "tsdb-ready")
|
|
testutil.Ok(t, err)
|
|
defer testutil.Ok(t, os.RemoveAll(dbDir))
|
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil)
|
|
testutil.Ok(t, err)
|
|
|
|
opts := &Options{
|
|
ListenAddress: ":9091",
|
|
ReadTimeout: 30 * time.Second,
|
|
MaxConnections: 512,
|
|
Context: nil,
|
|
TSDBDir: dbDir,
|
|
LocalStorage: &dbAdapter{db},
|
|
Storage: nil,
|
|
QueryEngine: nil,
|
|
ScrapeManager: nil,
|
|
RuleManager: nil,
|
|
Notifier: nil,
|
|
RoutePrefix: "/prometheus",
|
|
EnableAdminAPI: true,
|
|
ExternalURL: &url.URL{
|
|
Host: "localhost.localdomain:9090",
|
|
Scheme: "http",
|
|
},
|
|
}
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
webHandler := New(nil, opts)
|
|
go func() {
|
|
err := webHandler.Run(context.Background())
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
}
|
|
}()
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
// to be up before starting tests.
|
|
time.Sleep(5 * time.Second)
|
|
|
|
resp, err := http.Get("http://localhost:9091" + opts.RoutePrefix + "/-/healthy")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/-/ready")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/version")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/snapshot", "", strings.NewReader(""))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
// Set to ready.
|
|
webHandler.Ready()
|
|
|
|
resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/-/healthy")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/-/ready")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/version")
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/snapshot", "", strings.NewReader(""))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
|
|
resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
|
|
|
testutil.Ok(t, err)
|
|
testutil.Equals(t, http.StatusOK, resp.StatusCode)
|
|
}
|
|
|
|
func TestDebugHandler(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
prefix, url string
|
|
code int
|
|
}{
|
|
{"/", "/debug/pprof/cmdline", 200},
|
|
{"/foo", "/foo/debug/pprof/cmdline", 200},
|
|
|
|
{"/", "/debug/pprof/goroutine", 200},
|
|
{"/foo", "/foo/debug/pprof/goroutine", 200},
|
|
|
|
{"/", "/debug/pprof/foo", 404},
|
|
{"/foo", "/bar/debug/pprof/goroutine", 404},
|
|
} {
|
|
opts := &Options{
|
|
RoutePrefix: tc.prefix,
|
|
ListenAddress: "somehost:9090",
|
|
ExternalURL: &url.URL{
|
|
Host: "localhost.localdomain:9090",
|
|
Scheme: "http",
|
|
},
|
|
}
|
|
handler := New(nil, opts)
|
|
handler.Ready()
|
|
|
|
w := httptest.NewRecorder()
|
|
|
|
req, err := http.NewRequest("GET", tc.url, nil)
|
|
|
|
testutil.Ok(t, err)
|
|
|
|
handler.router.ServeHTTP(w, req)
|
|
|
|
testutil.Equals(t, tc.code, w.Code)
|
|
}
|
|
}
|
|
|
|
func TestHTTPMetrics(t *testing.T) {
|
|
t.Parallel()
|
|
handler := New(nil, &Options{
|
|
RoutePrefix: "/",
|
|
ListenAddress: "somehost:9090",
|
|
ExternalURL: &url.URL{
|
|
Host: "localhost.localdomain:9090",
|
|
Scheme: "http",
|
|
},
|
|
})
|
|
getReady := func() int {
|
|
t.Helper()
|
|
w := httptest.NewRecorder()
|
|
|
|
req, err := http.NewRequest("GET", "/-/ready", nil)
|
|
testutil.Ok(t, err)
|
|
|
|
handler.router.ServeHTTP(w, req)
|
|
return w.Code
|
|
}
|
|
|
|
code := getReady()
|
|
testutil.Equals(t, http.StatusServiceUnavailable, code)
|
|
counter := handler.metrics.requestCounter
|
|
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
|
|
|
handler.Ready()
|
|
for range [2]int{} {
|
|
code = getReady()
|
|
testutil.Equals(t, http.StatusOK, code)
|
|
}
|
|
testutil.Equals(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
|
|
testutil.Equals(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
|
}
|