mirror of
https://github.com/prometheus/prometheus
synced 2025-01-11 17:19:45 +00:00
Make scraping offset consistent.
To evenly distribute scraping load we currently rely on random jittering. This commit hashes over the target's identity and calculates a consistent offset. This also ensures that scrape intervals are constantly spaced between config/target changes.
This commit is contained in:
Fabian Reinartz
parent
938ebe78c2
commit
fe7e91e2eb
@ -18,7 +18,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@ -267,6 +266,30 @@ func (t *Target) String() string {
|
||||
return t.host()
|
||||
}
|
||||
|
||||
// fingerprint returns an identifying hash for the target.
|
||||
func (t *Target) fingerprint() model.Fingerprint {
|
||||
t.RLock()
|
||||
defer t.RUnlock()
|
||||
|
||||
return t.labels.Fingerprint()
|
||||
}
|
||||
|
||||
// offset returns the time until the next scrape cycle for the target.
|
||||
func (t *Target) offset(interval time.Duration) time.Duration {
|
||||
now := time.Now().UnixNano()
|
||||
|
||||
var (
|
||||
base = now % int64(interval)
|
||||
offset = uint64(t.fingerprint()) % uint64(interval)
|
||||
next = base + int64(offset)
|
||||
)
|
||||
|
||||
if next > int64(interval) {
|
||||
next -= int64(interval)
|
||||
}
|
||||
return time.Duration(next)
|
||||
}
|
||||
|
||||
func (t *Target) client() (*http.Client, error) {
|
||||
t.RLock()
|
||||
defer t.RUnlock()
|
||||
@ -366,14 +389,12 @@ func (t *Target) RunScraper(sampleAppender storage.SampleAppender) {
|
||||
|
||||
log.Debugf("Starting scraper for target %v...", t)
|
||||
|
||||
jitterTimer := time.NewTimer(time.Duration(float64(lastScrapeInterval) * rand.Float64()))
|
||||
select {
|
||||
case <-jitterTimer.C:
|
||||
case <-time.After(t.offset(lastScrapeInterval)):
|
||||
// Continue after scraping offset.
|
||||
case <-t.scraperStopping:
|
||||
jitterTimer.Stop()
|
||||
return
|
||||
}
|
||||
jitterTimer.Stop()
|
||||
|
||||
ticker := time.NewTicker(lastScrapeInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
@ -45,6 +45,52 @@ func TestTargetLabels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetOffset(t *testing.T) {
|
||||
interval := 10 * time.Second
|
||||
|
||||
offsets := make([]time.Duration, 10000)
|
||||
|
||||
// Calculate offsets for 10000 different targets.
|
||||
for i := range offsets {
|
||||
target := newTestTarget("example.com:80", 0, model.LabelSet{
|
||||
"label": model.LabelValue(fmt.Sprintf("%d", i)),
|
||||
})
|
||||
offsets[i] = target.offset(interval)
|
||||
}
|
||||
|
||||
// Put the offsets into buckets and validate that they are all
|
||||
// within bounds.
|
||||
bucketSize := 1 * time.Second
|
||||
buckets := make([]int, interval/bucketSize)
|
||||
|
||||
for _, offset := range offsets {
|
||||
if offset < 0 || offset >= interval {
|
||||
t.Fatalf("Offset %v out of bounds", offset)
|
||||
}
|
||||
|
||||
bucket := offset / bucketSize
|
||||
buckets[bucket]++
|
||||
}
|
||||
|
||||
t.Log(buckets)
|
||||
|
||||
// Calculate whether the the number of targets per bucket
|
||||
// does not differ more than a given tolerance.
|
||||
avg := len(offsets) / len(buckets)
|
||||
tolerance := 0.15
|
||||
|
||||
for _, bucket := range buckets {
|
||||
diff := bucket - avg
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
|
||||
if float64(diff)/float64(avg) > tolerance {
|
||||
t.Fatalf("Bucket out of tolerance bounds")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverwriteLabels(t *testing.T) {
|
||||
type test struct {
|
||||
metric string
|
||||
|
||||
@ -791,38 +791,38 @@ func AssetNames() []string {
|
||||
|
||||
// _bindata is a table, holding each asset generator, mapped to its name.
|
||||
var _bindata = map[string]func() (*asset, error){
|
||||
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
|
||||
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
|
||||
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
|
||||
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
|
||||
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
|
||||
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
|
||||
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
|
||||
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
|
||||
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
|
||||
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
|
||||
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
|
||||
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
|
||||
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
|
||||
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
|
||||
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
|
||||
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
|
||||
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
|
||||
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
|
||||
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
|
||||
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
|
||||
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
|
||||
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
|
||||
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
|
||||
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
|
||||
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
|
||||
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss,
|
||||
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
|
||||
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
|
||||
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
|
||||
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
|
||||
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
|
||||
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
|
||||
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
|
||||
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
|
||||
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
|
||||
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
|
||||
}
|
||||
|
||||
// AssetDir returns the file names below a certain
|
||||
@ -864,69 +864,70 @@ type bintree struct {
|
||||
Func func() (*asset, error)
|
||||
Children map[string]*bintree
|
||||
}
|
||||
|
||||
var _bintree = &bintree{nil, map[string]*bintree{
|
||||
"web": &bintree{nil, map[string]*bintree{
|
||||
"ui": &bintree{nil, map[string]*bintree{
|
||||
"static": &bintree{nil, map[string]*bintree{
|
||||
"css": &bintree{nil, map[string]*bintree{
|
||||
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
|
||||
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
|
||||
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
|
||||
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
|
||||
"prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}},
|
||||
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
|
||||
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
|
||||
}},
|
||||
"img": &bintree{nil, map[string]*bintree{
|
||||
"ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
|
||||
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
|
||||
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
|
||||
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
|
||||
"graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}},
|
||||
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
|
||||
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
|
||||
}},
|
||||
"vendor": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-3.3.1": &bintree{nil, map[string]*bintree{
|
||||
"css": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}},
|
||||
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
|
||||
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
|
||||
}},
|
||||
"fonts": &bintree{nil, map[string]*bintree{
|
||||
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}},
|
||||
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
|
||||
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
"bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
|
||||
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
|
||||
"bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}},
|
||||
}},
|
||||
"bootstrap3-typeahead": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
|
||||
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
|
||||
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
|
||||
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
|
||||
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
|
||||
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
|
||||
"jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}},
|
||||
}},
|
||||
"rickshaw": &bintree{nil, map[string]*bintree{
|
||||
"rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}},
|
||||
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
|
||||
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
|
||||
"vendor": &bintree{nil, map[string]*bintree{
|
||||
"d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}},
|
||||
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
|
||||
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
"templates": &bintree{nil, map[string]*bintree{
|
||||
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
|
||||
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
|
||||
"alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}},
|
||||
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
|
||||
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
|
||||
"status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
@ -979,4 +980,3 @@ func _filePath(dir, name string) string {
|
||||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user