Fix swapped constants, improve instrumentation
This commit is contained in:
Fabian Reinartz
parent
b150c5768c
commit
b0adfea8d5
|
|
@ -36,11 +36,7 @@ import (
|
|||
)
|
||||
|
||||
// Constants for instrumentation.
|
||||
const (
|
||||
namespace = "prometheus"
|
||||
|
||||
ruleTypeLabel = "rule_type"
|
||||
)
|
||||
const namespace = "prometheus"
|
||||
|
||||
var (
|
||||
evalDuration = prometheus.NewSummaryVec(
|
||||
|
|
@ -49,21 +45,23 @@ var (
|
|||
Name: "rule_evaluation_duration_seconds",
|
||||
Help: "The duration for a rule to execute.",
|
||||
},
|
||||
[]string{ruleTypeLabel},
|
||||
[]string{"rule_type"},
|
||||
)
|
||||
evalFailures = prometheus.NewCounter(
|
||||
evalFailures = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "rule_evaluation_failures_total",
|
||||
Help: "The total number of rule evaluation failures.",
|
||||
},
|
||||
[]string{"rule_type"},
|
||||
)
|
||||
evalTotal = prometheus.NewCounter(
|
||||
evalTotal = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "rule_evaluations_total",
|
||||
Help: "The total number of rule evaluations.",
|
||||
},
|
||||
[]string{"rule_type"},
|
||||
)
|
||||
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
|
|
@ -74,6 +72,11 @@ var (
|
|||
)
|
||||
|
||||
func init() {
|
||||
evalTotal.WithLabelValues(string(ruleTypeAlert))
|
||||
evalTotal.WithLabelValues(string(ruleTypeRecording))
|
||||
evalFailures.WithLabelValues(string(ruleTypeAlert))
|
||||
evalFailures.WithLabelValues(string(ruleTypeRecording))
|
||||
|
||||
prometheus.MustRegister(iterationDuration)
|
||||
prometheus.MustRegister(evalFailures)
|
||||
prometheus.MustRegister(evalDuration)
|
||||
|
|
@ -200,6 +203,16 @@ func (g *Group) copyState(from *Group) {
|
|||
}
|
||||
}
|
||||
|
||||
func typeForRule(r Rule) ruleType {
|
||||
switch r.(type) {
|
||||
case *AlertingRule:
|
||||
return ruleTypeAlert
|
||||
case *RecordingRule:
|
||||
return ruleTypeRecording
|
||||
}
|
||||
panic(fmt.Errorf("unknown rule type: %T", r))
|
||||
}
|
||||
|
||||
// eval runs a single evaluation cycle in which all rules are evaluated in parallel.
|
||||
// In the future a single group will be evaluated sequentially to properly handle
|
||||
// rule dependency.
|
||||
|
|
@ -210,13 +223,18 @@ func (g *Group) eval() {
|
|||
)
|
||||
|
||||
for _, rule := range g.rules {
|
||||
rtyp := string(typeForRule(rule))
|
||||
|
||||
wg.Add(1)
|
||||
// BUG(julius): Look at fixing thundering herd.
|
||||
go func(rule Rule) {
|
||||
defer wg.Done()
|
||||
|
||||
start := time.Now()
|
||||
evalTotal.Inc()
|
||||
defer func(t time.Time) {
|
||||
evalDuration.WithLabelValues(rtyp).Observe(float64(time.Since(t)) / float64(time.Second))
|
||||
}(time.Now())
|
||||
|
||||
evalTotal.WithLabelValues(rtyp).Inc()
|
||||
|
||||
vector, err := rule.eval(now, g.opts.QueryEngine)
|
||||
if err != nil {
|
||||
|
|
@ -225,26 +243,13 @@ func (g *Group) eval() {
|
|||
if _, ok := err.(promql.ErrQueryCanceled); !ok {
|
||||
log.Warnf("Error while evaluating rule %q: %s", rule, err)
|
||||
}
|
||||
evalFailures.Inc()
|
||||
}
|
||||
var rtyp ruleType
|
||||
|
||||
switch r := rule.(type) {
|
||||
case *AlertingRule:
|
||||
rtyp = ruleTypeRecording
|
||||
g.sendAlerts(r, now)
|
||||
|
||||
case *RecordingRule:
|
||||
rtyp = ruleTypeAlert
|
||||
|
||||
default:
|
||||
panic(fmt.Errorf("unknown rule type: %T", rule))
|
||||
evalFailures.WithLabelValues(rtyp).Inc()
|
||||
return
|
||||
}
|
||||
|
||||
evalDuration.WithLabelValues(string(rtyp)).Observe(
|
||||
float64(time.Since(start)) / float64(time.Second),
|
||||
)
|
||||
|
||||
if ar, ok := rule.(*AlertingRule); ok {
|
||||
g.sendAlerts(ar, now)
|
||||
}
|
||||
for _, s := range vector {
|
||||
g.opts.SampleAppender.Append(s)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -791,38 +791,38 @@ func AssetNames() []string {
|
|||
|
||||
// _bindata is a table, holding each asset generator, mapped to its name.
|
||||
var _bindata = map[string]func() (*asset, error){
|
||||
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
|
||||
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
|
||||
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
|
||||
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
|
||||
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
|
||||
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
|
||||
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
|
||||
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
|
||||
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
|
||||
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
|
||||
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
|
||||
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
|
||||
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
|
||||
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
|
||||
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
|
||||
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
|
||||
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
|
||||
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
|
||||
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
|
||||
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
|
||||
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
|
||||
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
|
||||
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
|
||||
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
|
||||
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
|
||||
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
|
||||
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
|
||||
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss,
|
||||
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
|
||||
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
|
||||
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
|
||||
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
|
||||
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
|
||||
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
|
||||
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
|
||||
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
|
||||
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
|
||||
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
|
||||
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
|
||||
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
|
||||
}
|
||||
|
||||
// AssetDir returns the file names below a certain
|
||||
|
|
@ -864,69 +864,70 @@ type bintree struct {
|
|||
Func func() (*asset, error)
|
||||
Children map[string]*bintree
|
||||
}
|
||||
|
||||
var _bintree = &bintree{nil, map[string]*bintree{
|
||||
"web": &bintree{nil, map[string]*bintree{
|
||||
"ui": &bintree{nil, map[string]*bintree{
|
||||
"static": &bintree{nil, map[string]*bintree{
|
||||
"css": &bintree{nil, map[string]*bintree{
|
||||
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
|
||||
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
|
||||
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
|
||||
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
|
||||
"prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}},
|
||||
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
|
||||
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
|
||||
}},
|
||||
"img": &bintree{nil, map[string]*bintree{
|
||||
"ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
|
||||
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
|
||||
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
|
||||
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
|
||||
"graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}},
|
||||
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
|
||||
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
|
||||
}},
|
||||
"vendor": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-3.3.1": &bintree{nil, map[string]*bintree{
|
||||
"css": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}},
|
||||
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
|
||||
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
|
||||
}},
|
||||
"fonts": &bintree{nil, map[string]*bintree{
|
||||
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
|
||||
"glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}},
|
||||
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
|
||||
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
"bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
|
||||
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
|
||||
"bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}},
|
||||
}},
|
||||
"bootstrap3-typeahead": &bintree{nil, map[string]*bintree{
|
||||
"bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}},
|
||||
}},
|
||||
"js": &bintree{nil, map[string]*bintree{
|
||||
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
|
||||
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
|
||||
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
|
||||
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
|
||||
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
|
||||
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
|
||||
"jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}},
|
||||
}},
|
||||
"rickshaw": &bintree{nil, map[string]*bintree{
|
||||
"rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}},
|
||||
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
|
||||
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
|
||||
"vendor": &bintree{nil, map[string]*bintree{
|
||||
"d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}},
|
||||
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
|
||||
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
"templates": &bintree{nil, map[string]*bintree{
|
||||
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
|
||||
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
|
||||
"alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}},
|
||||
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
|
||||
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
|
||||
"status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}},
|
||||
}},
|
||||
}},
|
||||
|
|
@ -979,4 +980,3 @@ func _filePath(dir, name string) string {
|
|||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue