From a31730e88bc7d3490346c40ebc3ce2d8032d0ced Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Thu, 22 Jan 2015 11:52:30 +0000 Subject: [PATCH] Make 2nd arg to delta optional. Add a deriv() function. The 2nd isCounter argument to delta is ugly, make it optional as the first step of deprecating it. This will makes delta only ever applied to gauges. Add a deriv function to calculate the least squares slope of a gauge. This is more useful for prediction than delta, as it isn't as heavily influenced by outliers at the boundaries. --- rules/ast/functions.go | 91 ++++++++++++++++++++++++++++++++++-------- rules/rules_test.go | 50 ++++++++++++++++------- 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/rules/ast/functions.go b/rules/ast/functions.go index 70833df2e..7b0bf95b8 100644 --- a/rules/ast/functions.go +++ b/rules/ast/functions.go @@ -28,37 +28,44 @@ import ( // Function represents a function of the expression language and is // used by function nodes. type Function struct { - name string - argTypes []ExprType - returnType ExprType - callFn func(timestamp clientmodel.Timestamp, args []Node) interface{} + name string + argTypes []ExprType + optionalArgs int + returnType ExprType + callFn func(timestamp clientmodel.Timestamp, args []Node) interface{} } // CheckArgTypes returns a non-nil error if the number or types of // passed in arg nodes do not match the function's expectations. func (function *Function) CheckArgTypes(args []Node) error { - if len(function.argTypes) != len(args) { + if len(function.argTypes) < len(args) { return fmt.Errorf( - "wrong number of arguments to function %v(): %v expected, %v given", + "too many arguments to function %v(): %v expected at most, %v given", function.name, len(function.argTypes), len(args), ) } - for idx, argType := range function.argTypes { + if len(function.argTypes) - function.optionalArgs > len(args) { + return fmt.Errorf( + "too few arguments to function %v(): %v expected at least, %v given", + function.name, len(function.argTypes)-function.optionalArgs, len(args), + ) + } + for idx, arg := range args { invalidType := false var expectedType string - if _, ok := args[idx].(ScalarNode); argType == ScalarType && !ok { + if _, ok := arg.(ScalarNode); function.argTypes[idx] == ScalarType && !ok { invalidType = true expectedType = "scalar" } - if _, ok := args[idx].(VectorNode); argType == VectorType && !ok { + if _, ok := arg.(VectorNode); function.argTypes[idx] == VectorType && !ok { invalidType = true expectedType = "vector" } - if _, ok := args[idx].(MatrixNode); argType == MatrixType && !ok { + if _, ok := arg.(MatrixNode); function.argTypes[idx] == MatrixType && !ok { invalidType = true expectedType = "matrix" } - if _, ok := args[idx].(StringNode); argType == StringType && !ok { + if _, ok := arg.(StringNode); function.argTypes[idx] == StringType && !ok { invalidType = true expectedType = "string" } @@ -78,10 +85,10 @@ func timeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} { return clientmodel.SampleValue(timestamp.Unix()) } -// === delta(matrix MatrixNode, isCounter ScalarNode) Vector === +// === delta(matrix MatrixNode, isCounter=0 ScalarNode) Vector === func deltaImpl(timestamp clientmodel.Timestamp, args []Node) interface{} { matrixNode := args[0].(MatrixNode) - isCounter := args[1].(ScalarNode).Eval(timestamp) > 0 + isCounter := len(args) >= 2 && args[1].(ScalarNode).Eval(timestamp) > 0 resultVector := Vector{} // If we treat these metrics as counters, we need to fetch all values @@ -406,6 +413,49 @@ func absentImpl(timestamp clientmodel.Timestamp, args []Node) interface{} { } } +// === deriv(node MatrixNode) Vector === +func derivImpl(timestamp clientmodel.Timestamp, args []Node) interface{} { + matrixNode := args[0].(MatrixNode) + resultVector := Vector{} + + matrixValue := matrixNode.Eval(timestamp) + for _, samples := range matrixValue { + // No sense in trying to compute a derivative without at least two points. + // Drop this vector element. + if len(samples.Values) < 2 { + continue + } + + // Least squares. + n := clientmodel.SampleValue(0) + sumY := clientmodel.SampleValue(0) + sumX := clientmodel.SampleValue(0) + sumXY := clientmodel.SampleValue(0) + sumX2 := clientmodel.SampleValue(0) + for _, sample := range samples.Values { + x := clientmodel.SampleValue(sample.Timestamp.UnixNano() / 1e9) + n += 1.0 + sumY += sample.Value + sumX += x + sumXY += x * sample.Value + sumX2 += x * x + } + numerator := sumXY - sumX*sumY/n + denominator := sumX2 - (sumX*sumX)/n + + resultValue := numerator / denominator + + resultSample := &Sample{ + Metric: samples.Metric, + Value: resultValue, + Timestamp: timestamp, + } + resultSample.Metric.Delete(clientmodel.MetricNameLabel) + resultVector = append(resultVector, resultSample) + } + return resultVector +} + var functions = map[string]*Function{ "abs": { name: "abs", @@ -444,10 +494,11 @@ var functions = map[string]*Function{ callFn: countScalarImpl, }, "delta": { - name: "delta", - argTypes: []ExprType{MatrixType, ScalarType}, - returnType: VectorType, - callFn: deltaImpl, + name: "delta", + argTypes: []ExprType{MatrixType, ScalarType}, + optionalArgs: 1, // The 2nd argument is deprecated. + returnType: VectorType, + callFn: deltaImpl, }, "drop_common_labels": { name: "drop_common_labels", @@ -509,6 +560,12 @@ var functions = map[string]*Function{ returnType: VectorType, callFn: topkImpl, }, + "deriv": { + name: "deriv", + argTypes: []ExprType{MatrixType}, + returnType: VectorType, + callFn: derivImpl, + }, } // GetFunction returns a predefined Function object for the given diff --git a/rules/rules_test.go b/rules/rules_test.go index 0f4bc7155..7244db999 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -260,13 +260,27 @@ func TestExpressions(t *testing.T) { fullRanges: 0, intervalRanges: 2, }, { - expr: `http_requests{job="api-server", group="canary"} + delta(http_requests{job="api-server"}[5m], 1)`, + expr: `http_requests{job="api-server", group="canary"} + rate(http_requests{job="api-server"}[5m]) * 5 * 60`, output: []string{ `{group="canary", instance="0", job="api-server"} => 330 @[%v]`, `{group="canary", instance="1", job="api-server"} => 440 @[%v]`, }, fullRanges: 4, intervalRanges: 0, + }, { + expr: `rate(http_requests[25m]) * 25 * 60`, + output: []string{ + `{group="canary", instance="0", job="api-server"} => 150 @[%v]`, + `{group="canary", instance="0", job="app-server"} => 350 @[%v]`, + `{group="canary", instance="1", job="api-server"} => 200 @[%v]`, + `{group="canary", instance="1", job="app-server"} => 400 @[%v]`, + `{group="production", instance="0", job="api-server"} => 50 @[%v]`, + `{group="production", instance="0", job="app-server"} => 249.99999999999997 @[%v]`, + `{group="production", instance="1", job="api-server"} => 100 @[%v]`, + `{group="production", instance="1", job="app-server"} => 300 @[%v]`, + }, + fullRanges: 8, + intervalRanges: 0, }, { expr: `delta(http_requests[25m], 1)`, output: []string{ @@ -368,38 +382,44 @@ func TestExpressions(t *testing.T) { intervalRanges: 8, }, { // Deltas should be adjusted for target interval vs. samples under target interval. - expr: `delta(http_requests{group="canary", instance="1", job="app-server"}[18m], 1)`, + expr: `delta(http_requests{group="canary", instance="1", job="app-server"}[18m])`, output: []string{`{group="canary", instance="1", job="app-server"} => 288 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, { - // Rates should transform per-interval deltas to per-second rates. - expr: `rate(http_requests{group="canary", instance="1", job="app-server"}[10m])`, + // Deltas should perform the same operation when 2nd argument is 0. + expr: `delta(http_requests{group="canary", instance="1", job="app-server"}[18m], 0)`, + output: []string{`{group="canary", instance="1", job="app-server"} => 288 @[%v]`}, + fullRanges: 1, + intervalRanges: 0, + }, { + // Rates should calculate per-second rates. + expr: `rate(http_requests{group="canary", instance="1", job="app-server"}[60m])`, output: []string{`{group="canary", instance="1", job="app-server"} => 0.26666666666666666 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, { - // Counter resets in middle of range are ignored by delta() if counter == 1. - expr: `delta(testcounter_reset_middle[50m], 1)`, - output: []string{`{} => 90 @[%v]`}, + // Deriv should return the same as rate in simple cases. + expr: `deriv(http_requests{group="canary", instance="1", job="app-server"}[60m])`, + output: []string{`{group="canary", instance="1", job="app-server"} => 0.26666666666666666 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, { - // Counter resets in middle of range are not ignored by delta() if counter == 0. - expr: `delta(testcounter_reset_middle[50m], 0)`, - output: []string{`{} => 50 @[%v]`}, + // Counter resets at in the middle of range are handled correctly by rate(). + expr: `rate(testcounter_reset_middle[60m])`, + output: []string{`{} => 0.03 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, { - // Counter resets at end of range are ignored by delta() if counter == 1. - expr: `delta(testcounter_reset_end[5m], 1)`, + // Counter resets at end of range are ignored by rate(). + expr: `rate(testcounter_reset_end[5m])`, output: []string{`{} => 0 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, { - // Counter resets at end of range are not ignored by delta() if counter == 0. - expr: `delta(testcounter_reset_end[5m], 0)`, - output: []string{`{} => -90 @[%v]`}, + // Deriv should return correct result. + expr: `deriv(testcounter_reset_middle[100m])`, + output: []string{`{} => 0.010606060606060607 @[%v]`}, fullRanges: 1, intervalRanges: 0, }, {