RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-26 00:23:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1571 from prometheus/ignoring

Browse Source 
Fix for #1503
This commit is contained in:
Brian Brazil 2016-04-26 16:02:12 +01:00
commit 04293424eb
9 changed files with 351 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
promql/ast.go
View File

@ -228,11 +228,14 @@ func (vmc VectorMatchCardinality) String() string {
type VectorMatching struct {
// The cardinality of the two vectors.
Card VectorMatchCardinality
// On contains the labels which define equality of a pair
// of elements from the vectors.
On model.LabelNames
// MatchingLabels contains the labels which define equality of a pair of
// elements from the vectors.
MatchingLabels model.LabelNames
// Ignoring excludes the given label names from matching,
// rather than only using them.
Ignoring bool
// Include contains additional labels that should be included in
// the result from the side with the higher cardinality.
// the result from the side with the lower cardinality.
Include model.LabelNames
}

75
promql/engine.go
View File

@ -728,7 +728,7 @@ func (ev *evaluator) vectorAnd(lhs, rhs vector, matching *VectorMatching) vector
if matching.Card != CardManyToMany {
panic("set operations must only use many-to-many matching")
}
sigf := signatureFunc(matching.On...)
sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...)
var result vector
// The set of signatures for the right-hand side vector.
@ -751,7 +751,7 @@ func (ev *evaluator) vectorOr(lhs, rhs vector, matching *VectorMatching) vector
if matching.Card != CardManyToMany {
panic("set operations must only use many-to-many matching")
}
sigf := signatureFunc(matching.On...)
sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...)
var result vector
leftSigs := map[uint64]struct{}{}
@ -773,7 +773,7 @@ func (ev *evaluator) vectorUnless(lhs, rhs vector, matching *VectorMatching) vec
if matching.Card != CardManyToMany {
panic("set operations must only use many-to-many matching")
}
sigf := signatureFunc(matching.On...)
sigf := signatureFunc(matching.Ignoring, matching.MatchingLabels...)
rightSigs := map[uint64]struct{}{}
for _, rs := range rhs {
@ -795,9 +795,8 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM
panic("many-to-many only allowed for set operators")
}
var (
result = vector{}
sigf = signatureFunc(matching.On...)
resultLabels = append(matching.On, matching.Include...)
result = vector{}
sigf = signatureFunc(matching.Ignoring, matching.MatchingLabels...)
)
// The control flow below handles one-to-one or many-to-one matching.
@ -851,7 +850,7 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM
} else if !keep {
continue
}
metric := resultMetric(ls.Metric, op, resultLabels...)
metric := resultMetric(ls.Metric, rs.Metric, op, matching)
insertedSigs, exists := matchedSigs[sig]
if matching.Card == CardOneToOne {
@ -863,7 +862,7 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM
// In many-to-one matching the grouping labels have to ensure a unique metric
// for the result vector. Check whether those labels have already been added for
// the same matching labels.
insertSig := model.SignatureForLabels(metric.Metric, matching.Include...)
insertSig := uint64(metric.Metric.Fingerprint())
if !exists {
insertedSigs = map[uint64]struct{}{}
matchedSigs[sig] = insertedSigs
@ -883,12 +882,16 @@ func (ev *evaluator) vectorBinop(op itemType, lhs, rhs vector, matching *VectorM
}
// signatureFunc returns a function that calculates the signature for a metric
// based on the provided labels.
func signatureFunc(labels ...model.LabelName) func(m metric.Metric) uint64 {
if len(labels) == 0 {
// based on the provided labels. If ignoring, then the given labels are ignored instead.
func signatureFunc(ignoring bool, labels ...model.LabelName) func(m metric.Metric) uint64 {
if len(labels) == 0 || ignoring {
return func(m metric.Metric) uint64 {
m.Del(model.MetricNameLabel)
return uint64(m.Metric.Fingerprint())
tmp := m.Metric.Clone()
for _, l := range labels {
delete(tmp, l)
}
delete(tmp, model.MetricNameLabel)
return uint64(tmp.Fingerprint())
}
}
return func(m metric.Metric) uint64 {
@ -898,19 +901,49 @@ func signatureFunc(labels ...model.LabelName) func(m metric.Metric) uint64 {
// resultMetric returns the metric for the given sample(s) based on the vector
// binary operation and the matching options.
func resultMetric(met metric.Metric, op itemType, labels ...model.LabelName) metric.Metric {
if len(labels) == 0 {
if shouldDropMetricName(op) {
met.Del(model.MetricNameLabel)
func resultMetric(lhs, rhs metric.Metric, op itemType, matching *VectorMatching) metric.Metric {
if shouldDropMetricName(op) {
lhs.Del(model.MetricNameLabel)
}
if len(matching.MatchingLabels)+len(matching.Include) == 0 {
return lhs
}
if matching.Ignoring {
if matching.Card == CardOneToOne {
for _, l := range matching.MatchingLabels {
lhs.Del(l)
}
}
return met
for _, ln := range matching.Include {
// Included labels from the `group_x` modifier are taken from the "one"-side.
value := rhs.Metric[ln]
if value != "" {
lhs.Set(ln, rhs.Metric[ln])
} else {
lhs.Del(ln)
}
}
return lhs
}
// As we definitely write, creating a new metric is the easiest solution.
m := model.Metric{}
for _, ln := range labels {
// Included labels from the `group_x` modifier are taken from the "many"-side.
if v, ok := met.Metric[ln]; ok {
if matching.Card == CardOneToOne {
for _, ln := range matching.MatchingLabels {
if v, ok := lhs.Metric[ln]; ok {
m[ln] = v
}
}
} else {
for k, v := range lhs.Metric {
m[k] = v
}
}
for _, ln := range matching.Include {
// Included labels from the `group_x` modifier are taken from the "one"-side .
if v, ok := rhs.Metric[ln]; ok {
m[ln] = v
} else {
delete(m, ln)
}
}
return metric.Metric{Metric: m, Copied: false}

2
promql/lex.go
View File

@ -172,6 +172,7 @@ const (
itemBy
itemWithout
itemOn
itemIgnoring
itemGroupLeft
itemGroupRight
itemBool
@ -209,6 +210,7 @@ var key = map[string]itemType{
"keeping_extra": itemKeepCommon,
"keep_common": itemKeepCommon,
"on": itemOn,
"ignoring": itemIgnoring,
"group_left": itemGroupLeft,
"group_right": itemGroupRight,
"bool": itemBool,

3
promql/lex_test.go
View File

@ -278,6 +278,9 @@ var tests = []struct {
}, {
input: "on",
expected: []item{{itemOn, 0, "on"}},
}, {
input: "ignoring",
expected: []item{{itemIgnoring, 0, "ignoring"}},
}, {
input: "group_left",
expected: []item{{itemGroupLeft, 0, "group_left"}},

33
promql/parse.go
View File

@ -461,27 +461,32 @@ func (p *parser) expr() Expr {
returnBool = true
}
// Parse ON clause.
if p.peek().typ == itemOn {
// Parse ON/IGNORING clause.
if p.peek().typ == itemOn || p.peek().typ == itemIgnoring {
if p.peek().typ == itemIgnoring {
vecMatching.Ignoring = true
}
p.next()
vecMatching.On = p.labels()
vecMatching.MatchingLabels = p.labels()
// Parse grouping.
if t := p.peek().typ; t == itemGroupLeft {
if t := p.peek().typ; t == itemGroupLeft || t == itemGroupRight {
p.next()
vecMatching.Card = CardManyToOne
vecMatching.Include = p.labels()
} else if t == itemGroupRight {
p.next()
vecMatching.Card = CardOneToMany
vecMatching.Include = p.labels()
if t == itemGroupLeft {
vecMatching.Card = CardManyToOne
} else {
vecMatching.Card = CardOneToMany
}
if p.peek().typ == itemLeftParen {
vecMatching.Include = p.labels()
}
}
}
for _, ln := range vecMatching.On {
for _, ln := range vecMatching.MatchingLabels {
for _, ln2 := range vecMatching.Include {
if ln == ln2 {
p.errorf("label %q must not occur in ON and INCLUDE clause at once", ln)
if ln == ln2 && !vecMatching.Ignoring {
p.errorf("label %q must not occur in ON and GROUP clause at once", ln)
}
}
}
@ -1047,7 +1052,7 @@ func (p *parser) checkType(node Node) (typ model.ValueType) {
}
if (lt != model.ValVector || rt != model.ValVector) && n.VectorMatching != nil {
if len(n.VectorMatching.On) > 0 {
if len(n.VectorMatching.MatchingLabels) > 0 {
p.errorf("vector matching only allowed between vectors")
}
n.VectorMatching = nil

156
promql/parse_test.go
View File

@ -250,6 +250,10 @@ var testExpr = []struct {
input: "1 offset 1d",
fail: true,
errMsg: "offset modifier must be preceded by an instant or range selector",
}, {
input: "a - on(b) ignoring(c) d",
fail: true,
errMsg: "parse error at char 11: no valid expression found",
},
// Vector binary operations.
{
@ -465,14 +469,14 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardOneToMany,
On: model.LabelNames{"baz", "buz"},
Include: model.LabelNames{"test"},
Card: CardOneToMany,
MatchingLabels: model.LabelNames{"baz", "buz"},
Include: model.LabelNames{"test"},
},
},
VectorMatching: &VectorMatching{
Card: CardOneToOne,
On: model.LabelNames{"foo"},
Card: CardOneToOne,
MatchingLabels: model.LabelNames{"foo"},
},
},
}, {
@ -492,8 +496,29 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardOneToOne,
On: model.LabelNames{"test", "blub"},
Card: CardOneToOne,
MatchingLabels: model.LabelNames{"test", "blub"},
},
},
}, {
input: "foo * on(test,blub) group_left bar",
expected: &BinaryExpr{
Op: itemMUL,
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
},
},
VectorMatching: &VectorMatching{
Card: CardManyToOne,
MatchingLabels: model.LabelNames{"test", "blub"},
},
},
}, {
@ -513,8 +538,30 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardManyToMany,
On: model.LabelNames{"test", "blub"},
Card: CardManyToMany,
MatchingLabels: model.LabelNames{"test", "blub"},
},
},
}, {
input: "foo and ignoring(test,blub) bar",
expected: &BinaryExpr{
Op: itemLAND,
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
},
},
VectorMatching: &VectorMatching{
Card: CardManyToMany,
MatchingLabels: model.LabelNames{"test", "blub"},
Ignoring: true,
},
},
}, {
@ -534,8 +581,8 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardManyToMany,
On: model.LabelNames{"bar"},
Card: CardManyToMany,
MatchingLabels: model.LabelNames{"bar"},
},
},
}, {
@ -555,9 +602,55 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardManyToOne,
On: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar"},
Card: CardManyToOne,
MatchingLabels: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar"},
},
},
}, {
input: "foo / ignoring(test,blub) group_left(blub) bar",
expected: &BinaryExpr{
Op: itemDIV,
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
},
},
VectorMatching: &VectorMatching{
Card: CardManyToOne,
MatchingLabels: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"blub"},
Ignoring: true,
},
},
}, {
input: "foo / ignoring(test,blub) group_left(bar) bar",
expected: &BinaryExpr{
Op: itemDIV,
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
},
},
VectorMatching: &VectorMatching{
Card: CardManyToOne,
MatchingLabels: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar"},
Ignoring: true,
},
},
}, {
@ -577,9 +670,32 @@ var testExpr = []struct {
},
},
VectorMatching: &VectorMatching{
Card: CardOneToMany,
On: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar", "foo"},
Card: CardOneToMany,
MatchingLabels: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar", "foo"},
},
},
}, {
input: "foo - ignoring(test,blub) group_right(bar,foo) bar",
expected: &BinaryExpr{
Op: itemSUB,
LHS: &VectorSelector{
Name: "foo",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
},
},
RHS: &VectorSelector{
Name: "bar",
LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
},
},
VectorMatching: &VectorMatching{
Card: CardOneToMany,
MatchingLabels: model.LabelNames{"test", "blub"},
Include: model.LabelNames{"bar", "foo"},
Ignoring: true,
},
},
}, {
@ -638,14 +754,10 @@ var testExpr = []struct {
input: "foo unless on(bar) group_right(baz) bar",
fail: true,
errMsg: "no grouping allowed for \"unless\" operation",
}, {
input: `http_requests{group="production"} / on(instance) group_left cpu_count{type="smp"}`,
fail: true,
errMsg: "unexpected identifier \"cpu_count\" in grouping opts, expected \"(\"",
}, {
input: `http_requests{group="production"} + on(instance) group_left(job,instance) cpu_count{type="smp"}`,
fail: true,
errMsg: "label \"instance\" must not occur in ON and INCLUDE clause at once",
errMsg: "label \"instance\" must not occur in ON and GROUP clause at once",
}, {
input: "foo + bool bar",
fail: true,

8
promql/printer.go
View File

@ -159,8 +159,12 @@ func (node *BinaryExpr) String() string {
matching := ""
vm := node.VectorMatching
if vm != nil && len(vm.On) > 0 {
matching = fmt.Sprintf(" ON(%s)", vm.On)
if vm != nil && len(vm.MatchingLabels) > 0 {
if vm.Ignoring {
matching = fmt.Sprintf(" IGNORING(%s)", vm.MatchingLabels)
} else {
matching = fmt.Sprintf(" ON(%s)", vm.MatchingLabels)
}
if vm.Card == CardManyToOne {
matching += fmt.Sprintf(" GROUP_LEFT(%s)", vm.Include)
}

6
promql/printer_test.go
View File

@ -36,6 +36,12 @@ func TestExprString(t *testing.T) {
{
in: `sum(task:errors:rate10s{job="s"}) WITHOUT (instance)`,
},
{
in: `a - ON(b) c`,
},
{
in: `a - IGNORING(b) c`,
},
{
in: `up > BOOL 0`,
},

120
promql/testdata/operators.test vendored
View File

@ -79,6 +79,14 @@ eval instant at 50m (http_requests{group="canary"} + 1) and on(instance) http_re
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m (http_requests{group="canary"} + 1) and ignoring(group, job) http_requests{instance="0", group="production"}
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
eval instant at 50m http_requests{group="canary"} or http_requests{group="production"}
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
@ -109,6 +117,14 @@ eval instant at 50m (http_requests{group="canary"} + 1) or on(instance) (http_re
vector_matching_a{l="x"} 10
vector_matching_a{l="y"} 20
eval instant at 50m (http_requests{group="canary"} + 1) or ignoring(l, group, job) (http_requests or cpu_count or vector_matching_a)
{group="canary", instance="0", job="api-server"} 301
{group="canary", instance="0", job="app-server"} 701
{group="canary", instance="1", job="api-server"} 401
{group="canary", instance="1", job="app-server"} 801
vector_matching_a{l="x"} 10
vector_matching_a{l="y"} 20
eval instant at 50m http_requests{group="canary"} unless http_requests{instance="0"}
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
@ -125,6 +141,18 @@ eval instant at 50m http_requests{group="canary"} / on(instance,job) http_reques
{instance="1", job="api-server"} 2
{instance="1", job="app-server"} 1.3333333333333333
eval instant at 50m http_requests{group="canary"} unless ignoring(group, instance) http_requests{instance="0"}
eval instant at 50m http_requests{group="canary"} unless ignoring(group) http_requests{instance="0"}
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
eval instant at 50m http_requests{group="canary"} / ignoring(group) http_requests{group="production"}
{instance="0", job="api-server"} 3
{instance="0", job="app-server"} 1.4
{instance="1", job="api-server"} 2
{instance="1", job="app-server"} 1.3333333333333333
# https://github.com/prometheus/prometheus/issues/1489
eval instant at 50m http_requests AND ON (dummy) vector(1)
http_requests{group="canary", instance="0", job="api-server"} 300
@ -136,6 +164,16 @@ eval instant at 50m http_requests AND ON (dummy) vector(1)
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
eval instant at 50m http_requests AND IGNORING (group, instance, job) vector(1)
http_requests{group="canary", instance="0", job="api-server"} 300
http_requests{group="canary", instance="0", job="app-server"} 700
http_requests{group="canary", instance="1", job="api-server"} 400
http_requests{group="canary", instance="1", job="app-server"} 800
http_requests{group="production", instance="0", job="api-server"} 100
http_requests{group="production", instance="0", job="app-server"} 500
http_requests{group="production", instance="1", job="api-server"} 200
http_requests{group="production", instance="1", job="app-server"} 600
# Comparisons.
eval instant at 50m SUM(http_requests) BY (job) > 1000
@ -170,3 +208,85 @@ eval instant at 50m 0 == bool 1
eval instant at 50m 1 == bool 1
1
# group_left/group_right.
clear
load 5m
node_var{instance="abc",job="node"} 2
node_role{instance="abc",job="node",role="prometheus"} 1
load 5m
node_cpu{instance="abc",job="node",mode="idle"} 3
node_cpu{instance="abc",job="node",mode="user"} 1
node_cpu{instance="def",job="node",mode="idle"} 8
node_cpu{instance="def",job="node",mode="user"} 2
load 5m
random{foo="bar"} 1
# Copy machine role to node variable.
eval instant at 5m node_role * on (instance) group_right (role) node_var
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_var * on (instance) group_left (role) node_role
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_var * ignoring (role) group_left (role) node_role
{instance="abc",job="node",role="prometheus"} 2
eval instant at 5m node_role * ignoring (role) group_right (role) node_var
{instance="abc",job="node",role="prometheus"} 2
# Copy machine role to node variable with instrumentation labels.
eval instant at 5m node_cpu * ignoring (role, mode) group_left (role) node_role
{instance="abc",job="node",mode="idle",role="prometheus"} 3
{instance="abc",job="node",mode="user",role="prometheus"} 1
eval instant at 5m node_cpu * on (instance) group_left (role) node_role
{instance="abc",job="node",mode="idle",role="prometheus"} 3
{instance="abc",job="node",mode="user",role="prometheus"} 1
# Ratio of total.
eval instant at 5m node_cpu / on (instance) group_left sum by (instance,job)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)
{job="node",mode="idle"} 0.7857142857142857
{job="node",mode="user"} 0.21428571428571427
eval instant at 5m sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu))
{} 1.0
eval instant at 5m node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu)
{instance="abc",job="node",mode="idle"} .75
{instance="abc",job="node",mode="user"} .25
{instance="def",job="node",mode="idle"} .80
{instance="def",job="node",mode="user"} .20
eval instant at 5m sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)
{job="node",mode="idle"} 0.7857142857142857
{job="node",mode="user"} 0.21428571428571427
eval instant at 5m sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu))
{} 1.0
# Copy over label from metric with no matching labels, without having to list cross-job target labels ('job' here).
eval instant at 5m node_cpu + on(dummy) group_left(foo) random*0
{instance="abc",job="node",mode="idle",foo="bar"} 3
{instance="abc",job="node",mode="user",foo="bar"} 1
{instance="def",job="node",mode="idle",foo="bar"} 8
{instance="def",job="node",mode="user",foo="bar"} 2