2013-02-07 10:49:04 +00:00
// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2013-01-07 22:24:26 +00:00
package ast
import (
"errors"
2014-06-06 09:55:53 +00:00
"flag"
2013-01-17 23:07:00 +00:00
"fmt"
2013-06-25 12:02:27 +00:00
"hash/fnv"
2013-01-07 22:24:26 +00:00
"math"
2013-01-17 23:07:00 +00:00
"sort"
2013-01-07 22:24:26 +00:00
"time"
2013-06-25 12:02:27 +00:00
clientmodel "github.com/prometheus/client_golang/model"
"github.com/prometheus/prometheus/stats"
"github.com/prometheus/prometheus/storage/metric"
2014-06-06 09:55:53 +00:00
"github.com/prometheus/prometheus/storage/local"
2013-01-07 22:24:26 +00:00
)
2014-06-06 09:55:53 +00:00
var defaultStalenessDelta = flag . Duration ( "defaultStalenessDelta" , 300 * time . Second , "Default staleness delta allowance in seconds during expression evaluations." )
2013-01-07 22:24:26 +00:00
// ----------------------------------------------------------------------------
// Raw data value types.
2014-02-13 17:48:56 +00:00
// Vector is basically only an alias for clientmodel.Samples, but the
// contract is that in a Vector, all Samples have the same timestamp.
2013-06-25 12:02:27 +00:00
type Vector clientmodel . Samples
2014-02-13 17:48:56 +00:00
// Matrix is a slice of SampleSets that implements sort.Interface and
// has a String method.
2013-06-25 12:02:27 +00:00
// BUG(julius): Pointerize this.
type Matrix [ ] metric . SampleSet
2013-01-07 22:24:26 +00:00
type groupedAggregation struct {
2013-06-25 12:02:27 +00:00
labels clientmodel . Metric
value clientmodel . SampleValue
2013-01-07 22:24:26 +00:00
groupCount int
}
// ----------------------------------------------------------------------------
// Enums.
2014-02-13 17:48:56 +00:00
// ExprType is an enum for the rule language expression types.
2013-01-07 22:24:26 +00:00
type ExprType int
2014-02-13 17:48:56 +00:00
// Possible language expression types.
2013-01-07 22:24:26 +00:00
const (
SCALAR ExprType = iota
VECTOR
MATRIX
STRING
)
2014-02-13 17:48:56 +00:00
// BinOpType is an enum for binary operator types.
2013-01-07 22:24:26 +00:00
type BinOpType int
2014-02-13 17:48:56 +00:00
// Possible binary operator types.
2013-01-07 22:24:26 +00:00
const (
ADD BinOpType = iota
SUB
MUL
DIV
MOD
NE
EQ
GT
LT
GE
LE
AND
OR
)
2014-02-13 17:48:56 +00:00
// AggrType is an enum for aggregation types.
2013-01-07 22:24:26 +00:00
type AggrType int
2014-02-13 17:48:56 +00:00
// Possible aggregation types.
2013-01-07 22:24:26 +00:00
const (
SUM AggrType = iota
AVG
MIN
MAX
2013-05-08 14:35:16 +00:00
COUNT
2013-01-07 22:24:26 +00:00
)
// ----------------------------------------------------------------------------
// Interfaces.
2014-02-13 17:48:56 +00:00
// Nodes is a slice of any mix of node types as all node types
// implement the Node interface.
2013-06-06 13:12:37 +00:00
type Nodes [ ] Node
2014-02-13 17:48:56 +00:00
// Node is the top-level interface for any kind of nodes. Each node
// type implements one of the ...Node interfaces, each of which embeds
// this Node interface.
2013-01-07 22:24:26 +00:00
type Node interface {
Type ( ) ExprType
2013-06-06 13:12:37 +00:00
Children ( ) Nodes
2013-01-07 22:24:26 +00:00
NodeTreeToDotGraph ( ) string
2013-06-06 13:12:37 +00:00
String ( ) string
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// ScalarNode is a Node for scalar values.
2013-01-07 22:24:26 +00:00
type ScalarNode interface {
Node
2014-02-13 17:48:56 +00:00
// Eval evaluates and returns the value of the scalar represented by this node.
2014-06-06 09:55:53 +00:00
Eval ( timestamp clientmodel . Timestamp ) clientmodel . SampleValue
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// VectorNode is a Node for vector values.
2013-01-07 22:24:26 +00:00
type VectorNode interface {
Node
2014-02-13 17:48:56 +00:00
// Eval evaluates the node recursively and returns the result
// as a Vector (i.e. a slice of Samples all at the given
// Timestamp).
2014-06-06 09:55:53 +00:00
Eval ( timestamp clientmodel . Timestamp ) Vector
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// MatrixNode is a Node for matrix values.
2013-01-07 22:24:26 +00:00
type MatrixNode interface {
Node
2014-02-13 17:48:56 +00:00
// Eval evaluates the node recursively and returns the result as a Matrix.
2014-06-06 09:55:53 +00:00
Eval ( timestamp clientmodel . Timestamp ) Matrix
2014-02-13 17:48:56 +00:00
// Eval evaluates the node recursively and returns the result
// as a Matrix that only contains the boundary values.
2014-06-06 09:55:53 +00:00
EvalBoundaries ( timestamp clientmodel . Timestamp ) Matrix
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// StringNode is a Node for string values.
2013-01-07 22:24:26 +00:00
type StringNode interface {
Node
2014-02-13 17:48:56 +00:00
// Eval evaluates and returns the value of the string
// represented by this node.
2014-06-06 09:55:53 +00:00
Eval ( timestamp clientmodel . Timestamp ) string
2013-01-07 22:24:26 +00:00
}
// ----------------------------------------------------------------------------
// ScalarNode types.
type (
2014-02-22 21:29:32 +00:00
// ScalarLiteral represents a numeric selector.
2013-01-07 22:24:26 +00:00
ScalarLiteral struct {
2013-06-25 12:02:27 +00:00
value clientmodel . SampleValue
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// ScalarFunctionCall represents a function with a numeric
// return type.
2013-01-07 22:24:26 +00:00
ScalarFunctionCall struct {
function * Function
2013-06-06 13:12:37 +00:00
args Nodes
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// ScalarArithExpr represents an arithmetic expression of
// numeric type.
2013-01-07 22:24:26 +00:00
ScalarArithExpr struct {
opType BinOpType
lhs ScalarNode
rhs ScalarNode
}
)
// ----------------------------------------------------------------------------
// VectorNode types.
type (
2014-02-22 21:29:32 +00:00
// A VectorSelector represents a metric name plus labelset.
VectorSelector struct {
2014-03-28 10:58:47 +00:00
labelMatchers metric . LabelMatchers
2014-06-06 09:55:53 +00:00
// The series iterators are populated at query analysis time.
iterators map [ clientmodel . Fingerprint ] storage_ng . SeriesIterator
metrics map [ clientmodel . Fingerprint ] clientmodel . Metric
2014-03-28 10:58:47 +00:00
// Fingerprints are populated from label matchers at query analysis time.
2014-06-06 09:55:53 +00:00
// TODO: do we still need these?
2013-06-25 12:02:27 +00:00
fingerprints clientmodel . Fingerprints
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// VectorFunctionCall represents a function with vector return
// type.
2013-01-07 22:24:26 +00:00
VectorFunctionCall struct {
function * Function
2013-06-06 13:12:37 +00:00
args Nodes
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// A VectorAggregation with vector return type.
2013-01-07 22:24:26 +00:00
VectorAggregation struct {
2013-12-13 18:20:42 +00:00
aggrType AggrType
groupBy clientmodel . LabelNames
keepExtraLabels bool
vector VectorNode
2013-01-07 22:24:26 +00:00
}
2014-04-08 14:14:35 +00:00
// VectorArithExpr represents an arithmetic expression of vector type. At
// least one of the two operand Nodes must be a VectorNode. The other may be
// a VectorNode or ScalarNode. Both criteria are checked at runtime.
2013-01-07 22:24:26 +00:00
VectorArithExpr struct {
opType BinOpType
2014-04-08 14:14:35 +00:00
lhs Node
2013-01-07 22:24:26 +00:00
rhs Node
}
)
// ----------------------------------------------------------------------------
// MatrixNode types.
type (
2014-02-22 21:29:32 +00:00
// A MatrixSelector represents a metric name plus labelset and
2014-02-13 17:48:56 +00:00
// timerange.
2014-02-22 21:29:32 +00:00
MatrixSelector struct {
2014-03-28 10:58:47 +00:00
labelMatchers metric . LabelMatchers
2014-06-06 09:55:53 +00:00
// The series iterators are populated at query analysis time.
iterators map [ clientmodel . Fingerprint ] storage_ng . SeriesIterator
metrics map [ clientmodel . Fingerprint ] clientmodel . Metric
// Fingerprints are populated from label matchers at query analysis time.
// TODO: do we still need these?
2013-06-25 12:02:27 +00:00
fingerprints clientmodel . Fingerprints
2013-03-26 11:33:48 +00:00
interval time . Duration
2013-01-07 22:24:26 +00:00
}
)
// ----------------------------------------------------------------------------
// StringNode types.
type (
2014-02-13 17:48:56 +00:00
// A StringLiteral is what you think it is.
2013-01-07 22:24:26 +00:00
StringLiteral struct {
str string
}
2014-02-13 17:48:56 +00:00
// StringFunctionCall represents a function with string return
// type.
2013-01-07 22:24:26 +00:00
StringFunctionCall struct {
function * Function
2013-06-06 13:12:37 +00:00
args Nodes
2013-01-07 22:24:26 +00:00
}
)
// ----------------------------------------------------------------------------
// Implementations.
2014-02-13 17:48:56 +00:00
// Type implements the Node interface.
func ( node ScalarLiteral ) Type ( ) ExprType { return SCALAR }
// Type implements the Node interface.
2013-01-07 22:24:26 +00:00
func ( node ScalarFunctionCall ) Type ( ) ExprType { return SCALAR }
2014-02-13 17:48:56 +00:00
// Type implements the Node interface.
func ( node ScalarArithExpr ) Type ( ) ExprType { return SCALAR }
// Type implements the Node interface.
2014-02-22 21:29:32 +00:00
func ( node VectorSelector ) Type ( ) ExprType { return VECTOR }
2014-02-13 17:48:56 +00:00
// Type implements the Node interface.
2013-01-07 22:24:26 +00:00
func ( node VectorFunctionCall ) Type ( ) ExprType { return VECTOR }
2014-02-13 17:48:56 +00:00
// Type implements the Node interface.
func ( node VectorAggregation ) Type ( ) ExprType { return VECTOR }
// Type implements the Node interface.
func ( node VectorArithExpr ) Type ( ) ExprType { return VECTOR }
// Type implements the Node interface.
2014-02-22 21:29:32 +00:00
func ( node MatrixSelector ) Type ( ) ExprType { return MATRIX }
2014-02-13 17:48:56 +00:00
// Type implements the Node interface.
func ( node StringLiteral ) Type ( ) ExprType { return STRING }
// Type implements the Node interface.
2013-01-07 22:24:26 +00:00
func ( node StringFunctionCall ) Type ( ) ExprType { return STRING }
2014-02-13 17:48:56 +00:00
// Children implements the Node interface and returns an empty slice.
func ( node ScalarLiteral ) Children ( ) Nodes { return Nodes { } }
// Children implements the Node interface and returns the args of the
// function call.
2013-06-06 13:12:37 +00:00
func ( node ScalarFunctionCall ) Children ( ) Nodes { return node . args }
2014-02-13 17:48:56 +00:00
// Children implements the Node interface and returns the LHS and the RHS
// of the expression.
func ( node ScalarArithExpr ) Children ( ) Nodes { return Nodes { node . lhs , node . rhs } }
// Children implements the Node interface and returns an empty slice.
2014-02-22 21:29:32 +00:00
func ( node VectorSelector ) Children ( ) Nodes { return Nodes { } }
2014-02-13 17:48:56 +00:00
// Children implements the Node interface and returns the args of the
// function call.
2013-06-06 13:12:37 +00:00
func ( node VectorFunctionCall ) Children ( ) Nodes { return node . args }
2014-02-13 17:48:56 +00:00
// Children implements the Node interface and returns the vector to be
// aggregated.
func ( node VectorAggregation ) Children ( ) Nodes { return Nodes { node . vector } }
// Children implements the Node interface and returns the LHS and the RHS
// of the expression.
func ( node VectorArithExpr ) Children ( ) Nodes { return Nodes { node . lhs , node . rhs } }
// Children implements the Node interface and returns an empty slice.
2014-02-22 21:29:32 +00:00
func ( node MatrixSelector ) Children ( ) Nodes { return Nodes { } }
2014-02-13 17:48:56 +00:00
// Children implements the Node interface and returns an empty slice.
func ( node StringLiteral ) Children ( ) Nodes { return Nodes { } }
// Children implements the Node interface and returns the args of the
// function call.
2013-06-06 13:12:37 +00:00
func ( node StringFunctionCall ) Children ( ) Nodes { return node . args }
2013-03-21 17:06:15 +00:00
2014-02-22 21:29:32 +00:00
// Eval implements the ScalarNode interface and returns the selector
2014-02-13 17:48:56 +00:00
// value.
2014-06-06 09:55:53 +00:00
func ( node * ScalarLiteral ) Eval ( timestamp clientmodel . Timestamp ) clientmodel . SampleValue {
2013-01-07 22:24:26 +00:00
return node . value
}
2014-02-13 17:48:56 +00:00
// Eval implements the ScalarNode interface and returns the result of
// the expression.
2014-06-06 09:55:53 +00:00
func ( node * ScalarArithExpr ) Eval ( timestamp clientmodel . Timestamp ) clientmodel . SampleValue {
lhs := node . lhs . Eval ( timestamp )
rhs := node . rhs . Eval ( timestamp )
2013-01-07 22:24:26 +00:00
return evalScalarBinop ( node . opType , lhs , rhs )
}
2014-02-13 17:48:56 +00:00
// Eval implements the ScalarNode interface and returns the result of
// the function call.
2014-06-06 09:55:53 +00:00
func ( node * ScalarFunctionCall ) Eval ( timestamp clientmodel . Timestamp ) clientmodel . SampleValue {
return node . function . callFn ( timestamp , node . args ) . ( clientmodel . SampleValue )
2013-01-07 22:24:26 +00:00
}
2013-06-25 12:02:27 +00:00
func ( node * VectorAggregation ) labelsToGroupingKey ( labels clientmodel . Metric ) uint64 {
summer := fnv . New64a ( )
for _ , label := range node . groupBy {
fmt . Fprint ( summer , labels [ label ] )
2013-01-07 22:24:26 +00:00
}
2013-06-25 12:02:27 +00:00
return summer . Sum64 ( )
2013-01-07 22:24:26 +00:00
}
2013-06-25 12:02:27 +00:00
func labelsToKey ( labels clientmodel . Metric ) uint64 {
pairs := metric . LabelPairs { }
2013-01-17 23:07:00 +00:00
for label , value := range labels {
2013-06-25 12:02:27 +00:00
pairs = append ( pairs , & metric . LabelPair {
Name : label ,
Value : value ,
} )
2013-01-17 23:07:00 +00:00
}
2013-06-25 12:02:27 +00:00
sort . Sort ( pairs )
summer := fnv . New64a ( )
for _ , pair := range pairs {
fmt . Fprint ( summer , pair . Name , pair . Value )
}
return summer . Sum64 ( )
2013-01-15 10:30:55 +00:00
}
2014-02-13 17:48:56 +00:00
// EvalVectorInstant evaluates a VectorNode with an instant query.
2014-06-06 09:55:53 +00:00
func EvalVectorInstant ( node VectorNode , timestamp clientmodel . Timestamp , storage storage_ng . Storage , queryStats * stats . TimerGroup ) ( Vector , error ) {
closer , err := prepareInstantQuery ( node , timestamp , storage , queryStats )
2013-03-21 17:06:15 +00:00
if err != nil {
2014-06-06 09:55:53 +00:00
return nil , err
2013-03-21 17:06:15 +00:00
}
2014-06-06 09:55:53 +00:00
defer closer . Close ( )
return node . Eval ( timestamp ) , nil
2013-03-21 17:06:15 +00:00
}
2014-02-13 17:48:56 +00:00
// EvalVectorRange evaluates a VectorNode with a range query.
2014-06-06 09:55:53 +00:00
func EvalVectorRange ( node VectorNode , start clientmodel . Timestamp , end clientmodel . Timestamp , interval time . Duration , storage storage_ng . Storage , queryStats * stats . TimerGroup ) ( Matrix , error ) {
2013-03-25 11:14:48 +00:00
// Explicitly initialize to an empty matrix since a nil Matrix encodes to
// null in JSON.
2013-05-16 05:38:31 +00:00
matrix := Matrix { }
2013-03-25 11:14:48 +00:00
2014-06-06 09:55:53 +00:00
prepareTimer := queryStats . GetTimer ( stats . TotalQueryPreparationTime ) . Start ( )
closer , err := prepareRangeQuery ( node , start , end , interval , storage , queryStats )
prepareTimer . Stop ( )
2013-03-21 17:06:15 +00:00
if err != nil {
2013-05-16 05:38:31 +00:00
return nil , err
2013-03-21 17:06:15 +00:00
}
2014-06-06 09:55:53 +00:00
defer closer . Close ( )
2013-06-03 15:07:03 +00:00
2013-01-17 23:07:00 +00:00
// TODO implement watchdog timer for long-running queries.
2013-06-03 15:07:03 +00:00
evalTimer := queryStats . GetTimer ( stats . InnerEvalTime ) . Start ( )
2013-06-25 12:02:27 +00:00
sampleSets := map [ uint64 ] * metric . SampleSet { }
2013-03-21 17:06:15 +00:00
for t := start ; t . Before ( end ) ; t = t . Add ( interval ) {
2014-06-06 09:55:53 +00:00
vector := node . Eval ( t )
2013-01-17 23:07:00 +00:00
for _ , sample := range vector {
2014-03-10 16:55:17 +00:00
samplePair := metric . SamplePair {
2013-01-17 23:07:00 +00:00
Value : sample . Value ,
Timestamp : sample . Timestamp ,
}
groupingKey := labelsToKey ( sample . Metric )
if sampleSets [ groupingKey ] == nil {
2013-06-25 12:02:27 +00:00
sampleSets [ groupingKey ] = & metric . SampleSet {
2013-01-17 23:07:00 +00:00
Metric : sample . Metric ,
2013-06-25 12:02:27 +00:00
Values : metric . Values { samplePair } ,
2013-01-17 23:07:00 +00:00
}
} else {
sampleSets [ groupingKey ] . Values = append ( sampleSets [ groupingKey ] . Values , samplePair )
}
}
}
2013-06-03 15:07:03 +00:00
evalTimer . Stop ( )
2013-01-17 23:07:00 +00:00
2013-06-03 15:07:03 +00:00
appendTimer := queryStats . GetTimer ( stats . ResultAppendTime ) . Start ( )
2013-01-17 23:07:00 +00:00
for _ , sampleSet := range sampleSets {
2013-03-28 16:05:06 +00:00
matrix = append ( matrix , * sampleSet )
2013-01-17 23:07:00 +00:00
}
2013-06-03 15:07:03 +00:00
appendTimer . Stop ( )
2013-05-16 05:38:31 +00:00
return matrix , nil
2013-01-15 10:30:55 +00:00
}
2013-06-25 12:02:27 +00:00
func labelIntersection ( metric1 , metric2 clientmodel . Metric ) clientmodel . Metric {
intersection := clientmodel . Metric { }
2013-01-07 22:24:26 +00:00
for label , value := range metric1 {
if metric2 [ label ] == value {
intersection [ label ] = value
}
}
return intersection
}
Use custom timestamp type for sample timestamps and related code.
So far we've been using Go's native time.Time for anything related to sample
timestamps. Since the range of time.Time is much bigger than what we need, this
has created two problems:
- there could be time.Time values which were out of the range/precision of the
time type that we persist to disk, therefore causing incorrectly ordered keys.
One bug caused by this was:
https://github.com/prometheus/prometheus/issues/367
It would be good to use a timestamp type that's more closely aligned with
what the underlying storage supports.
- sizeof(time.Time) is 192, while Prometheus should be ok with a single 64-bit
Unix timestamp (possibly even a 32-bit one). Since we store samples in large
numbers, this seriously affects memory usage. Furthermore, copying/working
with the data will be faster if it's smaller.
*MEMORY USAGE RESULTS*
Initial memory usage comparisons for a running Prometheus with 1 timeseries and
100,000 samples show roughly a 13% decrease in total (VIRT) memory usage. In my
tests, this advantage for some reason decreased a bit the more samples the
timeseries had (to 5-7% for millions of samples). This I can't fully explain,
but perhaps garbage collection issues were involved.
*WHEN TO USE THE NEW TIMESTAMP TYPE*
The new clientmodel.Timestamp type should be used whenever time
calculations are either directly or indirectly related to sample
timestamps.
For example:
- the timestamp of a sample itself
- all kinds of watermarks
- anything that may become or is compared to a sample timestamp (like the timestamp
passed into Target.Scrape()).
When to still use time.Time:
- for measuring durations/times not related to sample timestamps, like duration
telemetry exporting, timers that indicate how frequently to execute some
action, etc.
*NOTE ON OPERATOR OPTIMIZATION TESTS*
We don't use operator optimization code anymore, but it still lives in
the code as dead code. It still has tests, but I couldn't get all of them to
pass with the new timestamp format. I commented out the failing cases for now,
but we should probably remove the dead code soon. I just didn't want to do that
in the same change as this.
Change-Id: I821787414b0debe85c9fffaeb57abd453727af0f
2013-10-28 13:35:02 +00:00
func ( node * VectorAggregation ) groupedAggregationsToVector ( aggregations map [ uint64 ] * groupedAggregation , timestamp clientmodel . Timestamp ) Vector {
2013-01-07 22:24:26 +00:00
vector := Vector { }
for _ , aggregation := range aggregations {
2013-05-08 14:35:16 +00:00
switch node . aggrType {
case AVG :
2013-06-25 12:02:27 +00:00
aggregation . value = aggregation . value / clientmodel . SampleValue ( aggregation . groupCount )
2013-05-08 14:35:16 +00:00
case COUNT :
2013-06-25 12:02:27 +00:00
aggregation . value = clientmodel . SampleValue ( aggregation . groupCount )
2013-05-08 14:35:16 +00:00
default :
// For other aggregations, we already have the right value.
2013-01-07 22:24:26 +00:00
}
2013-06-25 12:02:27 +00:00
sample := & clientmodel . Sample {
2013-01-07 22:24:26 +00:00
Metric : aggregation . labels ,
Value : aggregation . value ,
2013-03-28 16:05:06 +00:00
Timestamp : timestamp ,
2013-01-07 22:24:26 +00:00
}
vector = append ( vector , sample )
}
return vector
}
2014-02-13 17:48:56 +00:00
// Eval implements the VectorNode interface and returns the aggregated
// Vector.
2014-06-06 09:55:53 +00:00
func ( node * VectorAggregation ) Eval ( timestamp clientmodel . Timestamp ) Vector {
vector := node . vector . Eval ( timestamp )
2013-06-25 12:02:27 +00:00
result := map [ uint64 ] * groupedAggregation { }
2013-01-07 22:24:26 +00:00
for _ , sample := range vector {
groupingKey := node . labelsToGroupingKey ( sample . Metric )
if groupedResult , ok := result [ groupingKey ] ; ok {
2013-12-13 18:20:42 +00:00
if node . keepExtraLabels {
groupedResult . labels = labelIntersection ( groupedResult . labels , sample . Metric )
}
2013-01-07 22:24:26 +00:00
switch node . aggrType {
case SUM :
groupedResult . value += sample . Value
case AVG :
groupedResult . value += sample . Value
groupedResult . groupCount ++
case MAX :
if groupedResult . value < sample . Value {
groupedResult . value = sample . Value
}
case MIN :
if groupedResult . value > sample . Value {
groupedResult . value = sample . Value
}
2013-05-08 14:35:16 +00:00
case COUNT :
groupedResult . groupCount ++
default :
panic ( "Unknown aggregation type" )
2013-01-07 22:24:26 +00:00
}
} else {
2013-12-13 18:20:42 +00:00
m := clientmodel . Metric { }
if node . keepExtraLabels {
m = sample . Metric
} else {
m [ clientmodel . MetricNameLabel ] = sample . Metric [ clientmodel . MetricNameLabel ]
for _ , l := range node . groupBy {
2014-01-24 15:05:30 +00:00
if v , ok := sample . Metric [ l ] ; ok {
m [ l ] = v
}
2013-12-13 18:20:42 +00:00
}
}
2013-01-07 22:24:26 +00:00
result [ groupingKey ] = & groupedAggregation {
2013-12-13 18:20:42 +00:00
labels : m ,
2013-01-07 22:24:26 +00:00
value : sample . Value ,
groupCount : 1 ,
}
}
}
2013-06-25 12:02:27 +00:00
2013-01-07 22:24:26 +00:00
return node . groupedAggregationsToVector ( result , timestamp )
}
2014-02-13 17:48:56 +00:00
// Eval implements the VectorNode interface and returns the value of
2014-02-22 21:29:32 +00:00
// the selector.
2014-06-06 09:55:53 +00:00
func ( node * VectorSelector ) Eval ( timestamp clientmodel . Timestamp ) Vector {
//// timer := v.stats.GetTimer(stats.GetValueAtTimeTime).Start()
samples := Vector { }
for fp , it := range node . iterators {
sampleCandidates := it . GetValueAtTime ( timestamp )
samplePair := chooseClosestSample ( sampleCandidates , timestamp )
if samplePair != nil {
samples = append ( samples , & clientmodel . Sample {
Metric : node . metrics [ fp ] , // TODO: need copy here because downstream can modify!
Value : samplePair . Value ,
Timestamp : timestamp ,
} )
}
}
//// timer.Stop()
return samples
}
// chooseClosestSample chooses the closest sample of a list of samples
// surrounding a given target time. If samples are found both before and after
// the target time, the sample value is interpolated between these. Otherwise,
// the single closest sample is returned verbatim.
func chooseClosestSample ( samples metric . Values , timestamp clientmodel . Timestamp ) * metric . SamplePair {
var closestBefore * metric . SamplePair
var closestAfter * metric . SamplePair
for _ , candidate := range samples {
delta := candidate . Timestamp . Sub ( timestamp )
// Samples before target time.
if delta < 0 {
// Ignore samples outside of staleness policy window.
if - delta > * defaultStalenessDelta {
continue
}
// Ignore samples that are farther away than what we've seen before.
if closestBefore != nil && candidate . Timestamp . Before ( closestBefore . Timestamp ) {
continue
}
sample := candidate
closestBefore = & sample
}
// Samples after target time.
if delta >= 0 {
// Ignore samples outside of staleness policy window.
if delta > * defaultStalenessDelta {
continue
}
// Ignore samples that are farther away than samples we've seen before.
if closestAfter != nil && candidate . Timestamp . After ( closestAfter . Timestamp ) {
continue
}
sample := candidate
closestAfter = & sample
}
}
switch {
case closestBefore != nil && closestAfter != nil :
return interpolateSamples ( closestBefore , closestAfter , timestamp )
case closestBefore != nil :
return closestBefore
default :
return closestAfter
}
}
// interpolateSamples interpolates a value at a target time between two
// provided sample pairs.
func interpolateSamples ( first , second * metric . SamplePair , timestamp clientmodel . Timestamp ) * metric . SamplePair {
dv := second . Value - first . Value
dt := second . Timestamp . Sub ( first . Timestamp )
dDt := dv / clientmodel . SampleValue ( dt )
offset := clientmodel . SampleValue ( timestamp . Sub ( first . Timestamp ) )
return & metric . SamplePair {
Value : first . Value + ( offset * dDt ) ,
Timestamp : timestamp ,
2013-01-07 22:24:26 +00:00
}
}
2014-02-13 17:48:56 +00:00
// Eval implements the VectorNode interface and returns the result of
// the function call.
2014-06-06 09:55:53 +00:00
func ( node * VectorFunctionCall ) Eval ( timestamp clientmodel . Timestamp ) Vector {
return node . function . callFn ( timestamp , node . args ) . ( Vector )
2013-01-07 22:24:26 +00:00
}
func evalScalarBinop ( opType BinOpType ,
2013-06-25 12:02:27 +00:00
lhs clientmodel . SampleValue ,
rhs clientmodel . SampleValue ) clientmodel . SampleValue {
2013-01-07 22:24:26 +00:00
switch opType {
case ADD :
return lhs + rhs
case SUB :
return lhs - rhs
case MUL :
return lhs * rhs
case DIV :
if rhs != 0 {
return lhs / rhs
}
2014-02-13 17:48:56 +00:00
return clientmodel . SampleValue ( math . Inf ( int ( rhs ) ) )
2013-01-07 22:24:26 +00:00
case MOD :
if rhs != 0 {
2013-06-25 12:02:27 +00:00
return clientmodel . SampleValue ( int ( lhs ) % int ( rhs ) )
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
return clientmodel . SampleValue ( math . Inf ( int ( rhs ) ) )
2013-01-07 22:24:26 +00:00
case EQ :
if lhs == rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
case NE :
if lhs != rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
case GT :
if lhs > rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
case LT :
if lhs < rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
case GE :
if lhs >= rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
case LE :
if lhs <= rhs {
return 1
}
2014-02-13 17:48:56 +00:00
return 0
2013-01-07 22:24:26 +00:00
}
panic ( "Not all enum values enumerated in switch" )
}
func evalVectorBinop ( opType BinOpType ,
2013-06-25 12:02:27 +00:00
lhs clientmodel . SampleValue ,
rhs clientmodel . SampleValue ) ( clientmodel . SampleValue , bool ) {
2013-01-07 22:24:26 +00:00
switch opType {
case ADD :
return lhs + rhs , true
case SUB :
return lhs - rhs , true
case MUL :
return lhs * rhs , true
case DIV :
if rhs != 0 {
return lhs / rhs , true
}
2014-02-13 17:48:56 +00:00
return clientmodel . SampleValue ( math . Inf ( int ( rhs ) ) ) , true
2013-01-07 22:24:26 +00:00
case MOD :
if rhs != 0 {
2013-06-25 12:02:27 +00:00
return clientmodel . SampleValue ( int ( lhs ) % int ( rhs ) ) , true
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
return clientmodel . SampleValue ( math . Inf ( int ( rhs ) ) ) , true
2013-01-07 22:24:26 +00:00
case EQ :
if lhs == rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case NE :
if lhs != rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case GT :
if lhs > rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case LT :
if lhs < rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case GE :
if lhs >= rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case LE :
if lhs <= rhs {
return lhs , true
}
2014-02-13 17:48:56 +00:00
return 0 , false
2013-01-07 22:24:26 +00:00
case AND :
return lhs , true
2013-01-17 22:45:18 +00:00
case OR :
return lhs , true // TODO: implement OR
2013-01-07 22:24:26 +00:00
}
panic ( "Not all enum values enumerated in switch" )
}
2013-06-25 12:02:27 +00:00
func labelsEqual ( labels1 , labels2 clientmodel . Metric ) bool {
2013-01-07 22:24:26 +00:00
if len ( labels1 ) != len ( labels2 ) {
return false
}
for label , value := range labels1 {
2013-06-25 12:02:27 +00:00
if labels2 [ label ] != value && label != clientmodel . MetricNameLabel {
2013-01-07 22:24:26 +00:00
return false
}
}
return true
}
2014-02-13 17:48:56 +00:00
// Eval implements the VectorNode interface and returns the result of
// the expression.
2014-06-06 09:55:53 +00:00
func ( node * VectorArithExpr ) Eval ( timestamp clientmodel . Timestamp ) Vector {
2013-01-07 22:24:26 +00:00
result := Vector { }
2014-04-08 14:14:35 +00:00
if node . lhs . Type ( ) == SCALAR && node . rhs . Type ( ) == VECTOR {
2014-06-06 09:55:53 +00:00
lhs := node . lhs . ( ScalarNode ) . Eval ( timestamp )
rhs := node . rhs . ( VectorNode ) . Eval ( timestamp )
2014-04-08 14:14:35 +00:00
for _ , rhsSample := range rhs {
value , keep := evalVectorBinop ( node . opType , lhs , rhsSample . Value )
if keep {
rhsSample . Value = value
result = append ( result , rhsSample )
}
}
return result
} else if node . lhs . Type ( ) == VECTOR && node . rhs . Type ( ) == SCALAR {
2014-06-06 09:55:53 +00:00
lhs := node . lhs . ( VectorNode ) . Eval ( timestamp )
rhs := node . rhs . ( ScalarNode ) . Eval ( timestamp )
2013-01-07 22:24:26 +00:00
for _ , lhsSample := range lhs {
value , keep := evalVectorBinop ( node . opType , lhsSample . Value , rhs )
if keep {
lhsSample . Value = value
result = append ( result , lhsSample )
}
}
return result
2014-04-08 14:14:35 +00:00
} else if node . lhs . Type ( ) == VECTOR && node . rhs . Type ( ) == VECTOR {
2014-06-06 09:55:53 +00:00
lhs := node . lhs . ( VectorNode ) . Eval ( timestamp )
rhs := node . rhs . ( VectorNode ) . Eval ( timestamp )
2013-01-07 22:24:26 +00:00
for _ , lhsSample := range lhs {
for _ , rhsSample := range rhs {
if labelsEqual ( lhsSample . Metric , rhsSample . Metric ) {
value , keep := evalVectorBinop ( node . opType , lhsSample . Value , rhsSample . Value )
if keep {
lhsSample . Value = value
result = append ( result , lhsSample )
}
}
}
}
return result
}
panic ( "Invalid vector arithmetic expression operands" )
}
2014-02-13 17:48:56 +00:00
// Eval implements the MatrixNode interface and returns the value of
2014-02-22 21:29:32 +00:00
// the selector.
2014-06-06 09:55:53 +00:00
func ( node * MatrixSelector ) Eval ( timestamp clientmodel . Timestamp ) Matrix {
2013-06-25 12:02:27 +00:00
interval := & metric . Interval {
2013-01-11 00:08:47 +00:00
OldestInclusive : timestamp . Add ( - node . interval ) ,
2013-03-28 16:05:06 +00:00
NewestInclusive : timestamp ,
2013-01-11 00:08:47 +00:00
}
2014-06-06 09:55:53 +00:00
//// timer := v.stats.GetTimer(stats.GetRangeValuesTime).Start()
sampleSets := [ ] metric . SampleSet { }
for fp , it := range node . iterators {
samplePairs := it . GetRangeValues ( * interval )
if len ( samplePairs ) == 0 {
continue
}
sampleSet := metric . SampleSet {
Metric : node . metrics [ fp ] , // TODO: need copy here because downstream can modify!
Values : samplePairs ,
}
sampleSets = append ( sampleSets , sampleSet )
2013-01-07 22:24:26 +00:00
}
2014-06-06 09:55:53 +00:00
//// timer.Stop()
return sampleSets
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// EvalBoundaries implements the MatrixNode interface and returns the
2014-02-22 21:29:32 +00:00
// boundary values of the selector.
2014-06-06 09:55:53 +00:00
func ( node * MatrixSelector ) EvalBoundaries ( timestamp clientmodel . Timestamp ) Matrix {
2013-06-25 12:02:27 +00:00
interval := & metric . Interval {
2013-01-07 22:24:26 +00:00
OldestInclusive : timestamp . Add ( - node . interval ) ,
2013-03-28 16:05:06 +00:00
NewestInclusive : timestamp ,
2013-01-07 22:24:26 +00:00
}
2014-06-06 09:55:53 +00:00
//// timer := v.stats.GetTimer(stats.GetBoundaryValuesTime).Start()
sampleSets := [ ] metric . SampleSet { }
for fp , it := range node . iterators {
samplePairs := it . GetBoundaryValues ( * interval )
if len ( samplePairs ) == 0 {
continue
}
sampleSet := metric . SampleSet {
Metric : node . metrics [ fp ] , // TODO: make copy of metric.
Values : samplePairs ,
}
sampleSets = append ( sampleSets , sampleSet )
2013-01-07 22:24:26 +00:00
}
2014-06-06 09:55:53 +00:00
//// timer.Stop()
return sampleSets
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// Len implements sort.Interface.
2013-01-15 10:30:55 +00:00
func ( matrix Matrix ) Len ( ) int {
2013-01-17 23:07:00 +00:00
return len ( matrix )
2013-01-15 10:30:55 +00:00
}
2014-02-13 17:48:56 +00:00
// Less implements sort.Interface.
2013-01-15 10:30:55 +00:00
func ( matrix Matrix ) Less ( i , j int ) bool {
2013-06-25 12:02:27 +00:00
return matrix [ i ] . Metric . String ( ) < matrix [ j ] . Metric . String ( )
2013-01-15 10:30:55 +00:00
}
2014-02-13 17:48:56 +00:00
// Swap implements sort.Interface.
2013-01-15 10:30:55 +00:00
func ( matrix Matrix ) Swap ( i , j int ) {
2013-01-17 23:07:00 +00:00
matrix [ i ] , matrix [ j ] = matrix [ j ] , matrix [ i ]
2013-01-15 10:30:55 +00:00
}
2014-02-13 17:48:56 +00:00
// Eval implements the StringNode interface and returns the value of
2014-02-22 21:29:32 +00:00
// the selector.
2014-06-06 09:55:53 +00:00
func ( node * StringLiteral ) Eval ( timestamp clientmodel . Timestamp ) string {
2013-01-07 22:24:26 +00:00
return node . str
}
2014-02-13 17:48:56 +00:00
// Eval implements the StringNode interface and returns the result of
// the function call.
2014-06-06 09:55:53 +00:00
func ( node * StringFunctionCall ) Eval ( timestamp clientmodel . Timestamp ) string {
return node . function . callFn ( timestamp , node . args ) . ( string )
2013-01-07 22:24:26 +00:00
}
// ----------------------------------------------------------------------------
// Constructors.
2014-02-13 17:48:56 +00:00
// NewScalarLiteral returns a ScalarLiteral with the given value.
2013-06-25 12:02:27 +00:00
func NewScalarLiteral ( value clientmodel . SampleValue ) * ScalarLiteral {
2013-01-07 22:24:26 +00:00
return & ScalarLiteral {
value : value ,
}
}
2014-02-22 21:29:32 +00:00
// NewVectorSelector returns a (not yet evaluated) VectorSelector with
2014-02-13 17:48:56 +00:00
// the given LabelSet.
2014-03-28 10:58:47 +00:00
func NewVectorSelector ( m metric . LabelMatchers ) * VectorSelector {
2014-02-22 21:29:32 +00:00
return & VectorSelector {
2014-03-28 10:58:47 +00:00
labelMatchers : m ,
2014-06-06 09:55:53 +00:00
iterators : map [ clientmodel . Fingerprint ] storage_ng . SeriesIterator { } ,
metrics : map [ clientmodel . Fingerprint ] clientmodel . Metric { } ,
2013-01-07 22:24:26 +00:00
}
}
2014-02-13 17:48:56 +00:00
// NewVectorAggregation returns a (not yet evaluated)
// VectorAggregation, aggregating the given VectorNode using the given
// AggrType, grouping by the given LabelNames.
2013-12-13 18:20:42 +00:00
func NewVectorAggregation ( aggrType AggrType , vector VectorNode , groupBy clientmodel . LabelNames , keepExtraLabels bool ) * VectorAggregation {
2013-01-07 22:24:26 +00:00
return & VectorAggregation {
2013-12-13 18:20:42 +00:00
aggrType : aggrType ,
groupBy : groupBy ,
keepExtraLabels : keepExtraLabels ,
vector : vector ,
2013-01-07 22:24:26 +00:00
}
}
2014-02-13 17:48:56 +00:00
// NewFunctionCall returns a (not yet evaluated) function call node
// (of type ScalarFunctionCall, VectorFunctionCall, or
// StringFunctionCall).
2013-06-06 13:12:37 +00:00
func NewFunctionCall ( function * Function , args Nodes ) ( Node , error ) {
2013-01-07 22:24:26 +00:00
if err := function . CheckArgTypes ( args ) ; err != nil {
return nil , err
}
switch function . returnType {
case SCALAR :
return & ScalarFunctionCall {
function : function ,
args : args ,
} , nil
case VECTOR :
return & VectorFunctionCall {
function : function ,
args : args ,
} , nil
case STRING :
return & StringFunctionCall {
function : function ,
args : args ,
} , nil
}
panic ( "Function with invalid return type" )
}
2013-06-06 13:12:37 +00:00
func nodesHaveTypes ( nodes Nodes , exprTypes [ ] ExprType ) bool {
2013-01-07 22:24:26 +00:00
for _ , node := range nodes {
2013-01-12 20:22:59 +00:00
correctType := false
2013-01-07 22:24:26 +00:00
for _ , exprType := range exprTypes {
if node . Type ( ) == exprType {
2013-01-11 00:09:31 +00:00
correctType = true
2013-01-07 22:24:26 +00:00
}
}
2013-01-12 20:22:59 +00:00
if ! correctType {
return false
}
2013-01-07 22:24:26 +00:00
}
2013-01-11 00:09:31 +00:00
return true
2013-01-07 22:24:26 +00:00
}
2014-02-13 17:48:56 +00:00
// NewArithExpr returns a (not yet evaluated) expression node (of type
// VectorArithExpr or ScalarArithExpr).
2013-01-07 22:24:26 +00:00
func NewArithExpr ( opType BinOpType , lhs Node , rhs Node ) ( Node , error ) {
2013-06-06 13:12:37 +00:00
if ! nodesHaveTypes ( Nodes { lhs , rhs } , [ ] ExprType { SCALAR , VECTOR } ) {
2014-02-13 17:48:56 +00:00
return nil , errors . New ( "binary operands must be of vector or scalar type" )
2013-01-07 22:24:26 +00:00
}
if opType == AND || opType == OR {
if lhs . Type ( ) == SCALAR || rhs . Type ( ) == SCALAR {
return nil , errors . New ( "AND and OR operators may only be used between vectors" )
}
}
if lhs . Type ( ) == VECTOR || rhs . Type ( ) == VECTOR {
return & VectorArithExpr {
opType : opType ,
2014-04-08 14:14:35 +00:00
lhs : lhs ,
2013-01-07 22:24:26 +00:00
rhs : rhs ,
} , nil
}
return & ScalarArithExpr {
opType : opType ,
lhs : lhs . ( ScalarNode ) ,
rhs : rhs . ( ScalarNode ) ,
} , nil
}
2014-02-22 21:29:32 +00:00
// NewMatrixSelector returns a (not yet evaluated) MatrixSelector with
// the given VectorSelector and Duration.
func NewMatrixSelector ( vector * VectorSelector , interval time . Duration ) * MatrixSelector {
return & MatrixSelector {
2014-03-28 10:58:47 +00:00
labelMatchers : vector . labelMatchers ,
interval : interval ,
2014-06-06 09:55:53 +00:00
iterators : map [ clientmodel . Fingerprint ] storage_ng . SeriesIterator { } ,
metrics : map [ clientmodel . Fingerprint ] clientmodel . Metric { } ,
2013-01-07 22:24:26 +00:00
}
}
2014-02-13 17:48:56 +00:00
// NewStringLiteral returns a StringLiteral with the given string as
// value.
2013-01-07 22:24:26 +00:00
func NewStringLiteral ( str string ) * StringLiteral {
return & StringLiteral {
str : str ,
}
}