2015-01-21 19:07:45 +00:00
// Copyright 2014 The Prometheus Authors
2014-09-19 16:18:44 +00:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2014-09-16 13:47:24 +00:00
package local
2014-06-06 09:55:53 +00:00
import (
2014-11-13 19:50:25 +00:00
"container/list"
2015-03-13 14:49:07 +00:00
"flag"
2015-03-04 12:40:18 +00:00
"fmt"
2014-06-06 09:55:53 +00:00
"io"
2014-10-22 17:21:23 +00:00
"sync"
2014-10-23 13:18:32 +00:00
"sync/atomic"
2014-06-06 09:55:53 +00:00
clientmodel "github.com/prometheus/client_golang/model"
"github.com/prometheus/prometheus/storage/metric"
)
2015-03-13 14:49:07 +00:00
var (
defaultChunkEncoding = flag . Int ( "storage.local.chunk-encoding-version" , 1 , "Which chunk encoding version to use for newly created chunks. Currently supported is 0 (delta encoding) and 1 (double-delta encoding)." )
)
type chunkEncoding byte
const (
delta chunkEncoding = iota
doubleDelta
)
2014-11-20 20:03:51 +00:00
// chunkDesc contains meta-data for a chunk. Many of its methods are
// goroutine-safe proxies for chunk methods.
2014-10-22 17:21:23 +00:00
type chunkDesc struct {
sync . Mutex
2015-05-20 17:13:06 +00:00
c chunk // nil if chunk is evicted.
rCnt int
2014-10-22 17:21:23 +00:00
chunkFirstTime clientmodel . Timestamp // Used if chunk is evicted.
chunkLastTime clientmodel . Timestamp // Used if chunk is evicted.
2014-11-13 19:50:25 +00:00
// evictListElement is nil if the chunk is not in the evict list.
// evictListElement is _not_ protected by the chunkDesc mutex.
// It must only be touched by the evict list handler in memorySeriesStorage.
evictListElement * list . Element
2014-10-22 17:21:23 +00:00
}
2014-11-20 20:03:51 +00:00
// newChunkDesc creates a new chunkDesc pointing to the provided chunk. The
// provided chunk is assumed to be not persisted yet. Therefore, the refCount of
// the new chunkDesc is 1 (preventing eviction prior to persisting).
2014-10-22 17:21:23 +00:00
func newChunkDesc ( c chunk ) * chunkDesc {
chunkOps . WithLabelValues ( createAndPin ) . Inc ( )
2014-10-23 13:18:32 +00:00
atomic . AddInt64 ( & numMemChunks , 1 )
2014-11-27 17:25:03 +00:00
numMemChunkDescs . Inc ( )
2015-05-20 17:13:06 +00:00
return & chunkDesc { c : c , rCnt : 1 }
2014-10-22 17:21:23 +00:00
}
func ( cd * chunkDesc ) add ( s * metric . SamplePair ) [ ] chunk {
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
return cd . c . add ( s )
2014-10-22 17:21:23 +00:00
}
2014-11-20 20:03:51 +00:00
// pin increments the refCount by one. Upon increment from 0 to 1, this
// chunkDesc is removed from the evict list. To enable the latter, the
// evictRequests channel has to be provided.
2014-11-13 19:50:25 +00:00
func ( cd * chunkDesc ) pin ( evictRequests chan <- evictRequest ) {
2014-10-22 17:21:23 +00:00
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . rCnt == 0 {
2014-11-13 19:50:25 +00:00
// Remove ourselves from the evict list.
evictRequests <- evictRequest { cd , false }
}
2015-05-20 17:13:06 +00:00
cd . rCnt ++
2014-10-22 17:21:23 +00:00
}
2014-11-20 20:03:51 +00:00
// unpin decrements the refCount by one. Upon decrement from 1 to 0, this
// chunkDesc is added to the evict list. To enable the latter, the evictRequests
// channel has to be provided.
2014-11-13 19:50:25 +00:00
func ( cd * chunkDesc ) unpin ( evictRequests chan <- evictRequest ) {
2014-10-22 17:21:23 +00:00
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . rCnt == 0 {
2014-10-22 17:21:23 +00:00
panic ( "cannot unpin already unpinned chunk" )
}
2015-05-20 17:13:06 +00:00
cd . rCnt --
if cd . rCnt == 0 {
2014-11-13 19:50:25 +00:00
// Add ourselves to the back of the evict list.
evictRequests <- evictRequest { cd , true }
2014-10-22 17:21:23 +00:00
}
}
2015-05-20 17:13:06 +00:00
func ( cd * chunkDesc ) refCount ( ) int {
2014-10-27 14:55:44 +00:00
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
return cd . rCnt
2014-10-27 14:55:44 +00:00
}
2014-10-22 17:21:23 +00:00
func ( cd * chunkDesc ) firstTime ( ) clientmodel . Timestamp {
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . c == nil {
2014-10-22 17:21:23 +00:00
return cd . chunkFirstTime
}
2015-05-20 17:13:06 +00:00
return cd . c . firstTime ( )
2014-10-22 17:21:23 +00:00
}
func ( cd * chunkDesc ) lastTime ( ) clientmodel . Timestamp {
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . c == nil {
2014-10-22 17:21:23 +00:00
return cd . chunkLastTime
}
2015-05-20 17:13:06 +00:00
return cd . c . newIterator ( ) . lastTimestamp ( )
2014-10-22 17:21:23 +00:00
}
func ( cd * chunkDesc ) isEvicted ( ) bool {
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
return cd . c == nil
2014-10-22 17:21:23 +00:00
}
func ( cd * chunkDesc ) contains ( t clientmodel . Timestamp ) bool {
return ! t . Before ( cd . firstTime ( ) ) && ! t . After ( cd . lastTime ( ) )
}
2015-05-20 17:13:06 +00:00
func ( cd * chunkDesc ) chunk ( ) chunk {
2014-12-03 17:07:23 +00:00
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
return cd . c
2014-12-03 17:07:23 +00:00
}
2014-10-22 17:21:23 +00:00
func ( cd * chunkDesc ) setChunk ( c chunk ) {
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . c != nil {
2014-10-22 17:21:23 +00:00
panic ( "chunk already set" )
}
2015-05-20 17:13:06 +00:00
cd . c = c
2014-10-22 17:21:23 +00:00
}
2014-11-20 20:03:51 +00:00
// maybeEvict evicts the chunk if the refCount is 0. It returns whether the chunk
2014-11-13 19:50:25 +00:00
// is now evicted, which includes the case that the chunk was evicted even
// before this method was called.
func ( cd * chunkDesc ) maybeEvict ( ) bool {
2014-10-22 17:21:23 +00:00
cd . Lock ( )
defer cd . Unlock ( )
2015-05-20 17:13:06 +00:00
if cd . c == nil {
2014-10-22 17:21:23 +00:00
return true
}
2015-05-20 17:13:06 +00:00
if cd . rCnt != 0 {
2014-11-13 19:50:25 +00:00
return false
2014-10-22 17:21:23 +00:00
}
2015-05-20 17:13:06 +00:00
cd . chunkFirstTime = cd . c . firstTime ( )
cd . chunkLastTime = cd . c . newIterator ( ) . lastTimestamp ( )
cd . c = nil
2014-10-22 17:21:23 +00:00
chunkOps . WithLabelValues ( evict ) . Inc ( )
2014-10-23 13:18:32 +00:00
atomic . AddInt64 ( & numMemChunks , - 1 )
2014-11-13 19:50:25 +00:00
return true
2014-10-22 17:21:23 +00:00
}
2014-09-16 13:47:24 +00:00
// chunk is the interface for all chunks. Chunks are generally not
// goroutine-safe.
2014-06-06 09:55:53 +00:00
type chunk interface {
2014-09-16 13:47:24 +00:00
// add adds a SamplePair to the chunks, performs any necessary
// re-encoding, and adds any necessary overflow chunks. It returns the
// new version of the original chunk, followed by overflow chunks, if
// any. The first chunk returned might be the same as the original one
// or a newly allocated version. In any case, take the returned chunk as
// the relevant one and discard the orginal chunk.
2015-03-13 14:49:07 +00:00
add ( sample * metric . SamplePair ) [ ] chunk
2014-06-06 09:55:53 +00:00
clone ( ) chunk
firstTime ( ) clientmodel . Timestamp
newIterator ( ) chunkIterator
marshal ( io . Writer ) error
unmarshal ( io . Reader ) error
2015-04-13 18:20:26 +00:00
unmarshalFromBuf ( [ ] byte )
2015-03-13 14:49:07 +00:00
encoding ( ) chunkEncoding
2014-06-06 09:55:53 +00:00
}
2014-09-16 13:47:24 +00:00
// A chunkIterator enables efficient access to the content of a chunk. It is
// generally not safe to use a chunkIterator concurrently with or after chunk
// mutation.
2014-06-06 09:55:53 +00:00
type chunkIterator interface {
2015-04-14 11:46:38 +00:00
// length returns the number of samples in the chunk.
length ( ) int
// Gets the timestamp of the n-th sample in the chunk.
2015-05-20 17:13:06 +00:00
timestampAtIndex ( int ) clientmodel . Timestamp
2015-04-14 11:46:38 +00:00
// Gets the last timestamp in the chunk.
2015-05-20 17:13:06 +00:00
lastTimestamp ( ) clientmodel . Timestamp
2015-04-14 11:46:38 +00:00
// Gets the sample value of the n-th sample in the chunk.
2015-05-20 17:13:06 +00:00
sampleValueAtIndex ( int ) clientmodel . SampleValue
2015-04-14 11:46:38 +00:00
// Gets the last sample value in the chunk.
2015-05-20 17:13:06 +00:00
lastSampleValue ( ) clientmodel . SampleValue
2014-09-16 13:47:24 +00:00
// Gets the two values that are immediately adjacent to a given time. In
// case a value exist at precisely the given time, only that single
// value is returned. Only the first or last value is returned (as a
// single value), if the given time is before or after the first or last
// value, respectively.
2015-05-20 17:13:06 +00:00
valueAtTime ( clientmodel . Timestamp ) metric . Values
2014-09-16 13:47:24 +00:00
// Gets all values contained within a given interval.
2015-05-20 17:13:06 +00:00
rangeValues ( metric . Interval ) metric . Values
2014-09-16 13:47:24 +00:00
// Whether a given timestamp is contained between first and last value
// in the chunk.
2014-06-06 09:55:53 +00:00
contains ( clientmodel . Timestamp ) bool
2015-04-14 11:46:38 +00:00
// values returns a channel, from which all sample values in the chunk
// can be received in order. The channel is closed after the last
// one. It is generally not safe to mutate the chunk while the channel
// is still open.
values ( ) <- chan * metric . SamplePair
2014-06-06 09:55:53 +00:00
}
2014-10-15 13:53:05 +00:00
func transcodeAndAdd ( dst chunk , src chunk , s * metric . SamplePair ) [ ] chunk {
2014-10-22 17:21:23 +00:00
chunkOps . WithLabelValues ( transcode ) . Inc ( )
2014-06-06 09:55:53 +00:00
head := dst
2014-10-15 13:53:05 +00:00
body := [ ] chunk { }
2015-04-14 11:46:38 +00:00
for v := range src . newIterator ( ) . values ( ) {
2014-06-06 09:55:53 +00:00
newChunks := head . add ( v )
body = append ( body , newChunks [ : len ( newChunks ) - 1 ] ... )
head = newChunks [ len ( newChunks ) - 1 ]
}
newChunks := head . add ( s )
2015-03-13 14:49:07 +00:00
return append ( body , newChunks ... )
2014-06-06 09:55:53 +00:00
}
2015-03-13 14:49:07 +00:00
// newChunk creates a new chunk according to the encoding set by the
// defaultChunkEncoding flag.
func newChunk ( ) chunk {
return newChunkForEncoding ( chunkEncoding ( * defaultChunkEncoding ) )
2014-06-06 09:55:53 +00:00
}
2015-03-13 14:49:07 +00:00
func newChunkForEncoding ( encoding chunkEncoding ) chunk {
switch encoding {
case delta :
2015-03-04 12:40:18 +00:00
return newDeltaEncodedChunk ( d1 , d0 , true , chunkLen )
2015-03-13 14:49:07 +00:00
case doubleDelta :
2015-03-04 12:40:18 +00:00
return newDoubleDeltaEncodedChunk ( d1 , d0 , true , chunkLen )
2014-06-06 09:55:53 +00:00
default :
2015-03-13 14:49:07 +00:00
panic ( fmt . Errorf ( "unknown chunk encoding: %v" , encoding ) )
2014-06-06 09:55:53 +00:00
}
}