524 lines
12 KiB
Go
524 lines
12 KiB
Go
// Copyright 2013 Prometheus Team
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package metric
|
|
|
|
import (
|
|
"code.google.com/p/goprotobuf/proto"
|
|
"github.com/prometheus/prometheus/coding"
|
|
"github.com/prometheus/prometheus/coding/indexable"
|
|
"github.com/prometheus/prometheus/model"
|
|
dto "github.com/prometheus/prometheus/model/generated"
|
|
"github.com/prometheus/prometheus/storage/raw/leveldb"
|
|
fixture "github.com/prometheus/prometheus/storage/raw/leveldb/test"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type (
|
|
curationState struct {
|
|
fingerprint string
|
|
groupSize int
|
|
recencyThreshold time.Duration
|
|
lastCurated time.Time
|
|
}
|
|
|
|
watermarkState struct {
|
|
fingerprint string
|
|
lastAppended time.Time
|
|
}
|
|
|
|
sample struct {
|
|
time time.Time
|
|
value model.SampleValue
|
|
}
|
|
|
|
sampleGroup struct {
|
|
fingerprint string
|
|
values []sample
|
|
}
|
|
|
|
in struct {
|
|
curationStates fixture.Pairs
|
|
watermarkStates fixture.Pairs
|
|
sampleGroups fixture.Pairs
|
|
recencyThreshold time.Duration
|
|
groupSize uint32
|
|
}
|
|
)
|
|
|
|
func (c curationState) Get() (key, value coding.Encoder) {
|
|
key = coding.NewProtocolBuffer(&dto.CurationKey{
|
|
Fingerprint: model.NewFingerprintFromRowKey(c.fingerprint).ToDTO(),
|
|
MinimumGroupSize: proto.Uint32(uint32(c.groupSize)),
|
|
OlderThan: proto.Int64(int64(c.recencyThreshold)),
|
|
})
|
|
|
|
value = coding.NewProtocolBuffer(&dto.CurationValue{
|
|
LastCompletionTimestamp: proto.Int64(c.lastCurated.Unix()),
|
|
})
|
|
|
|
return
|
|
}
|
|
|
|
func (w watermarkState) Get() (key, value coding.Encoder) {
|
|
key = coding.NewProtocolBuffer(model.NewFingerprintFromRowKey(w.fingerprint).ToDTO())
|
|
value = coding.NewProtocolBuffer(model.NewWatermarkFromTime(w.lastAppended).ToMetricHighWatermarkDTO())
|
|
return
|
|
}
|
|
|
|
func (s sampleGroup) Get() (key, value coding.Encoder) {
|
|
key = coding.NewProtocolBuffer(&dto.SampleKey{
|
|
Fingerprint: model.NewFingerprintFromRowKey(s.fingerprint).ToDTO(),
|
|
Timestamp: indexable.EncodeTime(s.values[0].time),
|
|
LastTimestamp: proto.Int64(s.values[len(s.values)-1].time.Unix()),
|
|
SampleCount: proto.Uint32(uint32(len(s.values))),
|
|
})
|
|
|
|
series := &dto.SampleValueSeries{}
|
|
|
|
for _, value := range s.values {
|
|
series.Value = append(series.Value, &dto.SampleValueSeries_Value{
|
|
Timestamp: proto.Int64(value.time.Unix()),
|
|
Value: value.value.ToDTO(),
|
|
})
|
|
}
|
|
|
|
value = coding.NewProtocolBuffer(series)
|
|
|
|
return
|
|
}
|
|
|
|
func TestCurator(t *testing.T) {
|
|
var (
|
|
scenarios = []struct {
|
|
in in
|
|
}{
|
|
{
|
|
in: in{
|
|
recencyThreshold: 1 * time.Hour,
|
|
groupSize: 5,
|
|
curationStates: fixture.Pairs{
|
|
curationState{
|
|
fingerprint: "0001-A-1-Z",
|
|
groupSize: 5,
|
|
recencyThreshold: 1 * time.Hour,
|
|
lastCurated: testInstant.Add(-1 * 30 * time.Minute),
|
|
},
|
|
curationState{
|
|
fingerprint: "0002-A-2-Z",
|
|
groupSize: 5,
|
|
recencyThreshold: 1 * time.Hour,
|
|
lastCurated: testInstant.Add(-1 * 90 * time.Minute),
|
|
},
|
|
// This rule should effectively be ignored.
|
|
curationState{
|
|
fingerprint: "0002-A-2-Z",
|
|
groupSize: 2,
|
|
recencyThreshold: 30 * time.Minute,
|
|
lastCurated: testInstant.Add(-1 * 90 * time.Minute),
|
|
},
|
|
},
|
|
watermarkStates: fixture.Pairs{
|
|
watermarkState{
|
|
fingerprint: "0001-A-1-Z",
|
|
lastAppended: testInstant.Add(-1 * 15 * time.Minute),
|
|
},
|
|
watermarkState{
|
|
fingerprint: "0002-A-2-Z",
|
|
lastAppended: testInstant.Add(-1 * 15 * time.Minute),
|
|
},
|
|
},
|
|
sampleGroups: fixture.Pairs{
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 90 * time.Minute),
|
|
value: 0,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 85 * time.Minute),
|
|
value: 1,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 80 * time.Minute),
|
|
value: 2,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 75 * time.Minute),
|
|
value: 3,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 70 * time.Minute),
|
|
value: 4,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 65 * time.Minute),
|
|
value: 0,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 60 * time.Minute),
|
|
value: 1,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 55 * time.Minute),
|
|
value: 2,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 50 * time.Minute),
|
|
value: 3,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 45 * time.Minute),
|
|
value: 4,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 40 * time.Minute),
|
|
value: 0,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 35 * time.Minute),
|
|
value: 1,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 30 * time.Minute),
|
|
value: 2,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 25 * time.Minute),
|
|
value: 0,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 35 * time.Minute),
|
|
value: 1,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0001-A-1-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 30 * time.Minute),
|
|
value: 2,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 90 * time.Minute),
|
|
value: 0,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 89 * time.Minute),
|
|
value: 1,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 88 * time.Minute),
|
|
value: 2,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 87 * time.Minute),
|
|
value: 3,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 86 * time.Minute),
|
|
value: 4,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 85 * time.Minute),
|
|
value: 5,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 84 * time.Minute),
|
|
value: 6,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 83 * time.Minute),
|
|
value: 7,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 82 * time.Minute),
|
|
value: 8,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 81 * time.Minute),
|
|
value: 9,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 80 * time.Minute),
|
|
value: 10,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 79 * time.Minute),
|
|
value: 11,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 78 * time.Minute),
|
|
value: 12,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 77 * time.Minute),
|
|
value: 13,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 76 * time.Minute),
|
|
value: 14,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 75 * time.Minute),
|
|
value: 15,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 74 * time.Minute),
|
|
value: 16,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 73 * time.Minute),
|
|
value: 17,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 72 * time.Minute),
|
|
value: 18,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 71 * time.Minute),
|
|
value: 19,
|
|
},
|
|
{
|
|
time: testInstant.Add(-1 * 70 * time.Minute),
|
|
value: 20,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 69 * time.Minute),
|
|
value: 21,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 68 * time.Minute),
|
|
value: 22,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 67 * time.Minute),
|
|
value: 23,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 66 * time.Minute),
|
|
value: 24,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 65 * time.Minute),
|
|
value: 25,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 64 * time.Minute),
|
|
value: 26,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 63 * time.Minute),
|
|
value: 27,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 62 * time.Minute),
|
|
value: 28,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 61 * time.Minute),
|
|
value: 29,
|
|
},
|
|
},
|
|
},
|
|
sampleGroup{
|
|
fingerprint: "0002-A-2-Z",
|
|
values: []sample{
|
|
{
|
|
time: testInstant.Add(-1 * 60 * time.Minute),
|
|
value: 30,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
)
|
|
|
|
for _, scenario := range scenarios {
|
|
curatorDirectory := fixture.NewPreparer(t).Prepare("curator", fixture.NewCassetteFactory(scenario.in.curationStates))
|
|
defer curatorDirectory.Close()
|
|
|
|
watermarkDirectory := fixture.NewPreparer(t).Prepare("watermark", fixture.NewCassetteFactory(scenario.in.watermarkStates))
|
|
defer watermarkDirectory.Close()
|
|
|
|
sampleDirectory := fixture.NewPreparer(t).Prepare("sample", fixture.NewCassetteFactory(scenario.in.sampleGroups))
|
|
defer sampleDirectory.Close()
|
|
|
|
curatorStates, err := leveldb.NewLevelDBPersistence(curatorDirectory.Path(), 0, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer curatorStates.Close()
|
|
|
|
watermarkStates, err := leveldb.NewLevelDBPersistence(watermarkDirectory.Path(), 0, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer watermarkStates.Close()
|
|
|
|
samples, err := leveldb.NewLevelDBPersistence(sampleDirectory.Path(), 0, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer samples.Close()
|
|
|
|
c := newCurator(scenario.in.recencyThreshold, scenario.in.groupSize, curatorStates, samples, watermarkStates)
|
|
c.run(testInstant)
|
|
}
|
|
}
|