RepoMirrors/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus synced 2024-12-27 00:53:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge "Remove unused labelname -> fingerprints index."

Browse Source
This commit is contained in:
Björn Rabenstein 2014-03-06 11:40:09 +01:00 committed by Gerrit Code Review
commit 0bb33b6525
7 changed files with 26 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
storage/metric/end_to_end_test.go
View File

@ -78,72 +78,6 @@ func GetFingerprintsForLabelSetTests(p MetricPersistence, t test.Tester) {
}
}
func GetFingerprintsForLabelNameTests(p MetricPersistence, t test.Tester) {
testAppendSamples(p, &clientmodel.Sample{
Value: 0,
Timestamp: 0,
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: "my_metric",
"request_type": "your_mom",
"language": "english",
},
}, t)
testAppendSamples(p, &clientmodel.Sample{
Value: 0,
Timestamp: 0,
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: "my_metric",
"request_type": "your_dad",
"sprache": "deutsch",
},
}, t)
b := clientmodel.MetricNameLabel
result, err := p.GetFingerprintsForLabelName(b)
if err != nil {
t.Error(err)
}
if len(result) != 2 {
t.Errorf("Expected two elements.")
}
b = clientmodel.LabelName("request_type")
result, err = p.GetFingerprintsForLabelName(b)
if err != nil {
t.Error(err)
}
if len(result) != 2 {
t.Errorf("Expected two elements.")
}
b = clientmodel.LabelName("language")
result, err = p.GetFingerprintsForLabelName(b)
if err != nil {
t.Error(err)
}
if len(result) != 1 {
t.Errorf("Expected one element.")
}
b = clientmodel.LabelName("sprache")
result, err = p.GetFingerprintsForLabelName(b)
if err != nil {
t.Error(err)
}
if len(result) != 1 {
t.Errorf("Expected one element.")
}
}
func GetMetricForFingerprintTests(p MetricPersistence, t test.Tester) {
testAppendSamples(p, &clientmodel.Sample{
Value: 0,
@ -388,18 +322,6 @@ func BenchmarkLevelDBGetFingerprintsForLabelSet(b *testing.B) {
}
}
var testLevelDBGetFingerprintsForLabelName = buildLevelDBTestPersistence("get_fingerprints_for_labelname", GetFingerprintsForLabelNameTests)
func TestLevelDBGetFingerprintsForLabelName(t *testing.T) {
testLevelDBGetFingerprintsForLabelName(t)
}
func BenchmarkLevelDBGetFingerprintsForLabelName(b *testing.B) {
for i := 0; i < b.N; i++ {
testLevelDBGetFingerprintsForLabelName(b)
}
}
var testLevelDBGetMetricForFingerprint = buildLevelDBTestPersistence("get_metric_for_fingerprint", GetMetricForFingerprintTests)
func TestLevelDBGetMetricForFingerprint(t *testing.T) {
@ -448,18 +370,6 @@ func BenchmarkMemoryGetFingerprintsForLabelSet(b *testing.B) {
}
}
var testMemoryGetFingerprintsForLabelName = buildMemoryTestPersistence(GetFingerprintsForLabelNameTests)
func TestMemoryGetFingerprintsForLabelName(t *testing.T) {
testMemoryGetFingerprintsForLabelName(t)
}
func BenchmarkMemoryGetFingerprintsForLabelName(b *testing.B) {
for i := 0; i < b.N; i++ {
testMemoryGetFingerprintsForLabelName(b)
}
}
var testMemoryGetMetricForFingerprint = buildMemoryTestPersistence(GetMetricForFingerprintTests)
func TestMemoryGetMetricForFingerprint(t *testing.T) {

135
storage/metric/index.go
View File

@ -96,93 +96,6 @@ func NewLevelDBFingerprintMetricIndex(o leveldb.LevelDBOptions) (*LevelDBFingerp
}, nil
}
// LabelNameFingerprintMapping is an in-memory map of LabelNames to
// Fingerprints.
type LabelNameFingerprintMapping map[clientmodel.LabelName]clientmodel.Fingerprints
// LabelNameFingerprintIndex models a database mapping LabelNames to
// Fingerprints.
type LabelNameFingerprintIndex interface {
raw.Database
raw.Pruner
IndexBatch(LabelNameFingerprintMapping) error
Lookup(clientmodel.LabelName) (fps clientmodel.Fingerprints, ok bool, err error)
Has(clientmodel.LabelName) (ok bool, err error)
}
// LevelDBLabelNameFingerprintIndex implements LabelNameFingerprintIndex using
// leveldb.
type LevelDBLabelNameFingerprintIndex struct {
*leveldb.LevelDBPersistence
}
// IndexBatch implements LabelNameFingerprintIndex.
func (i *LevelDBLabelNameFingerprintIndex) IndexBatch(b LabelNameFingerprintMapping) error {
batch := leveldb.NewBatch()
defer batch.Close()
for labelName, fingerprints := range b {
sort.Sort(fingerprints)
key := &dto.LabelName{
Name: proto.String(string(labelName)),
}
value := &dto.FingerprintCollection{}
for _, fingerprint := range fingerprints {
f := &dto.Fingerprint{}
dumpFingerprint(f, fingerprint)
value.Member = append(value.Member, f)
}
batch.Put(key, value)
}
return i.LevelDBPersistence.Commit(batch)
}
// Lookup implements LabelNameFingerprintIndex.
func (i *LevelDBLabelNameFingerprintIndex) Lookup(l clientmodel.LabelName) (fps clientmodel.Fingerprints, ok bool, err error) {
k := &dto.LabelName{}
dumpLabelName(k, l)
v := &dto.FingerprintCollection{}
ok, err = i.LevelDBPersistence.Get(k, v)
if err != nil {
return nil, false, err
}
if !ok {
return nil, false, nil
}
for _, m := range v.Member {
fp := &clientmodel.Fingerprint{}
loadFingerprint(fp, m)
fps = append(fps, fp)
}
return fps, true, nil
}
// Has implements LabelNameFingerprintIndex.
func (i *LevelDBLabelNameFingerprintIndex) Has(l clientmodel.LabelName) (ok bool, err error) {
return i.LevelDBPersistence.Has(&dto.LabelName{
Name: proto.String(string(l)),
})
}
// NewLevelLabelNameFingerprintIndex returns a LevelDBLabelNameFingerprintIndex
// ready to use.
func NewLevelLabelNameFingerprintIndex(o leveldb.LevelDBOptions) (*LevelDBLabelNameFingerprintIndex, error) {
s, err := leveldb.NewLevelDBPersistence(o)
if err != nil {
return nil, err
}
return &LevelDBLabelNameFingerprintIndex{
LevelDBPersistence: s,
}, nil
}
// LabelPairFingerprintMapping is an in-memory map of LabelPairs to
// Fingerprints.
type LabelPairFingerprintMapping map[LabelPair]clientmodel.Fingerprints
@ -557,7 +470,6 @@ func NewBufferedIndexer(i MetricIndexer, limit int) *BufferedIndexer {
// locking semantics to enforce this.
type TotalIndexer struct {
FingerprintToMetric FingerprintMetricIndex
LabelNameToFingerprint LabelNameFingerprintIndex
LabelPairToFingerprint LabelPairFingerprintIndex
MetricMembership MetricMembershipIndex
}
@ -578,45 +490,6 @@ func findUnindexed(i MetricMembershipIndex, b FingerprintMetricMapping) (Fingerp
return out, nil
}
func extendLabelNameIndex(i LabelNameFingerprintIndex, b FingerprintMetricMapping) (LabelNameFingerprintMapping, error) {
collection := map[clientmodel.LabelName]utility.Set{}
for fp, m := range b {
for l := range m {
set, ok := collection[l]
if !ok {
baseFps, _, err := i.Lookup(l)
if err != nil {
return nil, err
}
set = utility.Set{}
for _, baseFp := range baseFps {
set.Add(*baseFp)
}
collection[l] = set
}
set.Add(fp)
}
}
batch := LabelNameFingerprintMapping{}
for l, set := range collection {
fps := clientmodel.Fingerprints{}
for e := range set {
fp := e.(clientmodel.Fingerprint)
fps = append(fps, &fp)
}
batch[l] = fps
}
return batch, nil
}
func extendLabelPairIndex(i LabelPairFingerprintIndex, b FingerprintMetricMapping) (LabelPairFingerprintMapping, error) {
collection := map[LabelPair]utility.Set{}
@ -667,14 +540,6 @@ func (i *TotalIndexer) IndexMetrics(b FingerprintMetricMapping) error {
return err
}
labelNames, err := extendLabelNameIndex(i.LabelNameToFingerprint, unindexed)
if err != nil {
return err
}
if err := i.LabelNameToFingerprint.IndexBatch(labelNames); err != nil {
return err
}
labelPairs, err := extendLabelPairIndex(i.LabelPairToFingerprint, unindexed)
if err != nil {
return err

4
storage/metric/instrumentation.go
View File

@ -26,16 +26,13 @@ const (
result = "result"
appendFingerprints = "append_fingerprints"
appendLabelNameFingerprint = "append_label_name_fingerprint"
appendLabelPairFingerprint = "append_label_pair_fingerprint"
appendSample = "append_sample"
appendSamples = "append_samples"
findUnindexedMetrics = "find_unindexed_metrics"
flushMemory = "flush_memory"
getBoundaryValues = "get_boundary_values"
getFingerprintsForLabelName = "get_fingerprints_for_label_name"
getFingerprintsForLabelSet = "get_fingerprints_for_labelset"
getLabelNameFingerprints = "get_label_name_fingerprints"
getMetricForFingerprint = "get_metric_for_fingerprint"
getRangeValues = "get_range_values"
getValueAtTime = "get_value_at_time"
@ -50,7 +47,6 @@ const (
rebuildDiskFrontier = "rebuild_disk_frontier"
refreshHighWatermarks = "refresh_high_watermarks"
renderView = "render_view"
setLabelNameFingerprints = "set_label_name_fingerprints"
setLabelPairFingerprints = "set_label_pair_fingerprints"
writeMemory = "write_memory"

4
storage/metric/interface.go
View File

@ -32,10 +32,6 @@ type MetricPersistence interface {
// provided label set.
GetFingerprintsForLabelSet(clientmodel.LabelSet) (clientmodel.Fingerprints, error)
// Get all of the metric fingerprints that are associated for a given
// label name.
GetFingerprintsForLabelName(clientmodel.LabelName) (clientmodel.Fingerprints, error)
// Get the metric associated with the provided fingerprint.
GetMetricForFingerprint(*clientmodel.Fingerprint) (clientmodel.Metric, error)

59
storage/metric/leveldb.go
View File

@ -39,7 +39,6 @@ const sortConcurrency = 2
type LevelDBMetricPersistence struct {
CurationRemarks CurationRemarker
FingerprintToMetrics FingerprintMetricIndex
LabelNameToFingerprints LabelNameFingerprintIndex
LabelPairToFingerprints LabelPairFingerprintIndex
MetricHighWatermarks HighWatermarker
MetricMembershipIndex MetricMembershipIndex
@ -52,7 +51,6 @@ type LevelDBMetricPersistence struct {
//
// type FingerprintResolver interface {
// GetFingerprintForMetric(clientmodel.Metric) (*clientmodel.Fingerprint, bool, error)
// GetFingerprintsForLabelName(clientmodel.LabelName) (clientmodel.Fingerprints, bool, error)
// GetFingerprintsForLabelSet(LabelPair) (clientmodel.Fingerprints, bool, error)
// }
@ -69,7 +67,6 @@ var (
curationRemarksCacheSize = flag.Int("curationRemarksCacheSize", 5*1024*1024, "The size for the curation remarks cache (bytes).")
fingerprintsToLabelPairCacheSize = flag.Int("fingerprintsToLabelPairCacheSizeBytes", 25*1024*1024, "The size for the fingerprint to label pair index (bytes).")
highWatermarkCacheSize = flag.Int("highWatermarksByFingerprintSizeBytes", 5*1024*1024, "The size for the metric high watermarks (bytes).")
labelNameToFingerprintsCacheSize = flag.Int("labelNameToFingerprintsCacheSizeBytes", 25*1024*1024, "The size for the label name to metric fingerprint index (bytes).")
labelPairToFingerprintsCacheSize = flag.Int("labelPairToFingerprintsCacheSizeBytes", 25*1024*1024, "The size for the label pair to metric fingerprint index (bytes).")
metricMembershipIndexCacheSize = flag.Int("metricMembershipCacheSizeBytes", 5*1024*1024, "The size for the metric membership index (bytes).")
samplesByFingerprintCacheSize = flag.Int("samplesByFingerprintCacheSizeBytes", 50*1024*1024, "The size for the samples database (bytes).")
@ -83,7 +80,6 @@ func (l *LevelDBMetricPersistence) Close() {
var persistences = []raw.Database{
l.CurationRemarks,
l.FingerprintToMetrics,
l.LabelNameToFingerprints,
l.LabelPairToFingerprints,
l.MetricHighWatermarks,
l.MetricMembershipIndex,
@ -110,7 +106,7 @@ func (l *LevelDBMetricPersistence) Close() {
// NewLevelDBMetricPersistence returns a LevelDBMetricPersistence object ready
// to use.
func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistence, error) {
workers := utility.NewUncertaintyGroup(7)
workers := utility.NewUncertaintyGroup(6)
emission := &LevelDBMetricPersistence{}
@ -161,21 +157,6 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
workers.MayFail(err)
},
},
{
"Fingerprints by Label Name",
func() {
var err error
emission.LabelNameToFingerprints, err = NewLevelLabelNameFingerprintIndex(
leveldb.LevelDBOptions{
Name: "Fingerprints by Label Name",
Purpose: "Index",
Path: baseDirectory + "/fingerprints_by_label_name",
CacheSizeBytes: *labelNameToFingerprintsCacheSize,
},
)
workers.MayFail(err)
},
},
{
"Fingerprints by Label Name and Value Pair",
func() {
@ -238,7 +219,6 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc
emission.Indexer = &TotalIndexer{
FingerprintToMetric: emission.FingerprintToMetrics,
LabelNameToFingerprint: emission.LabelNameToFingerprints,
LabelPairToFingerprint: emission.LabelPairToFingerprints,
MetricMembership: emission.MetricMembershipIndex,
}
@ -442,20 +422,6 @@ func (l *LevelDBMetricPersistence) HasLabelPair(p *LabelPair) (value bool, err e
return l.LabelPairToFingerprints.Has(p)
}
// HasLabelName returns true if the given LabelName is present in the underlying
// LabelName index.
func (l *LevelDBMetricPersistence) HasLabelName(n clientmodel.LabelName) (value bool, err error) {
defer func(begin time.Time) {
duration := time.Since(begin)
recordOutcome(duration, err, map[string]string{operation: hasLabelName, result: success}, map[string]string{operation: hasLabelName, result: failure})
}(time.Now())
value, err = l.LabelNameToFingerprints.Has(n)
return
}
// GetFingerprintsForLabelSet returns the Fingerprints for the given LabelSet by
// querying the underlying LabelPairFingerprintIndex for each LabelPair
// contained in LabelSet. It implements the MetricPersistence interface.
@ -503,22 +469,6 @@ func (l *LevelDBMetricPersistence) GetFingerprintsForLabelSet(labelSet clientmod
return fps, nil
}
// GetFingerprintsForLabelName returns the Fingerprints for the given LabelName
// from the underlying LabelNameFingerprintIndex. It implements the
// MetricPersistence interface.
func (l *LevelDBMetricPersistence) GetFingerprintsForLabelName(labelName clientmodel.LabelName) (fps clientmodel.Fingerprints, err error) {
defer func(begin time.Time) {
duration := time.Since(begin)
recordOutcome(duration, err, map[string]string{operation: getFingerprintsForLabelName, result: success}, map[string]string{operation: getFingerprintsForLabelName, result: failure})
}(time.Now())
// TODO(matt): Update signature to work with ok.
fps, _, err = l.LabelNameToFingerprints.Lookup(labelName)
return fps, err
}
// GetMetricForFingerprint returns the Metric for the given Fingerprint from the
// underlying FingerprintMetricIndex. It implements the MetricPersistence
// interface.
@ -559,7 +509,6 @@ func (l *LevelDBMetricPersistence) GetAllValuesForLabel(labelName clientmodel.La
func (l *LevelDBMetricPersistence) Prune() {
l.CurationRemarks.Prune()
l.FingerprintToMetrics.Prune()
l.LabelNameToFingerprints.Prune()
l.LabelPairToFingerprints.Prune()
l.MetricHighWatermarks.Prune()
l.MetricMembershipIndex.Prune()
@ -580,11 +529,6 @@ func (l *LevelDBMetricPersistence) Sizes() (total uint64, err error) {
}
total += size
if size, err = l.LabelNameToFingerprints.Size(); err != nil {
return 0, err
}
total += size
if size, err = l.LabelPairToFingerprints.Size(); err != nil {
return 0, err
}
@ -613,7 +557,6 @@ func (l *LevelDBMetricPersistence) States() raw.DatabaseStates {
return raw.DatabaseStates{
l.CurationRemarks.State(),
l.FingerprintToMetrics.State(),
l.LabelNameToFingerprints.State(),
l.LabelPairToFingerprints.State(),
l.MetricHighWatermarks.State(),
l.MetricMembershipIndex.State(),

23
storage/metric/memory.go
View File

@ -177,7 +177,6 @@ type memorySeriesStorage struct {
wmCache *watermarkCache
fingerprintToSeries map[clientmodel.Fingerprint]stream
labelPairToFingerprints map[LabelPair]clientmodel.Fingerprints
labelNameToFingerprints map[clientmodel.LabelName]clientmodel.Fingerprints
}
// MemorySeriesOptions bundles options used by NewMemorySeriesStorage to create
@ -244,10 +243,6 @@ func (s *memorySeriesStorage) getOrCreateSeries(metric clientmodel.Metric, finge
labelPairValues := s.labelPairToFingerprints[labelPair]
labelPairValues = append(labelPairValues, fingerprint)
s.labelPairToFingerprints[labelPair] = labelPairValues
labelNameValues := s.labelNameToFingerprints[k]
labelNameValues = append(labelNameValues, fingerprint)
s.labelNameToFingerprints[k] = labelNameValues
}
}
return series
@ -302,7 +297,6 @@ func (s *memorySeriesStorage) dropSeries(fingerprint *clientmodel.Fingerprint) {
Value: v,
}
delete(s.labelPairToFingerprints, labelPair)
delete(s.labelNameToFingerprints, k)
}
delete(s.fingerprintToSeries, *fingerprint)
}
@ -357,21 +351,6 @@ func (s *memorySeriesStorage) GetFingerprintsForLabelSet(l clientmodel.LabelSet)
return fingerprints, nil
}
func (s *memorySeriesStorage) GetFingerprintsForLabelName(l clientmodel.LabelName) (clientmodel.Fingerprints, error) {
s.RLock()
defer s.RUnlock()
values, ok := s.labelNameToFingerprints[l]
if !ok {
return nil, nil
}
fingerprints := make(clientmodel.Fingerprints, len(values))
copy(fingerprints, values)
return fingerprints, nil
}
func (s *memorySeriesStorage) GetMetricForFingerprint(f *clientmodel.Fingerprint) (clientmodel.Metric, error) {
s.RLock()
defer s.RUnlock()
@ -453,7 +432,6 @@ func (s *memorySeriesStorage) Close() {
s.fingerprintToSeries = map[clientmodel.Fingerprint]stream{}
s.labelPairToFingerprints = map[LabelPair]clientmodel.Fingerprints{}
s.labelNameToFingerprints = map[clientmodel.LabelName]clientmodel.Fingerprints{}
}
func (s *memorySeriesStorage) GetAllValuesForLabel(labelName clientmodel.LabelName) (values clientmodel.LabelValues, err error) {
@ -478,7 +456,6 @@ func NewMemorySeriesStorage(o MemorySeriesOptions) *memorySeriesStorage {
return &memorySeriesStorage{
fingerprintToSeries: make(map[clientmodel.Fingerprint]stream),
labelPairToFingerprints: make(map[LabelPair]clientmodel.Fingerprints),
labelNameToFingerprints: make(map[clientmodel.LabelName]clientmodel.Fingerprints),
wmCache: o.WatermarkCache,
}
}

70
storage/metric/stochastic_test.go
View File

@ -66,29 +66,6 @@ func ReadEmptyTests(p MetricPersistence, t test.Tester) {
t.Error(err)
return
}
hasLabelName := func(x int) (success bool) {
labelName := clientmodel.LabelName(string(x))
fingerprints, err := p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
success = len(fingerprints) == 0
if !success {
t.Errorf("unexpected fingerprint length %d, got %d", 0, len(fingerprints))
}
return
}
err = quick.Check(hasLabelName, nil)
if err != nil {
t.Error(err)
return
}
}
func AppendSampleAsPureSparseAppendTests(p MetricPersistence, t test.Tester) {
@ -140,17 +117,7 @@ func AppendSampleAsSparseAppendWithReadsTests(p MetricPersistence, t test.Tester
return
}
fingerprints, err := p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != 1 {
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
fingerprints, err = p.GetFingerprintsForLabelSet(clientmodel.LabelSet{
fingerprints, err := p.GetFingerprintsForLabelSet(clientmodel.LabelSet{
labelName: labelValue,
})
if err != nil {
@ -345,31 +312,6 @@ func StochasticTests(persistenceMaker func() (MetricPersistence, test.Closer), t
t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
return
}
labelName := clientmodel.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex))
fingerprints, err = p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) == 0 {
t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
return
}
}
}
for sharedIndex := 0; sharedIndex < numberOfSharedLabels; sharedIndex++ {
labelName := clientmodel.LabelName(fmt.Sprintf("shared_label_%d", sharedIndex))
fingerprints, err := p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != numberOfMetrics {
t.Errorf("expected fingerprint count of %d, got %d", numberOfMetrics, len(fingerprints))
return
}
}
@ -390,16 +332,6 @@ func StochasticTests(persistenceMaker func() (MetricPersistence, test.Closer), t
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
fingerprints, err = p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != 1 {
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
}
metric := clientmodel.Metric{}