312 lines
9.2 KiB
Go
312 lines
9.2 KiB
Go
package index
|
|
|
|
import (
|
|
"flag"
|
|
"path"
|
|
|
|
clientmodel "github.com/prometheus/client_golang/model"
|
|
|
|
"github.com/prometheus/prometheus/storage/local/codec"
|
|
"github.com/prometheus/prometheus/storage/metric"
|
|
"github.com/prometheus/prometheus/utility"
|
|
)
|
|
|
|
const (
|
|
fingerprintToMetricDir = "archived_fingerprint_to_metric"
|
|
fingerprintTimeRangeDir = "archived_fingerprint_to_timerange"
|
|
labelNameToLabelValuesDir = "labelname_to_labelvalues"
|
|
labelPairToFingerprintsDir = "labelpair_to_fingerprints"
|
|
)
|
|
|
|
var (
|
|
fingerprintToMetricCacheSize = flag.Int("storage.fingerprintToMetricCacheSizeBytes", 25*1024*1024, "The size in bytes for the fingerprint to metric index cache.")
|
|
labelNameToLabelValuesCacheSize = flag.Int("storage.labelNameToLabelValuesCacheSizeBytes", 25*1024*1024, "The size in bytes for the label name to label values index cache.")
|
|
labelPairToFingerprintsCacheSize = flag.Int("storage.labelPairToFingerprintsCacheSizeBytes", 25*1024*1024, "The size in bytes for the label pair to fingerprints index cache.")
|
|
fingerprintTimeRangeCacheSize = flag.Int("storage.fingerprintTimeRangeCacheSizeBytes", 5*1024*1024, "The size in bytes for the metric time range index cache.")
|
|
)
|
|
|
|
// FingerprintMetricMapping is an in-memory map of fingerprints to metrics.
|
|
type FingerprintMetricMapping map[clientmodel.Fingerprint]clientmodel.Metric
|
|
|
|
// FingerprintMetricIndex models a database mapping fingerprints to metrics.
|
|
type FingerprintMetricIndex struct {
|
|
KeyValueStore
|
|
}
|
|
|
|
// IndexBatch indexes a batch of mappings from fingerprints to metrics.
|
|
func (i *FingerprintMetricIndex) IndexBatch(mapping FingerprintMetricMapping) error {
|
|
b := i.NewBatch()
|
|
|
|
for fp, m := range mapping {
|
|
b.Put(codec.CodableFingerprint(fp), codec.CodableMetric(m))
|
|
}
|
|
|
|
return i.Commit(b)
|
|
}
|
|
|
|
// UnindexBatch unindexes a batch of mappings from fingerprints to metrics.
|
|
func (i *FingerprintMetricIndex) UnindexBatch(mapping FingerprintMetricMapping) error {
|
|
b := i.NewBatch()
|
|
|
|
for fp, _ := range mapping {
|
|
b.Delete(codec.CodableFingerprint(fp))
|
|
}
|
|
|
|
return i.Commit(b)
|
|
}
|
|
|
|
// Lookup looks up a metric by fingerprint.
|
|
func (i *FingerprintMetricIndex) Lookup(fp clientmodel.Fingerprint) (metric clientmodel.Metric, ok bool, err error) {
|
|
ok, err = i.Get(codec.CodableFingerprint(fp), (*codec.CodableMetric)(&metric))
|
|
return
|
|
}
|
|
|
|
// NewFingerprintMetricIndex returns a FingerprintMetricIndex
|
|
// object ready to use.
|
|
func NewFingerprintMetricIndex(basePath string) (*FingerprintMetricIndex, error) {
|
|
fingerprintToMetricDB, err := NewLevelDB(LevelDBOptions{
|
|
Path: path.Join(basePath, fingerprintToMetricDir),
|
|
CacheSizeBytes: *fingerprintToMetricCacheSize,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &FingerprintMetricIndex{
|
|
KeyValueStore: fingerprintToMetricDB,
|
|
}, nil
|
|
}
|
|
|
|
// LabelNameLabelValuesMapping is an in-memory map of label names to
|
|
// label values.
|
|
type LabelNameLabelValuesMapping map[clientmodel.LabelName]clientmodel.LabelValues
|
|
|
|
// LabelNameLabelValuesIndex models a database mapping label names to
|
|
// label values.
|
|
type LabelNameLabelValuesIndex struct {
|
|
KeyValueStore
|
|
}
|
|
|
|
// IndexBatch implements LabelNameLabelValuesIndex.
|
|
func (i *LabelNameLabelValuesIndex) IndexBatch(b LabelNameLabelValuesMapping) error {
|
|
batch := i.NewBatch()
|
|
|
|
for name, values := range b {
|
|
if len(values) == 0 {
|
|
batch.Delete(codec.CodableLabelName(name))
|
|
} else {
|
|
batch.Put(codec.CodableLabelName(name), codec.CodableLabelValues(values))
|
|
}
|
|
}
|
|
|
|
return i.Commit(batch)
|
|
}
|
|
|
|
// Lookup looks up all label values for a given label name.
|
|
func (i *LabelNameLabelValuesIndex) Lookup(l clientmodel.LabelName) (values clientmodel.LabelValues, ok bool, err error) {
|
|
ok, err = i.Get(codec.CodableLabelName(l), (*codec.CodableLabelValues)(&values))
|
|
return
|
|
}
|
|
|
|
func (i *LabelNameLabelValuesIndex) Extend(m clientmodel.Metric) error {
|
|
b := make(LabelNameLabelValuesMapping, len(m))
|
|
for ln, lv := range m {
|
|
baseLVs, _, err := i.Lookup(ln)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lvSet := utility.Set{}
|
|
for _, baseLV := range baseLVs {
|
|
lvSet.Add(baseLV)
|
|
}
|
|
lvSet.Add(lv)
|
|
if len(lvSet) == len(baseLVs) {
|
|
continue
|
|
}
|
|
lvs := make(clientmodel.LabelValues, 0, len(lvSet))
|
|
for v := range lvSet {
|
|
lvs = append(lvs, v.(clientmodel.LabelValue))
|
|
}
|
|
b[ln] = lvs
|
|
}
|
|
return i.IndexBatch(b)
|
|
}
|
|
|
|
func (i *LabelNameLabelValuesIndex) Reduce(m LabelPairFingerprintsMapping) error {
|
|
b := make(LabelNameLabelValuesMapping, len(m))
|
|
for lp, fps := range m {
|
|
if len(fps) != 0 {
|
|
continue
|
|
}
|
|
ln := lp.Name
|
|
lv := lp.Value
|
|
baseValues, ok := b[ln]
|
|
if !ok {
|
|
var err error
|
|
baseValues, _, err = i.Lookup(ln)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
lvSet := utility.Set{}
|
|
for _, baseValue := range baseValues {
|
|
lvSet.Add(baseValue)
|
|
}
|
|
lvSet.Remove(lv)
|
|
if len(lvSet) == len(baseValues) {
|
|
continue
|
|
}
|
|
lvs := make(clientmodel.LabelValues, 0, len(lvSet))
|
|
for v := range lvSet {
|
|
lvs = append(lvs, v.(clientmodel.LabelValue))
|
|
}
|
|
b[ln] = lvs
|
|
}
|
|
return i.IndexBatch(b)
|
|
}
|
|
|
|
// NewLabelNameLabelValuesIndex returns a LabelNameLabelValuesIndex
|
|
// ready to use.
|
|
func NewLabelNameLabelValuesIndex(basePath string) (*LabelNameLabelValuesIndex, error) {
|
|
labelNameToLabelValuesDB, err := NewLevelDB(LevelDBOptions{
|
|
Path: path.Join(basePath, labelNameToLabelValuesDir),
|
|
CacheSizeBytes: *labelNameToLabelValuesCacheSize,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &LabelNameLabelValuesIndex{
|
|
KeyValueStore: labelNameToLabelValuesDB,
|
|
}, nil
|
|
}
|
|
|
|
// LabelPairFingerprintsMapping is an in-memory map of label pairs to
|
|
// fingerprints.
|
|
type LabelPairFingerprintsMapping map[metric.LabelPair]clientmodel.Fingerprints
|
|
|
|
// LabelPairFingerprintIndex models a database mapping label pairs to
|
|
// fingerprints.
|
|
type LabelPairFingerprintIndex struct {
|
|
KeyValueStore
|
|
}
|
|
|
|
// IndexBatch indexes a batch of mappings from label pairs to fingerprints.
|
|
func (i *LabelPairFingerprintIndex) IndexBatch(m LabelPairFingerprintsMapping) error {
|
|
batch := i.NewBatch()
|
|
|
|
for pair, fps := range m {
|
|
if len(fps) == 0 {
|
|
batch.Delete(codec.CodableLabelPair(pair))
|
|
} else {
|
|
batch.Put(codec.CodableLabelPair(pair), codec.CodableFingerprints(fps))
|
|
}
|
|
}
|
|
|
|
return i.Commit(batch)
|
|
}
|
|
|
|
// Lookup looks up all fingerprints for a given label pair.
|
|
func (i *LabelPairFingerprintIndex) Lookup(p metric.LabelPair) (fps clientmodel.Fingerprints, ok bool, err error) {
|
|
ok, err = i.Get((codec.CodableLabelPair)(p), (*codec.CodableFingerprints)(&fps))
|
|
return
|
|
}
|
|
|
|
func (i *LabelPairFingerprintIndex) Extend(m clientmodel.Metric, fp clientmodel.Fingerprint) error {
|
|
b := make(LabelPairFingerprintsMapping, len(m))
|
|
for ln, lv := range m {
|
|
lp := metric.LabelPair{Name: ln, Value: lv}
|
|
baseFPs, _, err := i.Lookup(lp)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fpSet := utility.Set{}
|
|
for _, baseFP := range baseFPs {
|
|
fpSet.Add(baseFP)
|
|
}
|
|
fpSet.Add(fp)
|
|
if len(fpSet) == len(baseFPs) {
|
|
continue
|
|
}
|
|
fps := make(clientmodel.Fingerprints, 0, len(fpSet))
|
|
for f := range fpSet {
|
|
fps = append(fps, f.(clientmodel.Fingerprint))
|
|
}
|
|
b[lp] = fps
|
|
|
|
}
|
|
return i.IndexBatch(b)
|
|
}
|
|
|
|
func (i *LabelPairFingerprintIndex) Reduce(m clientmodel.Metric, fp clientmodel.Fingerprint) (LabelPairFingerprintsMapping, error) {
|
|
b := make(LabelPairFingerprintsMapping, len(m))
|
|
for ln, lv := range m {
|
|
lp := metric.LabelPair{Name: ln, Value: lv}
|
|
baseFPs, _, err := i.Lookup(lp)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
fpSet := utility.Set{}
|
|
for _, baseFP := range baseFPs {
|
|
fpSet.Add(baseFP)
|
|
}
|
|
fpSet.Remove(fp)
|
|
if len(fpSet) == len(baseFPs) {
|
|
continue
|
|
}
|
|
fps := make(clientmodel.Fingerprints, 0, len(fpSet))
|
|
for f := range fpSet {
|
|
fps = append(fps, f.(clientmodel.Fingerprint))
|
|
}
|
|
b[lp] = fps
|
|
|
|
}
|
|
return b, i.IndexBatch(b)
|
|
}
|
|
|
|
// NewLabelPairFingerprintIndex returns a LabelPairFingerprintIndex
|
|
// object ready to use.
|
|
func NewLabelPairFingerprintIndex(basePath string) (*LabelPairFingerprintIndex, error) {
|
|
labelPairToFingerprintsDB, err := NewLevelDB(LevelDBOptions{
|
|
Path: path.Join(basePath, labelPairToFingerprintsDir),
|
|
CacheSizeBytes: *labelPairToFingerprintsCacheSize,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &LabelPairFingerprintIndex{
|
|
KeyValueStore: labelPairToFingerprintsDB,
|
|
}, nil
|
|
}
|
|
|
|
// FingerprintTimeRangeIndex models a database tracking the time ranges
|
|
// of metrics by their fingerprints.
|
|
type FingerprintTimeRangeIndex struct {
|
|
KeyValueStore
|
|
}
|
|
|
|
// Lookup returns the time range for the given fingerprint.
|
|
func (i *FingerprintTimeRangeIndex) Lookup(fp clientmodel.Fingerprint) (firstTime, lastTime clientmodel.Timestamp, ok bool, err error) {
|
|
var tr codec.CodableTimeRange
|
|
ok, err = i.Get(codec.CodableFingerprint(fp), &tr)
|
|
return tr.First, tr.Last, ok, err
|
|
}
|
|
|
|
// Has returns true if the given fingerprint is present.
|
|
func (i *FingerprintTimeRangeIndex) Has(fp clientmodel.Fingerprint) (ok bool, err error) {
|
|
return i.KeyValueStore.Has(codec.CodableFingerprint(fp))
|
|
}
|
|
|
|
// NewFingerprintTimeRangeIndex returns a FingerprintTimeRangeIndex object
|
|
// ready to use.
|
|
func NewFingerprintTimeRangeIndex(basePath string) (*FingerprintTimeRangeIndex, error) {
|
|
fingerprintTimeRangeDB, err := NewLevelDB(LevelDBOptions{
|
|
Path: path.Join(basePath, fingerprintTimeRangeDir),
|
|
CacheSizeBytes: *fingerprintTimeRangeCacheSize,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &FingerprintTimeRangeIndex{
|
|
KeyValueStore: fingerprintTimeRangeDB,
|
|
}, nil
|
|
}
|