From 1f7f89b4e3ce89abf923a126aee87e7042aefae9 Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sat, 11 May 2013 01:02:57 +0200 Subject: [PATCH] Simplify compaction and expose database sizes. This commit simplifies the way that compactions across a database's keyspace occur due to reading the LevelDB internals. Secondarily it introduces the database size estimation mechanisms. --- storage/metric/leveldb.go | 77 ++++++++++++++++++---------- storage/raw/index/leveldb/leveldb.go | 12 +++-- storage/raw/leveldb/leveldb.go | 29 ++++++++--- tools/pruner/main.go | 8 +-- 4 files changed, 83 insertions(+), 43 deletions(-) diff --git a/storage/metric/leveldb.go b/storage/metric/leveldb.go index b5d5b0d19..abe786c25 100644 --- a/storage/metric/leveldb.go +++ b/storage/metric/leveldb.go @@ -877,34 +877,57 @@ func (l *LevelDBMetricPersistence) ForEachSample(builder IteratorsForFingerprint panic("not implemented") } -// CompactKeyspace compacts each database's keyspace serially. An error may -// be returned if there are difficulties with the underlying database or if -// it's empty. +// CompactKeyspace compacts each database's keyspace serially. // // Beware that it would probably be imprudent to run this on a live user-facing // server due to latency implications. -func (l *LevelDBMetricPersistence) CompactKeyspaces() error { - if err := l.CurationRemarks.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact curation remarks: %s", err) - } - if err := l.fingerprintToMetrics.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact fingerprint to metric index: %s", err) - } - if err := l.labelNameToFingerprints.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact label name to fingerprint index: %s", err) - } - if err := l.labelSetToFingerprints.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact label pair to fingerprint index: %s", err) - } - if err := l.MetricHighWatermarks.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact metric high watermarks: %s", err) - } - if err := l.metricMembershipIndex.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact metric membership index: %s", err) - } - if err := l.MetricSamples.CompactKeyspace(); err != nil { - return fmt.Errorf("Could not compact metric samples database: %s", err) - } - - return nil +func (l *LevelDBMetricPersistence) CompactKeyspaces() { + l.CurationRemarks.CompactKeyspace() + l.fingerprintToMetrics.CompactKeyspace() + l.labelNameToFingerprints.CompactKeyspace() + l.labelSetToFingerprints.CompactKeyspace() + l.MetricHighWatermarks.CompactKeyspace() + l.metricMembershipIndex.CompactKeyspace() + l.MetricSamples.CompactKeyspace() +} + +func (l *LevelDBMetricPersistence) ApproximateSizes() (total uint64, err error) { + size := uint64(0) + + if size, err = l.CurationRemarks.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.fingerprintToMetrics.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.labelNameToFingerprints.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.labelSetToFingerprints.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.MetricHighWatermarks.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.metricMembershipIndex.ApproximateSize(); err != nil { + return 0, err + } + total += size + + if size, err = l.MetricSamples.ApproximateSize(); err != nil { + return 0, err + } + total += size + + return total, nil } diff --git a/storage/raw/index/leveldb/leveldb.go b/storage/raw/index/leveldb/leveldb.go index 24b62d648..fc71d9501 100644 --- a/storage/raw/index/leveldb/leveldb.go +++ b/storage/raw/index/leveldb/leveldb.go @@ -62,12 +62,14 @@ func (l *LevelDBMembershipIndex) Commit(batch raw.Batch) error { return l.persistence.Commit(batch) } -// CompactKeyspace compacts the entire database's keyspace. An error may be -// returned if there are difficulties with the underlying database or if it's -// empty. +// CompactKeyspace compacts the entire database's keyspace. // // Beware that it would probably be imprudent to run this on a live user-facing // server due to latency implications. -func (l *LevelDBMembershipIndex) CompactKeyspace() error { - return l.persistence.CompactKeyspace() +func (l *LevelDBMembershipIndex) CompactKeyspace() { + l.persistence.CompactKeyspace() +} + +func (l *LevelDBMembershipIndex) ApproximateSize() (uint64, error) { + return l.persistence.ApproximateSize() } diff --git a/storage/raw/leveldb/leveldb.go b/storage/raw/leveldb/leveldb.go index 4304c194d..089e1eb39 100644 --- a/storage/raw/leveldb/leveldb.go +++ b/storage/raw/leveldb/leveldb.go @@ -308,18 +308,27 @@ func (l *LevelDBPersistence) Commit(b raw.Batch) (err error) { return l.storage.Write(l.writeOptions, batch.batch) } -// CompactKeyspace compacts the entire database's keyspace. An error may be -// returned if there are difficulties with the underlying database or if it's -// empty. +// CompactKeyspace compacts the entire database's keyspace. // // Beware that it would probably be imprudent to run this on a live user-facing // server due to latency implications. -func (l *LevelDBPersistence) CompactKeyspace() error { +func (l *LevelDBPersistence) CompactKeyspace() { + + // Magic values per https://code.google.com/p/leveldb/source/browse/include/leveldb/db.h#131. + keyspace := levigo.Range{ + Start: nil, + Limit: nil, + } + + l.storage.CompactRange(keyspace) +} + +func (l *LevelDBPersistence) ApproximateSize() (uint64, error) { iterator := l.NewIterator(false) defer iterator.Close() if !iterator.SeekToFirst() { - return fmt.Errorf("could not seek to first key") + return 0, fmt.Errorf("could not seek to first key") } keyspace := levigo.Range{} @@ -327,14 +336,18 @@ func (l *LevelDBPersistence) CompactKeyspace() error { keyspace.Start = iterator.Key() if !iterator.SeekToLast() { - return fmt.Errorf("could not seek to last key") + return 0, fmt.Errorf("could not seek to last key") } keyspace.Limit = iterator.Key() - l.storage.CompactRange(keyspace) + sizes := l.storage.GetApproximateSizes([]levigo.Range{keyspace}) + total := uint64(0) + for _, size := range sizes { + total += size + } - return nil + return total, nil } // NewIterator creates a new levigoIterator, which follows the Iterator diff --git a/tools/pruner/main.go b/tools/pruner/main.go index 80b63059d..61acca5d1 100644 --- a/tools/pruner/main.go +++ b/tools/pruner/main.go @@ -42,8 +42,10 @@ func main() { start := time.Now() log.Printf("Starting compaction...") - if err := persistences.CompactKeyspaces(); err != nil { - log.Fatalf("Abording after %s", time.Since(start)) - } + size, _ := persistences.ApproximateSizes() + log.Printf("Original Size: %d", size) + persistences.CompactKeyspaces() log.Printf("Finished in %s", time.Since(start)) + size, _ = persistences.ApproximateSizes() + log.Printf("New Size: %d", size) }