diff --git a/Makefile.INCLUDE b/Makefile.INCLUDE index db73a729b..c4ca4e8d6 100644 --- a/Makefile.INCLUDE +++ b/Makefile.INCLUDE @@ -40,7 +40,7 @@ GOENV = TMPDIR=$(TMPDIR) GOROOT=$(GOROOT) GOPATH=$(GOPATH) GO = $(GOENV) $(GOCC) GOFMT = $(GOROOT)/bin/gofmt -LEVELDB_VERSION := 1.9.0 +LEVELDB_VERSION := 1.12.0 PROTOCOL_BUFFERS_VERSION := 2.5.0 SNAPPY_VERSION := 1.1.0 diff --git a/storage/metric/leveldb.go b/storage/metric/leveldb.go index 7de237e17..31848d68e 100644 --- a/storage/metric/leveldb.go +++ b/storage/metric/leveldb.go @@ -49,13 +49,13 @@ var ( // These flag values are back of the envelope, though they seem sensible. // Please re-evaluate based on your own needs. - curationRemarksCacheSize = flag.Int("curationRemarksCacheSize", 50*1024*1024, "The size for the curation remarks cache (bytes).") - fingerprintsToLabelPairCacheSize = flag.Int("fingerprintsToLabelPairCacheSizeBytes", 100*1024*1024, "The size for the fingerprint to label pair index (bytes).") - highWatermarkCacheSize = flag.Int("highWatermarksByFingerprintSizeBytes", 50*1024*1024, "The size for the metric high watermarks (bytes).") - labelNameToFingerprintsCacheSize = flag.Int("labelNameToFingerprintsCacheSizeBytes", 100*1024*1024, "The size for the label name to metric fingerprint index (bytes).") - labelPairToFingerprintsCacheSize = flag.Int("labelPairToFingerprintsCacheSizeBytes", 100*1024*1024, "The size for the label pair to metric fingerprint index (bytes).") - metricMembershipIndexCacheSize = flag.Int("metricMembershipCacheSizeBytes", 50*1024*1024, "The size for the metric membership index (bytes).") - samplesByFingerprintCacheSize = flag.Int("samplesByFingerprintCacheSizeBytes", 500*1024*1024, "The size for the samples database (bytes).") + curationRemarksCacheSize = flag.Int("curationRemarksCacheSize", 5*1024*1024, "The size for the curation remarks cache (bytes).") + fingerprintsToLabelPairCacheSize = flag.Int("fingerprintsToLabelPairCacheSizeBytes", 25*1024*1024, "The size for the fingerprint to label pair index (bytes).") + highWatermarkCacheSize = flag.Int("highWatermarksByFingerprintSizeBytes", 5*1024*1024, "The size for the metric high watermarks (bytes).") + labelNameToFingerprintsCacheSize = flag.Int("labelNameToFingerprintsCacheSizeBytes", 25*1024*1024, "The size for the label name to metric fingerprint index (bytes).") + labelPairToFingerprintsCacheSize = flag.Int("labelPairToFingerprintsCacheSizeBytes", 25*1024*1024, "The size for the label pair to metric fingerprint index (bytes).") + metricMembershipIndexCacheSize = flag.Int("metricMembershipCacheSizeBytes", 5*1024*1024, "The size for the metric membership index (bytes).") + samplesByFingerprintCacheSize = flag.Int("samplesByFingerprintCacheSizeBytes", 50*1024*1024, "The size for the samples database (bytes).") ) type leveldbOpener func() diff --git a/storage/raw/leveldb/leveldb.go b/storage/raw/leveldb/leveldb.go index d68bca53f..919aeedf8 100644 --- a/storage/raw/leveldb/leveldb.go +++ b/storage/raw/leveldb/leveldb.go @@ -29,7 +29,8 @@ import ( var ( leveldbFlushOnMutate = flag.Bool("leveldbFlushOnMutate", false, "Whether LevelDB should flush every operation to disk upon mutation before returning (bool).") leveldbUseSnappy = flag.Bool("leveldbUseSnappy", true, "Whether LevelDB attempts to use Snappy for compressing elements (bool).") - leveldbUseParanoidChecks = flag.Bool("leveldbUseParanoidChecks", true, "Whether LevelDB uses expensive checks (bool).") + leveldbUseParanoidChecks = flag.Bool("leveldbUseParanoidChecks", false, "Whether LevelDB uses expensive checks (bool).") + maximumOpenFiles = flag.Int("leveldb.maximumOpenFiles", 128, "The maximum number of files each LevelDB may maintain.") ) // LevelDBPersistence is a disk-backed sorted key-value store. @@ -168,7 +169,7 @@ func (i levigoIterator) GetError() (err error) { return i.iterator.GetError() } -func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) { +func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (*LevelDBPersistence, error) { options := levigo.NewOptions() options.SetCreateIfMissing(true) options.SetParanoidChecks(*leveldbUseParanoidChecks) @@ -184,18 +185,19 @@ func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilter filterPolicy := levigo.NewBloomFilter(bitsPerBloomFilterEncoded) options.SetFilterPolicy(filterPolicy) + options.SetMaxOpenFiles(*maximumOpenFiles) + storage, err := levigo.Open(storageRoot, options) if err != nil { - return + return nil, err } - var ( - readOptions = levigo.NewReadOptions() - writeOptions = levigo.NewWriteOptions() - ) + readOptions := levigo.NewReadOptions() + writeOptions := levigo.NewWriteOptions() writeOptions.SetSync(*leveldbFlushOnMutate) - p = &LevelDBPersistence{ + + return &LevelDBPersistence{ path: storageRoot, cache: cache, @@ -206,9 +208,7 @@ func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilter writeOptions: writeOptions, storage: storage, - } - - return + }, nil } func (l *LevelDBPersistence) Close() {