From ec3e98b3654d79f3447d763b84069618cb373751 Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Fri, 1 Feb 2013 13:35:07 +0100 Subject: [PATCH] Include Snappy in Runtime. Snappy should have been explicitly included in the runtime, for I erroneously thought that LevelDB bundled this into its runtime as-is. It turns out that this assumption is wrong, and I thought we had Snappy compression support all-along. --- Makefile | 3 ++- Makefile.TRAVIS | 25 +++++++++++++++++++++---- README.md | 1 + storage/raw/leveldb/leveldb.go | 11 +++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3ab203174..6c3153e12 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -TEST_ARTIFACTS = prometheus search_index +TEST_ARTIFACTS = prometheus prometheus.build search_index all: test @@ -21,6 +21,7 @@ test: build build: $(MAKE) -C model go build ./... + go build -o prometheus.build clean: rm -rf $(TEST_ARTIFACTS) diff --git a/Makefile.TRAVIS b/Makefile.TRAVIS index 45b7af8d0..4eae9649d 100644 --- a/Makefile.TRAVIS +++ b/Makefile.TRAVIS @@ -14,6 +14,7 @@ GO_VERSION := 1.0.3 LEVELDB_VERSION := 1.7.0 PROTOCOL_BUFFERS_VERSION := 2.4.1 +SNAPPY_VERSION := 1.0.5 OVERLAY_ROOT := ${HOME}/overlay_root @@ -24,7 +25,7 @@ export CFLAGS := $(CFLAGS) -I$(OVERLAY_ROOT)/include export CXXFLAGS := $(CXXFLAGS) -I$(OVERLAY_ROOT)/include export CPPFLAGS := $(CPPFLAGS) -I$(OVERLAY_ROOT)/include export LDFLAGS := $(LDFLAGS) -L$(OVERLAY_ROOT)/lib -export CGO_CFLAGS := $(CFLAGS) +export CGO_CFLAGS := $(CFLAGS) -lsnappy export CGO_LDFLAGS := $(LDFLAGS) GO_GET := go get -u -v -x @@ -114,7 +115,7 @@ instrumentation-stamp: go source leveldb: leveldb-stamp -leveldb-stamp: cc rsync leveldb-$(LEVELDB_VERSION).tar.gz overlay +leveldb-stamp: cc rsync leveldb-$(LEVELDB_VERSION).tar.gz snappy overlay tar xzvf leveldb-$(LEVELDB_VERSION).tar.gz $(MAKE) -C leveldb-$(LEVELDB_VERSION) rsync -av "leveldb-$(LEVELDB_VERSION)/include/" "$(OVERLAY_ROOT)/include/" @@ -128,7 +129,7 @@ leveldb-$(LEVELDB_VERSION).tar.gz: wget levigo: levigo-stamp -levigo-stamp: leveldb go source +levigo-stamp: leveldb go snappy source $(GO_GET) github.com/jmhodges/levigo touch $@ @@ -141,9 +142,23 @@ test: test-stamp test-stamp: preparation source cd ${GOPATH}/src/github.com/prometheus + $(MAKE) build $(MAKE) test touch $@ +snappy-$(SNAPPY_VERSION).tar.gz: wget + $(WGET) http://snappy.googlecode.com/files/snappy-$(SNAPPY_VERSION).tar.gz + tar xzvf snappy-$(SNAPPY_VERSION).tar.gz + cd snappy-$(SNAPPY_VERSION) && ./configure --prefix="$(OVERLAY_ROOT)" + $(MAKE) -C snappy-$(SNAPPY_VERSION) + $(MAKE) -C snappy-$(SNAPPY_VERSION) install + +snappy: snappy-stamp + tar xzvf snappy-$(SNAPPY_VERSION).tar.gz + +snappy-stamp: cc overlay rsync snappy-$(SNAPPY_VERSION).tar.gz + touch $@ + source: source-stamp source-stamp: @@ -156,8 +171,10 @@ clean: -rm -rf "$(OVERLAY_ROOT)" -rm -rf leveldb-$(LEVELDB_VERSION) -rm -rf protobuf-$(PROTOCOL_BUFFERS_VERSION) + -rm -rf snappy-$(SNAPPY_VERSION) -rm leveldb-$(LEVELDB_VERSION).tar.gz -rm protobuf-$(PROTOCOL_BUFFERS_VERSION).tar.bz2 + -rm snappy-$(SNAPPY_VERSION).tar.gz -.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync source test wget +.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync snappy source test wget diff --git a/README.md b/README.md index 977da119a..46187deba 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ action if some condition is observed to be true. 5. Levigo, a Go-wrapper around LevelDB's C library: (https://github.com/jmhodges/levigo). 6. GoRest, a RESTful style web-services framework: (http://code.google.com/p/gorest/). 7. Prometheus Client, Prometheus in Prometheus (https://github.com/matttproud/golang_instrumentation). + 8. Snappy, a compression library for LevelDB and Levigo (code.google.com/p/snappy/). ## Getting started diff --git a/storage/raw/leveldb/leveldb.go b/storage/raw/leveldb/leveldb.go index 34c414796..31dfaf84f 100644 --- a/storage/raw/leveldb/leveldb.go +++ b/storage/raw/leveldb/leveldb.go @@ -22,7 +22,9 @@ import ( ) var ( - leveldbFlushOnMutate = flag.Bool("leveldbFlushOnMutate", true, "Whether LevelDB should flush every operation to disk upon mutation before returning (bool).") + leveldbFlushOnMutate = flag.Bool("leveldbFlushOnMutate", true, "Whether LevelDB should flush every operation to disk upon mutation before returning (bool).") + leveldbUseSnappy = flag.Bool("leveldbUseSnappy", true, "Whether LevelDB attempts to use Snappy for compressing elements (bool).") + leveldbUseParanoidChecks = flag.Bool("leveldbUseParanoidChecks", true, "Whether LevelDB uses expensive checks (bool).") ) type LevelDBPersistence struct { @@ -44,7 +46,12 @@ type iteratorCloser struct { func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) { options := levigo.NewOptions() options.SetCreateIfMissing(true) - options.SetParanoidChecks(true) + options.SetParanoidChecks(*leveldbUseParanoidChecks) + compression := levigo.NoCompression + if *leveldbUseSnappy { + compression = levigo.SnappyCompression + } + options.SetCompression(compression) cache := levigo.NewLRUCache(cacheCapacity) options.SetCache(cache)