Merge pull request #290 from prometheus/snap

Improve usability of snapshots

compact.go

@@ -151,6 +151,13 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
+	// We do not include the most recently created block. This gives users a window
+	// of a full block size to piece-wise backup new data without having to care
+	// about data overlap.
+	if len(dirs) < 1 {
+		return nil, nil
+	}
+	dirs = dirs[:len(dirs)-1]
 
 	var dms []dirMeta
 

db.go

@@ -633,8 +633,9 @@ func (db *DB) EnableCompactions() {
 	level.Info(db.logger).Log("msg", "compactions enabled")
 }
 
-// Snapshot writes the current data to the directory.
-func (db *DB) Snapshot(dir string) error {
+// Snapshot writes the current data to the directory. If withHead is set to true it
+// will create a new block containing all data that's currently in the memory buffer/WAL.
+func (db *DB) Snapshot(dir string, withHead bool) error {
 	if dir == db.dir {
 		return errors.Errorf("cannot snapshot into base directory")
 	}
@@ -655,6 +656,9 @@ func (db *DB) Snapshot(dir string) error {
 			return errors.Wrapf(err, "error snapshotting block: %s", b.Dir())
 		}
 	}
+	if !withHead {
+		return nil
+	}
 	_, err := db.compactor.Write(dir, db.head, db.head.MinTime(), db.head.MaxTime())
 	return errors.Wrap(err, "snapshot head block")
 }

db_test.go

@@ -351,8 +351,9 @@ func TestDB_Snapshot(t *testing.T) {
 	// create snapshot
 	snap, err := ioutil.TempDir("", "snap")
 	testutil.Ok(t, err)
+
 	defer os.RemoveAll(snap)
-	testutil.Ok(t, db.Snapshot(snap))
+	testutil.Ok(t, db.Snapshot(snap, true))
 	testutil.Ok(t, db.Close())
 
 	// reopen DB from snapshot
@@ -417,8 +418,9 @@ Outer:
 		// create snapshot
 		snap, err := ioutil.TempDir("", "snap")
 		testutil.Ok(t, err)
+
 		defer os.RemoveAll(snap)
-		testutil.Ok(t, db.Snapshot(snap))
+		testutil.Ok(t, db.Snapshot(snap, true))
 		testutil.Ok(t, db.Close())
 
 		// reopen DB from snapshot
@@ -688,8 +690,9 @@ func TestTombstoneClean(t *testing.T) {
 		// create snapshot
 		snap, err := ioutil.TempDir("", "snap")
 		testutil.Ok(t, err)
+
 		defer os.RemoveAll(snap)
-		testutil.Ok(t, db.Snapshot(snap))
+		testutil.Ok(t, db.Snapshot(snap, true))
 		testutil.Ok(t, db.Close())
 
 		// reopen DB from snapshot
@@ -766,8 +769,9 @@ func TestDB_Retention(t *testing.T) {
 	// TODO(gouthamve): Add a method to compact headblock.
 	snap, err := ioutil.TempDir("", "snap")
 	testutil.Ok(t, err)
+
 	defer os.RemoveAll(snap)
-	testutil.Ok(t, db.Snapshot(snap))
+	testutil.Ok(t, db.Snapshot(snap, true))
 	testutil.Ok(t, db.Close())
 
 	// reopen DB from snapshot
@@ -785,7 +789,8 @@ func TestDB_Retention(t *testing.T) {
 	snap, err = ioutil.TempDir("", "snap")
 	testutil.Ok(t, err)
 	defer os.RemoveAll(snap)
-	testutil.Ok(t, db.Snapshot(snap))
+
+	testutil.Ok(t, db.Snapshot(snap, true))
 	testutil.Ok(t, db.Close())
 
 	// reopen DB from snapshot