From 9841283344a9fcff2afd3e775c628a297c0966ee Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 5 Feb 2020 11:49:48 -0500 Subject: [PATCH] rbd: add DiffIterate wrapper for rbd_diff_iterate2 The DiffIterate call accepts a data structure argument containing the parameters of the image to "diff" and a callback function. This callback is called in the C code, making use of the recently added callbacks helper. The callback itself is called with the offset and length of the differing area in the image as well as a data parameter so that a common function can distinguish or update different data for different calls if needed (compare to a void* in C). Signed-off-by: John Mulligan --- rbd/callback_shims.go | 13 + rbd/diff_iterate.go | 136 ++++++++++ rbd/diff_iterate_test.go | 522 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 671 insertions(+) create mode 100644 rbd/callback_shims.go create mode 100644 rbd/diff_iterate.go create mode 100644 rbd/diff_iterate_test.go diff --git a/rbd/callback_shims.go b/rbd/callback_shims.go new file mode 100644 index 0000000..ab040e7 --- /dev/null +++ b/rbd/callback_shims.go @@ -0,0 +1,13 @@ +package rbd + +/* + +#include + +extern int diffIterateCallback(uint64_t ofs, size_t len, int exists, int index); + +int callDiffIterateCallback(uint64_t ofs, size_t len, int exists, int index) { + return diffIterateCallback(ofs, len, exists, index); +} +*/ +import "C" diff --git a/rbd/diff_iterate.go b/rbd/diff_iterate.go new file mode 100644 index 0000000..5e53317 --- /dev/null +++ b/rbd/diff_iterate.go @@ -0,0 +1,136 @@ +package rbd + +/* +#cgo LDFLAGS: -lrbd +#undef _GNU_SOURCE +#include +#include +#include + +extern int callDiffIterateCallback(uint64_t ofs, size_t len, int exists, int index); + +// cgo is having trouble converting the callback from the librbd header +// to a unsafe.Pointer. This shim exists solely to help it along. +static inline int wrap_rbd_diff_iterate2( + rbd_image_t image, + const char *fromsnapname, + uint64_t ofs, uint64_t len, + uint8_t include_parent, uint8_t whole_object, + void *cb, + void *arg) { + return rbd_diff_iterate2(image, fromsnapname, ofs, len, include_parent, whole_object, cb, arg); +} +*/ +import "C" + +import ( + "unsafe" + + "github.com/ceph/go-ceph/internal/callbacks" +) + +var diffIterateCallbacks = callbacks.New() + +// DiffIncludeParent values control if the difference should include the parent +// image. +type DiffIncludeParent uint8 + +// DiffWholeObject values control if the diff extents should cover the whole +// object. +type DiffWholeObject uint8 + +// DiffIterateCallback defines the function signature needed for the +// DiffIterate callback. +// +// The function will be called with the arguments: offset, length, exists, and +// data. The offset and length correspond to the changed region of the image. +// The exists value is set to zero if the region is known to be zeros, +// otherwise it is set to 1. The data value is the extra data parameter that +// was set on the DiffIterateConfig and is meant to be used for passing +// arbitrary user-defined items to the callback function. +// +// The callback can trigger the iteration to terminate early by returning +// a non-zero error code. +type DiffIterateCallback func(uint64, uint64, int, interface{}) int + +// DiffIterateConfig is used to define the parameters of a DiffIterate call. +// Callback, Offset, and Length should always be specified when passed to +// DiffIterate. The other values are optional. +type DiffIterateConfig struct { + SnapName string + Offset uint64 + Length uint64 + IncludeParent DiffIncludeParent + WholeObject DiffWholeObject + Callback DiffIterateCallback + Data interface{} +} + +const ( + // ExcludeParent will exclude the parent from the diff. + ExcludeParent = DiffIncludeParent(0) + // IncludeParent will include the parent in the diff. + IncludeParent = DiffIncludeParent(1) + + // DisableWholeObject will not use the whole object in the diff. + DisableWholeObject = DiffWholeObject(0) + // EnableWholeObject will use the whole object in the diff. + EnableWholeObject = DiffWholeObject(1) +) + +// DiffIterate calls a callback on changed extents of an image. +// +// Calling DiffIterate will cause the callback specified in the +// DiffIterateConfig to be called as many times as there are changed +// regions in the image (controlled by the parameters as passed to librbd). +// +// See the documentation of DiffIterateCallback for a description of the +// arguments to the callback and the return behavior. +// +// Implements: +// int rbd_diff_iterate2(rbd_image_t image, +// const char *fromsnapname, +// uint64_t ofs, uint64_t len, +// uint8_t include_parent, uint8_t whole_object, +// int (*cb)(uint64_t, size_t, int, void *), +// void *arg); +func (image *Image) DiffIterate(config DiffIterateConfig) error { + + if err := image.validate(imageIsOpen); err != nil { + return err + } + if config.Callback == nil { + return RBDError(C.EINVAL) + } + + var cSnapName *C.char + if config.SnapName != NoSnapshot { + cSnapName = C.CString(config.SnapName) + defer C.free(unsafe.Pointer(cSnapName)) + } + + cbIndex := diffIterateCallbacks.Add(config) + defer diffIterateCallbacks.Remove(cbIndex) + + ret := C.wrap_rbd_diff_iterate2( + image.image, + cSnapName, + C.uint64_t(config.Offset), + C.uint64_t(config.Length), + C.uint8_t(config.IncludeParent), + C.uint8_t(config.WholeObject), + C.callDiffIterateCallback, + unsafe.Pointer(uintptr(cbIndex))) + + return getError(ret) +} + +//export diffIterateCallback +func diffIterateCallback( + offset C.uint64_t, length C.size_t, exists, index C.int) C.int { + + v := diffIterateCallbacks.Lookup(int(index)) + config := v.(DiffIterateConfig) + return C.int(config.Callback( + uint64(offset), uint64(length), int(exists), config.Data)) +} diff --git a/rbd/diff_iterate_test.go b/rbd/diff_iterate_test.go new file mode 100644 index 0000000..ff1d1b1 --- /dev/null +++ b/rbd/diff_iterate_test.go @@ -0,0 +1,522 @@ +package rbd + +import ( + "sync" + "testing" + "time" + + "github.com/ceph/go-ceph/rados" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffIterate(t *testing.T) { + conn := radosConnect(t) + defer conn.Shutdown() + + poolname := GetUUID() + err := conn.MakePool(poolname) + assert.NoError(t, err) + defer conn.DeletePool(poolname) + + ioctx, err := conn.OpenIOContext(poolname) + require.NoError(t, err) + defer ioctx.Destroy() + + t.Run("basic", func(t *testing.T) { + testDiffIterateBasic(t, ioctx) + }) + t.Run("twoAtOnce", func(t *testing.T) { + testDiffIterateTwoAtOnce(t, ioctx) + }) + t.Run("earlyExit", func(t *testing.T) { + testDiffIterateEarlyExit(t, ioctx) + }) + t.Run("snapshot", func(t *testing.T) { + testDiffIterateSnapshot(t, ioctx) + }) + t.Run("callbackData", func(t *testing.T) { + testDiffIterateCallbackData(t, ioctx) + }) + t.Run("badImage", func(t *testing.T) { + var gotCalled int + img := GetImage(ioctx, "bob") + err := img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: uint64(1 << 22), + Callback: func(o, l uint64, e int, x interface{}) int { + gotCalled++ + return 0 + }, + }) + assert.Error(t, err) + assert.EqualValues(t, 0, gotCalled) + }) + t.Run("missingCallback", func(t *testing.T) { + name := GetUUID() + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + err := CreateImage(ioctx, name, isize, options) + assert.NoError(t, err) + + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + assert.NoError(t, img.Remove()) + }() + + var gotCalled int + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: uint64(1 << 22), + }) + assert.Error(t, err) + assert.EqualValues(t, 0, gotCalled) + }) +} + +func testDiffIterateBasic(t *testing.T, ioctx *rados.IOContext) { + name := GetUUID() + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + err := CreateImage(ioctx, name, isize, options) + assert.NoError(t, err) + + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + assert.NoError(t, img.Remove()) + }() + + type diResult struct { + offset uint64 + length uint64 + } + calls := []diResult{} + + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + // Image is new, empty. Callback will not be called + assert.Len(t, calls, 0) + + _, err = img.WriteAt([]byte("sometimes you feel like a nut"), 0) + assert.NoError(t, err) + + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 29, calls[0].length) + } + + _, err = img.WriteAt([]byte("sometimes you don't"), 32) + assert.NoError(t, err) + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + if assert.NoError(t, err) { + assert.Len(t, calls, 1) + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 51, calls[0].length) + } + + // dirty a 2nd chunk + newOffset := 3145728 // 3MiB + _, err = img.WriteAt([]byte("alright, alright, alright"), int64(newOffset)) + assert.NoError(t, err) + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 2) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 51, calls[0].length) + assert.EqualValues(t, newOffset, calls[1].offset) + assert.EqualValues(t, 25, calls[1].length) + } + + // dirty a 3rd chunk + newOffset2 := 5242880 + 1024 // 5MiB + 1KiB + _, err = img.WriteAt([]byte("zowie!"), int64(newOffset2)) + assert.NoError(t, err) + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 3) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 51, calls[0].length) + assert.EqualValues(t, newOffset, calls[1].offset) + assert.EqualValues(t, 25, calls[1].length) + assert.EqualValues(t, newOffset2-1024, calls[2].offset) + assert.EqualValues(t, 6+1024, calls[2].length) + } +} + +// testDiffIterateTwoAtOnce aims to verify that multiple DiffIterate +// callbacks can be executed at the same time without error. +func testDiffIterateTwoAtOnce(t *testing.T, ioctx *rados.IOContext) { + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + + name1 := GetUUID() + err := CreateImage(ioctx, name1, isize, options) + assert.NoError(t, err) + + img1, err := OpenImage(ioctx, name1, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img1.Close()) + assert.NoError(t, img1.Remove()) + }() + + name2 := GetUUID() + err = CreateImage(ioctx, name2, isize, options) + assert.NoError(t, err) + + img2, err := OpenImage(ioctx, name2, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img2.Close()) + assert.NoError(t, img2.Remove()) + }() + + type diResult struct { + offset uint64 + length uint64 + } + + diffTest := func(wg *sync.WaitGroup, inbuf []byte, img *Image) { + _, err = img.WriteAt(inbuf[0:3], 0) + assert.NoError(t, err) + _, err = img.WriteAt(inbuf[3:6], 3145728) + assert.NoError(t, err) + _, err = img.WriteAt(inbuf[6:9], 5242880) + assert.NoError(t, err) + + calls := []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + time.Sleep(8 * time.Millisecond) + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 3) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 3, calls[0].length) + assert.EqualValues(t, 3145728, calls[1].offset) + assert.EqualValues(t, 3, calls[1].length) + assert.EqualValues(t, 5242880, calls[2].offset) + assert.EqualValues(t, 3, calls[2].length) + } + + wg.Done() + } + + wg := &sync.WaitGroup{} + wg.Add(1) + go diffTest(wg, []byte("foobarbaz"), img1) + wg.Add(1) + go diffTest(wg, []byte("abcdefghi"), img2) + wg.Wait() +} + +// testDiffIterateEarlyExit checks that returning an error from the callback +// function triggers the DiffIterate call to stop. +func testDiffIterateEarlyExit(t *testing.T, ioctx *rados.IOContext) { + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + + name := GetUUID() + err := CreateImage(ioctx, name, isize, options) + assert.NoError(t, err) + + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + assert.NoError(t, img.Remove()) + }() + + type diResult struct { + offset uint64 + length uint64 + } + + // "damage" the image + inbuf := []byte("xxxyyyzzz") + _, err = img.WriteAt(inbuf[0:3], 0) + assert.NoError(t, err) + _, err = img.WriteAt(inbuf[3:6], 3145728) + assert.NoError(t, err) + _, err = img.WriteAt(inbuf[6:9], 5242880) + assert.NoError(t, err) + + // if the offset is less than zero the callback will return an "error" and + // that will abort the DiffIterate call early and it will return the error + // code our callback used. + calls := []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + if o > 1 { + return -5 + } + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.Error(t, err) + if rbderr, ok := err.(RBDError); assert.True(t, ok) { + assert.EqualValues(t, -5, int(rbderr)) + } + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 3, calls[0].length) + } +} + +func testDiffIterateSnapshot(t *testing.T, ioctx *rados.IOContext) { + name := GetUUID() + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + err := CreateImage(ioctx, name, isize, options) + assert.NoError(t, err) + + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + assert.NoError(t, img.Remove()) + }() + + type diResult struct { + offset uint64 + length uint64 + } + calls := []diResult{} + + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + // Image is new, empty. Callback will not be called + assert.Len(t, calls, 0) + + _, err = img.WriteAt([]byte("sometimes you feel like a nut"), 0) + assert.NoError(t, err) + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 29, calls[0].length) + } + + ss1, err := img.CreateSnapshot("ss1") + assert.NoError(t, err) + defer func() { assert.NoError(t, ss1.Remove()) }() + + // there should be no differences between "now" and "ss1" + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + SnapName: "ss1", + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + assert.Len(t, calls, 0) + + // this next check was shamelessly cribbed from the pybind + // tests for diff_iterate out of the ceph tree + // it discards the current image, makes a 2nd snap, and compares + // the diff between snapshots 1 & 2. + _, err = img.Discard(0, isize) + assert.NoError(t, err) + + ss2, err := img.CreateSnapshot("ss2") + assert.NoError(t, err) + defer func() { assert.NoError(t, ss2.Remove()) }() + err = ss2.Set() // caution: this side-effects img! + assert.NoError(t, err) + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + SnapName: "ss1", + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 29, calls[0].length) + } +} + +func testDiffIterateCallbackData(t *testing.T, ioctx *rados.IOContext) { + name := GetUUID() + isize := uint64(1 << 23) // 8MiB + iorder := 20 // 1MiB + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, + options.SetUint64(RbdImageOptionOrder, uint64(iorder))) + err := CreateImage(ioctx, name, isize, options) + assert.NoError(t, err) + + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + assert.NoError(t, img.Remove()) + }() + + type diResult struct { + offset uint64 + length uint64 + } + calls := []diResult{} + + _, err = img.WriteAt([]byte("sometimes you feel like a nut"), 0) + assert.NoError(t, err) + + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + if v, ok := x.(int); ok { + assert.EqualValues(t, 77, v) + } else { + t.Fatalf("incorrect type") + } + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + Data: 77, + }) + assert.NoError(t, err) + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 29, calls[0].length) + } + + calls = []diResult{} + err = img.DiffIterate( + DiffIterateConfig{ + Offset: 0, + Length: isize, + Callback: func(o, l uint64, e int, x interface{}) int { + if v, ok := x.(string); ok { + assert.EqualValues(t, "bob", v) + } else { + t.Fatalf("incorrect type") + } + calls = append(calls, diResult{offset: o, length: l}) + return 0 + }, + Data: "bob", + }) + assert.NoError(t, err) + if assert.Len(t, calls, 1) { + assert.EqualValues(t, 0, calls[0].offset) + assert.EqualValues(t, 29, calls[0].length) + } +}