From 2be2128e338a58f0066258ba155dbc954840534b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Mon, 10 May 2021 10:04:36 -0400 Subject: [PATCH] rbd: add CreateMirrorPeerBootstrapToken & ImportMirrorPeerBootstrapToken functions * Add CreateMirrorPeerBootstrapToken implementing rbd_mirror_peer_bootstrap_create * Add ImportMirrorPeerBootstrapToken implementing rbd_mirror_peer_bootstrap_import These functions can be used to set up mirroring between pools. Basic tests included. The tests only verify that functions work, not that they actually mirror data. That is a job for another day. Signed-off-by: John Mulligan --- rbd/mirror.go | 69 ++++++++++++++++++++++++++++++++++ rbd/mirror_test.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/rbd/mirror.go b/rbd/mirror.go index 7f9d178..2a98dcf 100644 --- a/rbd/mirror.go +++ b/rbd/mirror.go @@ -514,3 +514,72 @@ func GetMirrorSiteName(conn *rados.Conn) (string, error) { // the C code sets the size including null byte return string(buf[:cSize-1]), nil } + +// CreateMirrorPeerBootstrapToken returns a token value, representing the +// cluster and pool associated with the given IO context, that can be provided +// to ImportMirrorPeerBootstrapToken in order to set up mirroring between +// pools. +// +// Implements: +// int rbd_mirror_peer_bootstrap_create( +// rados_ioctx_t io_ctx, char *token, size_t *max_len); +func CreateMirrorPeerBootstrapToken(ioctx *rados.IOContext) (string, error) { + var ( + cioctx = cephIoctx(ioctx) + err error + buf []byte + cSize C.size_t + ) + retry.WithSizes(1024, 1<<16, func(size int) retry.Hint { + cSize = C.size_t(size) + buf = make([]byte, cSize) + ret := C.rbd_mirror_peer_bootstrap_create( + cioctx, + (*C.char)(unsafe.Pointer(&buf[0])), + &cSize) + err = getErrorIfNegative(ret) + return retry.Size(int(cSize)).If(err == errRange) + }) + if err != nil { + return "", err + } + // the C code sets the size including null byte + return string(buf[:cSize-1]), nil +} + +// MirrorPeerDirection is used to indicate what direction data is mirrored. +type MirrorPeerDirection int + +const ( + // MirrorPeerDirectionRx is equivalent to RBD_MIRROR_PEER_DIRECTION_RX + MirrorPeerDirectionRx = MirrorPeerDirection(C.RBD_MIRROR_PEER_DIRECTION_RX) + // MirrorPeerDirectionTx is equivalent to RBD_MIRROR_PEER_DIRECTION_TX + MirrorPeerDirectionTx = MirrorPeerDirection(C.RBD_MIRROR_PEER_DIRECTION_TX) + // MirrorPeerDirectionRxTx is equivalent to RBD_MIRROR_PEER_DIRECTION_RX_TX + MirrorPeerDirectionRxTx = MirrorPeerDirection(C.RBD_MIRROR_PEER_DIRECTION_RX_TX) +) + +// ImportMirrorPeerBootstrapToken applies the provided bootstrap token to the +// pool associated with the IO context to create a mirroring relationship +// between pools. The direction parameter controls if data in the pool is a +// source, destination, or both. +// +// Implements: +// int rbd_mirror_peer_bootstrap_import( +// rados_ioctx_t io_ctx, rbd_mirror_peer_direction_t direction, +// const char *token); +func ImportMirrorPeerBootstrapToken( + ioctx *rados.IOContext, direction MirrorPeerDirection, token string) error { + // instead of taking a length, rbd_mirror_peer_bootstrap_import assumes a + // null terminated "c string". We don't use CString because we don't use + // Go's string type as we don't want to treat the token as something users + // should interpret. If we were doing CString we'd be doing a copy anyway. + cToken := C.CString(token) + defer C.free(unsafe.Pointer(cToken)) + + ret := C.rbd_mirror_peer_bootstrap_import( + cephIoctx(ioctx), + C.rbd_mirror_peer_direction_t(direction), + cToken) + return getError(ret) +} diff --git a/rbd/mirror_test.go b/rbd/mirror_test.go index 8ba1069..7080412 100644 --- a/rbd/mirror_test.go +++ b/rbd/mirror_test.go @@ -640,3 +640,97 @@ func TestMirrorSiteName(t *testing.T) { assert.Equal(t, "cluster_b", n2) }) } + +func TestMirrorBootstrapToken(t *testing.T) { + t.Run("ioctxNilCreate", func(t *testing.T) { + assert.Panics(t, func() { + CreateMirrorPeerBootstrapToken(nil) + }) + }) + t.Run("ioctxNilImport", func(t *testing.T) { + assert.Panics(t, func() { + ImportMirrorPeerBootstrapToken(nil, MirrorPeerDirectionRxTx, "") + }) + }) + t.Run("justCreate", func(t *testing.T) { + conn := radosConnect(t) + defer conn.Shutdown() + + poolName := GetUUID() + err := conn.MakePool(poolName) + require.NoError(t, err) + defer func() { + assert.NoError(t, conn.DeletePool(poolName)) + }() + + ioctx, err := conn.OpenIOContext(poolName) + assert.NoError(t, err) + defer func() { + ioctx.Destroy() + }() + + err = SetMirrorMode(ioctx, MirrorModeImage) + require.NoError(t, err) + + token, err := CreateMirrorPeerBootstrapToken(ioctx) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(token), 4) + }) + t.Run("roundTrip", func(t *testing.T) { + mconfig := mirrorConfig() + if mconfig == "" { + t.Skip("no mirror config env var set") + } + + conn1 := radosConnect(t) + defer conn1.Shutdown() + + poolName := GetUUID() + err := conn1.MakePool(poolName) + require.NoError(t, err) + defer func() { + assert.NoError(t, conn1.DeletePool(poolName)) + }() + + err = SetMirrorSiteName(conn1, "ceph_a") + require.NoError(t, err) + + ioctx1, err := conn1.OpenIOContext(poolName) + assert.NoError(t, err) + defer func() { + ioctx1.Destroy() + }() + + err = SetMirrorMode(ioctx1, MirrorModeImage) + require.NoError(t, err) + + token, err := CreateMirrorPeerBootstrapToken(ioctx1) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(token), 4) + + conn2 := radosConnectConfig(t, mconfig) + defer conn2.Shutdown() + err = conn2.MakePool(poolName) + require.NoError(t, err) + defer func() { + assert.NoError(t, conn2.DeletePool(poolName)) + }() + + err = SetMirrorSiteName(conn2, "ceph_b") + require.NoError(t, err) + + ioctx2, err := conn2.OpenIOContext(poolName) + assert.NoError(t, err) + defer func() { + ioctx2.Destroy() + }() + + err = SetMirrorMode(ioctx2, MirrorModeImage) + require.NoError(t, err) + + fmt.Printf("TOKEN: %s\n", string(token)) + err = ImportMirrorPeerBootstrapToken( + ioctx2, MirrorPeerDirectionRxTx, token) + assert.NoError(t, err) + }) +}