From 9041276dd9429ecdb2b7549f651bed5d7123c54b Mon Sep 17 00:00:00 2001 From: Nikhil-Ladha Date: Thu, 30 Mar 2023 11:30:41 +0530 Subject: [PATCH] rbd: Add locking rbd APIs Added new locking APIs for the rbd component named `rbd_lock_acquire`, `rbd_lock_break`, `rbd_lock_release`, `rbd_lock_get_owners`, `rbd_is_exclusive_lock_owner` along with supporting TCs. Fixes: #275 Signed-off-by: Nikhil-Ladha --- docs/api-status.json | 32 +++++++++- docs/api-status.md | 5 ++ rbd/locks.go | 140 +++++++++++++++++++++++++++++++++++++++++ rbd/locks_test.go | 146 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 rbd/locks.go create mode 100644 rbd/locks_test.go diff --git a/docs/api-status.json b/docs/api-status.json index 4269c34..6b1d653 100644 --- a/docs/api-status.json +++ b/docs/api-status.json @@ -1873,6 +1873,36 @@ "comment": "SparsifyWithProgress makes an image sparse by deallocating runs of zeros.\nThe sparseSize value will be used to find runs of zeros and must be\na power of two no less than 4096 and no larger than the image size.\nThe given progress callback will be called to report on the progress\nof sparse. The operation will be aborted if the progress callback returns\na non-zero value.\n\nImplements:\n\n\tint rbd_sparsify_with_progress(rbd_image_t image, size_t sparse_size,\n\t\t\t\t\t\t\t\t librbd_progress_fn_t cb, void *cbdata);\n", "added_in_version": "v0.21.0", "expected_stable_version": "v0.23.0" + }, + { + "name": "Image.LockAcquire", + "comment": "LockAcquire takes a lock on the given image as per the provided lock_mode.\n\nImplements:\n\n\tint rbd_lock_acquire(rbd_image_t image, rbd_lock_mode_t lock_mode);\n", + "added_in_version": "$NEXT_RELEASE", + "expected_stable_version": "$NEXT_RELEASE_STABLE" + }, + { + "name": "Image.LockBreak", + "comment": "LockBreak breaks the lock of lock_mode on the provided lock_owner.\n\nImplements:\n\n\tint rbd_lock_break(rbd_image_t image, rbd_lock_mode_t lock_mode,\n\t\t\t\t\t const char *lock_owner);\n", + "added_in_version": "$NEXT_RELEASE", + "expected_stable_version": "$NEXT_RELEASE_STABLE" + }, + { + "name": "Image.LockGetOwners", + "comment": "LockGetOwners fetches the list of lock owners.\n\nImplements:\n\n\tint rbd_lock_get_owners(rbd_image_t image, rbd_lock_mode_t *lock_mode,\n\t\t\t\t\t\t\tchar **lock_owners, size_t *max_lock_owners);\n", + "added_in_version": "$NEXT_RELEASE", + "expected_stable_version": "$NEXT_RELEASE_STABLE" + }, + { + "name": "Image.LockIsExclusiveOwner", + "comment": "LockIsExclusiveOwner gets the status of the image exclusive lock.\n\nImplements:\n\n\tint rbd_is_exclusive_lock_owner(rbd_image_t image, int *is_owner);\n", + "added_in_version": "$NEXT_RELEASE", + "expected_stable_version": "$NEXT_RELEASE_STABLE" + }, + { + "name": "Image.LockRelease", + "comment": "LockRelease releases a lock on the image.\n\nImplements:\n\n\tint rbd_lock_release(rbd_image_t image);\n", + "added_in_version": "$NEXT_RELEASE", + "expected_stable_version": "$NEXT_RELEASE_STABLE" } ] }, @@ -2159,4 +2189,4 @@ } ] } -} +} \ No newline at end of file diff --git a/docs/api-status.md b/docs/api-status.md index 5ef7d69..ea6184c 100644 --- a/docs/api-status.md +++ b/docs/api-status.md @@ -64,6 +64,11 @@ SetMirrorPeerSiteClientName | v0.21.0 | v0.23.0 | SetMirrorPeerSiteName | v0.21.0 | v0.23.0 | SetMirrorPeerSiteDirection | v0.21.0 | v0.23.0 | Image.SparsifyWithProgress | v0.21.0 | v0.23.0 | +Image.LockAcquire | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | +Image.LockBreak | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | +Image.LockGetOwners | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | +Image.LockIsExclusiveOwner | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | +Image.LockRelease | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | ### Deprecated APIs diff --git a/rbd/locks.go b/rbd/locks.go new file mode 100644 index 0000000..b4c9c09 --- /dev/null +++ b/rbd/locks.go @@ -0,0 +1,140 @@ +//go:build !nautilus && ceph_preview +// +build !nautilus,ceph_preview + +package rbd + +// #cgo LDFLAGS: -lrbd +// #include +// #include +// #include +import "C" + +import ( + "unsafe" +) + +// LockMode represents a group of configurable lock modes. +type LockMode C.rbd_lock_mode_t + +const ( + // LockModeExclusive is the representation of RBD_LOCK_MODE_EXCLUSIVE from librbd. + LockModeExclusive = LockMode(C.RBD_LOCK_MODE_EXCLUSIVE) + // LockModeShared is the representation of RBD_LOCK_MODE_SHARED from librbd. + LockModeShared = LockMode(C.RBD_LOCK_MODE_SHARED) +) + +// LockAcquire takes a lock on the given image as per the provided lock_mode. +// +// Implements: +// +// int rbd_lock_acquire(rbd_image_t image, rbd_lock_mode_t lock_mode); +func (image *Image) LockAcquire(lockMode LockMode) error { + if err := image.validate(imageIsOpen); err != nil { + return err + } + + ret := C.rbd_lock_acquire(image.image, C.rbd_lock_mode_t(lockMode)) + + return getError(ret) +} + +// LockBreak breaks the lock of lock_mode on the provided lock_owner. +// +// Implements: +// +// int rbd_lock_break(rbd_image_t image, rbd_lock_mode_t lock_mode, +// const char *lock_owner); +func (image *Image) LockBreak(lockMode LockMode, lockOwner string) error { + if err := image.validate(imageIsOpen); err != nil { + return err + } + + cLockOwner := C.CString(lockOwner) + defer C.free(unsafe.Pointer(cLockOwner)) + + ret := C.rbd_lock_break(image.image, C.rbd_lock_mode_t(lockMode), cLockOwner) + + return getError(ret) +} + +// LockOwner represents information about a lock owner. +type LockOwner struct { + Mode LockMode + Owner string +} + +// LockGetOwners fetches the list of lock owners. +// +// Implements: +// +// int rbd_lock_get_owners(rbd_image_t image, rbd_lock_mode_t *lock_mode, +// char **lock_owners, size_t *max_lock_owners); +func (image *Image) LockGetOwners() ([]*LockOwner, error) { + if err := image.validate(imageIsOpen); err != nil { + return nil, err + } + + var ( + maxLockOwners = C.size_t(8) + cLockOwners = make([]*C.char, 8) + lockMode LockMode + lockOwnersList []*LockOwner + ) + + for { + ret := C.rbd_lock_get_owners(image.image, (*C.rbd_lock_mode_t)(&lockMode), &cLockOwners[0], &maxLockOwners) + if ret >= 0 { + break + } else if ret == -C.ENOENT { + return nil, nil + } else if ret != -C.ERANGE { + return nil, getError(ret) + } + } + + defer C.rbd_lock_get_owners_cleanup(&cLockOwners[0], maxLockOwners) + + for i := 0; i < int(maxLockOwners); i++ { + lockOwnersList = append(lockOwnersList, &LockOwner{ + Mode: LockMode(lockMode), + Owner: C.GoString(cLockOwners[i]), + }) + } + + return lockOwnersList, nil +} + +// LockIsExclusiveOwner gets the status of the image exclusive lock. +// +// Implements: +// +// int rbd_is_exclusive_lock_owner(rbd_image_t image, int *is_owner); +func (image *Image) LockIsExclusiveOwner() (bool, error) { + if err := image.validate(imageIsOpen); err != nil { + return false, err + } + + cIsOwner := C.int(0) + + ret := C.rbd_is_exclusive_lock_owner(image.image, &cIsOwner) + if ret != 0 { + return false, getError(ret) + } + + return cIsOwner == 1, nil +} + +// LockRelease releases a lock on the image. +// +// Implements: +// +// int rbd_lock_release(rbd_image_t image); +func (image *Image) LockRelease() error { + if err := image.validate(imageIsOpen); err != nil { + return err + } + + ret := C.rbd_lock_release(image.image) + + return getError(ret) +} diff --git a/rbd/locks_test.go b/rbd/locks_test.go new file mode 100644 index 0000000..736d70d --- /dev/null +++ b/rbd/locks_test.go @@ -0,0 +1,146 @@ +//go:build !nautilus && ceph_preview +// +build !nautilus,ceph_preview + +package rbd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLocking(t *testing.T) { + conn := radosConnect(t) + require.NotNil(t, conn) + defer conn.Shutdown() + + poolname := GetUUID() + err := conn.MakePool(poolname) + require.NoError(t, err) + defer conn.DeletePool(poolname) + + ioctx, err := conn.OpenIOContext(poolname) + require.NoError(t, err) + defer ioctx.Destroy() + + // Connect another cluster + conn1 := radosConnect(t) + require.NotNil(t, conn1) + defer conn1.Shutdown() + + ioctx1, err := conn1.OpenIOContext(poolname) + require.NoError(t, err) + defer ioctx1.Destroy() + + options := NewRbdImageOptions() + defer options.Destroy() + assert.NoError(t, options.SetUint64(ImageOptionOrder, uint64(testImageOrder))) + + name := GetUUID() + err = CreateImage(ioctx, name, testImageSize, options) + assert.NoError(t, err) + + t.Run("acquireLock", func(t *testing.T) { + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + }() + + // Shared lock mode is not supported + // Ref: Check lock_acquire() logic in ceph codebase at https://github.com/ceph/ceph/blob/main/src/librbd/internal.cc + err = img.LockAcquire(LockModeShared) + assert.Error(t, err) + + err = img.LockAcquire(LockModeExclusive) + assert.NoError(t, err) + + isOwner, err := img.LockIsExclusiveOwner() + assert.NoError(t, err) + assert.True(t, isOwner) + + err = img.LockBreak(LockModeExclusive, "not owner") + assert.Error(t, err) + + err = img.LockRelease() + assert.NoError(t, err) + }) + + t.Run("listLock", func(t *testing.T) { + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + }() + + img1, err := OpenImage(ioctx1, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img1.Close()) + }() + + err = img1.LockAcquire(LockModeExclusive) + assert.NoError(t, err) + + err = img.LockAcquire(LockModeExclusive) + assert.Error(t, err) + + isOwner, err := img.LockIsExclusiveOwner() + assert.NoError(t, err) + assert.False(t, isOwner) + + // This logic of fetching the lock owners and breaking the lock using a + // different image is borrowed from the TCs in ceph codebase, check + // the TEST_F(TestLibRBD, BreakLock) at https://github.com/ceph/ceph/blob/main/src/test/librbd/test_librbd.cc + locksList, err := img.LockGetOwners() + assert.NoError(t, err) + assert.Equal(t, len(locksList), 1) + assert.Equal(t, locksList[0].Mode, LockModeExclusive) + + err = img.LockBreak(LockModeExclusive, locksList[0].Owner) + assert.NoError(t, err) + + err = img.LockAcquire(LockModeExclusive) + assert.NoError(t, err) + + err = img.LockRelease() + assert.NoError(t, err) + }) + + t.Run("emptyListLock", func(t *testing.T) { + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + defer func() { + assert.NoError(t, img.Close()) + }() + + locksList, err := img.LockGetOwners() + assert.NoError(t, err) + assert.Equal(t, len(locksList), 0) + }) + + t.Run("closedImage", func(t *testing.T) { + img, err := OpenImage(ioctx, name, NoSnapshot) + assert.NoError(t, err) + assert.NoError(t, img.Close()) + defer func() { + assert.NoError(t, img.Remove()) + }() + + err = img.LockAcquire(LockModeExclusive) + assert.Error(t, err) + + err = img.LockBreak(LockModeExclusive, "") + assert.Error(t, err) + + _, err = img.LockGetOwners() + assert.Error(t, err) + + _, err = img.LockIsExclusiveOwner() + assert.Error(t, err) + + err = img.LockRelease() + assert.Error(t, err) + }) +}