From 0102ce88701dd52208520b8c0ab2e505548ed3e1 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 8 Apr 2020 19:41:23 -0400 Subject: [PATCH] librbd: store mirror peer uuids in non-primary demoted snapshots This will allow a remote rbd-mirror process to have a snapshot to use for delta sync operations during failover. Signed-off-by: Jason Dillaman --- .../snapshot/CreateNonPrimaryRequest.cc | 66 +++++++++++++++ .../mirror/snapshot/CreateNonPrimaryRequest.h | 13 +-- .../test_mock_CreateNonPrimaryRequest.cc | 81 +++++++++++++++++++ 3 files changed, 154 insertions(+), 6 deletions(-) diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc index 7dffb92e97e..6d5588d7f2c 100644 --- a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -25,6 +25,19 @@ namespace snapshot { using librbd::util::create_context_callback; using librbd::util::create_rados_callback; +template +CreateNonPrimaryRequest::CreateNonPrimaryRequest( + I* image_ctx, bool demoted, const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_demoted(demoted), + m_primary_mirror_uuid(primary_mirror_uuid), + m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), + m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + template void CreateNonPrimaryRequest::send() { refresh_image(); @@ -110,6 +123,56 @@ void CreateNonPrimaryRequest::handle_get_mirror_image(int r) { m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." + uuid_gen.to_string(); + get_mirror_peers(); +} + +template +void CreateNonPrimaryRequest::get_mirror_peers() { + if (!m_demoted) { + create_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void CreateNonPrimaryRequest::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 20) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + create_snapshot(); } @@ -121,6 +184,9 @@ void CreateNonPrimaryRequest::create_snapshot() { (m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED : cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {}, m_primary_mirror_uuid, m_primary_snap_id}; + if (m_demoted) { + ns.mirror_peer_uuids = m_mirror_peer_uuids; + } ns.snap_seqs = m_snap_seqs; ns.complete = is_orphan(); ldout(cct, 20) << "ns=" << ns << dendl; diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h index 4a711343980..32cb46d1aad 100644 --- a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h @@ -43,12 +43,7 @@ public: uint64_t primary_snap_id, const SnapSeqs& snap_seqs, const ImageState &image_state, uint64_t *snap_id, - Context *on_finish) - : m_image_ctx(image_ctx), m_demoted(demoted), - m_primary_mirror_uuid(primary_mirror_uuid), - m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), - m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { - } + Context *on_finish); void send(); @@ -64,6 +59,9 @@ private: * v * GET_MIRROR_IMAGE * | + * v (skip if not needed) + * GET_MIRROR_PEERS + * | * v * CREATE_SNAPSHOT * | @@ -85,6 +83,9 @@ private: uint64_t *m_snap_id; Context *m_on_finish; + librados::IoCtx m_default_ns_ctx; + std::set m_mirror_peer_uuids; + std::string m_snap_name; bufferlist m_out_bl; diff --git a/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc index 25a85ce5319..a92be03304f 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc @@ -102,6 +102,14 @@ public: typedef WriteImageStateRequest MockWriteImageStateRequest; typedef util::Mock MockUtils; + void expect_clone_md_ctx(MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), clone()) + .WillOnce(Invoke([&mock_image_ctx]() { + get_mock_io_ctx(mock_image_ctx.md_ctx).get(); + return &get_mock_io_ctx(mock_image_ctx.md_ctx); + })); + } + void expect_refresh_image(MockTestImageCtx &mock_image_ctx, bool refresh_required, int r) { EXPECT_CALL(*mock_image_ctx.state, is_refresh_required()) @@ -132,6 +140,20 @@ public: .WillOnce(Return(result)); } + void expect_get_mirror_peers(MockTestImageCtx &mock_image_ctx, + const std::vector &peers, + int r) { + using ceph::encode; + bufferlist bl; + encode(peers, bl); + + EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_peer_list"), + _, _, _)) + .WillOnce(DoAll(WithArg<5>(CopyInBufferlist(bl)), + Return(r))); + } + void expect_create_snapshot(MockTestImageCtx &mock_image_ctx, int r) { EXPECT_CALL(*mock_image_ctx.operations, snap_create(_, _, _)) .WillOnce(WithArg<2>(CompleteContext( @@ -179,6 +201,38 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, Success) { ASSERT_EQ(0, ctx.wait()); } +TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, SuccessDemoted) { + REQUIRE_FORMAT_V2(); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + + expect_clone_md_ctx(mock_image_ctx); + expect_refresh_image(mock_image_ctx, true, 0); + expect_get_mirror_image( + mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0); + MockUtils mock_utils; + expect_can_create_non_primary_snapshot(mock_utils, true); + expect_get_mirror_peers(mock_image_ctx, + {{"uuid", cls::rbd::MIRROR_PEER_DIRECTION_TX, "ceph", + "mirror", "mirror uuid"}}, 0); + expect_create_snapshot(mock_image_ctx, 0); + MockWriteImageStateRequest mock_write_image_state_request; + expect_write_image_state(mock_image_ctx, mock_write_image_state_request, 0); + + C_SaferCond ctx; + auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true, + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, RefreshError) { REQUIRE_FORMAT_V2(); @@ -247,6 +301,33 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CanNotError) { ASSERT_EQ(-EINVAL, ctx.wait()); } +TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, GetMirrorPeersError) { + REQUIRE_FORMAT_V2(); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + InSequence seq; + + expect_clone_md_ctx(mock_image_ctx); + expect_refresh_image(mock_image_ctx, true, 0); + expect_get_mirror_image( + mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0); + MockUtils mock_utils; + expect_can_create_non_primary_snapshot(mock_utils, true); + expect_get_mirror_peers(mock_image_ctx, {}, -EPERM); + + C_SaferCond ctx; + auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true, + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CreateSnapshotError) { REQUIRE_FORMAT_V2();