diff --git a/src/test/journal/mock/MockJournaler.h b/src/test/journal/mock/MockJournaler.h index 2182c8db4fc..6af3087561f 100644 --- a/src/test/journal/mock/MockJournaler.h +++ b/src/test/journal/mock/MockJournaler.h @@ -138,6 +138,10 @@ struct MockJournaler { }; struct MockJournalerProxy { + MockJournalerProxy() { + MockJournaler::get_instance().construct(); + } + template MockJournalerProxy(IoCtxT &header_ioctx, const std::string &, const std::string &, const Settings&) { diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 4dff2f0a2c6..3d4c16a820d 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -448,6 +448,8 @@ public: const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *mirror_peer_client_meta, Context *on_finish) { return new MockBootstrapRequest(m_local_io_ctx, m_remote_io_ctx, @@ -462,14 +464,13 @@ public: local_mirror_uuid, remote_mirror_uuid, &mock_journaler, - &m_mirror_peer_client_meta, + client_state, mirror_peer_client_meta, on_finish, &m_do_resync); } librbd::ImageCtx *m_remote_image_ctx; librbd::ImageCtx *m_local_image_ctx = nullptr; librbd::MockTestImageCtx *m_local_test_image_ctx = nullptr; - librbd::journal::MirrorPeerClientMeta m_mirror_peer_client_meta; bool m_do_resync; }; @@ -495,22 +496,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, false, 0); // switch the state to replaying + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; client_data.client_meta = mirror_peer_client_meta; expect_journaler_update_client(mock_journaler, client_data, 0); @@ -520,10 +513,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(-EREMOTEIO, ctx.wait()); } @@ -550,22 +545,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; MockOpenLocalImageRequest mock_open_local_image_request; expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, @@ -596,10 +581,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -626,22 +615,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; MockOpenLocalImageRequest mock_open_local_image_request; expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, @@ -682,10 +661,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -712,22 +695,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; MockOpenLocalImageRequest mock_open_local_image_request; expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, @@ -756,10 +729,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -786,22 +763,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; MockOpenLocalImageRequest mock_open_local_image_request; expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, @@ -829,10 +796,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(-EEXIST, ctx.wait()); ASSERT_EQ(NULL, m_local_test_image_ctx); @@ -860,22 +831,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - mock_local_image_ctx.id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); mock_local_image_ctx.journal = &mock_journal; MockOpenLocalImageRequest mock_open_local_image_request; expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, @@ -890,10 +851,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); m_do_resync = false; request->send(); ASSERT_EQ(0, ctx.wait()); @@ -922,17 +887,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - client = {}; - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, -ENOENT); - - // register missing client in remote journal - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - expect_journaler_register_client(mock_journaler, client_data, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -942,7 +896,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { mock_local_image_ctx.journal = &mock_journal; librbd::util::s_image_id = mock_local_image_ctx.id; - mirror_peer_client_meta = {mock_local_image_ctx.id}; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; client_data.client_meta = mirror_peer_client_meta; client.data.clear(); @@ -968,10 +923,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.image_id = ""; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( &mock_instance_watcher, mock_journaler, "", mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -998,16 +956,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { expect_open_image(mock_open_image_request, m_remote_io_ctx, mock_remote_image_ctx.id, mock_remote_image_ctx, 0); - // lookup local peer in remote journal - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - "missing image id"}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - client_data.client_meta = mirror_peer_client_meta; - client.data.clear(); - ::encode(client_data, client.data); - expect_journaler_get_client(mock_journaler, "local mirror uuid", - client, 0); - // test if remote image is primary MockIsPrimaryRequest mock_is_primary_request; expect_is_primary(mock_is_primary_request, true, 0); @@ -1019,7 +967,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { // re-register the client expect_journaler_unregister_client(mock_journaler, 0); - mirror_peer_client_meta = {}; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = ""; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; client_data.client_meta = mirror_peer_client_meta; expect_journaler_register_client(mock_journaler, client_data, 0); @@ -1032,7 +981,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { mock_local_image_ctx.journal = &mock_journal; librbd::util::s_image_id = mock_local_image_ctx.id; - mirror_peer_client_meta = {mock_local_image_ctx.id}; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; client_data.client_meta = mirror_peer_client_meta; client.data.clear(); @@ -1057,10 +1006,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { C_SaferCond ctx; MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.image_id = "missing image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, "", + &mock_instance_watcher, mock_journaler, "missing image id", mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &ctx); + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc index 793811816e6..73658b3fb8b 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -4,8 +4,10 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "cls/rbd/cls_rbd_types.h" #include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "test/journal/mock/MockJournaler.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librbd/mock/MockImageCtx.h" @@ -20,10 +22,32 @@ struct MockTestImageCtx : public librbd::MockImageCtx { }; } // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal } // namespace librbd namespace rbd { namespace mirror { + +template <> +struct Threads { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + namespace image_replayer { template <> @@ -72,6 +96,7 @@ using ::testing::WithArg; class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture { public: + typedef Threads MockThreads; typedef PrepareRemoteImageRequest MockPrepareRemoteImageRequest; typedef GetMirrorImageIdRequest MockGetMirrorImageIdRequest; @@ -96,49 +121,160 @@ public: })), Return(r))); } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_register_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + ::encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } }; TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); + EXPECT_CALL(mock_remote_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + cls::journal::Client client; + client.state = cls::journal::CLIENT_STATE_DISCONNECTED; + ::encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, 0); + std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "local image id", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, client_state); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, 0); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, client_state); } TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "", -EINVAL); std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); ASSERT_EQ(std::string(""), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); } TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + InSequence seq; expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; @@ -146,16 +282,111 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { std::string remote_mirror_uuid; std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; C_SaferCond ctx; - auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx, + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, "global image id", + "local mirror uuid", + "", &remote_mirror_uuid, &remote_image_id, + &remote_journaler, + &client_state, &client_meta, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); } } // namespace image_replayer diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 9e2006c9d51..e63c7338a39 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -141,16 +141,28 @@ struct PrepareRemoteImageRequest { static PrepareRemoteImageRequest* s_instance; std::string *remote_mirror_uuid = nullptr; std::string *remote_image_id = nullptr; + cls::journal::ClientState *client_state; + ::journal::MockJournalerProxy **remote_journaler = nullptr; + librbd::journal::MirrorPeerClientMeta *client_meta = nullptr; Context *on_finish = nullptr; - static PrepareRemoteImageRequest* create(librados::IoCtx &, + static PrepareRemoteImageRequest* create(Threads *threads, + librados::IoCtx &, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + ::journal::MockJournalerProxy **remote_journaler, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *client_meta, Context *on_finish) { assert(s_instance != nullptr); s_instance->remote_mirror_uuid = remote_mirror_uuid; s_instance->remote_image_id = remote_image_id; + s_instance->remote_journaler = remote_journaler; + s_instance->client_state = client_state; + s_instance->client_meta = client_meta; s_instance->on_finish = on_finish; return s_instance; } @@ -179,6 +191,7 @@ struct BootstrapRequest { const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, ::journal::MockJournalerProxy *journaler, + cls::journal::ClientState *client_state, librbd::journal::MirrorPeerClientMeta *client_meta, Context *on_finish, bool *do_resync, rbd::mirror::ProgressContext *progress_ctx = nullptr) { @@ -422,6 +435,10 @@ public: int r) { EXPECT_CALL(mock_request, send()) .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() { + if (r >= 0) { + *mock_request.remote_journaler = new ::journal::MockJournalerProxy(); + } + *mock_request.remote_mirror_uuid = mirror_uuid; *mock_request.remote_image_id = image_id; mock_request.on_finish->complete(r); @@ -650,12 +667,14 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) { create_local_image(); librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_remote_journaler; MockThreads mock_threads(m_threads); expect_work_queue_repeatedly(mock_threads); expect_add_event_after_repeatedly(mock_threads); MockImageDeleter mock_image_deleter; MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; MockReplayStatusFormatter mock_replay_status_formatter; expect_get_or_send_update(mock_replay_status_formatter); @@ -664,6 +683,11 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) { expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0); expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, "", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + "remote image id", 0); + EXPECT_CALL(mock_remote_journaler, construct()); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); create_image_replayer(mock_threads, mock_image_deleter); diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index bf77e9db7bd..ddf3b5a3cea 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -440,10 +440,6 @@ void ImageReplayer::handle_prepare_local_image(int r) { } else if (r < 0) { on_start_fail(r, "error preparing local image for replay"); return; - } else if (m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { - dout(5) << "local image is primary" << dendl; - on_start_fail(0, "local image is primary"); - return; } // local image doesn't exist or is non-primary @@ -453,6 +449,11 @@ void ImageReplayer::handle_prepare_local_image(int r) { template void ImageReplayer::prepare_remote_image() { dout(20) << dendl; + if (m_peers.empty()) { + // technically nothing to bootstrap, but it handles the status update + bootstrap(); + return; + } // TODO need to support multiple remote images assert(!m_peers.empty()); @@ -461,8 +462,9 @@ void ImageReplayer::prepare_remote_image() { Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_prepare_remote_image>(this); auto req = PrepareRemoteImageRequest::create( - m_remote_image.io_ctx, m_global_image_id, &m_remote_image.mirror_uuid, - &m_remote_image.image_id, ctx); + m_threads, m_remote_image.io_ctx, m_global_image_id, m_local_mirror_uuid, + m_local_image_id, &m_remote_image.mirror_uuid, &m_remote_image.image_id, + &m_remote_journaler, &m_client_state, &m_client_meta, ctx); req->send(); } @@ -470,7 +472,11 @@ template void ImageReplayer::handle_prepare_remote_image(int r) { dout(20) << "r=" << r << dendl; - if (r == -ENOENT) { + assert(r < 0 ? m_remote_journaler == nullptr : m_remote_journaler != nullptr); + if (r < 0 && !m_local_image_id.empty() && + m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { + // local image is primary -- fall-through + } else if (r == -ENOENT) { dout(20) << "remote image does not exist" << dendl; // TODO need to support multiple remote images @@ -497,19 +503,16 @@ template void ImageReplayer::bootstrap() { dout(20) << dendl; - CephContext *cct = static_cast(m_local->cct()); - journal::Settings settings; - settings.commit_interval = cct->_conf->get_val( - "rbd_mirror_journal_commit_age"); - settings.max_fetch_bytes = cct->_conf->get_val( - "rbd_mirror_journal_max_fetch_bytes"); - - m_remote_journaler = new Journaler(m_threads->work_queue, - m_threads->timer, - &m_threads->timer_lock, - m_remote_image.io_ctx, - m_remote_image.image_id, - m_local_mirror_uuid, settings); + if (!m_local_image_id.empty() && + m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(5) << "local image is primary" << dendl; + on_start_fail(0, "local image is primary"); + return; + } else if (m_peers.empty()) { + dout(5) << "no peer clusters" << dendl; + on_start_fail(-ENOENT, "no peer clusters"); + return; + } Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_bootstrap>(this); @@ -519,8 +522,8 @@ void ImageReplayer::bootstrap() { &m_local_image_ctx, m_local_image_id, m_remote_image.image_id, m_global_image_id, m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, m_local_mirror_uuid, m_remote_image.mirror_uuid, - m_remote_journaler, &m_client_meta, ctx, &m_resync_requested, - &m_progress_cxt); + m_remote_journaler, &m_client_state, &m_client_meta, ctx, + &m_resync_requested, &m_progress_cxt); { Mutex::Locker locker(m_lock); diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index a66b02a24ab..55e44a6d01b 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -331,6 +331,8 @@ private: bool m_update_status_requested = false; Context *m_on_update_status_finish = nullptr; + cls::journal::ClientState m_client_state = + cls::journal::CLIENT_STATE_DISCONNECTED; librbd::journal::MirrorPeerClientMeta m_client_meta; ReplayEntry m_replay_entry; diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 1c521b274ac..2c24ef1f7d5 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -51,6 +51,7 @@ BootstrapRequest::BootstrapRequest( const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, + cls::journal::ClientState *client_state, MirrorPeerClientMeta *client_meta, Context *on_finish, bool *do_resync, @@ -64,8 +65,8 @@ BootstrapRequest::BootstrapRequest( m_timer(timer), m_timer_lock(timer_lock), m_local_mirror_uuid(local_mirror_uuid), m_remote_mirror_uuid(remote_mirror_uuid), m_journaler(journaler), - m_client_meta(client_meta), m_progress_ctx(progress_ctx), - m_do_resync(do_resync), + m_client_state(client_state), m_client_meta(client_meta), + m_progress_ctx(progress_ctx), m_do_resync(do_resync), m_lock(unique_lock_name("BootstrapRequest::m_lock", this)) { } @@ -172,77 +173,6 @@ void BootstrapRequest::handle_open_remote_image(int r) { return; } - get_client(); -} - -template -void BootstrapRequest::get_client() { - dout(20) << dendl; - - update_progress("GET_CLIENT"); - - Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_get_client>( - this); - m_journaler->get_client(m_local_mirror_uuid, &m_client, ctx); -} - -template -void BootstrapRequest::handle_get_client(int r) { - dout(20) << ": r=" << r << dendl; - - if (r == -ENOENT) { - dout(10) << ": client not registered" << dendl; - } else if (r < 0) { - derr << ": failed to retrieve client: " << cpp_strerror(r) << dendl; - m_ret_val = r; - close_remote_image(); - return; - } else if (decode_client_meta()) { - // skip registration if it already exists - is_primary(); - return; - } - - register_client(); -} - -template -void BootstrapRequest::register_client() { - dout(20) << dendl; - - update_progress("REGISTER_CLIENT"); - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ - m_local_image_id}; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - - librbd::journal::ClientData client_data{mirror_peer_client_meta}; - bufferlist client_data_bl; - ::encode(client_data, client_data_bl); - - Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_register_client>( - this); - m_journaler->register_client(client_data_bl, ctx); -} - -template -void BootstrapRequest::handle_register_client(int r) { - dout(20) << ": r=" << r << dendl; - - if (r < 0) { - derr << ": failed to register with remote journal: " << cpp_strerror(r) - << dendl; - m_ret_val = r; - close_remote_image(); - return; - } - - m_client = {}; - *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id); - m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; - is_primary(); } @@ -280,6 +210,12 @@ void BootstrapRequest::handle_is_primary(int r) { return; } + if (!m_client_meta->image_id.empty()) { + // have an image id -- use that to open the image since a deletion (resync) + // will leave the old image id registered in the peer + m_local_image_id = m_client_meta->image_id; + } + if (m_local_image_id.empty()) { update_client_image(); return; @@ -386,7 +322,7 @@ void BootstrapRequest::handle_open_local_image(int r) { return; } - if (m_client.state == cls::journal::CLIENT_STATE_DISCONNECTED) { + if (*m_client_state == cls::journal::CLIENT_STATE_DISCONNECTED) { dout(10) << ": client flagged disconnected -- skipping bootstrap" << dendl; // The caller is expected to detect disconnect initializing remote journal. m_ret_val = 0; @@ -424,6 +360,45 @@ void BootstrapRequest::handle_unregister_client(int r) { register_client(); } +template +void BootstrapRequest::register_client() { + dout(20) << dendl; + + update_progress("REGISTER_CLIENT"); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + m_local_image_id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + bufferlist client_data_bl; + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_register_client>( + this); + m_journaler->register_client(client_data_bl, ctx); +} + +template +void BootstrapRequest::handle_register_client(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": failed to register with remote journal: " << cpp_strerror(r) + << dendl; + m_ret_val = r; + close_remote_image(); + return; + } + + *m_client_state = cls::journal::CLIENT_STATE_CONNECTED; + *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id); + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + is_primary(); +} + template void BootstrapRequest::update_client_image() { dout(20) << dendl; @@ -769,36 +744,6 @@ void BootstrapRequest::handle_close_remote_image(int r) { finish(m_ret_val); } -template -bool BootstrapRequest::decode_client_meta() { - dout(20) << dendl; - - librbd::journal::ClientData client_data; - bufferlist::iterator it = m_client.data.begin(); - try { - ::decode(client_data, it); - } catch (const buffer::error &err) { - derr << ": failed to decode client meta data: " << err.what() << dendl; - return false; - } - - librbd::journal::MirrorPeerClientMeta *client_meta = - boost::get(&client_data.client_meta); - if (client_meta == nullptr) { - derr << ": unknown peer registration" << dendl; - return false; - } else if (!client_meta->image_id.empty()) { - // have an image id -- use that to open the image - m_local_image_id = client_meta->image_id; - } - - *m_client_meta = *client_meta; - - dout(20) << ": client found: image_id=" << m_local_image_id - << ", client_meta=" << *m_client_meta << dendl; - return true; -} - template void BootstrapRequest::update_progress(const std::string &description) { dout(20) << ": " << description << dendl; diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index 6c67222d4ab..8a46e7038d6 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -53,6 +53,7 @@ public: const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, + cls::journal::ClientState *client_state, MirrorPeerClientMeta *client_meta, Context *on_finish, bool *do_resync, @@ -62,8 +63,8 @@ public: local_image_id, remote_image_id, global_image_id, work_queue, timer, timer_lock, local_mirror_uuid, remote_mirror_uuid, - journaler, client_meta, on_finish, do_resync, - progress_ctx); + journaler, client_state, client_meta, on_finish, + do_resync, progress_ctx); } BootstrapRequest(librados::IoCtx &local_io_ctx, @@ -76,6 +77,7 @@ public: SafeTimer *timer, Mutex *timer_lock, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, + cls::journal::ClientState *client_state, MirrorPeerClientMeta *client_meta, Context *on_finish, bool *do_resync, ProgressContext *progress_ctx = nullptr); ~BootstrapRequest() override; @@ -97,14 +99,8 @@ private: * v * * OPEN_REMOTE_IMAGE * * * * * * * * * * * * * * * * * * * * | * - * v * - * GET_CLIENT * * * * * * * * * * * * * * * * * * * * * * - * | * * - * |/----------------------------------------------*---*---\ - * v (skip if not needed) * * | - * REGISTER_CLIENT * * * * * * * * * * * * * * * * * * * | - * | * * | - * v * * | + * |/--------------------------------------------------*---\ + * v * | * IS_PRIMARY * * * * * * * * * * * * * * * * * * * * * * | * | * * | * | (remote image primary, no local image id) * * | @@ -120,6 +116,9 @@ private: * | | . (image doesn't exist) * * | * | | . . > UNREGISTER_CLIENT * * * * * * * | * | | | * * | + * | | v * * | + * | | REGISTER_CLIENT * * * * * * * * | + * | | | * * | * | | \-----------------------*---*---/ * | | * * * | v (skip if not needed) * * @@ -159,6 +158,7 @@ private: std::string m_local_mirror_uuid; std::string m_remote_mirror_uuid; Journaler *m_journaler; + cls::journal::ClientState *m_client_state; MirrorPeerClientMeta *m_client_meta; ProgressContext *m_progress_ctx; bool *m_do_resync; @@ -179,12 +179,6 @@ private: void get_remote_tag_class(); void handle_get_remote_tag_class(int r); - void get_client(); - void handle_get_client(int r); - - void register_client(); - void handle_register_client(int r); - void open_remote_image(); void handle_open_remote_image(int r); @@ -200,6 +194,9 @@ private: void unregister_client(); void handle_unregister_client(int r); + void register_client(); + void handle_register_client(int r); + void create_local_image(); void handle_create_local_image(int r); @@ -218,8 +215,6 @@ private: void close_remote_image(); void handle_close_remote_image(int r); - bool decode_client_meta(); - void update_progress(const std::string &description); }; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index 2e620938ff4..35755a625a5 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -5,8 +5,13 @@ #include "include/rados/librados.hpp" #include "cls/rbd/cls_rbd_client.h" #include "common/errno.h" +#include "common/WorkQueue.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #define dout_context g_ceph_context @@ -20,6 +25,7 @@ namespace rbd { namespace mirror { namespace image_replayer { +using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; using librbd::util::create_rados_callback; @@ -38,7 +44,7 @@ void PrepareRemoteImageRequest::get_remote_mirror_uuid() { librados::AioCompletion *aio_comp = create_rados_callback< PrepareRemoteImageRequest, &PrepareRemoteImageRequest::handle_get_remote_mirror_uuid>(this); - int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); assert(r == 0); aio_comp->release(); } @@ -75,7 +81,8 @@ void PrepareRemoteImageRequest::get_remote_image_id() { Context *ctx = create_context_callback< PrepareRemoteImageRequest, &PrepareRemoteImageRequest::handle_get_remote_image_id>(this); - auto req = GetMirrorImageIdRequest::create(m_io_ctx, m_global_image_id, + auto req = GetMirrorImageIdRequest::create(m_remote_io_ctx, + m_global_image_id, m_remote_image_id, ctx); req->send(); } @@ -90,13 +97,125 @@ void PrepareRemoteImageRequest::handle_get_remote_image_id(int r) { return; } + get_client(); +} + +template +void PrepareRemoteImageRequest::get_client() { + dout(20) << dendl; + + journal::Settings settings; + settings.commit_interval = g_ceph_context->_conf->get_val( + "rbd_mirror_journal_commit_age"); + settings.max_fetch_bytes = g_ceph_context->_conf->get_val( + "rbd_mirror_journal_max_fetch_bytes"); + + assert(*m_remote_journaler == nullptr); + *m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, + &m_threads->timer_lock, m_remote_io_ctx, + *m_remote_image_id, m_local_mirror_uuid, + settings); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest, + &PrepareRemoteImageRequest::handle_get_client>(this)); + (*m_remote_journaler)->get_client(m_local_mirror_uuid, &m_client, ctx); +} + +template +void PrepareRemoteImageRequest::handle_get_client(int r) { + dout(20) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "client not registered" << dendl; + } else if (r < 0) { + derr << "failed to retrieve client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + *m_client_state = m_client.state; + if (decode_client_meta()) { + // skip registration if it already exists + finish(0); + return; + } + + register_client(); +} + +template +void PrepareRemoteImageRequest::register_client() { + dout(20) << dendl; + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + m_local_image_id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + bufferlist client_data_bl; + ::encode(client_data, client_data_bl); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest, + &PrepareRemoteImageRequest::handle_register_client>(this)); + (*m_remote_journaler)->register_client(client_data_bl, ctx); +} + +template +void PrepareRemoteImageRequest::handle_register_client(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + *m_client_state = cls::journal::CLIENT_STATE_CONNECTED; + *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id); + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + finish(0); } +template +bool PrepareRemoteImageRequest::decode_client_meta() { + dout(20) << dendl; + + librbd::journal::ClientData client_data; + bufferlist::iterator it = m_client.data.begin(); + try { + ::decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode client meta data: " << err.what() << dendl; + return false; + } + + librbd::journal::MirrorPeerClientMeta *client_meta = + boost::get(&client_data.client_meta); + if (client_meta == nullptr) { + derr << "unknown peer registration" << dendl; + return false; + } + + *m_client_meta = *client_meta; + dout(20) << "client found: client_meta=" << *m_client_meta << dendl; + return true; +} + template void PrepareRemoteImageRequest::finish(int r) { dout(20) << "r=" << r << dendl; + if (r < 0) { + delete *m_remote_journaler; + *m_remote_journaler = nullptr; + } + m_on_finish->complete(r); delete this; } diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h index 9943fd742a7..8d8f553d536 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -5,40 +5,68 @@ #define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H #include "include/buffer.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/TypeTraits.h" #include +namespace journal { class Journaler; } namespace librados { struct IoCtx; } namespace librbd { struct ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } struct Context; struct ContextWQ; namespace rbd { namespace mirror { + +template struct Threads; + namespace image_replayer { template class PrepareRemoteImageRequest { public: - static PrepareRemoteImageRequest *create(librados::IoCtx &io_ctx, + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + static PrepareRemoteImageRequest *create(Threads *threads, + librados::IoCtx &remote_io_ctx, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + Journaler **remote_journaler, + cls::journal::ClientState *client_state, + MirrorPeerClientMeta *client_meta, Context *on_finish) { - return new PrepareRemoteImageRequest(io_ctx, global_image_id, - remote_mirror_uuid, remote_image_id, - on_finish); + return new PrepareRemoteImageRequest(threads, remote_io_ctx, + global_image_id, local_mirror_uuid, + local_image_id, remote_mirror_uuid, + remote_image_id, remote_journaler, + client_state, client_meta, on_finish); } - PrepareRemoteImageRequest(librados::IoCtx &io_ctx, + PrepareRemoteImageRequest(Threads *threads, + librados::IoCtx &remote_io_ctx, const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, std::string *remote_mirror_uuid, std::string *remote_image_id, + Journaler **remote_journaler, + cls::journal::ClientState *client_state, + MirrorPeerClientMeta *client_meta, Context *on_finish) - : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + : m_threads(threads), m_remote_io_ctx(remote_io_ctx), + m_global_image_id(global_image_id), + m_local_mirror_uuid(local_mirror_uuid), m_local_image_id(local_image_id), m_remote_mirror_uuid(remote_mirror_uuid), m_remote_image_id(remote_image_id), - m_on_finish(on_finish) { + m_remote_journaler(remote_journaler), m_client_state(client_state), + m_client_meta(client_meta), m_on_finish(on_finish) { } void send(); @@ -56,18 +84,31 @@ private: * GET_REMOTE_IMAGE_ID * | * v + * GET_CLIENT + * | + * v (skip if not needed) + * REGISTER_CLIENT + * | + * v * * @endverbatim */ - librados::IoCtx &m_io_ctx; + Threads *m_threads; + librados::IoCtx &m_remote_io_ctx; std::string m_global_image_id; + std::string m_local_mirror_uuid; + std::string m_local_image_id; std::string *m_remote_mirror_uuid; std::string *m_remote_image_id; + Journaler **m_remote_journaler; + cls::journal::ClientState *m_client_state; + MirrorPeerClientMeta *m_client_meta; Context *m_on_finish; bufferlist m_out_bl; + cls::journal::Client m_client; void get_remote_mirror_uuid(); void handle_get_remote_mirror_uuid(int r); @@ -75,8 +116,15 @@ private: void get_remote_image_id(); void handle_get_remote_image_id(int r); + void get_client(); + void handle_get_client(int r); + + void register_client(); + void handle_register_client(int r); + void finish(int r); + bool decode_client_meta(); }; } // namespace image_replayer