RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-22 10:17:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request from dillaman/wip-21561

Browse Source 
rbd-mirror: primary image should register in remote, non-primary image's journal

Reviewed-by: Mykola Golub <to.my.trociny@gmail.com>
This commit is contained in:
Mykola Golub 2017-10-10 18:01:44 +03:00 committed by GitHub
commit 069c8dc2f4
10 changed files with 582 additions and 259 deletions

4
src/test/journal/mock/MockJournaler.h
View File

@ -138,6 +138,10 @@ struct MockJournaler {
};
struct MockJournalerProxy {
MockJournalerProxy() {
MockJournaler::get_instance().construct();
}
template <typename IoCtxT>
MockJournalerProxy(IoCtxT &header_ioctx, const std::string &,
const std::string &, const Settings&) {

154
src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
View File

@ -448,6 +448,8 @@ public:
const std::string &global_image_id,
const std::string &local_mirror_uuid,
const std::string &remote_mirror_uuid,
cls::journal::ClientState *client_state,
librbd::journal::MirrorPeerClientMeta *mirror_peer_client_meta,
Context *on_finish) {
return new MockBootstrapRequest(m_local_io_ctx,
m_remote_io_ctx,
@ -462,14 +464,13 @@ public:
local_mirror_uuid,
remote_mirror_uuid,
&mock_journaler,
&m_mirror_peer_client_meta,
client_state, mirror_peer_client_meta,
on_finish, &m_do_resync);
}
librbd::ImageCtx *m_remote_image_ctx;
librbd::ImageCtx *m_local_image_ctx = nullptr;
librbd::MockTestImageCtx *m_local_test_image_ctx = nullptr;
librbd::journal::MirrorPeerClientMeta m_mirror_peer_client_meta;
bool m_do_resync;
};
@ -495,22 +496,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, false, 0);
// switch the state to replaying
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
expect_journaler_update_client(mock_journaler, client_data, 0);
@ -520,10 +513,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(-EREMOTEIO, ctx.wait());
}
@ -550,22 +545,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
// open the local image
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
mock_local_image_ctx.journal = &mock_journal;
MockOpenLocalImageRequest mock_open_local_image_request;
expect_open_local_image(mock_open_local_image_request, m_local_io_ctx,
@ -596,10 +581,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(0, ctx.wait());
}
@ -626,22 +615,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
// open the local image
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
mock_local_image_ctx.journal = &mock_journal;
MockOpenLocalImageRequest mock_open_local_image_request;
expect_open_local_image(mock_open_local_image_request, m_local_io_ctx,
@ -682,10 +661,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(0, ctx.wait());
}
@ -712,22 +695,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
// open the local image
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
mock_local_image_ctx.journal = &mock_journal;
MockOpenLocalImageRequest mock_open_local_image_request;
expect_open_local_image(mock_open_local_image_request, m_local_io_ctx,
@ -756,10 +729,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(0, ctx.wait());
}
@ -786,22 +763,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
// open the local image
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
mock_local_image_ctx.journal = &mock_journal;
MockOpenLocalImageRequest mock_open_local_image_request;
expect_open_local_image(mock_open_local_image_request, m_local_io_ctx,
@ -829,10 +796,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(-EEXIST, ctx.wait());
ASSERT_EQ(NULL, m_local_test_image_ctx);
@ -860,22 +831,12 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
// open the local image
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
mock_local_image_ctx.journal = &mock_journal;
MockOpenLocalImageRequest mock_open_local_image_request;
expect_open_local_image(mock_open_local_image_request, m_local_io_ctx,
@ -890,10 +851,14 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
mock_local_image_ctx.id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
m_do_resync = false;
request->send();
ASSERT_EQ(0, ctx.wait());
@ -922,17 +887,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
client = {};
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, -ENOENT);
// register missing client in remote journal
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
expect_journaler_register_client(mock_journaler, client_data, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
@ -942,7 +896,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) {
mock_local_image_ctx.journal = &mock_journal;
librbd::util::s_image_id = mock_local_image_ctx.id;
mirror_peer_client_meta = {mock_local_image_ctx.id};
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.image_id = mock_local_image_ctx.id;
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
@ -968,10 +923,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
mirror_peer_client_meta.image_id = "";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, "",
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(0, ctx.wait());
}
@ -998,16 +956,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) {
expect_open_image(mock_open_image_request, m_remote_io_ctx,
mock_remote_image_ctx.id, mock_remote_image_ctx, 0);
// lookup local peer in remote journal
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
"missing image id"};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
::encode(client_data, client.data);
expect_journaler_get_client(mock_journaler, "local mirror uuid",
client, 0);
// test if remote image is primary
MockIsPrimaryRequest mock_is_primary_request;
expect_is_primary(mock_is_primary_request, true, 0);
@ -1019,7 +967,8 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) {
// re-register the client
expect_journaler_unregister_client(mock_journaler, 0);
mirror_peer_client_meta = {};
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.image_id = "";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
client_data.client_meta = mirror_peer_client_meta;
expect_journaler_register_client(mock_journaler, client_data, 0);
@ -1032,7 +981,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) {
mock_local_image_ctx.journal = &mock_journal;
librbd::util::s_image_id = mock_local_image_ctx.id;
mirror_peer_client_meta = {mock_local_image_ctx.id};
mirror_peer_client_meta.image_id = mock_local_image_ctx.id;
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
client_data.client_meta = mirror_peer_client_meta;
client.data.clear();
@ -1057,10 +1006,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) {
C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED;
mirror_peer_client_meta.image_id = "missing image id";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
MockBootstrapRequest *request = create_request(
&mock_instance_watcher, mock_journaler, "",
&mock_instance_watcher, mock_journaler, "missing image id",
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
"remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx);
request->send();
ASSERT_EQ(0, ctx.wait());
}

237
src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc
View File

@ -4,8 +4,10 @@
#include "test/rbd_mirror/test_mock_fixture.h"
#include "cls/rbd/cls_rbd_types.h"
#include "librbd/journal/TypeTraits.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h"
#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h"
#include "test/journal/mock/MockJournaler.h"
#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
#include "test/librbd/mock/MockImageCtx.h"
@ -20,10 +22,32 @@ struct MockTestImageCtx : public librbd::MockImageCtx {
};
} // anonymous namespace
namespace journal {
template <>
struct TypeTraits<MockTestImageCtx> {
typedef ::journal::MockJournalerProxy Journaler;
};
} // namespace journal
} // namespace librbd
namespace rbd {
namespace mirror {
template <>
struct Threads<librbd::MockTestImageCtx> {
Mutex &timer_lock;
SafeTimer *timer;
ContextWQ *work_queue;
Threads(Threads<librbd::ImageCtx> *threads)
: timer_lock(threads->timer_lock), timer(threads->timer),
work_queue(threads->work_queue) {
}
};
namespace image_replayer {
template <>
@ -72,6 +96,7 @@ using ::testing::WithArg;
class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture {
public:
typedef Threads<librbd::MockTestImageCtx> MockThreads;
typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest;
typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest;
@ -96,49 +121,160 @@ public:
})),
Return(r)));
}
void expect_journaler_get_client(::journal::MockJournaler &mock_journaler,
const std::string &client_id,
cls::journal::Client &client, int r) {
EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _))
.WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) {
*out_client = client;
})),
WithArg<2>(Invoke([this, r](Context *on_finish) {
m_threads->work_queue->queue(on_finish, r);
}))));
}
void expect_journaler_register_client(::journal::MockJournaler &mock_journaler,
const librbd::journal::ClientData &client_data,
int r) {
bufferlist bl;
::encode(client_data, bl);
EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _))
.WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) {
m_threads->work_queue->queue(on_finish, r);
})));
}
};
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
expect_get_mirror_image_id(mock_get_mirror_image_id_request,
"remote image id", 0);
EXPECT_CALL(mock_remote_journaler, construct());
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.image_id = "local image id";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
cls::journal::Client client;
client.state = cls::journal::CLIENT_STATE_DISCONNECTED;
::encode(client_data, client.data);
expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
client, 0);
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"local image id",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(0, ctx.wait());
ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
ASSERT_EQ(std::string("remote image id"), remote_image_id);
ASSERT_TRUE(remote_journaler != nullptr);
ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, client_state);
}
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
expect_get_mirror_image_id(mock_get_mirror_image_id_request,
"remote image id", 0);
EXPECT_CALL(mock_remote_journaler, construct());
cls::journal::Client client;
expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
client, -ENOENT);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.image_id = "local image id";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
expect_journaler_register_client(mock_remote_journaler, client_data, 0);
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"local image id",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(0, ctx.wait());
ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
ASSERT_EQ(std::string("remote image id"), remote_image_id);
ASSERT_TRUE(remote_journaler != nullptr);
ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, client_state);
}
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "", -EINVAL);
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
ASSERT_EQ(std::string(""), remote_mirror_uuid);
ASSERT_TRUE(remote_journaler == nullptr);
}
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
@ -146,16 +282,111 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) {
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(m_remote_io_ctx,
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
ASSERT_TRUE(remote_journaler == nullptr);
}
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
expect_get_mirror_image_id(mock_get_mirror_image_id_request,
"remote image id", 0);
EXPECT_CALL(mock_remote_journaler, construct());
cls::journal::Client client;
expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
client, -EINVAL);
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"local image id",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
ASSERT_EQ(std::string("remote image id"), remote_image_id);
ASSERT_TRUE(remote_journaler == nullptr);
}
TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) {
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
InSequence seq;
expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0);
MockGetMirrorImageIdRequest mock_get_mirror_image_id_request;
expect_get_mirror_image_id(mock_get_mirror_image_id_request,
"remote image id", 0);
EXPECT_CALL(mock_remote_journaler, construct());
cls::journal::Client client;
expect_journaler_get_client(mock_remote_journaler, "local mirror uuid",
client, -ENOENT);
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta;
mirror_peer_client_meta.image_id = "local image id";
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL);
std::string remote_mirror_uuid;
std::string remote_image_id;
journal::MockJournalerProxy *remote_journaler = nullptr;
cls::journal::ClientState client_state;
librbd::journal::MirrorPeerClientMeta client_meta;
C_SaferCond ctx;
auto req = MockPrepareRemoteImageRequest::create(&mock_threads,
m_remote_io_ctx,
"global image id",
"local mirror uuid",
"local image id",
&remote_mirror_uuid,
&remote_image_id,
&remote_journaler,
&client_state, &client_meta,
&ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid);
ASSERT_EQ(std::string("remote image id"), remote_image_id);
ASSERT_TRUE(remote_journaler == nullptr);
}
} // namespace image_replayer

26
src/test/rbd_mirror/test_mock_ImageReplayer.cc
View File

@ -141,16 +141,28 @@ struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> {
static PrepareRemoteImageRequest* s_instance;
std::string *remote_mirror_uuid = nullptr;
std::string *remote_image_id = nullptr;
cls::journal::ClientState *client_state;
::journal::MockJournalerProxy **remote_journaler = nullptr;
librbd::journal::MirrorPeerClientMeta *client_meta = nullptr;
Context *on_finish = nullptr;
static PrepareRemoteImageRequest* create(librados::IoCtx &,
static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads,
librados::IoCtx &,
const std::string &global_image_id,
const std::string &local_mirror_uuid,
const std::string &local_image_id,
std::string *remote_mirror_uuid,
std::string *remote_image_id,
::journal::MockJournalerProxy **remote_journaler,
cls::journal::ClientState *client_state,
librbd::journal::MirrorPeerClientMeta *client_meta,
Context *on_finish) {
assert(s_instance != nullptr);
s_instance->remote_mirror_uuid = remote_mirror_uuid;
s_instance->remote_image_id = remote_image_id;
s_instance->remote_journaler = remote_journaler;
s_instance->client_state = client_state;
s_instance->client_meta = client_meta;
s_instance->on_finish = on_finish;
return s_instance;
}
@ -179,6 +191,7 @@ struct BootstrapRequest<librbd::MockTestImageCtx> {
const std::string &local_mirror_uuid,
const std::string &remote_mirror_uuid,
::journal::MockJournalerProxy *journaler,
cls::journal::ClientState *client_state,
librbd::journal::MirrorPeerClientMeta *client_meta,
Context *on_finish, bool *do_resync,
rbd::mirror::ProgressContext *progress_ctx = nullptr) {
@ -422,6 +435,10 @@ public:
int r) {
EXPECT_CALL(mock_request, send())
.WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() {
if (r >= 0) {
*mock_request.remote_journaler = new ::journal::MockJournalerProxy();
}
*mock_request.remote_mirror_uuid = mirror_uuid;
*mock_request.remote_image_id = image_id;
mock_request.on_finish->complete(r);
@ -650,12 +667,14 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) {
create_local_image();
librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx);
journal::MockJournaler mock_remote_journaler;
MockThreads mock_threads(m_threads);
expect_work_queue_repeatedly(mock_threads);
expect_add_event_after_repeatedly(mock_threads);
MockImageDeleter mock_image_deleter;
MockPrepareLocalImageRequest mock_prepare_local_image_request;
MockPrepareRemoteImageRequest mock_prepare_remote_image_request;
MockReplayStatusFormatter mock_replay_status_formatter;
expect_get_or_send_update(mock_replay_status_formatter);
@ -664,6 +683,11 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) {
expect_wait_for_scheduled_deletion(mock_image_deleter, "global image id", 0);
expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
"", 0);
expect_send(mock_prepare_remote_image_request, "remote mirror uuid",
"remote image id", 0);
EXPECT_CALL(mock_remote_journaler, construct());
EXPECT_CALL(mock_remote_journaler, remove_listener(_));
expect_shut_down(mock_remote_journaler, 0);
create_image_replayer(mock_threads, mock_image_deleter);

47
src/tools/rbd_mirror/ImageReplayer.cc
View File

@ -440,10 +440,6 @@ void ImageReplayer<I>::handle_prepare_local_image(int r) {
} else if (r < 0) {
on_start_fail(r, "error preparing local image for replay");
return;
} else if (m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
dout(5) << "local image is primary" << dendl;
on_start_fail(0, "local image is primary");
return;
}
// local image doesn't exist or is non-primary
@ -453,6 +449,11 @@ void ImageReplayer<I>::handle_prepare_local_image(int r) {
template <typename I>
void ImageReplayer<I>::prepare_remote_image() {
dout(20) << dendl;
if (m_peers.empty()) {
// technically nothing to bootstrap, but it handles the status update
bootstrap();
return;
}
// TODO need to support multiple remote images
assert(!m_peers.empty());
@ -461,8 +462,9 @@ void ImageReplayer<I>::prepare_remote_image() {
Context *ctx = create_context_callback<
ImageReplayer, &ImageReplayer<I>::handle_prepare_remote_image>(this);
auto req = PrepareRemoteImageRequest<I>::create(
m_remote_image.io_ctx, m_global_image_id, &m_remote_image.mirror_uuid,
&m_remote_image.image_id, ctx);
m_threads, m_remote_image.io_ctx, m_global_image_id, m_local_mirror_uuid,
m_local_image_id, &m_remote_image.mirror_uuid, &m_remote_image.image_id,
&m_remote_journaler, &m_client_state, &m_client_meta, ctx);
req->send();
}
@ -470,7 +472,11 @@ template <typename I>
void ImageReplayer<I>::handle_prepare_remote_image(int r) {
dout(20) << "r=" << r << dendl;
if (r == -ENOENT) {
assert(r < 0 ? m_remote_journaler == nullptr : m_remote_journaler != nullptr);
if (r < 0 && !m_local_image_id.empty() &&
m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
// local image is primary -- fall-through
} else if (r == -ENOENT) {
dout(20) << "remote image does not exist" << dendl;
// TODO need to support multiple remote images
@ -497,19 +503,16 @@ template <typename I>
void ImageReplayer<I>::bootstrap() {
dout(20) << dendl;
CephContext *cct = static_cast<CephContext *>(m_local->cct());
journal::Settings settings;
settings.commit_interval = cct->_conf->get_val<double>(
"rbd_mirror_journal_commit_age");
settings.max_fetch_bytes = cct->_conf->get_val<uint64_t>(
"rbd_mirror_journal_max_fetch_bytes");
m_remote_journaler = new Journaler(m_threads->work_queue,
m_threads->timer,
&m_threads->timer_lock,
m_remote_image.io_ctx,
m_remote_image.image_id,
m_local_mirror_uuid, settings);
if (!m_local_image_id.empty() &&
m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) {
dout(5) << "local image is primary" << dendl;
on_start_fail(0, "local image is primary");
return;
} else if (m_peers.empty()) {
dout(5) << "no peer clusters" << dendl;
on_start_fail(-ENOENT, "no peer clusters");
return;
}
Context *ctx = create_context_callback<
ImageReplayer, &ImageReplayer<I>::handle_bootstrap>(this);
@ -519,8 +522,8 @@ void ImageReplayer<I>::bootstrap() {
&m_local_image_ctx, m_local_image_id, m_remote_image.image_id,
m_global_image_id, m_threads->work_queue, m_threads->timer,
&m_threads->timer_lock, m_local_mirror_uuid, m_remote_image.mirror_uuid,
m_remote_journaler, &m_client_meta, ctx, &m_resync_requested,
&m_progress_cxt);
m_remote_journaler, &m_client_state, &m_client_meta, ctx,
&m_resync_requested, &m_progress_cxt);
{
Mutex::Locker locker(m_lock);

2
src/tools/rbd_mirror/ImageReplayer.h
View File

@ -331,6 +331,8 @@ private:
bool m_update_status_requested = false;
Context *m_on_update_status_finish = nullptr;
cls::journal::ClientState m_client_state =
cls::journal::CLIENT_STATE_DISCONNECTED;
librbd::journal::MirrorPeerClientMeta m_client_meta;
ReplayEntry m_replay_entry;

153
src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
View File

@ -51,6 +51,7 @@ BootstrapRequest<I>::BootstrapRequest(
const std::string &local_mirror_uuid,
const std::string &remote_mirror_uuid,
Journaler *journaler,
cls::journal::ClientState *client_state,
MirrorPeerClientMeta *client_meta,
Context *on_finish,
bool *do_resync,
@ -64,8 +65,8 @@ BootstrapRequest<I>::BootstrapRequest(
m_timer(timer), m_timer_lock(timer_lock),
m_local_mirror_uuid(local_mirror_uuid),
m_remote_mirror_uuid(remote_mirror_uuid), m_journaler(journaler),
m_client_meta(client_meta), m_progress_ctx(progress_ctx),
m_do_resync(do_resync),
m_client_state(client_state), m_client_meta(client_meta),
m_progress_ctx(progress_ctx), m_do_resync(do_resync),
m_lock(unique_lock_name("BootstrapRequest::m_lock", this)) {
}
@ -172,77 +173,6 @@ void BootstrapRequest<I>::handle_open_remote_image(int r) {
return;
}
get_client();
}
template <typename I>
void BootstrapRequest<I>::get_client() {
dout(20) << dendl;
update_progress("GET_CLIENT");
Context *ctx = create_context_callback<
BootstrapRequest<I>, &BootstrapRequest<I>::handle_get_client>(
this);
m_journaler->get_client(m_local_mirror_uuid, &m_client, ctx);
}
template <typename I>
void BootstrapRequest<I>::handle_get_client(int r) {
dout(20) << ": r=" << r << dendl;
if (r == -ENOENT) {
dout(10) << ": client not registered" << dendl;
} else if (r < 0) {
derr << ": failed to retrieve client: " << cpp_strerror(r) << dendl;
m_ret_val = r;
close_remote_image();
return;
} else if (decode_client_meta()) {
// skip registration if it already exists
is_primary();
return;
}
register_client();
}
template <typename I>
void BootstrapRequest<I>::register_client() {
dout(20) << dendl;
update_progress("REGISTER_CLIENT");
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
m_local_image_id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
bufferlist client_data_bl;
::encode(client_data, client_data_bl);
Context *ctx = create_context_callback<
BootstrapRequest<I>, &BootstrapRequest<I>::handle_register_client>(
this);
m_journaler->register_client(client_data_bl, ctx);
}
template <typename I>
void BootstrapRequest<I>::handle_register_client(int r) {
dout(20) << ": r=" << r << dendl;
if (r < 0) {
derr << ": failed to register with remote journal: " << cpp_strerror(r)
<< dendl;
m_ret_val = r;
close_remote_image();
return;
}
m_client = {};
*m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id);
m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
is_primary();
}
@ -280,6 +210,12 @@ void BootstrapRequest<I>::handle_is_primary(int r) {
return;
}
if (!m_client_meta->image_id.empty()) {
// have an image id -- use that to open the image since a deletion (resync)
// will leave the old image id registered in the peer
m_local_image_id = m_client_meta->image_id;
}
if (m_local_image_id.empty()) {
update_client_image();
return;
@ -386,7 +322,7 @@ void BootstrapRequest<I>::handle_open_local_image(int r) {
return;
}
if (m_client.state == cls::journal::CLIENT_STATE_DISCONNECTED) {
if (*m_client_state == cls::journal::CLIENT_STATE_DISCONNECTED) {
dout(10) << ": client flagged disconnected -- skipping bootstrap" << dendl;
// The caller is expected to detect disconnect initializing remote journal.
m_ret_val = 0;
@ -424,6 +360,45 @@ void BootstrapRequest<I>::handle_unregister_client(int r) {
register_client();
}
template <typename I>
void BootstrapRequest<I>::register_client() {
dout(20) << dendl;
update_progress("REGISTER_CLIENT");
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
m_local_image_id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
bufferlist client_data_bl;
::encode(client_data, client_data_bl);
Context *ctx = create_context_callback<
BootstrapRequest<I>, &BootstrapRequest<I>::handle_register_client>(
this);
m_journaler->register_client(client_data_bl, ctx);
}
template <typename I>
void BootstrapRequest<I>::handle_register_client(int r) {
dout(20) << ": r=" << r << dendl;
if (r < 0) {
derr << ": failed to register with remote journal: " << cpp_strerror(r)
<< dendl;
m_ret_val = r;
close_remote_image();
return;
}
*m_client_state = cls::journal::CLIENT_STATE_CONNECTED;
*m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id);
m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
is_primary();
}
template <typename I>
void BootstrapRequest<I>::update_client_image() {
dout(20) << dendl;
@ -769,36 +744,6 @@ void BootstrapRequest<I>::handle_close_remote_image(int r) {
finish(m_ret_val);
}
template <typename I>
bool BootstrapRequest<I>::decode_client_meta() {
dout(20) << dendl;
librbd::journal::ClientData client_data;
bufferlist::iterator it = m_client.data.begin();
try {
::decode(client_data, it);
} catch (const buffer::error &err) {
derr << ": failed to decode client meta data: " << err.what() << dendl;
return false;
}
librbd::journal::MirrorPeerClientMeta *client_meta =
boost::get<librbd::journal::MirrorPeerClientMeta>(&client_data.client_meta);
if (client_meta == nullptr) {
derr << ": unknown peer registration" << dendl;
return false;
} else if (!client_meta->image_id.empty()) {
// have an image id -- use that to open the image
m_local_image_id = client_meta->image_id;
}
*m_client_meta = *client_meta;
dout(20) << ": client found: image_id=" << m_local_image_id
<< ", client_meta=" << *m_client_meta << dendl;
return true;
}
template <typename I>
void BootstrapRequest<I>::update_progress(const std::string &description) {
dout(20) << ": " << description << dendl;

31
src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
View File

@ -53,6 +53,7 @@ public:
const std::string &local_mirror_uuid,
const std::string &remote_mirror_uuid,
Journaler *journaler,
cls::journal::ClientState *client_state,
MirrorPeerClientMeta *client_meta,
Context *on_finish,
bool *do_resync,
@ -62,8 +63,8 @@ public:
local_image_id, remote_image_id,
global_image_id, work_queue, timer, timer_lock,
local_mirror_uuid, remote_mirror_uuid,
journaler, client_meta, on_finish, do_resync,
progress_ctx);
journaler, client_state, client_meta, on_finish,
do_resync, progress_ctx);
}
BootstrapRequest(librados::IoCtx &local_io_ctx,
@ -76,6 +77,7 @@ public:
SafeTimer *timer, Mutex *timer_lock,
const std::string &local_mirror_uuid,
const std::string &remote_mirror_uuid, Journaler *journaler,
cls::journal::ClientState *client_state,
MirrorPeerClientMeta *client_meta, Context *on_finish,
bool *do_resync, ProgressContext *progress_ctx = nullptr);
~BootstrapRequest() override;
@ -97,14 +99,8 @@ private:
* v *
* OPEN_REMOTE_IMAGE * * * * * * * * * * * * * * * * * * *
* | *
* v *
* GET_CLIENT * * * * * * * * * * * * * * * * * * * * * *
* | * *
* |/----------------------------------------------*---*---\
* v (skip if not needed) * * |
* REGISTER_CLIENT * * * * * * * * * * * * * * * * * * * |
* | * * |
* v * * |
* |/--------------------------------------------------*---\
* v * |
* IS_PRIMARY * * * * * * * * * * * * * * * * * * * * * * |
* | * * |
* | (remote image primary, no local image id) * * |
@ -120,6 +116,9 @@ private:
* | | . (image doesn't exist) * * |
* | | . . > UNREGISTER_CLIENT * * * * * * * |
* | | | * * |
* | | v * * |
* | | REGISTER_CLIENT * * * * * * * * |
* | | | * * |
* | | \-----------------------*---*---/
* | | * *
* | v (skip if not needed) * *
@ -159,6 +158,7 @@ private:
std::string m_local_mirror_uuid;
std::string m_remote_mirror_uuid;
Journaler *m_journaler;
cls::journal::ClientState *m_client_state;
MirrorPeerClientMeta *m_client_meta;
ProgressContext *m_progress_ctx;
bool *m_do_resync;
@ -179,12 +179,6 @@ private:
void get_remote_tag_class();
void handle_get_remote_tag_class(int r);
void get_client();
void handle_get_client(int r);
void register_client();
void handle_register_client(int r);
void open_remote_image();
void handle_open_remote_image(int r);
@ -200,6 +194,9 @@ private:
void unregister_client();
void handle_unregister_client(int r);
void register_client();
void handle_register_client(int r);
void create_local_image();
void handle_create_local_image(int r);
@ -218,8 +215,6 @@ private:
void close_remote_image();
void handle_close_remote_image(int r);
bool decode_client_meta();
void update_progress(const std::string &description);
};

123
src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc
View File

@ -5,8 +5,13 @@
#include "include/rados/librados.hpp"
#include "cls/rbd/cls_rbd_client.h"
#include "common/errno.h"
#include "common/WorkQueue.h"
#include "journal/Journaler.h"
#include "journal/Settings.h"
#include "librbd/ImageCtx.h"
#include "librbd/Utils.h"
#include "librbd/journal/Types.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h"
#define dout_context g_ceph_context
@ -20,6 +25,7 @@ namespace rbd {
namespace mirror {
namespace image_replayer {
using librbd::util::create_async_context_callback;
using librbd::util::create_context_callback;
using librbd::util::create_rados_callback;
@ -38,7 +44,7 @@ void PrepareRemoteImageRequest<I>::get_remote_mirror_uuid() {
librados::AioCompletion *aio_comp = create_rados_callback<
PrepareRemoteImageRequest<I>,
&PrepareRemoteImageRequest<I>::handle_get_remote_mirror_uuid>(this);
int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
assert(r == 0);
aio_comp->release();
}
@ -75,7 +81,8 @@ void PrepareRemoteImageRequest<I>::get_remote_image_id() {
Context *ctx = create_context_callback<
PrepareRemoteImageRequest<I>,
&PrepareRemoteImageRequest<I>::handle_get_remote_image_id>(this);
auto req = GetMirrorImageIdRequest<I>::create(m_io_ctx, m_global_image_id,
auto req = GetMirrorImageIdRequest<I>::create(m_remote_io_ctx,
m_global_image_id,
m_remote_image_id, ctx);
req->send();
}
@ -90,13 +97,125 @@ void PrepareRemoteImageRequest<I>::handle_get_remote_image_id(int r) {
return;
}
get_client();
}
template <typename I>
void PrepareRemoteImageRequest<I>::get_client() {
dout(20) << dendl;
journal::Settings settings;
settings.commit_interval = g_ceph_context->_conf->get_val<double>(
"rbd_mirror_journal_commit_age");
settings.max_fetch_bytes = g_ceph_context->_conf->get_val<uint64_t>(
"rbd_mirror_journal_max_fetch_bytes");
assert(*m_remote_journaler == nullptr);
*m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer,
&m_threads->timer_lock, m_remote_io_ctx,
*m_remote_image_id, m_local_mirror_uuid,
settings);
Context *ctx = create_async_context_callback(
m_threads->work_queue, create_context_callback<
PrepareRemoteImageRequest<I>,
&PrepareRemoteImageRequest<I>::handle_get_client>(this));
(*m_remote_journaler)->get_client(m_local_mirror_uuid, &m_client, ctx);
}
template <typename I>
void PrepareRemoteImageRequest<I>::handle_get_client(int r) {
dout(20) << "r=" << r << dendl;
if (r == -ENOENT) {
dout(10) << "client not registered" << dendl;
} else if (r < 0) {
derr << "failed to retrieve client: " << cpp_strerror(r) << dendl;
finish(r);
return;
}
*m_client_state = m_client.state;
if (decode_client_meta()) {
// skip registration if it already exists
finish(0);
return;
}
register_client();
}
template <typename I>
void PrepareRemoteImageRequest<I>::register_client() {
dout(20) << dendl;
librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
m_local_image_id};
mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
librbd::journal::ClientData client_data{mirror_peer_client_meta};
bufferlist client_data_bl;
::encode(client_data, client_data_bl);
Context *ctx = create_async_context_callback(
m_threads->work_queue, create_context_callback<
PrepareRemoteImageRequest<I>,
&PrepareRemoteImageRequest<I>::handle_register_client>(this));
(*m_remote_journaler)->register_client(client_data_bl, ctx);
}
template <typename I>
void PrepareRemoteImageRequest<I>::handle_register_client(int r) {
dout(20) << "r=" << r << dendl;
if (r < 0) {
derr << "failed to register with remote journal: " << cpp_strerror(r)
<< dendl;
finish(r);
return;
}
*m_client_state = cls::journal::CLIENT_STATE_CONNECTED;
*m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id);
m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
finish(0);
}
template <typename I>
bool PrepareRemoteImageRequest<I>::decode_client_meta() {
dout(20) << dendl;
librbd::journal::ClientData client_data;
bufferlist::iterator it = m_client.data.begin();
try {
::decode(client_data, it);
} catch (const buffer::error &err) {
derr << "failed to decode client meta data: " << err.what() << dendl;
return false;
}
librbd::journal::MirrorPeerClientMeta *client_meta =
boost::get<librbd::journal::MirrorPeerClientMeta>(&client_data.client_meta);
if (client_meta == nullptr) {
derr << "unknown peer registration" << dendl;
return false;
}
*m_client_meta = *client_meta;
dout(20) << "client found: client_meta=" << *m_client_meta << dendl;
return true;
}
template <typename I>
void PrepareRemoteImageRequest<I>::finish(int r) {
dout(20) << "r=" << r << dendl;
if (r < 0) {
delete *m_remote_journaler;
*m_remote_journaler = nullptr;
}
m_on_finish->complete(r);
delete this;
}

64
src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h
View File

@ -5,40 +5,68 @@
#define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H
#include "include/buffer.h"
#include "cls/journal/cls_journal_types.h"
#include "librbd/journal/TypeTraits.h"
#include <string>
namespace journal { class Journaler; }
namespace librados { struct IoCtx; }
namespace librbd { struct ImageCtx; }
namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
struct Context;
struct ContextWQ;
namespace rbd {
namespace mirror {
template <typename> struct Threads;
namespace image_replayer {
template <typename ImageCtxT = librbd::ImageCtx>
class PrepareRemoteImageRequest {
public:
static PrepareRemoteImageRequest *create(librados::IoCtx &io_ctx,
typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits;
typedef typename TypeTraits::Journaler Journaler;
typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta;
static PrepareRemoteImageRequest *create(Threads<ImageCtxT> *threads,
librados::IoCtx &remote_io_ctx,
const std::string &global_image_id,
const std::string &local_mirror_uuid,
const std::string &local_image_id,
std::string *remote_mirror_uuid,
std::string *remote_image_id,
Journaler **remote_journaler,
cls::journal::ClientState *client_state,
MirrorPeerClientMeta *client_meta,
Context *on_finish) {
return new PrepareRemoteImageRequest(io_ctx, global_image_id,
remote_mirror_uuid, remote_image_id,
on_finish);
return new PrepareRemoteImageRequest(threads, remote_io_ctx,
global_image_id, local_mirror_uuid,
local_image_id, remote_mirror_uuid,
remote_image_id, remote_journaler,
client_state, client_meta, on_finish);
}
PrepareRemoteImageRequest(librados::IoCtx &io_ctx,
PrepareRemoteImageRequest(Threads<ImageCtxT> *threads,
librados::IoCtx &remote_io_ctx,
const std::string &global_image_id,
const std::string &local_mirror_uuid,
const std::string &local_image_id,
std::string *remote_mirror_uuid,
std::string *remote_image_id,
Journaler **remote_journaler,
cls::journal::ClientState *client_state,
MirrorPeerClientMeta *client_meta,
Context *on_finish)
: m_io_ctx(io_ctx), m_global_image_id(global_image_id),
: m_threads(threads), m_remote_io_ctx(remote_io_ctx),
m_global_image_id(global_image_id),
m_local_mirror_uuid(local_mirror_uuid), m_local_image_id(local_image_id),
m_remote_mirror_uuid(remote_mirror_uuid),
m_remote_image_id(remote_image_id),
m_on_finish(on_finish) {
m_remote_journaler(remote_journaler), m_client_state(client_state),
m_client_meta(client_meta), m_on_finish(on_finish) {
}
void send();
@ -56,18 +84,31 @@ private:
* GET_REMOTE_IMAGE_ID
* |
* v
* GET_CLIENT
* |
* v (skip if not needed)
* REGISTER_CLIENT
* |
* v
* <finish>
* @endverbatim
*/
librados::IoCtx &m_io_ctx;
Threads<ImageCtxT> *m_threads;
librados::IoCtx &m_remote_io_ctx;
std::string m_global_image_id;
std::string m_local_mirror_uuid;
std::string m_local_image_id;
std::string *m_remote_mirror_uuid;
std::string *m_remote_image_id;
Journaler **m_remote_journaler;
cls::journal::ClientState *m_client_state;
MirrorPeerClientMeta *m_client_meta;
Context *m_on_finish;
bufferlist m_out_bl;
cls::journal::Client m_client;
void get_remote_mirror_uuid();
void handle_get_remote_mirror_uuid(int r);
@ -75,8 +116,15 @@ private:
void get_remote_image_id();
void handle_get_remote_image_id(int r);
void get_client();
void handle_get_client(int r);
void register_client();
void handle_register_client(int r);
void finish(int r);
bool decode_client_meta();
};
} // namespace image_replayer