librbd: store mirror peer uuids in non-primary demoted snapshots

This will allow a remote rbd-mirror process to have a snapshot to use for
delta sync operations during failover.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
This commit is contained in:
Jason Dillaman 2020-04-08 19:41:23 -04:00
parent eed00eb179
commit 0102ce8870
3 changed files with 154 additions and 6 deletions

View File

@ -25,6 +25,19 @@ namespace snapshot {
using librbd::util::create_context_callback;
using librbd::util::create_rados_callback;
template <typename I>
CreateNonPrimaryRequest<I>::CreateNonPrimaryRequest(
I* image_ctx, bool demoted, const std::string &primary_mirror_uuid,
uint64_t primary_snap_id, const SnapSeqs& snap_seqs,
const ImageState &image_state, uint64_t *snap_id, Context *on_finish)
: m_image_ctx(image_ctx), m_demoted(demoted),
m_primary_mirror_uuid(primary_mirror_uuid),
m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs),
m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) {
m_default_ns_ctx.dup(m_image_ctx->md_ctx);
m_default_ns_ctx.set_namespace("");
}
template <typename I>
void CreateNonPrimaryRequest<I>::send() {
refresh_image();
@ -110,6 +123,56 @@ void CreateNonPrimaryRequest<I>::handle_get_mirror_image(int r) {
m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." +
uuid_gen.to_string();
get_mirror_peers();
}
template <typename I>
void CreateNonPrimaryRequest<I>::get_mirror_peers() {
if (!m_demoted) {
create_snapshot();
return;
}
CephContext *cct = m_image_ctx->cct;
ldout(cct, 20) << dendl;
librados::ObjectReadOperation op;
cls_client::mirror_peer_list_start(&op);
auto aio_comp = create_rados_callback<
CreateNonPrimaryRequest<I>,
&CreateNonPrimaryRequest<I>::handle_get_mirror_peers>(this);
m_out_bl.clear();
int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
ceph_assert(r == 0);
aio_comp->release();
}
template <typename I>
void CreateNonPrimaryRequest<I>::handle_get_mirror_peers(int r) {
CephContext *cct = m_image_ctx->cct;
ldout(cct, 20) << "r=" << r << dendl;
std::vector<cls::rbd::MirrorPeer> peers;
if (r == 0) {
auto iter = m_out_bl.cbegin();
r = cls_client::mirror_peer_list_finish(&iter, &peers);
}
if (r < 0) {
lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r)
<< dendl;
finish(r);
return;
}
for (auto &peer : peers) {
if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
continue;
}
m_mirror_peer_uuids.insert(peer.uuid);
}
create_snapshot();
}
@ -121,6 +184,9 @@ void CreateNonPrimaryRequest<I>::create_snapshot() {
(m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED :
cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {},
m_primary_mirror_uuid, m_primary_snap_id};
if (m_demoted) {
ns.mirror_peer_uuids = m_mirror_peer_uuids;
}
ns.snap_seqs = m_snap_seqs;
ns.complete = is_orphan();
ldout(cct, 20) << "ns=" << ns << dendl;

View File

@ -43,12 +43,7 @@ public:
uint64_t primary_snap_id,
const SnapSeqs& snap_seqs,
const ImageState &image_state, uint64_t *snap_id,
Context *on_finish)
: m_image_ctx(image_ctx), m_demoted(demoted),
m_primary_mirror_uuid(primary_mirror_uuid),
m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs),
m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) {
}
Context *on_finish);
void send();
@ -64,6 +59,9 @@ private:
* v
* GET_MIRROR_IMAGE
* |
* v (skip if not needed)
* GET_MIRROR_PEERS
* |
* v
* CREATE_SNAPSHOT
* |
@ -85,6 +83,9 @@ private:
uint64_t *m_snap_id;
Context *m_on_finish;
librados::IoCtx m_default_ns_ctx;
std::set<std::string> m_mirror_peer_uuids;
std::string m_snap_name;
bufferlist m_out_bl;

View File

@ -102,6 +102,14 @@ public:
typedef WriteImageStateRequest<MockTestImageCtx> MockWriteImageStateRequest;
typedef util::Mock MockUtils;
void expect_clone_md_ctx(MockTestImageCtx &mock_image_ctx) {
EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), clone())
.WillOnce(Invoke([&mock_image_ctx]() {
get_mock_io_ctx(mock_image_ctx.md_ctx).get();
return &get_mock_io_ctx(mock_image_ctx.md_ctx);
}));
}
void expect_refresh_image(MockTestImageCtx &mock_image_ctx,
bool refresh_required, int r) {
EXPECT_CALL(*mock_image_ctx.state, is_refresh_required())
@ -132,6 +140,20 @@ public:
.WillOnce(Return(result));
}
void expect_get_mirror_peers(MockTestImageCtx &mock_image_ctx,
const std::vector<cls::rbd::MirrorPeer> &peers,
int r) {
using ceph::encode;
bufferlist bl;
encode(peers, bl);
EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx),
exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_peer_list"),
_, _, _))
.WillOnce(DoAll(WithArg<5>(CopyInBufferlist(bl)),
Return(r)));
}
void expect_create_snapshot(MockTestImageCtx &mock_image_ctx, int r) {
EXPECT_CALL(*mock_image_ctx.operations, snap_create(_, _, _))
.WillOnce(WithArg<2>(CompleteContext(
@ -179,6 +201,38 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, Success) {
ASSERT_EQ(0, ctx.wait());
}
TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, SuccessDemoted) {
REQUIRE_FORMAT_V2();
librbd::ImageCtx *ictx;
ASSERT_EQ(0, open_image(m_image_name, &ictx));
MockTestImageCtx mock_image_ctx(*ictx);
InSequence seq;
expect_clone_md_ctx(mock_image_ctx);
expect_refresh_image(mock_image_ctx, true, 0);
expect_get_mirror_image(
mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid",
cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0);
MockUtils mock_utils;
expect_can_create_non_primary_snapshot(mock_utils, true);
expect_get_mirror_peers(mock_image_ctx,
{{"uuid", cls::rbd::MIRROR_PEER_DIRECTION_TX, "ceph",
"mirror", "mirror uuid"}}, 0);
expect_create_snapshot(mock_image_ctx, 0);
MockWriteImageStateRequest mock_write_image_state_request;
expect_write_image_state(mock_image_ctx, mock_write_image_state_request, 0);
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
"mirror_uuid", 123, {{1, 2}}, {},
nullptr, &ctx);
req->send();
ASSERT_EQ(0, ctx.wait());
}
TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, RefreshError) {
REQUIRE_FORMAT_V2();
@ -247,6 +301,33 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CanNotError) {
ASSERT_EQ(-EINVAL, ctx.wait());
}
TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, GetMirrorPeersError) {
REQUIRE_FORMAT_V2();
librbd::ImageCtx *ictx;
ASSERT_EQ(0, open_image(m_image_name, &ictx));
MockTestImageCtx mock_image_ctx(*ictx);
InSequence seq;
expect_clone_md_ctx(mock_image_ctx);
expect_refresh_image(mock_image_ctx, true, 0);
expect_get_mirror_image(
mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid",
cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0);
MockUtils mock_utils;
expect_can_create_non_primary_snapshot(mock_utils, true);
expect_get_mirror_peers(mock_image_ctx, {}, -EPERM);
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
"mirror_uuid", 123, {{1, 2}}, {},
nullptr, &ctx);
req->send();
ASSERT_EQ(-EPERM, ctx.wait());
}
TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CreateSnapshotError) {
REQUIRE_FORMAT_V2();