librbd: store mirror peer uuids in non-primary demoted snapshots

This will allow a remote rbd-mirror process to have a snapshot to use for
delta sync operations during failover.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc

@@ -25,6 +25,19 @@ namespace snapshot {
 using librbd::util::create_context_callback;
 using librbd::util::create_rados_callback;
 
+template <typename I>
+CreateNonPrimaryRequest<I>::CreateNonPrimaryRequest(
+    I* image_ctx, bool demoted, const std::string &primary_mirror_uuid,
+    uint64_t primary_snap_id, const SnapSeqs& snap_seqs,
+    const ImageState &image_state, uint64_t *snap_id, Context *on_finish)
+  : m_image_ctx(image_ctx), m_demoted(demoted),
+    m_primary_mirror_uuid(primary_mirror_uuid),
+    m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs),
+    m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) {
+  m_default_ns_ctx.dup(m_image_ctx->md_ctx);
+  m_default_ns_ctx.set_namespace("");
+}
+
 template <typename I>
 void CreateNonPrimaryRequest<I>::send() {
   refresh_image();
@@ -110,6 +123,56 @@ void CreateNonPrimaryRequest<I>::handle_get_mirror_image(int r) {
   m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." +
     uuid_gen.to_string();
 
+  get_mirror_peers();
+}
+
+template <typename I>
+void CreateNonPrimaryRequest<I>::get_mirror_peers() {
+  if (!m_demoted) {
+    create_snapshot();
+    return;
+  }
+
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  librados::ObjectReadOperation op;
+  cls_client::mirror_peer_list_start(&op);
+
+  auto aio_comp = create_rados_callback<
+    CreateNonPrimaryRequest<I>,
+    &CreateNonPrimaryRequest<I>::handle_get_mirror_peers>(this);
+  m_out_bl.clear();
+  int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void CreateNonPrimaryRequest<I>::handle_get_mirror_peers(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 20) << "r=" << r << dendl;
+
+  std::vector<cls::rbd::MirrorPeer> peers;
+  if (r == 0) {
+    auto iter = m_out_bl.cbegin();
+    r = cls_client::mirror_peer_list_finish(&iter, &peers);
+  }
+
+  if (r < 0) {
+    lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r)
+               << dendl;
+    finish(r);
+    return;
+  }
+
+  for (auto &peer : peers) {
+    if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
+      continue;
+    }
+    m_mirror_peer_uuids.insert(peer.uuid);
+  }
+
   create_snapshot();
 }
 
@@ -121,6 +184,9 @@ void CreateNonPrimaryRequest<I>::create_snapshot() {
     (m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED :
                  cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {},
     m_primary_mirror_uuid, m_primary_snap_id};
+  if (m_demoted) {
+    ns.mirror_peer_uuids = m_mirror_peer_uuids;
+  }
   ns.snap_seqs = m_snap_seqs;
   ns.complete = is_orphan();
   ldout(cct, 20) << "ns=" << ns << dendl;

src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h

@@ -43,12 +43,7 @@ public:
                           uint64_t primary_snap_id,
                           const SnapSeqs& snap_seqs,
                           const ImageState &image_state, uint64_t *snap_id,
-                          Context *on_finish)
-    : m_image_ctx(image_ctx), m_demoted(demoted),
-      m_primary_mirror_uuid(primary_mirror_uuid),
-      m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs),
-      m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) {
-  }
+                          Context *on_finish);
 
   void send();
 
@@ -64,6 +59,9 @@ private:
    *    v
    * GET_MIRROR_IMAGE
    *    |
+   *    v (skip if not needed)
+   * GET_MIRROR_PEERS
+   *    |
    *    v
    * CREATE_SNAPSHOT
    *    |
@@ -85,6 +83,9 @@ private:
   uint64_t *m_snap_id;
   Context *m_on_finish;
 
+  librados::IoCtx m_default_ns_ctx;
+  std::set<std::string> m_mirror_peer_uuids;
+
   std::string m_snap_name;
 
   bufferlist m_out_bl;

src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc

@@ -102,6 +102,14 @@ public:
   typedef WriteImageStateRequest<MockTestImageCtx> MockWriteImageStateRequest;
   typedef util::Mock MockUtils;
 
+  void expect_clone_md_ctx(MockTestImageCtx &mock_image_ctx) {
+    EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx), clone())
+      .WillOnce(Invoke([&mock_image_ctx]() {
+                         get_mock_io_ctx(mock_image_ctx.md_ctx).get();
+                         return &get_mock_io_ctx(mock_image_ctx.md_ctx);
+                       }));
+  }
+
   void expect_refresh_image(MockTestImageCtx &mock_image_ctx,
                             bool refresh_required, int r) {
     EXPECT_CALL(*mock_image_ctx.state, is_refresh_required())
@@ -132,6 +140,20 @@ public:
       .WillOnce(Return(result));
   }
 
+  void expect_get_mirror_peers(MockTestImageCtx &mock_image_ctx,
+                               const std::vector<cls::rbd::MirrorPeer> &peers,
+                               int r) {
+    using ceph::encode;
+    bufferlist bl;
+    encode(peers, bl);
+
+    EXPECT_CALL(get_mock_io_ctx(mock_image_ctx.md_ctx),
+                exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_peer_list"),
+                     _, _, _))
+      .WillOnce(DoAll(WithArg<5>(CopyInBufferlist(bl)),
+                      Return(r)));
+  }
+
   void expect_create_snapshot(MockTestImageCtx &mock_image_ctx, int r) {
     EXPECT_CALL(*mock_image_ctx.operations, snap_create(_, _, _))
       .WillOnce(WithArg<2>(CompleteContext(
@@ -179,6 +201,38 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, Success) {
   ASSERT_EQ(0, ctx.wait());
 }
 
+TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, SuccessDemoted) {
+  REQUIRE_FORMAT_V2();
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+
+  expect_clone_md_ctx(mock_image_ctx);
+  expect_refresh_image(mock_image_ctx, true, 0);
+  expect_get_mirror_image(
+    mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid",
+                     cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0);
+  MockUtils mock_utils;
+  expect_can_create_non_primary_snapshot(mock_utils, true);
+  expect_get_mirror_peers(mock_image_ctx,
+                          {{"uuid", cls::rbd::MIRROR_PEER_DIRECTION_TX, "ceph",
+                            "mirror", "mirror uuid"}}, 0);
+  expect_create_snapshot(mock_image_ctx, 0);
+  MockWriteImageStateRequest mock_write_image_state_request;
+  expect_write_image_state(mock_image_ctx, mock_write_image_state_request, 0);
+
+  C_SaferCond ctx;
+  auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
+                                             "mirror_uuid", 123, {{1, 2}}, {},
+                                             nullptr, &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+}
+
 TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, RefreshError) {
   REQUIRE_FORMAT_V2();
 
@@ -247,6 +301,33 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CanNotError) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
+TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, GetMirrorPeersError) {
+  REQUIRE_FORMAT_V2();
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+
+  expect_clone_md_ctx(mock_image_ctx);
+  expect_refresh_image(mock_image_ctx, true, 0);
+  expect_get_mirror_image(
+    mock_image_ctx, {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, "gid",
+                     cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, 0);
+  MockUtils mock_utils;
+  expect_can_create_non_primary_snapshot(mock_utils, true);
+  expect_get_mirror_peers(mock_image_ctx, {}, -EPERM);
+
+  C_SaferCond ctx;
+  auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
+                                             "mirror_uuid", 123, {{1, 2}}, {},
+                                             nullptr, &ctx);
+  req->send();
+  ASSERT_EQ(-EPERM, ctx.wait());
+}
+
 TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CreateSnapshotError) {
   REQUIRE_FORMAT_V2();