rbd-mirror: new image sync point create/prune state machines

These are responsible for creating/removing snapshots in the
remote image and updating the client registration to account
for the snapshots.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

src/tools/Makefile-client.am

@@ -89,7 +89,9 @@ librbd_mirror_internal_la_SOURCES = \
 	tools/rbd_mirror/Threads.cc \
 	tools/rbd_mirror/types.cc \
 	tools/rbd_mirror/image_sync/ObjectCopyRequest.cc \
-	tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc
+	tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc \
+	tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc \
+	tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc
 noinst_LTLIBRARIES += librbd_mirror_internal.la
 noinst_HEADERS += \
 	tools/rbd_mirror/ClusterWatcher.h \
@@ -100,7 +102,9 @@ noinst_HEADERS += \
 	tools/rbd_mirror/Threads.h \
 	tools/rbd_mirror/types.h \
 	tools/rbd_mirror/image_sync/ObjectCopyRequest.h \
-	tools/rbd_mirror/image_sync/SnapshotCopyRequest.h
+	tools/rbd_mirror/image_sync/SnapshotCopyRequest.h \
+	tools/rbd_mirror/image_sync/SyncPointCreateRequest.h \
+	tools/rbd_mirror/image_sync/SyncPointPruneRequest.h
 
 rbd_mirror_SOURCES = \
 	tools/rbd_mirror/main.cc

src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc

@@ -0,0 +1,162 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "SyncPointCreateRequest.h"
+#include "include/uuid.h"
+#include "common/errno.h"
+#include "journal/Journaler.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointCreateRequest: " \
+                           << this << " " << __func__
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+namespace {
+
+static const std::string SNAP_NAME_PREFIX(".rbd-mirror");
+
+} // anonymous namespace
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+SyncPointCreateRequest<I>::SyncPointCreateRequest(I *remote_image_ctx,
+                                                  const std::string &mirror_uuid,
+                                                  Journaler *journaler,
+                                                  MirrorPeerClientMeta *client_meta,
+                                                  Context *on_finish)
+  : m_remote_image_ctx(remote_image_ctx), m_mirror_uuid(mirror_uuid),
+    m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish),
+    m_client_meta_copy(*client_meta) {
+  assert(m_client_meta->sync_points.size() < 2);
+
+  // initialize the updated client meta with the new sync point
+  m_client_meta_copy.sync_points.emplace_back();
+  if (m_client_meta_copy.sync_points.size() > 1) {
+    m_client_meta_copy.sync_points.back().from_snap_name =
+      m_client_meta_copy.sync_points.front().snap_name;
+  }
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send() {
+  send_update_client();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_update_client() {
+  uuid_d uuid_gen;
+  uuid_gen.generate_random();
+
+  MirrorPeerSyncPoint &sync_point = m_client_meta_copy.sync_points.back();
+  sync_point.snap_name = SNAP_NAME_PREFIX + "." + m_mirror_uuid + "." +
+                         uuid_gen.to_string();
+
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": sync_point=" << sync_point << dendl;
+
+  bufferlist client_data_bl;
+  librbd::journal::ClientData client_data(m_client_meta_copy);
+  ::encode(client_data, client_data_bl);
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>, &SyncPointCreateRequest<I>::handle_update_client>(
+      this);
+  m_journaler->update_client(client_data_bl, ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_update_client(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  // update provided meta structure to reflect reality
+  *m_client_meta = m_client_meta_copy;
+
+  send_refresh_image();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_refresh_image() {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>, &SyncPointCreateRequest<I>::handle_refresh_image>(
+      this);
+  m_remote_image_ctx->state->refresh(ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_refresh_image(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "remote image refresh failed: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_create_snap();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_create_snap() {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  MirrorPeerSyncPoint &sync_point = m_client_meta_copy.sync_points.back();
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>, &SyncPointCreateRequest<I>::handle_create_snap>(
+      this);
+  m_remote_image_ctx->operations->snap_create(
+    sync_point.snap_name.c_str(), ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_create_snap(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r == -EEXIST) {
+    send_update_client();
+    return;
+  } else if (r < 0) {
+    lderr(cct) << "failed to create snapshot: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::finish(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_sync::SyncPointCreateRequest<librbd::ImageCtx>;

src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h

@@ -0,0 +1,90 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H
+#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H
+
+#include "librbd/Journal.h"
+#include "librbd/journal/Types.h"
+#include <string>
+
+class Context;
+namespace journal { class Journaler; }
+namespace librbd { class ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class SyncPointCreateRequest {
+public:
+  typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits;
+  typedef typename TypeTraits::Journaler Journaler;
+  typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta;
+  typedef librbd::journal::MirrorPeerSyncPoint MirrorPeerSyncPoint;
+
+  static SyncPointCreateRequest* create(ImageCtxT *remote_image_ctx,
+                                        const std::string &mirror_uuid,
+                                        Journaler *journaler,
+                                        MirrorPeerClientMeta *client_meta,
+                                        Context *on_finish) {
+    return new SyncPointCreateRequest(remote_image_ctx, mirror_uuid, journaler,
+                                      client_meta, on_finish);
+  }
+
+  SyncPointCreateRequest(ImageCtxT *remote_image_ctx,
+                         const std::string &mirror_uuid, Journaler *journaler,
+                         MirrorPeerClientMeta *client_meta, Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * UPDATE_CLIENT < . .
+   *    |              .
+   *    v              .
+   * REFRESH_IMAGE     .
+   *    |              . (repeat on EEXIST)
+   *    v              .
+   * CREATE_SNAP . . . .
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ImageCtxT *m_remote_image_ctx;
+  std::string m_mirror_uuid;
+  Journaler *m_journaler;
+  MirrorPeerClientMeta *m_client_meta;
+  Context *m_on_finish;
+
+  MirrorPeerClientMeta m_client_meta_copy;
+
+  void send_update_client();
+  void handle_update_client(int r);
+
+  void send_refresh_image();
+  void handle_refresh_image(int r);
+
+  void send_create_snap();
+  void handle_create_snap(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_sync::SyncPointCreateRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H

src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc

@@ -0,0 +1,202 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "SyncPointPruneRequest.h"
+#include "common/errno.h"
+#include "journal/Journaler.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include <set>
+
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointPruneRequest: " \
+                           << this << " " << __func__
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+SyncPointPruneRequest<I>::SyncPointPruneRequest(I *remote_image_ctx,
+                                                bool sync_complete,
+                                                Journaler *journaler,
+                                                MirrorPeerClientMeta *client_meta,
+                                                Context *on_finish)
+  : m_remote_image_ctx(remote_image_ctx), m_sync_complete(sync_complete),
+    m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish),
+    m_client_meta_copy(*client_meta) {
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send() {
+  if (m_client_meta->sync_points.empty()) {
+    send_remove_snap();
+    return;
+  }
+
+  if (m_sync_complete) {
+    // if sync is complete, we can remove the master sync point
+    auto it = m_client_meta_copy.sync_points.begin();
+    MirrorPeerSyncPoint &sync_point = *it;
+
+    ++it;
+    if (it == m_client_meta_copy.sync_points.end() ||
+        it->from_snap_name != sync_point.snap_name) {
+      m_snap_names.push_back(sync_point.snap_name);
+    }
+
+    if (!sync_point.from_snap_name.empty()) {
+      m_snap_names.push_back(sync_point.from_snap_name);
+    }
+  } else {
+    // if we have more than one sync point, trim the extras off
+    std::set<std::string> snap_names;
+    for (auto it = m_client_meta_copy.sync_points.rbegin();
+         it != m_client_meta_copy.sync_points.rend(); ++it) {
+      MirrorPeerSyncPoint &sync_point =
+        m_client_meta_copy.sync_points.back();
+      if (&sync_point == &m_client_meta_copy.sync_points.front()) {
+        break;
+      }
+
+      if (snap_names.count(sync_point.snap_name) == 0) {
+        snap_names.insert(sync_point.snap_name);
+        m_snap_names.push_back(sync_point.snap_name);
+      }
+
+      MirrorPeerSyncPoint &front_sync_point =
+        m_client_meta_copy.sync_points.front();
+      if (!sync_point.from_snap_name.empty() &&
+          snap_names.count(sync_point.from_snap_name) == 0 &&
+          sync_point.from_snap_name != front_sync_point.snap_name) {
+        snap_names.insert(sync_point.from_snap_name);
+        m_snap_names.push_back(sync_point.from_snap_name);
+      }
+    }
+  }
+
+  send_remove_snap();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_remove_snap() {
+  if (m_snap_names.empty()) {
+    send_refresh_image();
+    return;
+  }
+
+  std::string snap_name = m_snap_names.front();
+
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": snap_name=" << snap_name << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointPruneRequest<I>, &SyncPointPruneRequest<I>::handle_remove_snap>(
+      this);
+  m_remote_image_ctx->operations->snap_remove(snap_name.c_str(), ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_remove_snap(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  assert(!m_snap_names.empty());
+  std::string snap_name = m_snap_names.front();
+  m_snap_names.pop_front();
+
+  if (r == -ENOENT) {
+    r = 0;
+  }
+  if (r < 0) {
+    lderr(cct) << "failed to remove snapshot '" << snap_name << "': "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_remove_snap();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_refresh_image() {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointPruneRequest<I>, &SyncPointPruneRequest<I>::handle_refresh_image>(
+      this);
+  m_remote_image_ctx->state->refresh(ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_refresh_image(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "remote image refresh failed: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_update_client();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_update_client() {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  if (m_sync_complete) {
+    m_client_meta_copy.sync_points.pop_front();
+  } else {
+    while (m_client_meta_copy.sync_points.size() > 1) {
+      m_client_meta_copy.sync_points.pop_back();
+    }
+  }
+
+  bufferlist client_data_bl;
+  librbd::journal::ClientData client_data(m_client_meta_copy);
+  ::encode(client_data, client_data_bl);
+
+  Context *ctx = create_context_callback<
+    SyncPointPruneRequest<I>, &SyncPointPruneRequest<I>::handle_update_client>(
+      this);
+  m_journaler->update_client(client_data_bl, ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_update_client(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  // update provided meta structure to reflect reality
+  *m_client_meta = m_client_meta_copy;
+  finish(0);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::finish(int r) {
+  CephContext *cct = m_remote_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::ImageCtx>;

src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h

@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H
+#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H
+
+#include "librbd/Journal.h"
+#include "librbd/journal/Types.h"
+#include <list>
+#include <string>
+
+class Context;
+namespace journal { class Journaler; }
+namespace librbd { class ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class SyncPointPruneRequest {
+public:
+  typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits;
+  typedef typename TypeTraits::Journaler Journaler;
+  typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta;
+  typedef librbd::journal::MirrorPeerSyncPoint MirrorPeerSyncPoint;
+
+  static SyncPointPruneRequest* create(ImageCtxT *remote_image_ctx,
+                                       bool sync_complete,
+                                       Journaler *journaler,
+                                       MirrorPeerClientMeta *client_meta,
+                                       Context *on_finish) {
+    return new SyncPointPruneRequest(remote_image_ctx, sync_complete, journaler,
+                                      client_meta, on_finish);
+  }
+
+  SyncPointPruneRequest(ImageCtxT *remote_image_ctx, bool sync_complete,
+                        Journaler *journaler, MirrorPeerClientMeta *client_meta,
+                        Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    |    . . . . .
+   *    |    .       .
+   *    v    v       . (repeat if from snap
+   * REMOVE_SNAP . . .  unused by other sync)
+   *    |
+   *    v
+   * REFRESH_IMAGE
+   *    |
+   *    v
+   * UPDATE_CLIENT
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ImageCtxT *m_remote_image_ctx;
+  bool m_sync_complete;
+  Journaler *m_journaler;
+  MirrorPeerClientMeta *m_client_meta;
+  Context *m_on_finish;
+
+  MirrorPeerClientMeta m_client_meta_copy;
+  std::list<std::string> m_snap_names;
+
+  void send_remove_snap();
+  void handle_remove_snap(int r);
+
+  void send_refresh_image();
+  void handle_refresh_image(int r);
+
+  void send_update_client();
+  void handle_update_client(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H