librbd: async journal reset state machine

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

src/librbd/CMakeLists.txt

@@ -65,6 +65,7 @@ set(librbd_internal_srcs
   journal/PromoteRequest.cc
   journal/RemoveRequest.cc
   journal/Replay.cc
+  journal/ResetRequest.cc
   journal/StandardPolicy.cc
   journal/Utils.cc
   managed_lock/AcquireRequest.cc

src/librbd/Journal.cc

@@ -20,6 +20,7 @@
 #include "librbd/journal/DemoteRequest.h"
 #include "librbd/journal/OpenRequest.h"
 #include "librbd/journal/RemoveRequest.h"
+#include "librbd/journal/ResetRequest.h"
 #include "librbd/journal/Replay.h"
 #include "librbd/journal/PromoteRequest.h"
 
@@ -405,54 +406,17 @@ int Journal<I>::reset(librados::IoCtx &io_ctx, const std::string &image_id) {
   CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
   ldout(cct, 5) << __func__ << ": image=" << image_id << dendl;
 
-  Journaler journaler(io_ctx, image_id, IMAGE_CLIENT_ID, {});
+  ThreadPool *thread_pool;
+  ContextWQ *op_work_queue;
+  ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
 
   C_SaferCond cond;
-  journaler.init(&cond);
-  BOOST_SCOPE_EXIT_ALL(&journaler) {
-    journaler.shut_down();
-  };
+  auto req = journal::ResetRequest<I>::create(io_ctx, image_id, IMAGE_CLIENT_ID,
+                                              Journal<>::LOCAL_MIRROR_UUID,
+                                              op_work_queue, &cond);
+  req->send();
 
-  int r = cond.wait();
-  if (r == -ENOENT) {
-    return 0;
-  } else if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to initialize journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  uint8_t order, splay_width;
-  int64_t pool_id;
-  journaler.get_metadata(&order, &splay_width, &pool_id);
-
-  std::string pool_name;
-  if (pool_id != -1) {
-    librados::Rados rados(io_ctx);
-    r = rados.pool_reverse_lookup(pool_id, &pool_name);
-    if (r < 0) {
-      lderr(cct) << __func__ << ": "
-                 << "failed to lookup data pool: " << cpp_strerror(r) << dendl;
-      return r;
-    }
-  }
-
-  C_SaferCond ctx1;
-  journaler.remove(true, &ctx1);
-  r = ctx1.wait();
-  if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to reset journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  r = create(io_ctx, image_id, order, splay_width, pool_name);
-  if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to create journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-  return 0;
+  return cond.wait();
 }
 
 template <typename I>

src/librbd/journal/ResetRequest.cc

@@ -0,0 +1,162 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/journal/ResetRequest.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/Timer.h"
+#include "common/WorkQueue.h"
+#include "journal/Journaler.h"
+#include "journal/Settings.h"
+#include "include/assert.h"
+#include "librbd/Journal.h"
+#include "librbd/Utils.h"
+#include "librbd/journal/CreateRequest.h"
+#include "librbd/journal/RemoveRequest.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::journal::ResetRequest: " << this << " " \
+                           << __func__ << ": "
+
+namespace librbd {
+namespace journal {
+
+using util::create_async_context_callback;
+using util::create_context_callback;
+
+template<typename I>
+void ResetRequest<I>::send() {
+   init_journaler();
+}
+
+template<typename I>
+void ResetRequest<I>::init_journaler() {
+  ldout(m_cct, 10) << dendl;
+
+  m_journaler = new Journaler(m_io_ctx, m_image_id, m_client_id, {});
+  Context *ctx = create_context_callback<
+     ResetRequest<I>, &ResetRequest<I>::handle_init_journaler>(this);
+  m_journaler->init(ctx);
+}
+
+template<typename I>
+void ResetRequest<I>::handle_init_journaler(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    ldout(m_cct, 5) << "journal does not exist" << dendl;
+    m_ret_val = r;
+  } else if (r < 0) {
+    lderr(m_cct) << "failed to init journaler: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+  } else {
+    int64_t pool_id;
+    m_journaler->get_metadata(&m_order, &m_splay_width, &pool_id);
+
+    if (pool_id != -1) {
+      librados::Rados rados(m_io_ctx);
+      r = rados.pool_reverse_lookup(pool_id, &m_object_pool_name);
+      if (r < 0) {
+        lderr(m_cct) << "failed to lookup data pool: " << cpp_strerror(r)
+                     << dendl;
+        m_ret_val = r;
+      }
+    }
+  }
+
+  shut_down_journaler();
+}
+
+template<typename I>
+void ResetRequest<I>::shut_down_journaler() {
+  ldout(m_cct, 10) << dendl;
+
+  Context *ctx = create_async_context_callback(
+    m_op_work_queue, create_context_callback<
+      ResetRequest<I>, &ResetRequest<I>::handle_journaler_shutdown>(this));
+  m_journaler->shut_down(ctx);
+}
+
+template<typename I>
+void ResetRequest<I>::handle_journaler_shutdown(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  delete m_journaler;
+  if (r < 0) {
+    lderr(m_cct) << "failed to shut down journaler: " << cpp_strerror(r)
+                 << dendl;
+    if (m_ret_val == 0) {
+      m_ret_val = r;
+    }
+  }
+
+  if (m_ret_val < 0) {
+    finish(m_ret_val);
+    return;
+  }
+
+  remove_journal();
+}
+
+template<typename I>
+void ResetRequest<I>::remove_journal() {
+  ldout(m_cct, 10) << dendl;
+
+  Context *ctx = create_context_callback<
+    ResetRequest<I>, &ResetRequest<I>::handle_remove_journal>(this);
+  auto req = RemoveRequest<I>::create(m_io_ctx, m_image_id, m_client_id,
+                                      m_op_work_queue, ctx);
+  req->send();
+}
+
+template<typename I>
+void ResetRequest<I>::handle_remove_journal(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to remove journal: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  create_journal();
+}
+
+template<typename I>
+void ResetRequest<I>::create_journal() {
+  ldout(m_cct, 10) << dendl;
+
+  Context *ctx = create_context_callback<
+    ResetRequest<I>, &ResetRequest<I>::handle_create_journal>(this);
+  journal::TagData tag_data(m_mirror_uuid);
+  auto req = CreateRequest<I>::create(m_io_ctx, m_image_id, m_order,
+                                      m_splay_width, m_object_pool_name,
+                                      cls::journal::Tag::TAG_CLASS_NEW,
+                                      tag_data, m_client_id, m_op_work_queue,
+                                      ctx);
+  req->send();
+}
+
+template<typename I>
+void ResetRequest<I>::handle_create_journal(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to create journal: " << cpp_strerror(r) << dendl;
+  }
+  finish(r);
+}
+
+template<typename I>
+void ResetRequest<I>::finish(int r) {
+   ldout(m_cct, 10) << "r=" << r << dendl;
+
+   m_on_finish->complete(r);
+   delete this;
+}
+
+} // namespace journal
+} // namespace librbd
+
+template class librbd::journal::ResetRequest<librbd::ImageCtx>;

src/librbd/journal/ResetRequest.h

@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_JOURNAL_RESET_REQUEST_H
+#define CEPH_LIBRBD_JOURNAL_RESET_REQUEST_H
+
+#include "include/int_types.h"
+#include "include/buffer.h"
+#include "include/rados/librados.hpp"
+#include "include/rbd/librbd.hpp"
+#include "common/Mutex.h"
+#include "librbd/journal/TypeTraits.h"
+#include <string>
+
+class Context;
+class ContextWQ;
+class SafeTimer;
+
+namespace journal { class Journaler; }
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace journal {
+
+template<typename ImageCtxT = ImageCtx>
+class ResetRequest {
+public:
+  static ResetRequest *create(librados::IoCtx &io_ctx,
+                              const std::string &image_id,
+                              const std::string &client_id,
+                              const std::string &mirror_uuid,
+                              ContextWQ *op_work_queue, Context *on_finish) {
+    return new ResetRequest(io_ctx, image_id, client_id, mirror_uuid,
+                            op_work_queue, on_finish);
+  }
+
+  ResetRequest(librados::IoCtx &io_ctx, const std::string &image_id,
+               const std::string &client_id, const std::string &mirror_uuid,
+               ContextWQ *op_work_queue, Context *on_finish)
+    : m_io_ctx(io_ctx), m_image_id(image_id), m_client_id(client_id),
+      m_mirror_uuid(mirror_uuid), m_op_work_queue(op_work_queue),
+      m_on_finish(on_finish),
+      m_cct(reinterpret_cast<CephContext *>(m_io_ctx.cct())) {
+  }
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * INIT_JOURNALER
+   *    |
+   *    v
+   * SHUT_DOWN_JOURNALER
+   *    |
+   *    v
+   * REMOVE_JOURNAL
+   *    |
+   *    v
+   * CREATE_JOURNAL
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+  typedef typename TypeTraits<ImageCtxT>::Journaler Journaler;
+
+  librados::IoCtx &m_io_ctx;
+  std::string m_image_id;
+  std::string m_client_id;
+  std::string m_mirror_uuid;
+  ContextWQ *m_op_work_queue;
+  Context *m_on_finish;
+
+  CephContext *m_cct;
+  Journaler *m_journaler = nullptr;
+  int m_ret_val = 0;
+
+  uint8_t m_order = 0;
+  uint8_t m_splay_width = 0;
+  std::string m_object_pool_name;
+
+  void init_journaler();
+  void handle_init_journaler(int r);
+
+  void shut_down_journaler();
+  void handle_journaler_shutdown(int r);
+
+  void remove_journal();
+  void handle_remove_journal(int r);
+
+  void create_journal();
+  void handle_create_journal(int r);
+
+  void finish(int r);
+
+};
+
+} // namespace journal
+} // namespace librbd
+
+extern template class librbd::journal::ResetRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_JOURNAL_REMOVE_REQUEST_H

src/test/librbd/CMakeLists.txt

@@ -51,6 +51,7 @@ set(unittest_librbd_srcs
   journal/test_mock_OpenRequest.cc
   journal/test_mock_PromoteRequest.cc
   journal/test_mock_Replay.cc
+  journal/test_mock_ResetRequest.cc
   managed_lock/test_mock_AcquireRequest.cc
   managed_lock/test_mock_BreakRequest.cc
   managed_lock/test_mock_GetLockerRequest.cc

src/test/librbd/journal/test_mock_ResetRequest.cc

@@ -0,0 +1,264 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_mock_fixture.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/journal/mock/MockJournaler.h"
+#include "common/Mutex.h"
+#include "cls/journal/cls_journal_types.h"
+#include "librbd/journal/CreateRequest.h"
+#include "librbd/journal/RemoveRequest.h"
+#include "librbd/journal/ResetRequest.h"
+
+namespace librbd {
+
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+  MockTestImageCtx(librbd::ImageCtx& image_ctx) : MockImageCtx(image_ctx) {
+  }
+};
+
+} // anonymous namespace
+
+namespace journal {
+
+template <>
+struct TypeTraits<MockTestImageCtx> {
+  typedef ::journal::MockJournalerProxy Journaler;
+};
+
+template <>
+struct CreateRequest<MockTestImageCtx> {
+  static CreateRequest* s_instance;
+  Context* on_finish = nullptr;
+
+  static CreateRequest* create(IoCtx &ioctx, const std::string &imageid,
+                               uint8_t order, uint8_t splay_width,
+                               const std::string &object_pool,
+                               uint64_t tag_class, TagData &tag_data,
+                               const std::string &client_id,
+                               ContextWQ *op_work_queue, Context *on_finish) {
+    assert(s_instance != nullptr);
+    s_instance->on_finish = on_finish;
+    return s_instance;
+  }
+
+  MOCK_METHOD0(send, void());
+
+  CreateRequest() {
+    s_instance = this;
+  }
+};
+
+template <>
+struct RemoveRequest<MockTestImageCtx> {
+  static RemoveRequest* s_instance;
+  Context* on_finish = nullptr;
+
+  static RemoveRequest* create(IoCtx &ioctx, const std::string &image_id,
+                               const std::string &client_id,
+                               ContextWQ *op_work_queue, Context *on_finish) {
+    assert(s_instance != nullptr);
+    s_instance->on_finish = on_finish;
+    return s_instance;
+  }
+
+  MOCK_METHOD0(send, void());
+
+  RemoveRequest() {
+    s_instance = this;
+  }
+};
+
+CreateRequest<MockTestImageCtx>* CreateRequest<MockTestImageCtx>::s_instance = nullptr;
+RemoveRequest<MockTestImageCtx>* RemoveRequest<MockTestImageCtx>::s_instance = nullptr;
+
+} // namespace journal
+} // namespace librbd
+
+#include "librbd/journal/ResetRequest.cc"
+
+namespace librbd {
+namespace journal {
+
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::WithArg;
+
+class TestMockJournalResetRequest : public TestMockFixture {
+public:
+  typedef ResetRequest<MockTestImageCtx> MockResetRequest;
+  typedef CreateRequest<MockTestImageCtx> MockCreateRequest;
+  typedef RemoveRequest<MockTestImageCtx> MockRemoveRequest;
+
+  void expect_construct_journaler(::journal::MockJournaler &mock_journaler) {
+    EXPECT_CALL(mock_journaler, construct());
+  }
+
+  void expect_init_journaler(::journal::MockJournaler &mock_journaler, int r) {
+    EXPECT_CALL(mock_journaler, init(_))
+      .WillOnce(CompleteContext(r, static_cast<ContextWQ*>(NULL)));
+  }
+
+  void expect_get_metadata(::journal::MockJournaler& mock_journaler) {
+    EXPECT_CALL(mock_journaler, get_metadata(_, _, _))
+      .WillOnce(Invoke([](uint8_t* order, uint8_t* splay_width,
+                          int64_t* pool_id) {
+                  *order = 24;
+                  *splay_width = 4;
+                  *pool_id = -1;
+                }));
+  }
+
+  void expect_shut_down_journaler(::journal::MockJournaler &mock_journaler,
+                                  int r) {
+    EXPECT_CALL(mock_journaler, shut_down(_))
+      .WillOnce(CompleteContext(r, static_cast<ContextWQ*>(NULL)));
+  }
+
+  void expect_remove(MockRemoveRequest& mock_remove_request, int r) {
+    EXPECT_CALL(mock_remove_request, send())
+      .WillOnce(Invoke([&mock_remove_request, r]() {
+                  mock_remove_request.on_finish->complete(r);
+                }));
+  }
+
+  void expect_create(MockCreateRequest& mock_create_request, int r) {
+    EXPECT_CALL(mock_create_request, send())
+      .WillOnce(Invoke([&mock_create_request, r]() {
+                  mock_create_request.on_finish->complete(r);
+                }));
+  }
+};
+
+TEST_F(TestMockJournalResetRequest, Success) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  InSequence seq;
+  ::journal::MockJournaler mock_journaler;
+  expect_construct_journaler(mock_journaler);
+  expect_init_journaler(mock_journaler, 0);
+  expect_get_metadata(mock_journaler);
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  MockRemoveRequest mock_remove_request;
+  expect_remove(mock_remove_request, 0);
+
+  MockCreateRequest mock_create_request;
+  expect_create(mock_create_request, 0);
+
+  C_SaferCond ctx;
+  auto req = MockResetRequest::create(m_ioctx, "image id",
+                                      Journal<>::IMAGE_CLIENT_ID,
+                                      Journal<>::LOCAL_MIRROR_UUID,
+                                      ictx->op_work_queue , &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+}
+
+TEST_F(TestMockJournalResetRequest, InitError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  InSequence seq;
+  ::journal::MockJournaler mock_journaler;
+  expect_construct_journaler(mock_journaler);
+  expect_init_journaler(mock_journaler, -EINVAL);
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  C_SaferCond ctx;
+  auto req = MockResetRequest::create(m_ioctx, "image id",
+                                      Journal<>::IMAGE_CLIENT_ID,
+                                      Journal<>::LOCAL_MIRROR_UUID,
+                                      ictx->op_work_queue , &ctx);
+  req->send();
+  ASSERT_EQ(-EINVAL, ctx.wait());
+}
+
+TEST_F(TestMockJournalResetRequest, ShutDownError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  InSequence seq;
+  ::journal::MockJournaler mock_journaler;
+  expect_construct_journaler(mock_journaler);
+  expect_init_journaler(mock_journaler, 0);
+  expect_get_metadata(mock_journaler);
+  expect_shut_down_journaler(mock_journaler, -EINVAL);
+
+  C_SaferCond ctx;
+  auto req = MockResetRequest::create(m_ioctx, "image id",
+                                      Journal<>::IMAGE_CLIENT_ID,
+                                      Journal<>::LOCAL_MIRROR_UUID,
+                                      ictx->op_work_queue , &ctx);
+  req->send();
+  ASSERT_EQ(-EINVAL, ctx.wait());
+}
+
+TEST_F(TestMockJournalResetRequest, RemoveError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  InSequence seq;
+  ::journal::MockJournaler mock_journaler;
+  expect_construct_journaler(mock_journaler);
+  expect_init_journaler(mock_journaler, 0);
+  expect_get_metadata(mock_journaler);
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  MockRemoveRequest mock_remove_request;
+  expect_remove(mock_remove_request, -EINVAL);
+
+  C_SaferCond ctx;
+  auto req = MockResetRequest::create(m_ioctx, "image id",
+                                      Journal<>::IMAGE_CLIENT_ID,
+                                      Journal<>::LOCAL_MIRROR_UUID,
+                                      ictx->op_work_queue , &ctx);
+  req->send();
+  ASSERT_EQ(-EINVAL, ctx.wait());
+}
+
+TEST_F(TestMockJournalResetRequest, CreateError) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  InSequence seq;
+  ::journal::MockJournaler mock_journaler;
+  expect_construct_journaler(mock_journaler);
+  expect_init_journaler(mock_journaler, 0);
+  expect_get_metadata(mock_journaler);
+  expect_shut_down_journaler(mock_journaler, 0);
+
+  MockRemoveRequest mock_remove_request;
+  expect_remove(mock_remove_request, 0);
+
+  MockCreateRequest mock_create_request;
+  expect_create(mock_create_request, -EINVAL);
+
+  C_SaferCond ctx;
+  auto req = MockResetRequest::create(m_ioctx, "image id",
+                                      Journal<>::IMAGE_CLIENT_ID,
+                                      Journal<>::LOCAL_MIRROR_UUID,
+                                      ictx->op_work_queue , &ctx);
+  req->send();
+  ASSERT_EQ(-EINVAL, ctx.wait());
+}
+
+} // namespace journal
+} // namespace librbd