test: unit tests for librbd IO work queue failure path

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

src/librbd/io/ImageRequestWQ.cc

@@ -174,7 +174,7 @@ void ImageRequestWQ<I>::aio_read(AioCompletion *c, uint64_t off, uint64_t len,
     trace.event("start");
   }
 
-  c->init_time(&m_image_ctx, AIO_TYPE_READ);
+  c->init_time(util::get_image_ctx(&m_image_ctx), AIO_TYPE_READ);
   ldout(cct, 20) << "ictx=" << &m_image_ctx << ", "
                  << "completion=" << c << ", off=" << off << ", "
                  << "len=" << len << ", " << "flags=" << op_flags << dendl;
@@ -215,7 +215,7 @@ void ImageRequestWQ<I>::aio_write(AioCompletion *c, uint64_t off, uint64_t len,
     trace.event("init");
   }
 
-  c->init_time(&m_image_ctx, AIO_TYPE_WRITE);
+  c->init_time(util::get_image_ctx(&m_image_ctx), AIO_TYPE_WRITE);
   ldout(cct, 20) << "ictx=" << &m_image_ctx << ", "
                  << "completion=" << c << ", off=" << off << ", "
                  << "len=" << len << ", flags=" << op_flags << dendl;
@@ -252,7 +252,7 @@ void ImageRequestWQ<I>::aio_discard(AioCompletion *c, uint64_t off,
     trace.event("init");
   }
 
-  c->init_time(&m_image_ctx, AIO_TYPE_DISCARD);
+  c->init_time(util::get_image_ctx(&m_image_ctx), AIO_TYPE_DISCARD);
   ldout(cct, 20) << "ictx=" << &m_image_ctx << ", "
                  << "completion=" << c << ", off=" << off << ", len=" << len
                  << dendl;
@@ -287,7 +287,7 @@ void ImageRequestWQ<I>::aio_flush(AioCompletion *c, bool native_async) {
     trace.event("init");
   }
 
-  c->init_time(&m_image_ctx, AIO_TYPE_FLUSH);
+  c->init_time(util::get_image_ctx(&m_image_ctx), AIO_TYPE_FLUSH);
   ldout(cct, 20) << "ictx=" << &m_image_ctx << ", "
                  << "completion=" << c << dendl;
 
@@ -320,7 +320,7 @@ void ImageRequestWQ<I>::aio_writesame(AioCompletion *c, uint64_t off,
     trace.event("init");
   }
 
-  c->init_time(&m_image_ctx, AIO_TYPE_WRITESAME);
+  c->init_time(util::get_image_ctx(&m_image_ctx), AIO_TYPE_WRITESAME);
   ldout(cct, 20) << "ictx=" << &m_image_ctx << ", "
                  << "completion=" << c << ", off=" << off << ", "
                  << "len=" << len << ", data_len = " << bl.length() << ", "
@@ -575,6 +575,7 @@ int ImageRequestWQ<I>::start_in_flight_io(AioCompletion *c) {
     CephContext *cct = m_image_ctx.cct;
     lderr(cct) << "IO received on closed image" << dendl;
 
+    c->get();
     c->fail(-ESHUTDOWN);
     return false;
   }

src/librbd/io/ImageRequestWQ.h

@@ -24,7 +24,7 @@ class ReadResult;
 
 template <typename ImageCtxT = librbd::ImageCtx>
 class ImageRequestWQ
-  : protected ThreadPool::PointerWQ<ImageRequest<ImageCtxT> > {
+  : public ThreadPool::PointerWQ<ImageRequest<ImageCtxT> > {
 public:
   ImageRequestWQ(ImageCtxT *image_ctx, const string &name, time_t ti,
                  ThreadPool *tp);

src/test/librbd/CMakeLists.txt

@@ -37,6 +37,7 @@ set(unittest_librbd_srcs
   image/test_mock_RefreshRequest.cc
   image/test_mock_RemoveRequest.cc
   io/test_mock_ImageRequest.cc
+  io/test_mock_ImageRequestWQ.cc
   journal/test_mock_OpenRequest.cc
   journal/test_mock_PromoteRequest.cc
   journal/test_mock_Replay.cc

src/test/librbd/io/test_mock_ImageRequestWQ.cc

@@ -0,0 +1,279 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_mock_fixture.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librbd/mock/exclusive_lock/MockPolicy.h"
+#include "librbd/io/ImageRequestWQ.h"
+#include "librbd/io/ImageRequest.h"
+
+namespace librbd {
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+  MockTestImageCtx(ImageCtx &image_ctx) : MockImageCtx(image_ctx) {
+  }
+};
+
+} // anonymous namespace
+
+namespace io {
+
+template <>
+struct ImageRequest<librbd::MockTestImageCtx> {
+  static ImageRequest* s_instance;
+  AioCompletion *aio_comp;
+
+  static ImageRequest* create_write_request(librbd::MockTestImageCtx &image_ctx,
+                                            AioCompletion *aio_comp,
+                                            Extents &&image_extents,
+                                            bufferlist &&bl, int op_flags,
+                                            const ZTracer::Trace &parent_trace) {
+    assert(s_instance != nullptr);
+    s_instance->aio_comp = aio_comp;
+    return s_instance;
+  }
+  static void aio_write(librbd::MockTestImageCtx *ictx, AioCompletion *c,
+                        Extents &&image_extents, bufferlist &&bl, int op_flags,
+                        const ZTracer::Trace &parent_trace) {
+  }
+
+
+  MOCK_CONST_METHOD0(is_write_op, bool());
+  MOCK_CONST_METHOD0(start_op, void());
+  MOCK_CONST_METHOD0(send, void());
+  MOCK_CONST_METHOD1(fail, void(int));
+
+  ImageRequest() {
+    s_instance = this;
+  }
+};
+
+} // namespace io
+
+namespace util {
+
+inline ImageCtx *get_image_ctx(MockTestImageCtx *image_ctx) {
+  return image_ctx->image_ctx;
+}
+
+} // namespace util
+
+} // namespace librbd
+
+template <>
+struct ThreadPool::PointerWQ<librbd::io::ImageRequest<librbd::MockTestImageCtx>> {
+  typedef librbd::io::ImageRequest<librbd::MockTestImageCtx> ImageRequest;
+  static PointerWQ* s_instance;
+
+  Mutex m_lock;
+
+  PointerWQ(const std::string &name, time_t, int, ThreadPool *)
+    : m_lock(name) {
+    s_instance = this;
+  }
+  virtual ~PointerWQ() {
+  }
+
+  MOCK_METHOD0(drain, void());
+  MOCK_METHOD0(empty, bool());
+  MOCK_METHOD0(signal, void());
+  MOCK_METHOD0(process_finish, void());
+
+  MOCK_METHOD0(front, ImageRequest*());
+  MOCK_METHOD1(requeue, void(ImageRequest*));
+
+  MOCK_METHOD0(dequeue, void*());
+  MOCK_METHOD1(queue, void(ImageRequest*));
+
+  void register_work_queue() {
+    // no-op
+  }
+  Mutex &get_pool_lock() {
+    return m_lock;
+  }
+
+  void* invoke_dequeue() {
+    Mutex::Locker locker(m_lock);
+    return _void_dequeue();
+  }
+  void invoke_process(ImageRequest *image_request) {
+    process(image_request);
+  }
+
+  virtual void *_void_dequeue() {
+    return dequeue();
+  }
+  virtual void process(ImageRequest *req) = 0;
+
+};
+
+ThreadPool::PointerWQ<librbd::io::ImageRequest<librbd::MockTestImageCtx>>*
+  ThreadPool::PointerWQ<librbd::io::ImageRequest<librbd::MockTestImageCtx>>::s_instance = nullptr;
+librbd::io::ImageRequest<librbd::MockTestImageCtx>*
+  librbd::io::ImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+#include "librbd/io/ImageRequestWQ.cc"
+
+namespace librbd {
+namespace io {
+
+using ::testing::_;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::WithArg;
+
+struct TestMockIoImageRequestWQ : public TestMockFixture {
+  typedef ImageRequestWQ<librbd::MockTestImageCtx> MockImageRequestWQ;
+  typedef ImageRequest<librbd::MockTestImageCtx> MockImageRequest;
+
+  void expect_is_write_op(MockImageRequest &image_request, bool write_op) {
+    EXPECT_CALL(image_request, is_write_op()).WillOnce(Return(write_op));
+  }
+
+  void expect_signal(MockImageRequestWQ &image_request_wq) {
+    EXPECT_CALL(image_request_wq, signal());
+  }
+
+  void expect_queue(MockImageRequestWQ &image_request_wq) {
+    EXPECT_CALL(image_request_wq, queue(_));
+  }
+
+  void expect_front(MockImageRequestWQ &image_request_wq,
+                    MockImageRequest *image_request) {
+    EXPECT_CALL(image_request_wq, front()).WillOnce(Return(image_request));
+  }
+
+  void expect_is_refresh_request(MockTestImageCtx &mock_image_ctx,
+                                 bool required) {
+    EXPECT_CALL(*mock_image_ctx.state, is_refresh_required()).WillOnce(
+      Return(required));
+  }
+
+  void expect_dequeue(MockImageRequestWQ &image_request_wq,
+                      MockImageRequest *image_request) {
+    EXPECT_CALL(image_request_wq, dequeue()).WillOnce(Return(image_request));
+  }
+
+  void expect_get_exclusive_lock_policy(MockTestImageCtx &mock_image_ctx,
+                                        librbd::exclusive_lock::MockPolicy &policy) {
+    EXPECT_CALL(mock_image_ctx,
+                get_exclusive_lock_policy()).WillOnce(Return(&policy));
+  }
+
+  void expect_may_auto_request_lock(librbd::exclusive_lock::MockPolicy &policy,
+                                    bool value) {
+    EXPECT_CALL(policy, may_auto_request_lock()).WillOnce(Return(value));
+  }
+
+  void expect_acquire_lock(MockExclusiveLock &mock_exclusive_lock,
+                           Context **on_finish) {
+    EXPECT_CALL(mock_exclusive_lock, acquire_lock(_))
+      .WillOnce(Invoke([on_finish](Context *ctx) {
+                    *on_finish = ctx;
+                  }));
+  }
+
+  void expect_process_finish(MockImageRequestWQ &mock_image_request_wq) {
+    EXPECT_CALL(mock_image_request_wq, process_finish()).Times(1);
+  }
+
+  void expect_fail(MockImageRequest &mock_image_request, int r) {
+    EXPECT_CALL(mock_image_request, fail(r))
+      .WillOnce(Invoke([&mock_image_request](int r) {
+                    mock_image_request.aio_comp->get();
+                    mock_image_request.aio_comp->fail(r);
+                  }));
+  }
+
+  void expect_refresh(MockTestImageCtx &mock_image_ctx, Context **on_finish) {
+    EXPECT_CALL(*mock_image_ctx.state, refresh(_))
+      .WillOnce(Invoke([on_finish](Context *ctx) {
+                    *on_finish = ctx;
+                  }));
+  }
+};
+
+TEST_F(TestMockIoImageRequestWQ, AcquireLockError) {
+  REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+
+  InSequence seq;
+  MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr);
+  expect_signal(mock_image_request_wq);
+  mock_image_request_wq.set_require_lock(DIRECTION_WRITE, true);
+
+  auto mock_image_request = new MockImageRequest();
+  expect_is_write_op(*mock_image_request, true);
+  expect_queue(mock_image_request_wq);
+  auto *aio_comp = new librbd::io::AioCompletion();
+  mock_image_request_wq.aio_write(aio_comp, 0, 0, {}, 0);
+
+  librbd::exclusive_lock::MockPolicy mock_exclusive_lock_policy;
+  expect_front(mock_image_request_wq, mock_image_request);
+  expect_is_refresh_request(mock_image_ctx, false);
+  expect_is_write_op(*mock_image_request, true);
+  expect_dequeue(mock_image_request_wq, mock_image_request);
+  expect_get_exclusive_lock_policy(mock_image_ctx, mock_exclusive_lock_policy);
+  expect_may_auto_request_lock(mock_exclusive_lock_policy, true);
+  Context *on_acquire = nullptr;
+  expect_acquire_lock(mock_exclusive_lock, &on_acquire);
+  ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr);
+  ASSERT_TRUE(on_acquire != nullptr);
+
+  expect_process_finish(mock_image_request_wq);
+  expect_fail(*mock_image_request, -EPERM);
+  expect_is_write_op(*mock_image_request, true);
+  expect_signal(mock_image_request_wq);
+  on_acquire->complete(-EPERM);
+
+  ASSERT_EQ(0, aio_comp->wait_for_complete());
+  ASSERT_EQ(-EPERM, aio_comp->get_return_value());
+  aio_comp->release();
+}
+
+TEST_F(TestMockIoImageRequestWQ, RefreshError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr);
+
+  auto mock_image_request = new MockImageRequest();
+  expect_is_write_op(*mock_image_request, true);
+  expect_queue(mock_image_request_wq);
+  auto *aio_comp = new librbd::io::AioCompletion();
+  mock_image_request_wq.aio_write(aio_comp, 0, 0, {}, 0);
+
+  expect_front(mock_image_request_wq, mock_image_request);
+  expect_is_refresh_request(mock_image_ctx, true);
+  expect_is_write_op(*mock_image_request, true);
+  expect_dequeue(mock_image_request_wq, mock_image_request);
+  Context *on_refresh = nullptr;
+  expect_refresh(mock_image_ctx, &on_refresh);
+  ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr);
+  ASSERT_TRUE(on_refresh != nullptr);
+
+  expect_process_finish(mock_image_request_wq);
+  expect_fail(*mock_image_request, -EPERM);
+  expect_is_write_op(*mock_image_request, true);
+  expect_signal(mock_image_request_wq);
+  on_refresh->complete(-EPERM);
+
+  ASSERT_EQ(0, aio_comp->wait_for_complete());
+  ASSERT_EQ(-EPERM, aio_comp->get_return_value());
+  aio_comp->release();
+}
+
+} // namespace io
+} // namespace librbd

src/test/librbd/mock/MockImageCtx.h

@@ -83,6 +83,7 @@ struct MockImageCtx {
       io_work_queue(new io::MockImageRequestWQ()),
       op_work_queue(new MockContextWQ()),
       readahead_max_bytes(image_ctx.readahead_max_bytes),
+      event_socket(image_ctx.event_socket),
       parent(NULL), operations(new MockOperations()),
       state(new MockImageState()),
       image_watcher(NULL), object_map(NULL),
@@ -103,7 +104,8 @@ struct MockImageCtx {
           image_ctx.journal_max_concurrent_object_sets),
       mirroring_resync_after_disconnect(
           image_ctx.mirroring_resync_after_disconnect),
-      mirroring_replay_delay(image_ctx.mirroring_replay_delay)
+      mirroring_replay_delay(image_ctx.mirroring_replay_delay),
+      non_blocking_aio(image_ctx.non_blocking_aio)
   {
     md_ctx.dup(image_ctx.md_ctx);
     data_ctx.dup(image_ctx.data_ctx);
@@ -191,6 +193,8 @@ struct MockImageCtx {
   MOCK_METHOD0(notify_update, void());
   MOCK_METHOD1(notify_update, void(Context *));
 
+  MOCK_CONST_METHOD0(get_exclusive_lock_policy, exclusive_lock::Policy*());
+
   MOCK_CONST_METHOD0(get_journal_policy, journal::Policy*());
   MOCK_CONST_METHOD1(set_journal_policy, void(journal::Policy*));
 
@@ -265,6 +269,8 @@ struct MockImageCtx {
   MockReadahead readahead;
   uint64_t readahead_max_bytes;
 
+  EventSocket &event_socket;
+
   MockImageCtx *parent;
   MockOperations *operations;
   MockImageState *state;
@@ -290,6 +296,7 @@ struct MockImageCtx {
   int journal_max_concurrent_object_sets;
   bool mirroring_resync_after_disconnect;
   int mirroring_replay_delay;
+  bool non_blocking_aio;
 };
 
 } // namespace librbd

src/test/librbd/mock/exclusive_lock/MockPolicy.h

@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_LIBRBD_MOCK_EXCLUSIVE_LOCK_POLICY_H
+#define CEPH_TEST_LIBRBD_MOCK_EXCLUSIVE_LOCK_POLICY_H
+
+#include "librbd/exclusive_lock/Policy.h"
+#include <gmock/gmock.h>
+
+namespace librbd {
+namespace exclusive_lock {
+
+struct MockPolicy : public Policy {
+
+  MOCK_METHOD0(may_auto_request_lock, bool());
+  MOCK_METHOD1(lock_requested, int(bool));
+
+};
+
+} // namespace exclusive_lock
+} // librbd
+
+#endif