From b0e1eb52459717dd0fea93d68e7ed46d2238b40e Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Fri, 26 Feb 2016 16:11:07 +0200 Subject: [PATCH] test: ImageReplayer on_stop/on_start error tests Signed-off-by: Mykola Golub --- src/test/rbd_mirror/test_ImageReplayer.cc | 252 +++++++++++++++++++++- 1 file changed, 242 insertions(+), 10 deletions(-) diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index c0bdef56195..b1e2ccf48f8 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -84,9 +84,8 @@ public: m_remote_pool_name = get_temp_pool_name(); EXPECT_EQ("", create_one_pool_pp(m_remote_pool_name, m_remote_cluster)); - int64_t remote_pool_id = - m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); - EXPECT_GE(remote_pool_id, 0); + m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); + EXPECT_GE(m_remote_pool_id, 0); EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), m_remote_ioctx)); @@ -101,13 +100,6 @@ public: m_threads = new rbd::mirror::Threads(reinterpret_cast( m_local_ioctx.cct())); - m_replayer = new rbd::mirror::ImageReplayer( - m_threads, - rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), - rbd::mirror::RadosRef(new librados::Rados(m_remote_ioctx)), - m_client_id, m_local_ioctx.get_id(), remote_pool_id, m_remote_image_id); - - bootstrap(); } ~TestImageReplayer() @@ -119,6 +111,14 @@ public: EXPECT_EQ(0, m_local_cluster.pool_delete(m_local_pool_name.c_str())); } + template + void create_replayer() { + m_replayer = new ImageReplayerT(m_threads, + rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), + rbd::mirror::RadosRef(new librados::Rados(m_remote_ioctx)), + m_client_id, m_local_ioctx.get_id(), m_remote_pool_id, m_remote_image_id); + } + void start(rbd::mirror::ImageReplayer::BootstrapParams *bootstap_params = nullptr) { @@ -148,6 +148,8 @@ public: void bootstrap() { + create_replayer<>(); + rbd::mirror::ImageReplayer::BootstrapParams bootstap_params(m_local_pool_name, m_image_name); start(&bootstap_params); @@ -326,6 +328,7 @@ public: std::string m_local_pool_name, m_remote_pool_name; librados::IoCtx m_local_ioctx, m_remote_ioctx; std::string m_image_name; + int64_t m_remote_pool_id; std::string m_remote_image_id; rbd::mirror::ImageReplayer *m_replayer; C_WatchCtx *m_watch_ctx; @@ -335,8 +338,105 @@ public: int TestImageReplayer::_image_number; +TEST_F(TestImageReplayer, Bootstrap) +{ + bootstrap(); +} + +TEST_F(TestImageReplayer, BootstrapErrorInvalidPool) +{ + create_replayer<>(); + + rbd::mirror::ImageReplayer::BootstrapParams + bootstap_params("INVALID_LOCAL_POOL_NAME", m_image_name); + C_SaferCond cond; + m_replayer->start(&cond, &bootstap_params); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) +{ + int order = 0; + EXPECT_EQ(0, librbd::create(m_local_ioctx, m_image_name.c_str(), 1 << 22, + false, 0, &order, 0, 0)); + + create_replayer<>(); + rbd::mirror::ImageReplayer::BootstrapParams + bootstap_params(m_local_pool_name, m_image_name); + C_SaferCond cond; + m_replayer->start(&cond, &bootstap_params); + ASSERT_EQ(-EEXIST, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapErrorNoJournal) +{ + // disable remote journal journaling + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + uint64_t features; + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0, librbd::update_features(ictx, RBD_FEATURE_JOURNALING, false)); + close_image(ictx); + + create_replayer<>(); + rbd::mirror::ImageReplayer::BootstrapParams + bootstap_params(m_local_pool_name, m_image_name); + C_SaferCond cond; + m_replayer->start(&cond, &bootstap_params); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TEST_F(TestImageReplayer, StartInterrupted) +{ + create_replayer<>(); + rbd::mirror::ImageReplayer::BootstrapParams + bootstap_params(m_local_pool_name, m_image_name); + C_SaferCond start_cond, stop_cond; + m_replayer->start(&start_cond, &bootstap_params); + m_replayer->stop(&stop_cond); + int r = start_cond.wait(); + printf("start returned %d\n", r); + // TODO: improve the test to avoid this race // TODO: improve the test to avoid this race + ASSERT_TRUE(r == -EINTR || r == 0); + ASSERT_EQ(0, stop_cond.wait()); +} + +TEST_F(TestImageReplayer, ErrorJournalReset) +{ + bootstrap(); + + ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); + + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-EEXIST, cond.wait()); +} + +TEST_F(TestImageReplayer, ErrorNoJournal) +{ + bootstrap(); + + // disable remote journal journaling + // (reset before disabling, so it does not fail with EBUSY) + ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + uint64_t features; + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0, librbd::update_features(ictx, RBD_FEATURE_JOURNALING, false)); + close_image(ictx); + + rbd::mirror::ImageReplayer::BootstrapParams + bootstap_params(m_local_pool_name, m_image_name); + C_SaferCond cond; + m_replayer->start(&cond, &bootstap_params); + ASSERT_EQ(-ENOENT, cond.wait()); +} + TEST_F(TestImageReplayer, StartStop) { + bootstrap(); + start(); wait_for_replay_complete(); stop(); @@ -344,6 +444,8 @@ TEST_F(TestImageReplayer, StartStop) TEST_F(TestImageReplayer, WriteAndStartReplay) { + bootstrap(); + // Write to remote image and start replay librbd::ImageCtx *ictx; @@ -369,6 +471,8 @@ TEST_F(TestImageReplayer, WriteAndStartReplay) TEST_F(TestImageReplayer, StartReplayAndWrite) { + bootstrap(); + // Start replay and write to remote image librbd::ImageCtx *ictx; @@ -400,3 +504,131 @@ TEST_F(TestImageReplayer, StartReplayAndWrite) stop(); } + +class ImageReplayer : public rbd::mirror::ImageReplayer { +public: + ImageReplayer(rbd::mirror::Threads *threads, + rbd::mirror::RadosRef local, rbd::mirror::RadosRef remote, + const std::string &client_id, int64_t local_pool_id, + int64_t remote_pool_id, const std::string &remote_image_id) + : rbd::mirror::ImageReplayer(threads, local, remote, client_id, + local_pool_id, remote_pool_id, remote_image_id) + {} + + void set_error(const std::string &state, int r) { + m_errors[state] = r; + } + + int get_error(const std::string &state) const { + std::map::const_iterator i = m_errors.find(state); + return i == m_errors.end() ? 0 : i->second; + } + +protected: + virtual void on_start_get_registered_client_status_finish(int r, + const std::set ®istered_clients, + const BootstrapParams &bootstrap_params) { + rbd::mirror::ImageReplayer::on_start_get_registered_client_status_finish( + get_error("on_start_get_registered_client_status"), registered_clients, + bootstrap_params); + } + + virtual void on_start_bootstrap_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_start_bootstrap_finish( + get_error("on_start_bootstrap")); + } + + virtual void on_start_remote_journaler_init_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_start_remote_journaler_init_finish( + get_error("on_start_remote_journaler_init")); + } + + virtual void on_start_local_image_open_finish(int r) { + int test_r = get_error("on_start_local_image_open"); + if (!test_r) { + rbd::mirror::ImageReplayer::on_start_local_image_open_finish(r); + return; + } + + // The image open error was imitated, so we need to close the image back + // before propagating the error. + ASSERT_EQ(0, r); + set_error("on_start_local_image_open", 0); + FunctionContext *ctx = new FunctionContext( + [this, test_r](int r) { + on_start_local_image_open_finish(test_r); + }); + close_local_image(ctx); + } + + virtual void on_start_local_image_lock_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_start_local_image_lock_finish( + get_error("on_start_local_image_lock")); + } + + virtual void on_start_wait_for_local_journal_ready_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_start_wait_for_local_journal_ready_finish( + get_error("on_start_wait_for_local_journal_ready")); + } + + virtual void on_stop_journal_replay_shut_down_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_stop_journal_replay_shut_down_finish( + get_error("on_stop_journal_replay_shut_down")); + } + + virtual void on_stop_local_image_close_finish(int r) { + ASSERT_EQ(0, r); + rbd::mirror::ImageReplayer::on_stop_local_image_close_finish( + get_error("on_stop_local_image_close")); + } + +private: + std::map m_errors; +}; + +#define TEST_ON_START_ERROR(state) \ +TEST_F(TestImageReplayer, Error_on_start_##state) \ +{ \ + create_replayer(); \ + reinterpret_cast(m_replayer)-> \ + set_error("on_start_" #state, -1); \ + rbd::mirror::ImageReplayer::BootstrapParams \ + bootstap_params(m_local_pool_name, m_image_name); \ + C_SaferCond cond; \ + m_replayer->start(&cond, &bootstap_params); \ + ASSERT_EQ(-1, cond.wait()); \ +} + +#define TEST_ON_STOP_ERROR(state) \ +TEST_F(TestImageReplayer, Error_on_stop_##state) \ +{ \ + create_replayer(); \ + reinterpret_cast(m_replayer)-> \ + set_error("on_stop_" #state, -1); \ + rbd::mirror::ImageReplayer::BootstrapParams \ + bootstap_params(m_local_pool_name, m_image_name); \ + start(&bootstap_params); \ + /* TODO: investigate: without wait below I observe: */ \ + /* librbd/journal/Replay.cc: 70: FAILED assert(m_op_events.empty()) */\ + wait_for_replay_complete(); \ + C_SaferCond cond; \ + m_replayer->stop(&cond); \ + ASSERT_EQ(0, cond.wait()); \ +} + +TEST_ON_START_ERROR(get_registered_client_status); +TEST_ON_START_ERROR(bootstrap); +TEST_ON_START_ERROR(remote_journaler_init); +TEST_ON_START_ERROR(local_image_open); +TEST_ON_START_ERROR(local_image_lock); +TEST_ON_START_ERROR(wait_for_local_journal_ready); + +TEST_ON_STOP_ERROR(journal_replay_shut_down); +TEST_ON_STOP_ERROR(local_image_close); +TEST_ON_STOP_ERROR(no_error); +