Merge pull request #33411 from trociny/wip-44159

rbd-mirror: improve detection of blacklisted state

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

src/test/rbd_mirror/test_mock_PoolReplayer.cc

@@ -186,6 +186,7 @@ struct LeaderWatcher<librbd::MockTestImageCtx> {
     return s_instance;
   }
 
+  MOCK_METHOD0(is_blacklisted, bool());
   MOCK_METHOD0(is_leader, bool());
   MOCK_METHOD0(release_leader, void());
 
@@ -409,6 +410,12 @@ public:
                 }));
   }
 
+  void expect_leader_watcher_is_blacklisted(
+      MockLeaderWatcher &mock_leader_watcher, bool blacklisted) {
+    EXPECT_CALL(mock_leader_watcher, is_blacklisted())
+      .WillRepeatedly(Return(blacklisted));
+  }
+
   void expect_namespace_replayer_is_blacklisted(
       MockNamespaceReplayer &mock_namespace_replayer,
       bool blacklisted) {
@@ -543,6 +550,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
 
   auto mock_leader_watcher = new MockLeaderWatcher();
   expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
+  expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false);
 
   InSequence seq;
 
@@ -605,6 +613,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
   auto mock_leader_watcher = new MockLeaderWatcher();
   expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
   expect_leader_watcher_list_instances(*mock_leader_watcher);
+  expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false);
 
   InSequence seq;
 
@@ -692,6 +701,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
   auto mock_leader_watcher = new MockLeaderWatcher();
   expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
   expect_leader_watcher_list_instances(*mock_leader_watcher);
+  expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false);
 
   auto& mock_cluster = get_mock_cluster();
   auto mock_local_rados_client = mock_cluster.do_create_rados_client(
@@ -809,6 +819,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
   auto mock_leader_watcher = new MockLeaderWatcher();
   expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
   expect_leader_watcher_list_instances(*mock_leader_watcher);
+  expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false);
 
   auto& mock_cluster = get_mock_cluster();
   auto mock_local_rados_client = mock_cluster.do_create_rados_client(

src/tools/rbd_mirror/InstanceReplayer.cc

@@ -56,6 +56,12 @@ InstanceReplayer<I>::~InstanceReplayer() {
   ceph_assert(m_image_replayers.empty());
 }
 
+template <typename I>
+bool InstanceReplayer<I>::is_blacklisted() const {
+  std::lock_guard locker{m_lock};
+  return m_blacklisted;
+}
+
 template <typename I>
 int InstanceReplayer<I>::init() {
   C_SaferCond init_ctx;
@@ -327,6 +333,7 @@ void InstanceReplayer<I>::start_image_replayer(
   } else if (image_replayer->is_blacklisted()) {
     derr << "global_image_id=" << global_image_id << ": blacklisted detected "
          << "during image replay" << dendl;
+    m_blacklisted = true;
     return;
   } else if (image_replayer->is_finished()) {
     // TODO temporary until policy integrated

src/tools/rbd_mirror/InstanceReplayer.h

@@ -52,6 +52,8 @@ public:
                    PoolMetaCache* pool_meta_cache);
   ~InstanceReplayer();
 
+  bool is_blacklisted() const;
+
   int init();
   void shut_down();
 
@@ -102,13 +104,14 @@ private:
   journal::CacheManagerHandler *m_cache_manager_handler;
   PoolMetaCache* m_pool_meta_cache;
 
-  ceph::mutex m_lock;
+  mutable ceph::mutex m_lock;
   AsyncOpTracker m_async_op_tracker;
   std::map<std::string, ImageReplayer<ImageCtxT> *> m_image_replayers;
   Peers m_peers;
   Context *m_image_state_check_task = nullptr;
   Context *m_on_shut_down = nullptr;
   bool m_manual_stop = false;
+  bool m_blacklisted = false;
 
   void wait_for_ops();
   void handle_wait_for_ops(int r);

src/tools/rbd_mirror/LeaderWatcher.cc

@@ -248,6 +248,12 @@ void LeaderWatcher<I>::handle_wait_for_tasks() {
   m_work_queue->queue(ctx, 0);
 }
 
+template <typename I>
+bool LeaderWatcher<I>::is_blacklisted() const {
+  std::lock_guard locker{m_lock};
+  return m_blacklisted;
+}
+
 template <typename I>
 bool LeaderWatcher<I>::is_leader() const {
   std::lock_guard locker{m_lock};
@@ -1003,7 +1009,7 @@ void LeaderWatcher<I>::handle_notify(uint64_t notify_id, uint64_t handle,
     auto iter = bl.cbegin();
     decode(notify_message, iter);
   } catch (const buffer::error &err) {
-    derr << ": error decoding image notification: " << err.what() << dendl;
+    derr << "error decoding image notification: " << err.what() << dendl;
     ctx->complete(0);
     return;
   }
@@ -1015,9 +1021,13 @@ template <typename I>
 void LeaderWatcher<I>::handle_rewatch_complete(int r) {
   dout(5) << "r=" << r << dendl;
 
-  if (r != -EBLACKLISTED) {
-    m_leader_lock->reacquire_lock(nullptr);
+  if (r == -EBLACKLISTED) {
+    dout(1) << "blacklisted detected" << dendl;
+    m_blacklisted = true;
+    return;
   }
+
+  m_leader_lock->reacquire_lock(nullptr);
 }
 
 template <typename I>

src/tools/rbd_mirror/LeaderWatcher.h

@@ -44,6 +44,7 @@ public:
   void init(Context *on_finish);
   void shut_down(Context *on_finish);
 
+  bool is_blacklisted() const;
   bool is_leader() const;
   bool is_releasing_leader() const;
   bool get_leader_instance_id(std::string *instance_id) const;
@@ -215,6 +216,8 @@ private:
   Instances<ImageCtxT> *m_instances = nullptr;
   librbd::managed_lock::Locker m_locker;
 
+  bool m_blacklisted = false;
+
   AsyncOpTracker m_timer_op_tracker;
   Context *m_timer_task = nullptr;
   C_TimerGate *m_timer_gate = nullptr;

src/tools/rbd_mirror/NamespaceReplayer.cc

@@ -72,7 +72,8 @@ NamespaceReplayer<I>::NamespaceReplayer(
 template <typename I>
 bool NamespaceReplayer<I>::is_blacklisted() const {
   std::lock_guard locker{m_lock};
-  return (m_local_pool_watcher &&
+  return m_instance_replayer->is_blacklisted() ||
+         (m_local_pool_watcher &&
           m_local_pool_watcher->is_blacklisted()) ||
          (m_remote_pool_watcher &&
           m_remote_pool_watcher->is_blacklisted());

src/tools/rbd_mirror/PoolReplayer.cc

@@ -590,7 +590,8 @@ void PoolReplayer<I>::run() {
 
     std::unique_lock locker{m_lock};
 
-    if (m_default_namespace_replayer->is_blacklisted()) {
+    if (m_leader_watcher->is_blacklisted() ||
+        m_default_namespace_replayer->is_blacklisted()) {
       m_blacklisted = true;
       m_stopping = true;
     }