Merge pull request #5143 from trociny/fix-mds_metadata

mon: fix mds metadata

Reviewed-by: Joao Eduardo Luis <joao@suse.de>
Reviewed-by: Kefu Chai <kchai@redhat.com>

qa/workunits/cephtool/test.sh

@@ -579,6 +579,7 @@ function test_mon_misc()
   ceph_watch_wait "$mymsg"
 
   ceph mon_metadata a
+  ceph node ls
 }
 
 function check_mds_active()
@@ -884,6 +885,25 @@ function test_mon_mds()
   ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
 }
 
+function test_mon_mds_metadata()
+{
+  local nmons=$(ceph tell 'mon.*' version | grep -c 'version')
+  test "$nmons" -gt 0
+
+  ceph mds dump |
+  sed -nEe "s/^([0-9]+):.*'([a-z])' mds\\.([0-9]+)\\..*/\\1 \\2 \\3/p" |
+  while read gid id rank; do
+    ceph mds metadata ${gid} | grep '"hostname":'
+    ceph mds metadata ${id} | grep '"hostname":'
+    ceph mds metadata ${rank} | grep '"hostname":'
+
+    local n=$(ceph tell 'mon.*' mds metadata ${id} | grep -c '"hostname":')
+    test "$n" -eq "$nmons"
+  done
+
+  expect_false ceph mds metadata UNKNOWN
+}
+
 function test_mon_mon()
 {
   # print help message
@@ -1642,6 +1662,7 @@ OSD_TESTS+=" osd_bench"
 
 MDS_TESTS+=" mds_tell"
 MDS_TESTS+=" mon_mds"
+MDS_TESTS+=" mon_mds_metadata"
 
 TESTS+=$MON_TESTS
 TESTS+=$OSD_TESTS

src/mon/MDSMonitor.cc

@@ -138,6 +138,11 @@ void MDSMonitor::update_from_paxos(bool *need_bootstrap)
   update_logger();
 }
 
+void MDSMonitor::init()
+{
+  (void)load_metadata(pending_metadata);
+}
+
 void MDSMonitor::create_pending()
 {
   pending_mdsmap = mdsmap;
@@ -175,8 +180,8 @@ void MDSMonitor::encode_pending(MonitorDBStore::TransactionRef t)
       i != pending_daemon_health_rm.end(); ++i) {
     t->erase(MDS_HEALTH_PREFIX, stringify(*i));
   }
-  remove_from_metadata(t);
   pending_daemon_health_rm.clear();
+  remove_from_metadata(t);
 }
 
 version_t MDSMonitor::get_trim_to()
@@ -1757,42 +1762,31 @@ void MDSMonitor::update_metadata(mds_gid_t gid,
   if (metadata.empty()) {
     return;
   }
-  bufferlist bl;
-  int err = mon->store->get(MDS_METADATA_PREFIX, "last_metadata", bl);
-  map<mds_gid_t, Metadata> last_metadata;
-  if (!err) {
-    bufferlist::iterator iter = bl.begin();
-    ::decode(last_metadata, iter);
-    bl.clear();
-  }
-  last_metadata[gid] = metadata;
+  pending_metadata[gid] = metadata;
 
   MonitorDBStore::TransactionRef t = paxos->get_pending_transaction();
-  ::encode(last_metadata, bl);
+  bufferlist bl;
+  ::encode(pending_metadata, bl);
   t->put(MDS_METADATA_PREFIX, "last_metadata", bl);
   paxos->trigger_propose();
 }
 
 void MDSMonitor::remove_from_metadata(MonitorDBStore::TransactionRef t)
 {
+  bool update = false;
+  for (map<mds_gid_t, Metadata>::iterator i = pending_metadata.begin();
+       i != pending_metadata.end(); ) {
+    if (pending_mdsmap.get_state_gid(i->first) == MDSMap::STATE_NULL) {
+      pending_metadata.erase(i++);
+      update = true;
+    } else {
+      ++i;
+    }
+  }
+  if (!update)
+    return;
   bufferlist bl;
-  int err = mon->store->get(MDS_METADATA_PREFIX, "last_metadata", bl);
-  map<mds_gid_t, Metadata> last_metadata;
-  if (err) {
-    return;
-  }
-  bufferlist::iterator iter = bl.begin();
-  ::decode(last_metadata, iter);
-  bl.clear();
-
-  if (pending_daemon_health_rm.empty()) {
-    return;
-  }
-  for (std::set<uint64_t>::const_iterator to_remove = pending_daemon_health_rm.begin();
-       to_remove != pending_daemon_health_rm.end(); ++to_remove) {
-    last_metadata.erase(mds_gid_t(*to_remove));
-  }
-  ::encode(last_metadata, bl);
+  ::encode(pending_metadata, bl);
   t->put(MDS_METADATA_PREFIX, "last_metadata", bl);
 }
 
@@ -1852,7 +1846,10 @@ int MDSMonitor::print_nodes(Formatter *f)
       continue;
     }
     const mds_gid_t gid = it->first;
-    assert(mdsmap.get_state_gid(gid) != MDSMap::STATE_NULL);
+    if (mdsmap.get_state_gid(gid) == MDSMap::STATE_NULL) {
+      dout(5) << __func__ << ": GID " << gid << " not existent" << dendl;
+      continue;
+    }
     const MDSMap::mds_info_t& mds_info = mdsmap.get_info_gid(gid);
     mdses[hostname->second].push_back(mds_info.rank);
   }

src/mon/MDSMonitor.h

@@ -74,6 +74,7 @@ class MDSMonitor : public PaxosService {
   // service methods
   void create_initial();
   void update_from_paxos(bool *need_bootstrap);
+  void init();
   void create_pending(); 
   void encode_pending(MonitorDBStore::TransactionRef t);
   // we don't require full versions; don't encode any.
@@ -147,6 +148,8 @@ private:
   std::map<uint64_t, MDSHealth> pending_daemon_health;
   std::set<uint64_t> pending_daemon_health_rm;
 
+  map<mds_gid_t, Metadata> pending_metadata;
+
   int _check_pool(const int64_t pool_id, std::stringstream *ss) const;
   mds_gid_t gid_from_arg(const std::string& arg, std::ostream& err);
 };

src/mon/Paxos.cc

@@ -1483,7 +1483,6 @@ void Paxos::propose_pending()
 
   bufferlist bl;
   pending_proposal->encode(bl);
-  pending_proposal.reset();
 
   dout(10) << __func__ << " " << (last_committed + 1)
 	   << " " << bl.length() << " bytes" << dendl;
@@ -1493,6 +1492,8 @@ void Paxos::propose_pending()
   f.flush(*_dout);
   *_dout << dendl;
 
+  pending_proposal.reset();
+
   committing_finishers.swap(pending_finishers);
   state = STATE_UPDATING;
   begin(bl);