Merge pull request #16460 from liewegas/wip-mgr-metadata

mon: add mgr metdata commands, and overall 'versions' command for all daemon versions

Reviewed-by: Kefu Chai <kchai@redhat.com>

doc/man/8/ceph.rst

@@ -442,6 +442,63 @@ Usage::
 
 	ceph mon_status
 
+mgr
+---
+
+Ceph manager daemon configuration and management.
+
+Subcommand ``dump`` dumps the latest MgrMap, which describes the active
+and standby manager daemons.
+
+Usage::
+
+  ceph mgr dump
+
+Subcommand ``fail`` will mark a manager daemon as failed, removing it
+from the manager map.  If it is the active manager daemon a standby
+will take its place.
+
+Usage::
+
+  ceph mgr fail <name>
+
+Subcommand ``module ls`` will list currently enabled manager modules (plugins).
+
+Usage::
+
+  ceph mgr module ls
+
+Subcommand ``module enable`` will enable a manager module.  Available modules are included in MgrMap and visible via ``mgr dump``.
+
+Usage::
+
+  ceph mgr module enable <module>
+
+Subcommand ``module disable`` will disable an active manager module.
+
+Usage::
+
+  ceph mgr module disable <module>
+
+Subcommand ``metadata`` will report metadata about all manager daemons or, if the name is specified, a single manager daemon.
+
+Usage::
+
+  ceph mgr metadata [name]
+
+Subcommand ``versions`` will report a count of running daemon versions.
+
+Usage::
+
+  ceph mgr versions
+
+Subcommand ``count-metadata`` will report a count of any daemon metadata field.
+
+Usage::
+
+  ceph mgr count-metadata <field>
+
+
 osd
 ---
 

doc/release-notes.rst

@@ -160,10 +160,13 @@ Major Changes from Kraken
   * *CLI changes*:
 
     - The ``ceph -s`` or ``ceph status`` command has a fresh look.
-    - ``ceph {osd,mds,mon} versions`` summarizes versions of running daemons.
-    - ``ceph {osd,mds,mon} count-metadata <property>`` similarly
+    - ``ceph mgr metadata`` will dump metadata associated with each mgr
+      daemon.
+    - ``ceph versions`` or ``ceph {osd,mds,mon,mgr} versions``
+      summarize versions of running daemons.
+    - ``ceph {osd,mds,mon,mgr} count-metadata <property>`` similarly
       tabulates any other daemon metadata visible via the ``ceph
-      {osd,mds,mon} metadata`` commands.
+      {osd,mds,mon,mgr} metadata`` commands.
     - ``ceph features`` summarizes features and releases of connected
       clients and daemons.
     - ``ceph osd require-osd-release <release>`` replaces the old
@@ -317,6 +320,10 @@ Upgrade from Jewel or Kraken
 
 #. Do not create any new erasure-code pools while upgrading the monitors.
 
+#. You can monitor the progress of your upgrade at each stage with the
+   ``ceph versions`` command, which will tell you what ceph version is
+   running for each type of daemon.
+
 #. Set the ``noout`` flag for the duration of the upgrade. (Optional
    but recommended.)::
 
@@ -368,7 +375,7 @@ Upgrade from Jewel or Kraken
      # systemctl restart ceph-osd.target
 
    You can monitor the progress of the OSD upgrades with the new
-   ``ceph osd versions`` command.::
+   ``ceph versions`` or ``ceph osd versions`` command.::
 
      # ceph osd versions
      {

qa/workunits/cephtool/test.sh

@@ -749,6 +749,12 @@ function test_mon_misc()
   ceph mon count-metadata ceph_version
   ceph mon versions
 
+  ceph mgr metadata
+  ceph mgr versions
+  ceph mgr count-metadata ceph_version
+
+  ceph versions
+
   ceph node ls
 }
 

src/messages/MMgrBeacon.h

@@ -23,7 +23,7 @@
 
 class MMgrBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 4;
+  static const int HEAD_VERSION = 5;
   static const int COMPAT_VERSION = 1;
 
 protected:
@@ -33,6 +33,7 @@ protected:
   std::string name;
   uuid_d fsid;
   std::set<std::string> available_modules;
+  map<string,string> metadata; ///< misc metadata about this osd
 
   // Only populated during activation
   std::vector<MonCommand> command_descs;
@@ -46,10 +47,11 @@ public:
 
   MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_,
              entity_addr_t server_addr_, bool available_,
-	     const std::set<std::string>& module_list)
+	     const std::set<std::string>& module_list,
+	     map<string,string>&& metadata)
     : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION),
       gid(gid_), server_addr(server_addr_), available(available_), name(name_),
-      fsid(fsid_), available_modules(module_list)
+      fsid(fsid_), available_modules(module_list), metadata(std::move(metadata))
   {
   }
 
@@ -59,6 +61,9 @@ public:
   const std::string& get_name() const { return name; }
   const uuid_d& get_fsid() const { return fsid; }
   std::set<std::string>& get_available_modules() { return available_modules; }
+  const std::map<std::string,std::string>& get_metadata() const {
+    return metadata;
+  }
 
   void set_command_descs(const std::vector<MonCommand> &cmds)
   {
@@ -92,6 +97,7 @@ public:
     ::encode(fsid, payload);
     ::encode(available_modules, payload);
     ::encode(command_descs, payload);
+    ::encode(metadata, payload);
   }
   void decode_payload() override {
     bufferlist::iterator p = payload.begin();
@@ -109,6 +115,9 @@ public:
     if (header.version >= 4) {
       ::decode(command_descs, p);
     }
+    if (header.version >= 5) {
+      ::decode(metadata, p);
+    }
   }
 };
 

src/mgr/MgrStandby.cc

@@ -161,12 +161,16 @@ void MgrStandby::send_beacon()
   dout(10) << "sending beacon as gid " << monc.get_global_id()
 	   << " modules " << modules << dendl;
 
+  map<string,string> metadata;
+  collect_sys_info(&metadata, g_ceph_context);
+
   MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(),
 				 monc.get_global_id(),
                                  g_conf->name.get_id(),
                                  addr,
                                  available,
-				 modules);
+				 modules,
+				 std::move(metadata));
 
   if (available && !available_in_map) {
     // We are informing the mon that we are done initializing: inform

src/mgr/ServiceMap.h

@@ -47,10 +47,23 @@ struct ServiceMap {
 	return "no daemons active";
       }
       std::ostringstream ss;
-      ss << daemons.size() << (daemons.size() > 1 ? "daemonss" : "daemon")
+      ss << daemons.size() << (daemons.size() > 1 ? " daemons" : " daemon")
 	 << " active";
       return ss.str();
     }
+
+    void count_metadata(const string& field,
+			std::map<std::string,int> *out) const {
+      for (auto& p : daemons) {
+	auto q = p.second.metadata.find(field);
+	if (q == p.second.metadata.end()) {
+	  (*out)["unknown"]++;
+	} else {
+	  (*out)[q->second]++;
+	}
+      }
+    }
+
   };
 
   epoch_t epoch = 0;

src/mon/MDSMonitor.cc

@@ -1863,19 +1863,24 @@ int MDSMonitor::load_metadata(map<mds_gid_t, Metadata>& m)
   return 0;
 }
 
-void MDSMonitor::count_metadata(const string& field, Formatter *f)
+void MDSMonitor::count_metadata(const string& field, map<string,int> *out)
 {
-  map<string,int> by_val;
   map<mds_gid_t,Metadata> meta;
   load_metadata(meta);
   for (auto& p : meta) {
     auto q = p.second.find(field);
     if (q == p.second.end()) {
-      by_val["unknown"]++;
+      (*out)["unknown"]++;
     } else {
-      by_val[q->second]++;
+      (*out)[q->second]++;
     }
   }
+}
+
+void MDSMonitor::count_metadata(const string& field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
   f->open_object_section(field.c_str());
   for (auto& p : by_val) {
     f->dump_int(p.first.c_str(), p.second);

src/mon/MDSMonitor.h

@@ -139,6 +139,9 @@ class MDSMonitor : public PaxosService {
   void remove_from_metadata(MonitorDBStore::TransactionRef t);
   int load_metadata(map<mds_gid_t, Metadata>& m);
   void count_metadata(const string& field, Formatter *f);
+public:
+  void count_metadata(const string& field, map<string,int> *out);
+protected:
 
   // MDS daemon GID to latest health state from that GID
   std::map<uint64_t, MDSHealth> pending_daemon_health;

src/mon/MgrMap.h

@@ -95,6 +95,29 @@ public:
     return true;
   }
 
+  bool have_name(const string& name) const {
+    if (active_name == name) {
+      return true;
+    }
+    for (auto& p : standbys) {
+      if (p.second.name == name) {
+	return true;
+      }
+    }
+    return false;
+  }
+
+  std::set<std::string> get_all_names() const {
+    std::set<std::string> ls;
+    if (active_name.size()) {
+      ls.insert(active_name);
+    }
+    for (auto& p : standbys) {
+      ls.insert(p.second.name);
+    }
+    return ls;
+  }
+
   void encode(bufferlist& bl, uint64_t features) const
   {
     ENCODE_START(2, 1, bl);

src/mon/MgrMonitor.cc

@@ -25,6 +25,8 @@
 
 #include "MgrMonitor.h"
 
+#define MGR_METADATA_PREFIX "mgr_metadata"
+
 #define dout_subsys ceph_subsys_mon
 #undef dout_prefix
 #define dout_prefix _prefix(_dout, mon, map)
@@ -35,7 +37,6 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon,
 		<< ").mgr e" << mgrmap.get_epoch() << " ";
 }
 
-
 // Prefix for mon store of active mgr's command descriptions
 const static std::string command_descs_prefix = "mgr_command_descs";
 
@@ -138,6 +139,17 @@ void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t)
   put_version(t, pending_map.epoch, bl);
   put_last_committed(t, pending_map.epoch);
 
+  for (auto& p : pending_metadata) {
+    dout(10) << __func__ << " set metadata for " << p.first << dendl;
+    t->put(MGR_METADATA_PREFIX, p.first, p.second);
+  }
+  for (auto& name : pending_metadata_rm) {
+    dout(10) << __func__ << " rm metadata for " << name << dendl;
+    t->erase(MGR_METADATA_PREFIX, name);
+  }
+  pending_metadata.clear();
+  pending_metadata_rm.clear();
+
   health_check_map_t next;
   if (pending_map.active_gid == 0) {
     auto level = should_warn_about_mgr_down();
@@ -329,6 +341,8 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
     pending_map.active_gid = m->get_gid();
     pending_map.active_name = m->get_name();
     pending_map.available_modules = m->get_available_modules();
+    ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
+    pending_metadata_rm.erase(m->get_name());
 
     mon->clog->info() << "Activating manager daemon "
                       << pending_map.active_name;
@@ -353,6 +367,8 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
                          << " started";
       pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
 					    m->get_available_modules()};
+      ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
+      pending_metadata_rm.erase(m->get_name());
       updated = true;
     }
   }
@@ -592,6 +608,8 @@ void MgrMonitor::drop_active()
     last_beacon.erase(pending_map.active_gid);
   }
 
+  pending_metadata_rm.insert(pending_map.active_name);
+  pending_metadata.erase(pending_map.active_name);
   pending_map.active_name = "";
   pending_map.active_gid = 0;
   pending_map.available = false;
@@ -604,11 +622,12 @@ void MgrMonitor::drop_active()
 
 void MgrMonitor::drop_standby(uint64_t gid)
 {
+  pending_metadata_rm.insert(pending_map.standbys[gid].name);
+  pending_metadata.erase(pending_map.standbys[gid].name);
   pending_map.standbys.erase(gid);
   if (last_beacon.count(gid) > 0) {
     last_beacon.erase(gid);
   }
-
 }
 
 bool MgrMonitor::preprocess_command(MonOpRequestRef op)
@@ -665,6 +684,58 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op)
     }
     f->close_section();
     f->flush(rdata);
+  } else if (prefix == "mgr metadata") {
+    string name;
+    cmd_getval(g_ceph_context, cmdmap, "id", name);
+    if (name.size() > 0 && !map.have_name(name)) {
+      ss << "mgr." << name << " does not exist";
+      r = -ENOENT;
+      goto reply;
+    }
+    string format;
+    cmd_getval(g_ceph_context, cmdmap, "format", format);
+    boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
+    if (name.size()) {
+      f->open_object_section("mgr_metadata");
+      f->dump_string("id", name);
+      r = dump_metadata(name, f.get(), &ss);
+      if (r < 0)
+        goto reply;
+      f->close_section();
+    } else {
+      r = 0;
+      f->open_array_section("mgr_metadata");
+      for (auto& i : map.get_all_names()) {
+	f->open_object_section("mgr");
+	f->dump_string("id", i);
+	r = dump_metadata(i, f.get(), NULL);
+	if (r == -EINVAL || r == -ENOENT) {
+	  // Drop error, continue to get other daemons' metadata
+	  dout(4) << "No metadata for mgr." << i << dendl;
+	  r = 0;
+	} else if (r < 0) {
+	  // Unexpected error
+	  goto reply;
+	}
+	f->close_section();
+      }
+      f->close_section();
+    }
+    f->flush(rdata);
+  } else if (prefix == "mgr versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    count_metadata("ceph_version", f.get());
+    f->flush(rdata);
+    r = 0;
+  } else if (prefix == "mgr count-metadata") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    string field;
+    cmd_getval(g_ceph_context, cmdmap, "property", field);
+    count_metadata(field, f.get());
+    f->flush(rdata);
+    r = 0;
   } else {
     return false;
   }
@@ -807,4 +878,59 @@ void MgrMonitor::on_shutdown()
   cancel_timer();
 }
 
+int MgrMonitor::load_metadata(const string& name, std::map<string, string>& m,
+			      ostream *err)
+{
+  bufferlist bl;
+  int r = mon->store->get(MGR_METADATA_PREFIX, name, bl);
+  if (r < 0)
+    return r;
+  try {
+    bufferlist::iterator p = bl.begin();
+    ::decode(m, p);
+  }
+  catch (buffer::error& e) {
+    if (err)
+      *err << "mgr." << name << " metadata is corrupt";
+    return -EIO;
+  }
+  return 0;
+}
+
+void MgrMonitor::count_metadata(const string& field, std::map<string,int> *out)
+{
+  std::set<string> ls = map.get_all_names();
+  for (auto& name : ls) {
+    std::map<string,string> meta;
+    load_metadata(name, meta, nullptr);
+    auto p = meta.find(field);
+    if (p == meta.end()) {
+      (*out)["unknown"]++;
+    } else {
+      (*out)[p->second]++;
+    }
+  }
+}
+
+void MgrMonitor::count_metadata(const string& field, Formatter *f)
+{
+  std::map<string,int> by_val;
+  count_metadata(field, &by_val);
+  f->open_object_section(field.c_str());
+  for (auto& p : by_val) {
+    f->dump_int(p.first.c_str(), p.second);
+  }
+  f->close_section();
+}
+
+int MgrMonitor::dump_metadata(const string& name, Formatter *f, ostream *err)
+{
+  std::map<string,string> m;
+  if (int r = load_metadata(name, m, err))
+    return r;
+  for (auto& p : m) {
+    f->dump_string(p.first.c_str(), p.second);
+  }
+  return 0;
+}
 

src/mon/MgrMonitor.h

@@ -14,6 +14,9 @@
 #ifndef CEPH_MGRMONITOR_H
 #define CEPH_MGRMONITOR_H
 
+#include <map>
+#include <set>
+
 #include "include/Context.h"
 #include "MgrMap.h"
 #include "PaxosService.h"
@@ -25,6 +28,9 @@ class MgrMonitor: public PaxosService
   MgrMap pending_map;
   bool ever_had_active_mgr = false;
 
+  std::map<std::string, bufferlist> pending_metadata;
+  std::set<std::string>             pending_metadata_rm;
+
   utime_t first_seen_inactive;
 
   std::map<uint64_t, ceph::coarse_mono_clock::time_point> last_beacon;
@@ -97,6 +103,12 @@ public:
     return command_descs;
   }
 
+  int load_metadata(const string& name, std::map<string, string>& m,
+		    ostream *err);
+  int dump_metadata(const string& name, Formatter *f, ostream *err);
+  void count_metadata(const string& field, Formatter *f);
+  void count_metadata(const string& field, std::map<string,int> *out);
+
   friend class C_Updated;
 };
 

src/mon/MonCommands.h

@@ -279,6 +279,10 @@ COMMAND("mon count-metadata name=property,type=CephString",
 COMMAND("mon versions",
 	"check running versions of monitors",
 	"mon", "r", "cli,rest")
+COMMAND("versions",
+	"check running versions of ceph daemons",
+	"mon", "r", "cli,rest")
+
 
 
 /*
@@ -1017,3 +1021,12 @@ COMMAND("mgr module enable "						\
 COMMAND("mgr module disable "						\
 	"name=module,type=CephString",
 	"disable mgr module", "mgr", "rw", "cli,rest")
+COMMAND("mgr metadata name=id,type=CephString,req=false",
+	"dump metadata for all daemons or a specific daemon",
+	"mgr", "r", "cli,rest")
+COMMAND("mgr count-metadata name=property,type=CephString",
+	"count ceph-mgr daemons by metadata field property",
+	"mgr", "r", "cli,rest")
+COMMAND("mgr versions", \
+	"check running versions of ceph-mgr daemons",
+	"mgr", "r", "cli,rest")

src/mon/Monitor.cc

@@ -3564,6 +3564,65 @@ void Monitor::handle_command(MonOpRequestRef op)
     rdata.append(ds);
     rs = "";
     r = 0;
+  } else if (prefix == "versions") {
+    if (!f)
+      f.reset(Formatter::create("json-pretty"));
+    map<string,int> overall;
+    f->open_object_section("version");
+    map<string,int> mon, mgr, osd, mds;
+
+    count_metadata("ceph_version", &mon);
+    f->open_object_section("mon");
+    for (auto& p : mon) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    mgrmon()->count_metadata("ceph_version", &mgr);
+    f->open_object_section("mgr");
+    for (auto& p : mgr) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    osdmon()->count_metadata("ceph_version", &osd);
+    f->open_object_section("osd");
+    for (auto& p : osd) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    mdsmon()->count_metadata("ceph_version", &mds);
+    f->open_object_section("mds");
+    for (auto& p : mon) {
+      f->dump_int(p.first.c_str(), p.second);
+      overall[p.first] += p.second;
+    }
+    f->close_section();
+
+    for (auto& p : mgrstatmon()->get_service_map().services) {
+      f->open_object_section(p.first.c_str());
+      map<string,int> m;
+      p.second.count_metadata("ceph_version", &m);
+      for (auto& q : m) {
+	f->dump_int(q.first.c_str(), q.second);
+	overall[q.first] += q.second;
+      }
+      f->close_section();
+    }
+
+    f->open_object_section("overall");
+    for (auto& p : overall) {
+      f->dump_int(p.first.c_str(), p.second);
+    }
+    f->close_section();
+    f->close_section();
+    f->flush(rdata);
+    rs = "";
+    r = 0;
   }
 
  out:
@@ -5011,17 +5070,22 @@ int Monitor::get_mon_metadata(int mon, Formatter *f, ostream& err)
   return 0;
 }
 
-void Monitor::count_metadata(const string& field, Formatter *f)
+void Monitor::count_metadata(const string& field, map<string,int> *out)
 {
-  map<string,int> by_val;
   for (auto& p : mon_metadata) {
     auto q = p.second.find(field);
     if (q == p.second.end()) {
-      by_val["unknown"]++;
+      (*out)["unknown"]++;
     } else {
-      by_val[q->second]++;
+      (*out)[q->second]++;
     }
   }
+}
+
+void Monitor::count_metadata(const string& field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
   f->open_object_section(field.c_str());
   for (auto& p : by_val) {
     f->dump_int(p.first.c_str(), p.second);

src/mon/Monitor.h

@@ -901,6 +901,7 @@ public:
   void update_mon_metadata(int from, Metadata&& m);
   int load_metadata();
   void count_metadata(const string& field, Formatter *f);
+  void count_metadata(const string& field, map<string,int> *out);
 
   // features
   static CompatSet get_initial_supported_features();

src/mon/OSDMonitor.cc

@@ -1226,21 +1226,26 @@ int OSDMonitor::load_metadata(int osd, map<string, string>& m, ostream *err)
   return 0;
 }
 
-void OSDMonitor::count_metadata(const string& field, Formatter *f)
+void OSDMonitor::count_metadata(const string& field, map<string,int> *out)
 {
-  map<string,int> by_val;
   for (int osd = 0; osd < osdmap.get_max_osd(); ++osd) {
     if (osdmap.is_up(osd)) {
       map<string,string> meta;
       load_metadata(osd, meta, nullptr);
       auto p = meta.find(field);
       if (p == meta.end()) {
-	by_val["unknown"]++;
+	(*out)["unknown"]++;
       } else {
-	by_val[p->second]++;
+	(*out)[p->second]++;
       }
     }
   }
+}
+
+void OSDMonitor::count_metadata(const string& field, Formatter *f)
+{
+  map<string,int> by_val;
+  count_metadata(field, &by_val);
   f->open_object_section(field.c_str());
   for (auto& p : by_val) {
     f->dump_int(p.first.c_str(), p.second);

src/mon/OSDMonitor.h

@@ -428,6 +428,9 @@ private:
 
   int load_metadata(int osd, map<string, string>& m, ostream *err);
   void count_metadata(const string& field, Formatter *f);
+public:
+  void count_metadata(const string& field, map<string,int> *out);
+protected:
   int get_osd_objectstore_type(int osd, std::string *type);
   bool is_pool_currently_all_bluestore(int64_t pool_id, const pg_pool_t &pool,
 				       ostream *err);