Merge pull request #15958 from liewegas/wip-mgr-modules

mgr,mon: enable/disable mgr modules via 'ceph mgr module ...' commands

Reviewed-by: John Spray <john.spray@redhat.com>

doc/mgr/administrator.rst

@@ -76,12 +76,6 @@ OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python
 :Type: String
 :Default: ``"<library dir>/mgr"``
 
-``mgr modules``
-
-:Description: List of python modules to load
-:Type: String
-:Default: ``"rest"`` (Load the REST API module only)
-
 ``mgr data``
 
 :Description: Path to load daemon data (such as keyring)

doc/mgr/dashboard.rst

@@ -22,11 +22,9 @@ If the port is not configured, the web app will bind to port ``7000``.
 Setting the IP to ``::`` makes the dashboard bind to all available IPv4 and IPv6
 addresses.
 
-In addition, make sure that the *dashboard* module is enabled in the ceph.conf
-configuration file:
+In addition, make sure that the *dashboard* module is enabled with::
 
-  [mgr]
-  mgr_modules = dashboard
+  ceph mgr module enable dashboard
 
 Please note that the dashboard will *only* start on the manager which is active
 at that moment. Query the Ceph cluster status to see which manager is active.

doc/mgr/plugins.rst

@@ -21,13 +21,10 @@ Installing a plugin
 -------------------
 
 Once your module is present in the location set by the
-``mgr module path`` configuration setting, add its name
-to the ``mgr modules`` configuration setting and restart the ceph-mgr
-daemon to load it.
+``mgr module path`` configuration setting, you can enable it
+via the ``ceph mgr module enable`` command::
 
-If you're working within a Ceph vstart cluster then your module
-should be found in the default pybind/mgr location, and you only
-have to add it to ``mgr modules`` to get it loaded.
+  ceph mgr module enable mymodule
 
 Note that the MgrModule interface is not stable, so any modules maintained
 outside of the Ceph tree are liable to break when run against any newer

doc/release-notes.rst

@@ -196,6 +196,11 @@ Major Changes from Kraken
       log.
     - ``ceph mgr dump`` will dump the MgrMap, including the currently active
       ceph-mgr daemon and any standbys.
+    - ``ceph mgr module ls`` will list active ceph-mgr modules.
+    - ``ceph mgr module {enable,disable} <name>`` will enable or
+      disable the named mgr module.  The module must be present in the
+      configured `mgr_module_path` on the host(s) where `ceph-mgr` is
+      running.
     - ``ceph osd crush swap-bucket <src> <dest>`` will swap the
       contents of two CRUSH buckets in the hierarchy while preserving
       the buckets' ids.  This allows an entire subtree of devices to

qa/workunits/cephtool/test.sh

@@ -712,6 +712,12 @@ function test_mon_misc()
   ceph_watch_wait "$mymsg"
 
   ceph mgr dump
+  ceph mgr module ls
+  ceph mgr module enable restful
+  expect_false ceph mgr module enable foodne
+  ceph mgr module enable foodne --force
+  ceph mgr module disable foodne
+  ceph mgr module disable foodnebizbangbash
 
   ceph mon metadata a
   ceph mon metadata

src/common/config_opts.h

@@ -1729,7 +1729,7 @@ OPTION(rgw_shard_warning_threshold, OPT_DOUBLE, 90) // pct of safe max
 OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled
 
 OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from
-OPTION(mgr_modules, OPT_STR, "restful status")  // Which modules to load
+OPTION(mgr_initial_modules, OPT_STR, "restful status")  // Which modules to load
 OPTION(mgr_data, OPT_STR, "/var/lib/ceph/mgr/$cluster-$id") // where to find keyring etc
 OPTION(mgr_tick_period, OPT_INT, 2)  // How frequently to tick
 OPTION(mgr_stats_period, OPT_INT, 5) // How frequently clients send stats

src/messages/MMgrBeacon.h

@@ -22,7 +22,7 @@
 
 class MMgrBeacon : public PaxosServiceMessage {
 
-  static const int HEAD_VERSION = 2;
+  static const int HEAD_VERSION = 3;
   static const int COMPAT_VERSION = 1;
 
 protected:
@@ -31,6 +31,7 @@ protected:
   bool available;
   std::string name;
   uuid_d fsid;
+  std::set<std::string> available_modules;
 
 public:
   MMgrBeacon()
@@ -40,10 +41,11 @@ public:
   }
 
   MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_,
-             entity_addr_t server_addr_, bool available_)
+             entity_addr_t server_addr_, bool available_,
+	     const std::set<std::string>& module_list)
     : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION),
       gid(gid_), server_addr(server_addr_), available(available_), name(name_),
-      fsid(fsid_)
+      fsid(fsid_), available_modules(module_list)
   {
   }
 
@@ -52,6 +54,7 @@ public:
   bool get_available() const { return available; }
   const std::string& get_name() const { return name; }
   const uuid_d& get_fsid() const { return fsid; }
+  std::set<std::string>& get_available_modules() { return available_modules; }
 
 private:
   ~MMgrBeacon() override {}
@@ -62,7 +65,8 @@ public:
 
   void print(ostream& out) const override {
     out << get_type_name() << " mgr." << name << "(" << fsid << ","
-	<< gid << ", " << server_addr << ", " << available << ")";
+	<< gid << ", " << server_addr << ", " << available
+	<< ")";
   }
 
   void encode_payload(uint64_t features) override {
@@ -72,6 +76,7 @@ public:
     ::encode(available, payload);
     ::encode(name, payload);
     ::encode(fsid, payload);
+    ::encode(available_modules, payload);
   }
   void decode_payload() override {
     bufferlist::iterator p = payload.begin();
@@ -83,6 +88,9 @@ public:
     if (header.version >= 2) {
       ::decode(fsid, p);
     }
+    if (header.version >= 3) {
+      ::decode(available_modules, p);
+    }
   }
 };
 

src/mgr/ClusterState.cc

@@ -22,8 +22,15 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "mgr " << __func__ << " "
 
-ClusterState::ClusterState(MonClient *monc_, Objecter *objecter_)
-  : monc(monc_), objecter(objecter_), lock("ClusterState"), pgservice(pg_map)
+ClusterState::ClusterState(
+  MonClient *monc_,
+  Objecter *objecter_,
+  const MgrMap& mgrmap)
+  : monc(monc_),
+    objecter(objecter_),
+    lock("ClusterState"),
+    mgr_map(mgrmap),
+    pgservice(pg_map)
 {}
 
 void ClusterState::set_objecter(Objecter *objecter_)
@@ -40,6 +47,12 @@ void ClusterState::set_fsmap(FSMap const &new_fsmap)
   fsmap = new_fsmap;
 }
 
+void ClusterState::set_mgr_map(MgrMap const &new_mgrmap)
+{
+  Mutex::Locker l(lock);
+  mgr_map = new_mgrmap;
+}
+
 void ClusterState::load_digest(MMgrDigest *m)
 {
   health_json = std::move(m->health_json);

src/mgr/ClusterState.h

@@ -15,6 +15,7 @@
 #define CLUSTER_STATE_H_
 
 #include "mds/FSMap.h"
+#include "mon/MgrMap.h"
 #include "common/Mutex.h"
 
 #include "osdc/Objecter.h"
@@ -38,6 +39,8 @@ protected:
   FSMap fsmap;
   mutable Mutex lock;
 
+  MgrMap mgr_map;
+
   set<int64_t> existing_pools; ///< pools that exist, as of PGMap epoch
   PGMap pg_map;
   PGMap::Incremental pending_inc;
@@ -57,10 +60,11 @@ public:
   const bufferlist &get_health() const {return health_json;}
   const bufferlist &get_mon_status() const {return mon_status_json;}
 
-  ClusterState(MonClient *monc_, Objecter *objecter_);
+  ClusterState(MonClient *monc_, Objecter *objecter_, const MgrMap& mgrmap);
 
   void set_objecter(Objecter *objecter_);
   void set_fsmap(FSMap const &new_fsmap);
+  void set_mgr_map(MgrMap const &new_mgrmap);
 
   void notify_osdmap(const OSDMap &osd_map);
 
@@ -76,6 +80,13 @@ public:
     std::forward<Callback>(cb)(fsmap, std::forward<Args>(args)...);
   }
 
+  template<typename Callback, typename...Args>
+  void with_mgrmap(Callback&& cb, Args&&...args) const
+  {
+    Mutex::Locker l(lock);
+    std::forward<Callback>(cb)(mgr_map, std::forward<Args>(args)...);
+  }
+
   template<typename Callback, typename...Args>
   auto with_pgmap(Callback&& cb, Args&&...args) const ->
     decltype(cb(pg_map, std::forward<Args>(args)...))

src/mgr/Mgr.cc

@@ -39,7 +39,8 @@
 #define dout_prefix *_dout << "mgr " << __func__ << " "
 
 
-Mgr::Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_,
+Mgr::Mgr(MonClient *monc_, const MgrMap& mgrmap,
+	 Messenger *clientm_, Objecter *objecter_,
 	 Client* client_, LogChannelRef clog_, LogChannelRef audit_clog_) :
   monc(monc_),
   objecter(objecter_),
@@ -50,7 +51,7 @@ Mgr::Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_,
   finisher(g_ceph_context, "Mgr", "mgr-fin"),
   py_modules(daemon_state, cluster_state, *monc, clog_, *objecter, *client,
              finisher),
-  cluster_state(monc, nullptr),
+  cluster_state(monc, nullptr, mgrmap),
   server(monc, finisher, daemon_state, cluster_state, py_modules,
          clog_, audit_clog_),
   initialized(false),
@@ -567,6 +568,23 @@ void Mgr::handle_fs_map(MFSMap* m)
   daemon_state.cull(CEPH_ENTITY_TYPE_MDS, names_exist);
 }
 
+bool Mgr::got_mgr_map(const MgrMap& m)
+{
+  Mutex::Locker l(lock);
+  dout(10) << m << dendl;
+
+  set<string> old_modules;
+  cluster_state.with_mgrmap([&](const MgrMap& m) {
+      old_modules = m.modules;
+    });
+  if (m.modules != old_modules) {
+    derr << "mgrmap module list changed to (" << m.modules << "), respawn"
+	 << dendl;
+    return true;
+  }
+  cluster_state.set_mgr_map(m);
+  return false;
+}
 
 void Mgr::handle_mgr_digest(MMgrDigest* m)
 {

src/mgr/Mgr.h

@@ -72,7 +72,8 @@ protected:
   bool initializing;
 
 public:
-  Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_,
+  Mgr(MonClient *monc_, const MgrMap& mgrmap,
+      Messenger *clientm_, Objecter *objecter_,
       Client *client_, LogChannelRef clog_, LogChannelRef audit_clog_);
   ~Mgr();
 
@@ -84,6 +85,8 @@ public:
   void handle_osd_map();
   void handle_log(MLog *m);
 
+  bool got_mgr_map(const MgrMap& m);
+
   bool ms_dispatch(Message *m);
 
   void tick();

src/mgr/MgrStandby.cc

@@ -148,16 +148,20 @@ void MgrStandby::send_beacon()
 {
   assert(lock.is_locked_by_me());
   dout(1) << state_str() << dendl;
-  dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl;
 
+  set<string> modules;
+  PyModules::list_modules(&modules);
   bool available = active_mgr != nullptr && active_mgr->is_initialized();
   auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
+  dout(10) << "sending beacon as gid " << monc.get_global_id()
+	   << " modules " << modules << dendl;
+
   MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(),
 				 monc.get_global_id(),
                                  g_conf->name.get_id(),
                                  addr,
-                                 available);
-                                 
+                                 available,
+				 modules);
   monc.send_mon_message(m);
 }
 
@@ -282,12 +286,16 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap)
   if (active_in_map) {
     if (!active_mgr) {
       dout(1) << "Activating!" << dendl;
-      active_mgr.reset(new Mgr(&monc, client_messenger.get(), &objecter,
+      active_mgr.reset(new Mgr(&monc, map, client_messenger.get(), &objecter,
 			       &client, clog, audit_clog));
       active_mgr->background_init();
       dout(1) << "I am now active" << dendl;
     } else {
       dout(10) << "I was already active" << dendl;
+      bool need_respawn = active_mgr->got_mgr_map(map);
+      if (need_respawn) {
+	respawn();
+      }
     }
   } else {
     if (active_mgr != nullptr) {

src/mgr/PyModules.cc

@@ -15,7 +15,6 @@
 #include "PyState.h"
 #include "Gil.h"
 
-#include <boost/tokenizer.hpp>
 #include "common/errno.h"
 #include "include/stringify.h"
 
@@ -363,8 +362,11 @@ int PyModules::init()
   std::list<std::string> failed_modules;
 
   // Load python code
-  boost::tokenizer<> tok(g_conf->mgr_modules);
-  for(const auto& module_name : tok) {
+  set<string> ls;
+  cluster_state.with_mgrmap([&](const MgrMap& m) {
+      ls = m.modules;
+    });
+  for (const auto& module_name : ls) {
     dout(1) << "Loading python module '" << module_name << "'" << dendl;
     auto mod = std::unique_ptr<MgrPyModule>(new MgrPyModule(module_name, sys_path, pMainThreadState));
     int r = mod->load();
@@ -672,3 +674,32 @@ PyObject *PyModules::get_context()
   return capsule;
 }
 
+static void _list_modules(
+  const std::string path,
+  std::set<std::string> *modules)
+{
+  DIR *dir = opendir(path.c_str());
+  if (!dir) {
+    return;
+  }
+  struct dirent *entry = NULL;
+  while ((entry = readdir(dir)) != NULL) {
+    string n(entry->d_name);
+    string fn = path + "/" + n;
+    struct stat st;
+    int r = ::stat(fn.c_str(), &st);
+    if (r == 0 && S_ISDIR(st.st_mode)) {
+      string initfn = fn + "/module.py";
+      r = ::stat(initfn.c_str(), &st);
+      if (r == 0) {
+	modules->insert(n);
+      }
+    }
+  }
+  closedir(dir);
+}
+
+void PyModules::list_modules(std::set<std::string> *modules)
+{
+  _list_modules(g_conf->mgr_module_path, modules);
+}

src/mgr/PyModules.h

@@ -101,6 +101,8 @@ public:
 
   void log(const std::string &handle,
            int level, const std::string &record);
+
+  static void list_modules(std::set<std::string> *modules);
 };
 
 #endif

src/mon/MgrMap.h

@@ -25,9 +25,11 @@ class StandbyInfo
 public:
   uint64_t gid;
   std::string name;
+  std::set<std::string> available_modules;
 
-  StandbyInfo(uint64_t gid_, const std::string &name_)
-    : gid(gid_), name(name_)
+  StandbyInfo(uint64_t gid_, const std::string &name_,
+	      std::set<std::string>& am)
+    : gid(gid_), name(name_), available_modules(am)
   {}
 
   StandbyInfo()
@@ -36,17 +38,21 @@ public:
 
   void encode(bufferlist& bl) const
   {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(gid, bl);
     ::encode(name, bl);
+    ::encode(available_modules, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& p)
   {
-    DECODE_START(1, p);
+    DECODE_START(2, p);
     ::decode(gid, p);
     ::decode(name, p);
+    if (struct_v >= 2) {
+      ::decode(available_modules, p);
+    }
     DECODE_FINISH(p);
   }
 };
@@ -68,33 +74,54 @@ public:
 
   std::map<uint64_t, StandbyInfo> standbys;
 
+  std::set<std::string> modules;
+  std::set<std::string> available_modules;
+
   epoch_t get_epoch() const { return epoch; }
   entity_addr_t get_active_addr() const { return active_addr; }
   uint64_t get_active_gid() const { return active_gid; }
   bool get_available() const { return available; }
   const std::string &get_active_name() const { return active_name; }
 
+  bool all_support_module(const std::string& module) {
+    if (!available_modules.count(module)) {
+      return false;
+    }
+    for (auto& p : standbys) {
+      if (!p.second.available_modules.count(module)) {
+	return false;
+      }
+    }
+    return true;
+  }
+
   void encode(bufferlist& bl, uint64_t features) const
   {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(epoch, bl);
     ::encode(active_addr, bl, features);
     ::encode(active_gid, bl);
     ::encode(available, bl);
     ::encode(active_name, bl);
     ::encode(standbys, bl);
+    ::encode(modules, bl);
+    ::encode(available_modules, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& p)
   {
-    DECODE_START(1, p);
+    DECODE_START(2, p);
     ::decode(epoch, p);
     ::decode(active_addr, p);
     ::decode(active_gid, p);
     ::decode(available, p);
     ::decode(active_name, p);
     ::decode(standbys, p);
+    if (struct_v >= 2) {
+      ::decode(modules, p);
+      ::decode(available_modules, p);
+    }
     DECODE_FINISH(p);
   }
 
@@ -109,7 +136,22 @@ public:
       f->open_object_section("standby");
       f->dump_int("gid", i.second.gid);
       f->dump_string("name", i.second.name);
+      f->open_array_section("available_modules");
+      for (auto& j : i.second.available_modules) {
+	f->dump_string("module", j);
+      }
       f->close_section();
+      f->close_section();
+    }
+    f->close_section();
+    f->open_array_section("modules");
+    for (auto& i : modules) {
+      f->dump_string("module", i);
+    }
+    f->close_section();
+    f->open_array_section("available_modules");
+    for (auto& j : available_modules) {
+      f->dump_string("module", j);
     }
     f->close_section();
   }

src/mon/MgrMonitor.cc

@@ -11,6 +11,8 @@
  * Foundation.  See file COPYING.
  */
 
+#include <boost/tokenizer.hpp>
+
 #include "messages/MMgrBeacon.h"
 #include "messages/MMgrMap.h"
 #include "messages/MMgrDigest.h"
@@ -35,6 +37,11 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon,
 
 void MgrMonitor::create_initial()
 {
+  boost::tokenizer<> tok(g_conf->mgr_initial_modules);
+  for (auto& m : tok) {
+    pending_map.modules.insert(m);
+  }
+  dout(10) << __func__ << " initial modules " << pending_map.modules << dendl;
 }
 
 void MgrMonitor::update_from_paxos(bool *need_bootstrap)
@@ -206,6 +213,12 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
       pending_map.available = m->get_available();
       updated = true;
     }
+    if (pending_map.available_modules != m->get_available_modules()) {
+      dout(4) << "available_modules " << m->get_available_modules()
+	      << " (was " << pending_map.available_modules << ")" << dendl;
+      pending_map.available_modules = m->get_available_modules();
+      updated = true;
+    }
   } else if (pending_map.active_gid == 0) {
     // There is no currently active daemon, select this one.
     if (pending_map.standbys.count(m->get_gid())) {
@@ -217,14 +230,26 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
 	    << pending_map.active_name << ")" << dendl;
     pending_map.active_gid = m->get_gid();
     pending_map.active_name = m->get_name();
+    pending_map.available_modules = m->get_available_modules();
 
     updated = true;
   } else {
     if (pending_map.standbys.count(m->get_gid()) > 0) {
       dout(10) << "from existing standby " << m->get_gid() << dendl;
+      if (pending_map.standbys[m->get_gid()].available_modules !=
+	  m->get_available_modules()) {
+	dout(10) << "existing standby " << m->get_gid() << " available_modules "
+		 << m->get_available_modules() << " (was "
+		 << pending_map.standbys[m->get_gid()].available_modules << ")"
+		 << dendl;
+	pending_map.standbys[m->get_gid()].available_modules =
+	  m->get_available_modules();
+	updated = true;
+      }
     } else {
       dout(10) << "new standby " << m->get_gid() << dendl;
-      pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name()};
+      pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
+					    m->get_available_modules()};
       updated = true;
     }
   }
@@ -500,6 +525,13 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op)
       f->dump_object("mgrmap", m);
     }
     f->flush(rdata);
+  } else if (prefix == "mgr module ls") {
+    f->open_array_section("modules");
+    for (auto& p : map.modules) {
+      f->dump_string("module", p);
+    }
+    f->close_section();
+    f->flush(rdata);
   } else {
     return false;
   }
@@ -531,6 +563,10 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
     return true;
   }
 
+  string format;
+  cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
+  boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
   string prefix;
   cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
 
@@ -578,10 +614,36 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
     if (changed && pending_map.active_gid == 0) {
       promote_standby();
     }
+  } else if (prefix == "mgr module enable") {
+    string module;
+    cmd_getval(g_ceph_context, cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    string force;
+    cmd_getval(g_ceph_context, cmdmap, "force", force);
+    if (!pending_map.all_support_module(module) &&
+	force != "--force") {
+      ss << "all mgr daemons do not support module '" << module << "', pass "
+	 << "--force to force enablement";
+      r = -ENOENT;
+      goto out;
+    }
+    pending_map.modules.insert(module);
+  } else if (prefix == "mgr module disable") {
+    string module;
+    cmd_getval(g_ceph_context, cmdmap, "module", module);
+    if (module.empty()) {
+      r = -EINVAL;
+      goto out;
+    }
+    pending_map.modules.erase(module);
   } else {
     r = -ENOSYS;
   }
 
+out:
   dout(4) << __func__ << " done, r=" << r << dendl;
   /* Compose response */
   string rs;

src/mon/MonCommands.h

@@ -955,3 +955,12 @@ COMMAND("mgr dump "				     \
 	"mgr", "r", "cli,rest")
 COMMAND("mgr fail name=who,type=CephString", \
 	"treat the named manager daemon as failed", "mgr", "rw", "cli,rest")
+COMMAND("mgr module ls",
+	"list active mgr modules", "mgr", "r", "cli,rest")
+COMMAND("mgr module enable "						\
+	"name=module,type=CephString "					\
+	"name=force,type=CephChoices,strings=--force,req=false",
+	"enable mgr module", "mgr", "rw", "cli,rest")
+COMMAND("mgr module disable "						\
+	"name=module,type=CephString",
+	"disable mgr module", "mgr", "rw", "cli,rest")

src/pybind/mgr/dashboard/README.rst

@@ -24,12 +24,13 @@ If you had already enabled the module, restart ceph-mgr after installing depende
 Enabling
 ========
 
-Add this to your ceph.conf on nodes where you run ceph-mgr:
+Enable the module with::
 
-::
+  ceph mgr module enable dashboard
 
-    [mgr]                                                                           
-        mgr modules = dashboard
+You can see currently enabled modules with::
+
+  ceph mgr module ls
 
 If you use any other ceph-mgr modules, make sure they're in the list too.
 

src/test/mgr/mgr-dashboard-smoke.sh

@@ -25,7 +25,9 @@ function run() {
         FSID=$(uuidgen) 
         export CEPH_ARGS
         CEPH_ARGS+="--fsid=$FSID --auth-supported=none "
-        CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON"
+        CEPH_ARGS+="--mon-initial-members=a --mon-host=$MON "
+        CEPH_ARGS+="--mgr-initial-modules=dashbaord "
+	CEPH_ARGS+="--mon-host=$MON"
         run_mon $dir a --public-addr $MON || return 1
     )
 
@@ -33,7 +35,6 @@ function run() {
     export CEPH_ARGS="--mon_host $MON "
     ceph config-key put mgr/x/dashboard/server_port 7001
     MGR_ARGS+="--mgr_module_path=${CEPH_ROOT}/src/pybind/mgr "
-    MGR_ARGS+="--mgr_modules=dashboard "
     run_mgr $dir x ${MGR_ARGS} || return 1
 
     tries=0

src/vstart.sh

@@ -458,7 +458,6 @@ $CMDSDEBUG
         mds root ino gid = `id -g`
 $extra_conf
 [mgr]
-        mgr modules = restful status dashboard
         mgr data = $CEPH_DEV_DIR/mgr.\$id
         mgr module path = $MGR_PYTHON_PATH
         mon reweight min pgs per osd = 4
@@ -497,6 +496,7 @@ $COSDMEMSTORE
 $COSDSHORT
 $extra_conf
 [mon]
+        mgr initial modules = restful status dashboard
         mon pg warn min per osd = 3
         mon osd allow primary affinity = true
         mon reweight min pgs per osd = 4