mon/MonClient: one-shot mon connection on started to fetch config

This is not particularly efficient, but it works:

 - connect to the monitor to get the monmap and config
 - tear it all down
 - proceed with normal startup (which presumably involves reconnecting
   to the mon all over again).

This allows us to set config options that may affect the mon communication
itself, like ms_type.

Signed-off-by: Sage Weil <sage@redhat.com>

src/ceph_fuse.cc

@@ -80,7 +80,8 @@ int main(int argc, const char **argv, const char *envp[]) {
   env_to_vec(args);
 
   std::map<std::string,std::string> defaults = {
-    { "pid_file", "" }
+    { "pid_file", "" },
+    { "chdir", "/" }  // FUSE will chdir("/"); be ready.
   };
 
   auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
@@ -107,10 +108,6 @@ int main(int argc, const char **argv, const char *envp[]) {
   int newargc;
   vec_to_argv(argv[0], args, &newargc, &newargv);
 
-  // FUSE will chdir("/"); be ready.
-  g_ceph_context->_conf->set_val("chdir", "/");
-  g_ceph_context->_conf->apply_changes(nullptr);
-
   // check for 32-bit arch
 #ifndef __LP64__
     cerr << std::endl;

src/ceph_mgr.cc

@@ -47,14 +47,16 @@ int main(int argc, const char **argv)
   argv_to_vec(argc, argv, args);
   env_to_vec(args);
 
-  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_MGR,
+  map<string,string> defaults = {
+    { "keyring", "$mgr_data/keyring" }
+  };
+  auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_MGR,
 			 CODE_ENVIRONMENT_DAEMON, 0,
 			 "mgr_data");
-  // For consumption by KeyRing::from_ceph_context in MonClient
-  g_conf->set_val_or_die("keyring", "$mgr_data/keyring");
 
   // Handle --help
-  if ((args.size() == 1 && (std::string(args[0]) == "--help" || std::string(args[0]) == "-h"))) {
+  if ((args.size() == 1 && (std::string(args[0]) == "--help" ||
+			    std::string(args[0]) == "-h"))) {
     usage();
   }
 

src/ceph_mon.cc

@@ -242,6 +242,9 @@ int main(int argc, const char **argv)
     }
   }
 
+  // don't try to get config from mon cluster during startup
+  flags |= CINIT_FLAG_NO_MON_CONFIG;
+
   auto cct = global_init(&defaults, args,
 			 CEPH_ENTITY_TYPE_MON, CODE_ENVIRONMENT_DAEMON,
 			 flags, "mon_data");

src/ceph_osd.cc

@@ -328,18 +328,14 @@ int main(int argc, const char **argv)
   }
   if (mkfs) {
     common_init_finish(g_ceph_context);
-    MonClient mc(g_ceph_context);
-    if (mc.build_initial_monmap() < 0)
-      return -1;
-    if (mc.get_monmap_privately() < 0)
-      return -1;
 
-    if (mc.monmap.fsid.is_zero()) {
+    if (g_conf->get_val<uuid_d>("fsid").is_zero()) {
       derr << "must specify cluster fsid" << dendl;
       forker.exit(-EINVAL);
     }
 
-    int err = OSD::mkfs(g_ceph_context, store, data_path, mc.monmap.fsid,
+    int err = OSD::mkfs(g_ceph_context, store, data_path,
+			g_conf->get_val<uuid_d>("fsid"),
                         whoami);
     if (err < 0) {
       derr << TEXT_RED << " ** ERROR: error creating empty object store in "
@@ -347,7 +343,9 @@ int main(int argc, const char **argv)
       forker.exit(1);
     }
     dout(0) << "created object store " << data_path
-	    << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl;
+	    << " for osd." << whoami
+	    << " fsid " << g_conf->get_val<uuid_d>("fsid")
+	    << dendl;
   }
   if (mkfs || mkkey) {
     forker.exit(0);

src/common/ceph_argparse.cc

@@ -484,6 +484,9 @@ CephInitParameters ceph_argparse_early_args
     else if (ceph_argparse_witharg(args, i, &val, "--conf", "-c", (char*)NULL)) {
       *conf_file_list = val;
     }
+    else if (ceph_argparse_flag(args, i, "--no-mon-config", (char*)NULL)) {
+      iparams.no_mon_config = true;
+    }
     else if (ceph_argparse_witharg(args, i, &val, "--cluster", (char*)NULL)) {
       *cluster = val;
     }

src/common/ceph_argparse.h

@@ -37,6 +37,8 @@ public:
 
   uint32_t module_type;
   EntityName name;
+
+  bool no_mon_config = false;
 };
 
 /////////////////////// Functions ///////////////////////

src/common/common_init.h

@@ -26,7 +26,7 @@ enum common_init_flags_t {
   // Set up defaults that make sense for an unprivileged daemon
   CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS = 0x1,
 
-  // By default, don't read a configuration file
+  // By default, don't read a configuration file OR contact mons
   CINIT_FLAG_NO_DEFAULT_CONFIG_FILE = 0x2,
 
   // Don't close stderr (in daemonize)
@@ -37,6 +37,9 @@ enum common_init_flags_t {
 
   // don't drop privileges
   CINIT_FLAG_DEFER_DROP_PRIVILEGES = 0x10,
+
+  // do'nt contact mons for config
+  CINIT_FLAG_NO_MON_CONFIG = 0x20,
 };
 
 /*

src/global/global_init.cc

@@ -26,6 +26,7 @@
 #include "global/signal_handler.h"
 #include "include/compat.h"
 #include "include/str_list.h"
+#include "mon/MonClient.h"
 
 #include <pwd.h>
 #include <grp.h>
@@ -88,8 +89,15 @@ void global_pre_init(
 {
   std::string conf_file_list;
   std::string cluster = "";
-  CephInitParameters iparams = ceph_argparse_early_args(args, module_type,
-							&cluster, &conf_file_list);
+
+  CephInitParameters iparams = ceph_argparse_early_args(
+    args, module_type,
+    &cluster, &conf_file_list);
+  if (flags & (CINIT_FLAG_NO_DEFAULT_CONFIG_FILE|
+	       CINIT_FLAG_NO_MON_CONFIG)) {
+    iparams.no_mon_config = true;
+  }
+
   CephContext *cct = common_preinit(iparams, code_env, flags);
   cct->_conf->cluster = cluster;
   global_init_set_globals(cct);
@@ -130,6 +138,15 @@ void global_pre_init(
 
   conf->parse_argv(args); // argv override
 
+  if (!iparams.no_mon_config) {
+    MonClient mc_bootstrap(g_ceph_context);
+    if (mc_bootstrap.get_monmap_and_config() < 0) {
+      derr << "failed to fetch mon config (--no-mon-config to skip)" << dendl;
+      cct->_log->flush();
+      _exit(1);
+    }
+  }
+
   // Now we're ready to complain about config file parse errors
   g_conf->complain_about_parse_errors(g_ceph_context);
 }

src/libcephfs.cc

@@ -77,6 +77,13 @@ public:
 
     int ret;
 
+    {
+      MonClient mc_bootstrap(cct);
+      ret = mc_bootstrap.get_monmap_and_config();
+      if (ret < 0)
+	return ret;
+    }
+
     //monmap
     monclient = new MonClient(cct);
     ret = -CEPHFS_ERROR_MON_MAP_BUILD; //defined in libcephfs.h;

src/librados/RadosClient.cc

@@ -248,6 +248,13 @@ int librados::RadosClient::connect()
     return -EISCONN;
   state = CONNECTING;
 
+  {
+    MonClient mc_bootstrap(cct);
+    err = mc_bootstrap.get_monmap_and_config();
+    if (err < 0)
+      return err;
+  }
+
   // get monmap
   err = monclient.build_initial_monmap();
   if (err < 0)

src/mon/MonClient.cc

@@ -22,6 +22,7 @@
 #include "messages/MMonGetVersionReply.h"
 #include "messages/MMonMap.h"
 #include "messages/MConfig.h"
+#include "messages/MGetConfig.h"
 #include "messages/MAuth.h"
 #include "messages/MLogAck.h"
 #include "messages/MAuthReply.h"
@@ -92,67 +93,71 @@ int MonClient::get_monmap()
   return 0;
 }
 
-int MonClient::get_monmap_privately()
+int MonClient::get_monmap_and_config()
 {
   ldout(cct, 10) << __func__ << dendl;
-  Mutex::Locker l(monc_lock);
+  assert(!messenger);
 
-  bool temp_msgr = false;
-  Messenger* smessenger = NULL;
-  if (!messenger) {
-    messenger = smessenger = Messenger::create_client_messenger(cct, "temp_mon_client");
-    if (NULL == messenger) {
-        return -1;
-    }
-    messenger->add_dispatcher_head(this);
-    smessenger->start();
-    temp_msgr = true;
+  utime_t interval;
+  interval.set_from_double(cct->_conf->mon_client_hunt_interval * 10);
+
+  cct->init_crypto();
+
+  int r = build_initial_monmap();
+  if (r < 0) {
+    lderr(cct) << __func__ << " cannot identify monitors to contact" << dendl;
+    goto out;
   }
 
-  int attempt = 10;
+  messenger = Messenger::create_client_messenger(
+    cct, "temp_mon_client");
+  assert(messenger);
+  messenger->add_dispatcher_head(this);
+  messenger->start();
 
-  ldout(cct, 10) << "have " << monmap.epoch << " fsid " << monmap.fsid << dendl;
-
-  std::random_device rd;
-  std::mt19937 rng(rd());
-  assert(monmap.size() > 0);
-  std::uniform_int_distribution<unsigned> ranks(0, monmap.size() - 1);
-  while (monmap.fsid.is_zero()) {
-    auto rank = ranks(rng);
-    auto& pending_con = _add_conn(rank, 0);
-    auto con = pending_con.get_con();
-    ldout(cct, 10) << "querying mon." << monmap.get_name(rank) << " "
-		   << con->get_peer_addr() << dendl;
-    con->send_message(new MMonGetMap);
-
-    if (--attempt == 0)
-      break;
-
-    utime_t interval;
-    interval.set_from_double(cct->_conf->mon_client_hunt_interval);
-    map_cond.WaitInterval(monc_lock, interval);
-
-    if (monmap.fsid.is_zero() && con) {
-      con->mark_down();  // nope, clean that connection up
+  r = init();
+  if (r < 0) {
+    goto out_msgr;
+  }
+  r = authenticate(cct->_conf->client_mount_timeout);
+  if (r < 0) {
+    goto out_shutdown;
+  }
+  if (!monmap.persistent_features.contains_all(
+	ceph::features::mon::FEATURE_MIMIC)) {
+    ldout(cct,10) << __func__ << " pre-mimic monitor, no config to fetch"
+		  << dendl;
+  } else {
+    Mutex::Locker l(monc_lock);
+    while (!got_config) {
+      ldout(cct,20) << __func__ << " waiting for config" << dendl;
+      map_cond.WaitInterval(monc_lock, interval);
     }
   }
-
-  if (temp_msgr) {
-    pending_cons.clear();
-    monc_lock.Unlock();
-    messenger->shutdown();
-    if (smessenger)
-      smessenger->wait();
-    delete messenger;
-    messenger = 0;
-    monc_lock.Lock();
+  if (got_config) {
+    ldout(cct,10) << __func__ << " success" << dendl;
+    r = 0;
+  } else {
+    lderr(cct) << __func__ << " failed to get config" << dendl;
+    r = -EIO;
   }
 
-  pending_cons.clear();
+out_shutdown:
+  shutdown();
 
-  if (!monmap.fsid.is_zero())
-    return 0;
-  return -1;
+out_msgr:
+  messenger->shutdown();
+  messenger->wait();
+  delete messenger;
+  messenger = nullptr;
+
+  if (!monmap.fsid.is_zero()) {
+    cct->_conf->set_val("fsid", stringify(monmap.fsid));
+  }
+
+out:
+  cct->shutdown_crypto();
+  return r;
 }
 
 
@@ -360,6 +365,8 @@ void MonClient::handle_config(MConfig *m)
   ldout(cct,10) << __func__ << " " << *m << dendl;
   cct->_conf->set_mon_vals(cct, m->config);
   m->put();
+  got_config = true;
+  map_cond.Signal();
 }
 
 // ----------------------
@@ -449,6 +456,7 @@ void MonClient::shutdown()
   if (initialized) {
     finisher.wait_for_empty();
     finisher.stop();
+    initialized = false;
   }
   monc_lock.Lock();
   timer.shutdown();

src/mon/MonClient.h

@@ -190,6 +190,7 @@ private:
   bool want_monmap;
   Cond map_cond;
   bool passthrough_monmap = false;
+  bool got_config = false;
 
   // authenticate
   std::unique_ptr<AuthClientHandler> auth;
@@ -344,7 +345,7 @@ public:
 
   int build_initial_monmap();
   int get_monmap();
-  int get_monmap_privately();
+  int get_monmap_and_config();
   /**
    * If you want to see MonMap messages, set this and
    * the MonClient will tell the Messenger it hasn't

src/tools/ceph_conf.cc

@@ -178,7 +178,8 @@ int main(int argc, const char **argv)
   vector<const char*> orig_args = args;
 
   global_pre_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON,
-		  CINIT_FLAG_NO_DAEMON_ACTIONS);
+		  CINIT_FLAG_NO_DAEMON_ACTIONS |
+		  CINIT_FLAG_NO_MON_CONFIG);
   std::unique_ptr<CephContext,
 		  std::function<void(CephContext*)> > cct_deleter{
       g_ceph_context,