diff --git a/src/ceph_fuse.cc b/src/ceph_fuse.cc index 35e3509bf70..2b3dfb86e7b 100644 --- a/src/ceph_fuse.cc +++ b/src/ceph_fuse.cc @@ -80,7 +80,8 @@ int main(int argc, const char **argv, const char *envp[]) { env_to_vec(args); std::map defaults = { - { "pid_file", "" } + { "pid_file", "" }, + { "chdir", "/" } // FUSE will chdir("/"); be ready. }; auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, @@ -107,10 +108,6 @@ int main(int argc, const char **argv, const char *envp[]) { int newargc; vec_to_argv(argv[0], args, &newargc, &newargv); - // FUSE will chdir("/"); be ready. - g_ceph_context->_conf->set_val("chdir", "/"); - g_ceph_context->_conf->apply_changes(nullptr); - // check for 32-bit arch #ifndef __LP64__ cerr << std::endl; diff --git a/src/ceph_mgr.cc b/src/ceph_mgr.cc index 5e8f6798e39..77e3933666c 100644 --- a/src/ceph_mgr.cc +++ b/src/ceph_mgr.cc @@ -47,14 +47,16 @@ int main(int argc, const char **argv) argv_to_vec(argc, argv, args); env_to_vec(args); - auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_MGR, + map defaults = { + { "keyring", "$mgr_data/keyring" } + }; + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_MGR, CODE_ENVIRONMENT_DAEMON, 0, "mgr_data"); - // For consumption by KeyRing::from_ceph_context in MonClient - g_conf->set_val_or_die("keyring", "$mgr_data/keyring"); // Handle --help - if ((args.size() == 1 && (std::string(args[0]) == "--help" || std::string(args[0]) == "-h"))) { + if ((args.size() == 1 && (std::string(args[0]) == "--help" || + std::string(args[0]) == "-h"))) { usage(); } diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 8683000af9c..d0791955f0f 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -242,6 +242,9 @@ int main(int argc, const char **argv) } } + // don't try to get config from mon cluster during startup + flags |= CINIT_FLAG_NO_MON_CONFIG; + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_MON, CODE_ENVIRONMENT_DAEMON, flags, "mon_data"); diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index c7f887ae87a..37feada6e1b 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -328,18 +328,14 @@ int main(int argc, const char **argv) } if (mkfs) { common_init_finish(g_ceph_context); - MonClient mc(g_ceph_context); - if (mc.build_initial_monmap() < 0) - return -1; - if (mc.get_monmap_privately() < 0) - return -1; - if (mc.monmap.fsid.is_zero()) { + if (g_conf->get_val("fsid").is_zero()) { derr << "must specify cluster fsid" << dendl; forker.exit(-EINVAL); } - int err = OSD::mkfs(g_ceph_context, store, data_path, mc.monmap.fsid, + int err = OSD::mkfs(g_ceph_context, store, data_path, + g_conf->get_val("fsid"), whoami); if (err < 0) { derr << TEXT_RED << " ** ERROR: error creating empty object store in " @@ -347,7 +343,9 @@ int main(int argc, const char **argv) forker.exit(1); } dout(0) << "created object store " << data_path - << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl; + << " for osd." << whoami + << " fsid " << g_conf->get_val("fsid") + << dendl; } if (mkfs || mkkey) { forker.exit(0); diff --git a/src/common/ceph_argparse.cc b/src/common/ceph_argparse.cc index a89de95f6e4..5f86a002cc3 100644 --- a/src/common/ceph_argparse.cc +++ b/src/common/ceph_argparse.cc @@ -484,6 +484,9 @@ CephInitParameters ceph_argparse_early_args else if (ceph_argparse_witharg(args, i, &val, "--conf", "-c", (char*)NULL)) { *conf_file_list = val; } + else if (ceph_argparse_flag(args, i, "--no-mon-config", (char*)NULL)) { + iparams.no_mon_config = true; + } else if (ceph_argparse_witharg(args, i, &val, "--cluster", (char*)NULL)) { *cluster = val; } diff --git a/src/common/ceph_argparse.h b/src/common/ceph_argparse.h index 900b18f31a7..ba9462acfc1 100644 --- a/src/common/ceph_argparse.h +++ b/src/common/ceph_argparse.h @@ -37,6 +37,8 @@ public: uint32_t module_type; EntityName name; + + bool no_mon_config = false; }; /////////////////////// Functions /////////////////////// diff --git a/src/common/common_init.h b/src/common/common_init.h index f66c4df4211..bd004b0e60e 100644 --- a/src/common/common_init.h +++ b/src/common/common_init.h @@ -26,7 +26,7 @@ enum common_init_flags_t { // Set up defaults that make sense for an unprivileged daemon CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS = 0x1, - // By default, don't read a configuration file + // By default, don't read a configuration file OR contact mons CINIT_FLAG_NO_DEFAULT_CONFIG_FILE = 0x2, // Don't close stderr (in daemonize) @@ -37,6 +37,9 @@ enum common_init_flags_t { // don't drop privileges CINIT_FLAG_DEFER_DROP_PRIVILEGES = 0x10, + + // do'nt contact mons for config + CINIT_FLAG_NO_MON_CONFIG = 0x20, }; /* diff --git a/src/global/global_init.cc b/src/global/global_init.cc index daaaf955668..716104c71f8 100644 --- a/src/global/global_init.cc +++ b/src/global/global_init.cc @@ -26,6 +26,7 @@ #include "global/signal_handler.h" #include "include/compat.h" #include "include/str_list.h" +#include "mon/MonClient.h" #include #include @@ -88,8 +89,15 @@ void global_pre_init( { std::string conf_file_list; std::string cluster = ""; - CephInitParameters iparams = ceph_argparse_early_args(args, module_type, - &cluster, &conf_file_list); + + CephInitParameters iparams = ceph_argparse_early_args( + args, module_type, + &cluster, &conf_file_list); + if (flags & (CINIT_FLAG_NO_DEFAULT_CONFIG_FILE| + CINIT_FLAG_NO_MON_CONFIG)) { + iparams.no_mon_config = true; + } + CephContext *cct = common_preinit(iparams, code_env, flags); cct->_conf->cluster = cluster; global_init_set_globals(cct); @@ -130,6 +138,15 @@ void global_pre_init( conf->parse_argv(args); // argv override + if (!iparams.no_mon_config) { + MonClient mc_bootstrap(g_ceph_context); + if (mc_bootstrap.get_monmap_and_config() < 0) { + derr << "failed to fetch mon config (--no-mon-config to skip)" << dendl; + cct->_log->flush(); + _exit(1); + } + } + // Now we're ready to complain about config file parse errors g_conf->complain_about_parse_errors(g_ceph_context); } diff --git a/src/libcephfs.cc b/src/libcephfs.cc index df687e5db3a..9f1a27eb73f 100644 --- a/src/libcephfs.cc +++ b/src/libcephfs.cc @@ -77,6 +77,13 @@ public: int ret; + { + MonClient mc_bootstrap(cct); + ret = mc_bootstrap.get_monmap_and_config(); + if (ret < 0) + return ret; + } + //monmap monclient = new MonClient(cct); ret = -CEPHFS_ERROR_MON_MAP_BUILD; //defined in libcephfs.h; diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index 3fcd2523daf..efbd6b563fb 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -248,6 +248,13 @@ int librados::RadosClient::connect() return -EISCONN; state = CONNECTING; + { + MonClient mc_bootstrap(cct); + err = mc_bootstrap.get_monmap_and_config(); + if (err < 0) + return err; + } + // get monmap err = monclient.build_initial_monmap(); if (err < 0) diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index cee5bb20615..75a66b8d098 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -22,6 +22,7 @@ #include "messages/MMonGetVersionReply.h" #include "messages/MMonMap.h" #include "messages/MConfig.h" +#include "messages/MGetConfig.h" #include "messages/MAuth.h" #include "messages/MLogAck.h" #include "messages/MAuthReply.h" @@ -92,67 +93,71 @@ int MonClient::get_monmap() return 0; } -int MonClient::get_monmap_privately() +int MonClient::get_monmap_and_config() { ldout(cct, 10) << __func__ << dendl; - Mutex::Locker l(monc_lock); + assert(!messenger); - bool temp_msgr = false; - Messenger* smessenger = NULL; - if (!messenger) { - messenger = smessenger = Messenger::create_client_messenger(cct, "temp_mon_client"); - if (NULL == messenger) { - return -1; - } - messenger->add_dispatcher_head(this); - smessenger->start(); - temp_msgr = true; + utime_t interval; + interval.set_from_double(cct->_conf->mon_client_hunt_interval * 10); + + cct->init_crypto(); + + int r = build_initial_monmap(); + if (r < 0) { + lderr(cct) << __func__ << " cannot identify monitors to contact" << dendl; + goto out; } - int attempt = 10; + messenger = Messenger::create_client_messenger( + cct, "temp_mon_client"); + assert(messenger); + messenger->add_dispatcher_head(this); + messenger->start(); - ldout(cct, 10) << "have " << monmap.epoch << " fsid " << monmap.fsid << dendl; - - std::random_device rd; - std::mt19937 rng(rd()); - assert(monmap.size() > 0); - std::uniform_int_distribution ranks(0, monmap.size() - 1); - while (monmap.fsid.is_zero()) { - auto rank = ranks(rng); - auto& pending_con = _add_conn(rank, 0); - auto con = pending_con.get_con(); - ldout(cct, 10) << "querying mon." << monmap.get_name(rank) << " " - << con->get_peer_addr() << dendl; - con->send_message(new MMonGetMap); - - if (--attempt == 0) - break; - - utime_t interval; - interval.set_from_double(cct->_conf->mon_client_hunt_interval); - map_cond.WaitInterval(monc_lock, interval); - - if (monmap.fsid.is_zero() && con) { - con->mark_down(); // nope, clean that connection up + r = init(); + if (r < 0) { + goto out_msgr; + } + r = authenticate(cct->_conf->client_mount_timeout); + if (r < 0) { + goto out_shutdown; + } + if (!monmap.persistent_features.contains_all( + ceph::features::mon::FEATURE_MIMIC)) { + ldout(cct,10) << __func__ << " pre-mimic monitor, no config to fetch" + << dendl; + } else { + Mutex::Locker l(monc_lock); + while (!got_config) { + ldout(cct,20) << __func__ << " waiting for config" << dendl; + map_cond.WaitInterval(monc_lock, interval); } } - - if (temp_msgr) { - pending_cons.clear(); - monc_lock.Unlock(); - messenger->shutdown(); - if (smessenger) - smessenger->wait(); - delete messenger; - messenger = 0; - monc_lock.Lock(); + if (got_config) { + ldout(cct,10) << __func__ << " success" << dendl; + r = 0; + } else { + lderr(cct) << __func__ << " failed to get config" << dendl; + r = -EIO; } - pending_cons.clear(); +out_shutdown: + shutdown(); - if (!monmap.fsid.is_zero()) - return 0; - return -1; +out_msgr: + messenger->shutdown(); + messenger->wait(); + delete messenger; + messenger = nullptr; + + if (!monmap.fsid.is_zero()) { + cct->_conf->set_val("fsid", stringify(monmap.fsid)); + } + +out: + cct->shutdown_crypto(); + return r; } @@ -360,6 +365,8 @@ void MonClient::handle_config(MConfig *m) ldout(cct,10) << __func__ << " " << *m << dendl; cct->_conf->set_mon_vals(cct, m->config); m->put(); + got_config = true; + map_cond.Signal(); } // ---------------------- @@ -449,6 +456,7 @@ void MonClient::shutdown() if (initialized) { finisher.wait_for_empty(); finisher.stop(); + initialized = false; } monc_lock.Lock(); timer.shutdown(); diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 7ca35016892..6caf57360e4 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -190,6 +190,7 @@ private: bool want_monmap; Cond map_cond; bool passthrough_monmap = false; + bool got_config = false; // authenticate std::unique_ptr auth; @@ -344,7 +345,7 @@ public: int build_initial_monmap(); int get_monmap(); - int get_monmap_privately(); + int get_monmap_and_config(); /** * If you want to see MonMap messages, set this and * the MonClient will tell the Messenger it hasn't diff --git a/src/tools/ceph_conf.cc b/src/tools/ceph_conf.cc index 40cbf946d4b..12399c0099b 100644 --- a/src/tools/ceph_conf.cc +++ b/src/tools/ceph_conf.cc @@ -178,7 +178,8 @@ int main(int argc, const char **argv) vector orig_args = args; global_pre_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_DAEMON, - CINIT_FLAG_NO_DAEMON_ACTIONS); + CINIT_FLAG_NO_DAEMON_ACTIONS | + CINIT_FLAG_NO_MON_CONFIG); std::unique_ptr > cct_deleter{ g_ceph_context,