Merge pull request #38999 from tchaikov/wip-crimson-bootstrap

crimson/osd: fetch_config() before mkfs

Reviewed-by: Radoslaw Zarzynski <rzarzyns@redhat.com>

src/crimson/mon/MonClient.cc

@@ -254,8 +254,7 @@ Connection::do_auth_single(Connection::request_t what)
     if (!m) {
       ceph_assert(closed);
       logger().info("do_auth: connection closed");
-      return seastar::make_ready_future<std::optional<Connection::auth_result_t>>(
-	std::make_optional(auth_result_t::canceled));
+      return std::make_optional(auth_result_t::canceled);
     }
     logger().info(
       "do_auth: mon {} => {} returns {}: {}",
@@ -264,19 +263,22 @@ Connection::do_auth_single(Connection::request_t what)
     auto p = m->result_bl.cbegin();
     auto ret = auth->handle_response(m->result, p,
 				     nullptr, nullptr);
-    if (ret != 0 && ret != -EAGAIN) {
+    std::optional<Connection::auth_result_t> auth_result;
+    switch (ret) {
+    case -EAGAIN:
+      auth_result = std::nullopt;
+      break;
+    case 0:
+      auth_result = auth_result_t::success;
+      break;
+    default:
+      auth_result = auth_result_t::failure;
       logger().error(
-	"do_auth: got error {} on mon {}",
-	ret,
-	conn->get_peer_addr());
+        "do_auth: got error {} on mon {}",
+        ret, conn->get_peer_addr());
+      break;
     }
-    return seastar::make_ready_future<std::optional<Connection::auth_result_t>>(
-      ret == -EAGAIN
-      ? std::nullopt
-      : std::make_optional(ret == 0
-	 ? auth_result_t::success
-	 : auth_result_t::failure
-      ));
+    return auth_result;
   });
 }
 
@@ -894,7 +896,11 @@ seastar::future<> Client::handle_log_ack(Ref<MLogAck> m)
 
 seastar::future<> Client::handle_config(Ref<MConfig> m)
 {
-  return crimson::common::local_conf().set_mon_vals(m->config);
+  return crimson::common::local_conf().set_mon_vals(m->config).then([this] {
+    if (config_updated) {
+      config_updated->set_value();
+    }
+  });
 }
 
 std::vector<unsigned> Client::get_random_mons(unsigned n) const
@@ -957,10 +963,17 @@ seastar::future<> Client::reopen_session(int rank)
   pending_conns.reserve(mons.size());
   return seastar::parallel_for_each(mons, [this](auto rank) {
     // TODO: connect to multiple addrs
-    auto peer = monmap.get_addrs(rank).pick_addr(msgr.get_myaddr().get_type());
+    // connect to v2 addresses if we have not bound to any address, otherwise
+    // use the advertised msgr protocol
+    uint32_t type = msgr.get_myaddr().get_type();
+    if (type == entity_addr_t::TYPE_NONE) {
+      type = entity_addr_t::TYPE_MSGR2;
+    }
+    auto peer = monmap.get_addrs(rank).pick_addr(type);
     if (peer == entity_addr_t{}) {
       // crimson msgr only uses the first bound addr
-      logger().warn("mon.{} does not have an addr compatible with me", rank);
+      logger().warn("mon.{} does not have an addr compatible with my type: {}",
+		    rank, msgr.get_myaddr().get_type());
       return seastar::now();
     }
     logger().info("connecting to mon.{}", rank);
@@ -1103,6 +1116,13 @@ seastar::future<> Client::renew_subs()
   });
 }
 
+seastar::future<> Client::wait_for_config()
+{
+  assert(!config_updated);
+  config_updated = seastar::promise<>();
+  return config_updated->get_future();
+}
+
 void Client::print(std::ostream& out) const
 {
   out << "mon." << entity_name;

src/crimson/mon/MonClient.h

@@ -90,6 +90,7 @@ public:
   void sub_unwant(const std::string& what);
   bool sub_want_increment(const std::string& what, version_t start, unsigned flags);
   seastar::future<> renew_subs();
+  seastar::future<> wait_for_config();
 
   void print(std::ostream&) const;
 private:
@@ -173,6 +174,7 @@ private:
     seastar::promise<> pr;
   };
   std::deque<pending_msg_t> pending_messages;
+  std::optional<seastar::promise<>> config_updated;
 };
 
 inline std::ostream& operator<<(std::ostream& out, const Client& client) {

src/crimson/net/SocketMessenger.cc

@@ -38,6 +38,11 @@ SocketMessenger::SocketMessenger(const entity_name_t& myname,
     nonce{nonce}
 {}
 
+SocketMessenger::~SocketMessenger()
+{
+  ceph_assert(!listener);
+}
+
 seastar::future<> SocketMessenger::set_myaddrs(const entity_addrvec_t& addrs)
 {
   assert(seastar::this_shard_id() == master_sid);

src/crimson/net/SocketMessenger.h

@@ -54,7 +54,7 @@ class SocketMessenger final : public Messenger {
   SocketMessenger(const entity_name_t& myname,
                   const std::string& logic_name,
                   uint32_t nonce);
-  ~SocketMessenger() override { ceph_assert(!listener); }
+  ~SocketMessenger() override;
 
   seastar::future<> set_myaddrs(const entity_addrvec_t& addr) override;
 

src/crimson/os/cyanstore/cyan_store.cc

@@ -95,7 +95,7 @@ seastar::future<> CyanStore::mkfs(uuid_d new_osd_fsid)
     } else if (r < 0) {
       throw std::runtime_error("read_meta");
     } else {
-      logger().info("{} already has fsid {}", __func__, fsid_str);
+      logger().info("mkfs already has fsid {}", fsid_str);
       if (!osd_fsid.parse(fsid_str.c_str())) {
         throw std::runtime_error("failed to parse fsid");
       } else if (osd_fsid != new_osd_fsid) {

src/crimson/osd/main.cc

@@ -16,6 +16,7 @@
 #include "common/ceph_argparse.h"
 #include "crimson/common/buffer_io.h"
 #include "crimson/common/config_proxy.h"
+#include "crimson/mon/MonClient.h"
 #include "crimson/net/Messenger.h"
 #include "global/pidfile.h"
 
@@ -117,6 +118,43 @@ uint64_t get_nonce()
   }
 }
 
+seastar::future<> fetch_config()
+{
+  // i don't have any client before joining the cluster, so no need to have
+  // a proper auth handler
+  class DummyAuthHandler : public crimson::common::AuthHandler {
+  public:
+    void handle_authentication(const EntityName& name,
+                               const AuthCapsInfo& caps)
+    {}
+  };
+  auto auth_handler = std::make_unique<DummyAuthHandler>();
+  auto msgr = crimson::net::Messenger::create(entity_name_t::CLIENT(),
+                                              "temp_mon_client",
+                                              get_nonce());
+  auto monc = std::make_unique<crimson::mon::Client>(*msgr, *auth_handler);
+  msgr->set_auth_client(monc.get());
+  return msgr->start({monc.get()}).then([monc=monc.get()] {
+    return monc->start();
+  }).then([monc=monc.get()] {
+    monc->sub_want("config", 0, 0);
+    return monc->renew_subs();
+  }).then([monc=monc.get()] {
+    // wait for monmap and config
+    return monc->wait_for_config();
+  }).then([monc=monc.get()] {
+    return local_conf().set_val("fsid", monc->get_fsid().to_string());
+  }).then([monc=monc.get(), msgr=msgr.get()] {
+    msgr->stop();
+    return monc->stop();
+  }).then([msgr=msgr.get()] {
+    return msgr->shutdown();
+  }).then([msgr=std::move(msgr),
+           auth_handler=std::move(auth_handler),
+           monc=std::move(monc)]
+  {});
+}
+
 int main(int argc, char* argv[])
 {
   seastar::app_template app;
@@ -198,6 +236,7 @@ int main(int argc, char* argv[])
             seastar::engine().exit(1);
           }).get();
         }
+        fetch_config().get();
         if (config.count("mkfs")) {
           osd.invoke_on(
 	    0,

src/mon/MonClient.cc

@@ -31,7 +31,6 @@
 #include "messages/MMonGetVersionReply.h"
 #include "messages/MMonMap.h"
 #include "messages/MConfig.h"
-#include "messages/MGetConfig.h"
 #include "messages/MAuth.h"
 #include "messages/MLogAck.h"
 #include "messages/MAuthReply.h"

src/vstart.sh

@@ -913,7 +913,7 @@ EOF
             echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json
             ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json
             rm $CEPH_DEV_DIR/osd$osd/new.json
-            $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args
+            prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args
 
             local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
             cat > $key_fn<<EOF