Merge PR #36533 into master

* refs/pull/36533/head:
	mon: store mon updates in ceph context for future MonMap instantiation

Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>

src/common/ceph_context.cc

@@ -45,6 +45,9 @@
 #include "common/PluginRegistry.h"
 #include "common/valgrind.h"
 #include "include/spinlock.h"
+#if !(defined(WITH_SEASTAR) && !defined(WITH_ALIEN))
+#include "mon/MonMap.h"
+#endif
 
 // for CINIT_FLAGS
 #include "common/common_init.h"
@@ -991,5 +994,14 @@ void CephContext::notify_post_fork()
   for (auto &&t : _fork_watchers)
     t->handle_post_fork();
 }
+
+void CephContext::set_mon_addrs(const MonMap& mm) {
+  std::vector<entity_addrvec_t> mon_addrs;
+  for (auto& i : mm.mon_info) {
+    mon_addrs.push_back(i.second.public_addrs);
+  }
+
+  set_mon_addrs(mon_addrs);
+}
 }
 #endif	// WITH_SEASTAR

src/common/ceph_context.h

@@ -33,6 +33,7 @@
 
 #include "common/cmdparse.h"
 #include "common/code_environment.h"
+#include "msg/msg_types.h"
 #if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
 #include "crimson/common/config_proxy.h"
 #include "crimson/common/perf_counters_collection.h"
@@ -48,6 +49,7 @@
 class AdminSocket;
 class CryptoHandler;
 class CryptoRandom;
+class MonMap;
 
 namespace ceph::common {
   class CephContextServiceThread;
@@ -261,6 +263,21 @@ public:
   void notify_pre_fork();
   void notify_post_fork();
 
+  /**
+   * update CephContext with a copy of the passed in MonMap mon addrs
+   *
+   * @param mm MonMap to extract and update mon addrs
+   */
+  void set_mon_addrs(const MonMap& mm);
+  void set_mon_addrs(const std::vector<entity_addrvec_t>& in) {
+    auto ptr = std::make_shared<std::vector<entity_addrvec_t>>(in);
+    atomic_store_explicit(&_mon_addrs, std::move(ptr), std::memory_order_relaxed);
+  }
+  std::shared_ptr<std::vector<entity_addrvec_t>> get_mon_addrs() const {
+    auto ptr = atomic_load_explicit(&_mon_addrs, std::memory_order_relaxed);
+    return ptr;
+  }
+
 private:
 
 
@@ -278,6 +295,8 @@ private:
 
   int _crypto_inited;
 
+  std::shared_ptr<std::vector<entity_addrvec_t>> _mon_addrs;
+
   /* libcommon service thread.
    * SIGHUP wakes this thread, which then reopens logfiles */
   friend class CephContextServiceThread;

src/mon/MonClient.cc

@@ -424,6 +424,8 @@ void MonClient::handle_monmap(MMonMap *m)
     }
   }
 
+  cct->set_mon_addrs(monmap);
+
   sub.got("monmap", monmap.get_epoch());
   map_cond.notify_all();
   want_monmap = false;

src/mon/MonMap.cc

@@ -372,7 +372,6 @@ void MonMap::dump_summary(Formatter *f) const
   f->dump_unsigned("num_mons", ranks.size());
 }
 
-
 // an ambiguous mon addr may be legacy or may be msgr2--we aren' sure.
 // when that happens we need to try them both (unless we can
 // reasonably infer from the port number which it is).
@@ -461,9 +460,29 @@ void MonMap::_add_ambiguous_addr(const string& name,
   }
 }
 
+int
+MonMap::init_with_addrs(const std::vector<entity_addrvec_t>& addrs,
+                        bool for_mkfs,
+                        std::string_view prefix)
+{
+  char id = 'a';
+  for (auto& addr : addrs) {
+    string name{prefix};
+    name += id++;
+    if (addr.v.size() == 1) {
+      _add_ambiguous_addr(name, addr.front(), 0, 0, for_mkfs);
+    } else {
+      // they specified an addrvec, so let's assume they also specified
+      // the addr *type* and *port*.  (we could possibly improve this?)
+      add(name, addr, 0);
+    }
+  }
+  return 0;
+}
+
 int MonMap::init_with_ips(const std::string& ips,
 			  bool for_mkfs,
-			  const std::string &prefix)
+			  std::string_view prefix)
 {
   vector<entity_addrvec_t> addrs;
   if (!parse_ip_port_vec(
@@ -473,27 +492,12 @@ int MonMap::init_with_ips(const std::string& ips,
   }
   if (addrs.empty())
     return -ENOENT;
-  for (unsigned i=0; i<addrs.size(); i++) {
-    char n[2];
-    n[0] = 'a' + i;
-    n[1] = 0;
-    string name;
-    name = prefix;
-    name += n;
-    if (addrs[i].v.size() == 1) {
-      _add_ambiguous_addr(name, addrs[i].front(), 0, 0, for_mkfs);
-    } else {
-      // they specified an addrvec, so let's assume they also specified
-      // the addr *type* and *port*.  (we could possibly improve this?)
-      add(name, addrs[i], 0);
-    }
-  }
-  return 0;
+  return init_with_addrs(addrs, for_mkfs, prefix);
 }
 
 int MonMap::init_with_hosts(const std::string& hostlist,
 			    bool for_mkfs,
-			    const std::string& prefix)
+			    std::string_view prefix)
 {
   // maybe they passed us a DNS-resolvable name
   char *hosts = resolve_addrs(hostlist.c_str());
@@ -509,19 +513,8 @@ int MonMap::init_with_hosts(const std::string& hostlist,
     return -EINVAL;
   if (addrs.empty())
     return -ENOENT;
-  for (unsigned i=0; i<addrs.size(); i++) {
-    char n[2];
-    n[0] = 'a' + i;
-    n[1] = 0;
-    string name = prefix;
-    name += n;
-    if (addrs[i].v.size() == 1) {
-      _add_ambiguous_addr(name, addrs[i].front(), 0, 0, for_mkfs);
-    } else {
-      // they specified an addrvec, so let's assume they also specified
-      // the addr *type* and *port*.  (we could possibly improve this?)
-      add(name, addrs[i], 0);
-    }
+  if (!init_with_addrs(addrs, for_mkfs, prefix)) {
+    return -EINVAL;
   }
   calc_legacy_ranks();
   return 0;
@@ -826,6 +819,13 @@ int MonMap::init_with_dns_srv(CephContext* cct,
 int MonMap::build_initial(CephContext *cct, bool for_mkfs, ostream& errout)
 {
   const auto& conf = cct->_conf;
+
+  // cct?
+  auto addrs = cct->get_mon_addrs();
+  if (addrs != nullptr && (addrs->size() > 0)) {
+    return init_with_addrs(*addrs, for_mkfs, "noname-");
+  }
+
   // file?
   if (const auto monmap = conf.get_val<std::string>("monmap");
       !monmap.empty()) {

src/mon/MonMap.h

@@ -30,7 +30,7 @@
 
 
 #ifdef WITH_SEASTAR
-namespace ceph::common {
+namespace crimson::common {
   class ConfigProxy;
 }
 #endif
@@ -456,6 +456,18 @@ public:
 
   static void generate_test_instances(std::list<MonMap*>& o);
 protected:
+  /**
+   * build a monmap from a list of entity_addrvec_t's
+   *
+   * Give mons dummy names.
+   *
+   * @param addrs  list of entity_addrvec_t's
+   * @param prefix prefix to prepend to generated mon names
+   * @return 0 for success, -errno on error
+   */
+  int init_with_addrs(const std::vector<entity_addrvec_t>& addrs,
+        bool for_mkfs,
+        std::string_view prefix);
   /**
    * build a monmap from a list of ips
    *
@@ -467,7 +479,7 @@ protected:
    */
   int init_with_ips(const std::string& ips,
 		    bool for_mkfs,
-		    const std::string &prefix);
+		    std::string_view prefix);
   /**
    * build a monmap from a list of hostnames
    *
@@ -479,7 +491,7 @@ protected:
    */
   int init_with_hosts(const std::string& hostlist,
 		      bool for_mkfs,
-		      const std::string& prefix);
+		      std::string_view prefix);
   int init_with_config_file(const ConfigProxy& conf, std::ostream& errout);
 #if WITH_SEASTAR
   seastar::future<> read_monmap(const std::string& monmap);

src/test/libcephfs/CMakeLists.txt

@@ -9,6 +9,7 @@ if(${WITH_CEPHFS})
     acl.cc
     main.cc
     deleg.cc
+    monconfig.cc
   )
   target_link_libraries(ceph_test_libcephfs
     ceph-common

src/test/libcephfs/monconfig.cc

@@ -0,0 +1,99 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "common/ceph_context.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+class MonConfig : public ::testing::Test
+{
+  protected:
+    struct ceph_mount_info *ca;
+
+    void SetUp() override {
+      ASSERT_EQ(0, ceph_create(&ca, NULL));
+      ASSERT_EQ(0, ceph_conf_read_file(ca, NULL));
+      ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+    }
+
+    void TearDown() override {
+      ceph_shutdown(ca);
+    }
+
+    // Helper to remove/unset all possible mon information from ConfigProxy
+    void clear_mon_config(CephContext *cct) {
+      auto& conf = cct->_conf;
+      // Clear safe_to_start_threads, allowing updates to config values
+      conf._clear_safe_to_start_threads();
+      ASSERT_EQ(0, conf.set_val("monmap", "", nullptr));
+      ASSERT_EQ(0, conf.set_val("mon_host", "", nullptr));
+      ASSERT_EQ(0, conf.set_val("mon_dns_srv_name", "", nullptr));
+      conf.set_safe_to_start_threads();
+    }
+
+    // Helper to test basic operation on a mount
+    void use_mount(struct ceph_mount_info *mnt, string name_prefix) {
+      char name[20];
+      snprintf(name, sizeof(name), "%s.%d", name_prefix.c_str(), getpid());
+      int fd = ceph_open(mnt, name, O_CREAT|O_RDWR, 0644);
+      ASSERT_LE(0, fd);
+
+      ceph_close(mnt, fd);
+    }
+};
+
+TEST_F(MonConfig, MonAddrsMissing) {
+  CephContext *cct;
+
+  // Test mount failure when there is no known mon config source
+  cct = ceph_get_mount_context(ca);
+  ASSERT_NE(nullptr, cct);
+  clear_mon_config(cct);
+
+  ASSERT_EQ(-ENOENT, ceph_mount(ca, NULL));
+}
+
+TEST_F(MonConfig, MonAddrsInConfigProxy) {
+  // Test a successful mount with default mon config source in ConfigProxy
+  ASSERT_EQ(0, ceph_mount(ca, NULL));
+
+  use_mount(ca, "foo");
+}
+
+TEST_F(MonConfig, MonAddrsInCct) {
+  struct ceph_mount_info *cb;
+  CephContext *cct;
+
+  // Perform mount to bootstrap mon addrs in CephContext
+  ASSERT_EQ(0, ceph_mount(ca, NULL));
+
+  // Reuse bootstrapped CephContext, clearing ConfigProxy mon addr sources
+  cct = ceph_get_mount_context(ca);
+  ASSERT_NE(nullptr, cct);
+  clear_mon_config(cct);
+  ASSERT_EQ(0, ceph_create_with_context(&cb, cct));
+
+  // Test a successful mount with only mon values in CephContext
+  ASSERT_EQ(0, ceph_mount(cb, NULL));
+
+  use_mount(ca, "bar");
+  use_mount(cb, "bar");
+
+  ceph_shutdown(cb);
+}