mon: Monitor: mon debug cli via admin socket

Purpose: allow a dev to test certain portions of the monitors on-the-fly
that would otherwise be nearly impossible without requiring shutting
them off.

Description:

  Currently we are adding support to debug monitors on-the-fly via the
  ceph tool cli, via the admin socket.

  An initial attempt was made to make this interface available via
  over-the-network ceph cli, but that would require a quorum to exist,
  or significant changes to be made to the handling of commands to allow
  this interface to be an exception to the rule.

  Therefore, this interface will live on the admin sockets for the time
  being; potentially forever, because the jury is still out on whether
  there's added value making this available via any other medium that's
  not the admin socket.

  The work means to test the newly added monmap features. This new
  interface will allow one to set, unset and list supported features.

  This will make it easier to implement functional tests for the
  features themselves, for quorum features that are expected to be set
  on-quorum, as well as to test upgrade paths from version A to version
  B.

Signed-off-by: Joao Eduardo Luis <joao@suse.de>

src/common/bit_str.cc

@@ -14,12 +14,14 @@
 #include "common/bit_str.h"
 #include "common/Formatter.h"
 #include "include/assert.h"
+#include "common/debug.h"
 
 static void _dump_bit_str(
     uint64_t bits,
     std::ostream *out,
     ceph::Formatter *f,
-    std::function<const char*(uint64_t)> func)
+    std::function<const char*(uint64_t)> func,
+    bool dump_bit_val)
 {
   uint64_t b = bits;
   int cnt = 0;
@@ -32,9 +34,17 @@ static void _dump_bit_str(
         if (outted)
           *out << ",";
         *out << func(r);
+        if (dump_bit_val) {
+          *out << "(" << r << ")";
+        }
       } else {
         assert(f != NULL);
-        f->dump_stream("bit_flag") << func(r);
+        if (dump_bit_val) {
+          f->dump_stream("bit_flag") << func(r)
+                                     << "(" << r << ")";
+        } else {
+          f->dump_stream("bit_flag") << func(r);
+        }
       }
       outted = true;
     }
@@ -47,15 +57,17 @@ static void _dump_bit_str(
 void print_bit_str(
     uint64_t bits,
     std::ostream &out,
-    std::function<const char*(uint64_t)> func)
+    std::function<const char*(uint64_t)> func,
+    bool dump_bit_val)
 {
-  _dump_bit_str(bits, &out, NULL, func);
+  _dump_bit_str(bits, &out, NULL, func, dump_bit_val);
 }
 
 void dump_bit_str(
     uint64_t bits,
     ceph::Formatter *f,
-    std::function<const char*(uint64_t)> func)
+    std::function<const char*(uint64_t)> func,
+    bool dump_bit_val)
 {
-  _dump_bit_str(bits, NULL, f, func);
+  _dump_bit_str(bits, NULL, f, func, dump_bit_val);
 }

src/common/bit_str.h

@@ -25,11 +25,13 @@ namespace ceph {
 extern void print_bit_str(
     uint64_t bits,
     std::ostream &out,
-    std::function<const char*(uint64_t)> func);
+    std::function<const char*(uint64_t)> func,
+    bool dump_bit_val = false);
 
 extern void dump_bit_str(
     uint64_t bits,
     ceph::Formatter *f,
-    std::function<const char*(uint64_t)> func);
+    std::function<const char*(uint64_t)> func,
+    bool dump_bit_val = false);
 
 #endif /* CEPH_COMMON_BIT_STR_H */

src/mon/MonCommands.h

@@ -438,6 +438,22 @@ COMMAND("mon remove " \
 COMMAND("mon rm " \
 	"name=name,type=CephString", \
 	"remove monitor named <name>", "mon", "rw", "cli,rest")
+COMMAND("mon debug set_feature" \
+        "name=feature_type,type=CephChoices,strings=persistent|optional " \
+        "name=feature_name,type=CephString " \
+        "name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+        "set provided feature on mon map", \
+        "mon", "rw", "cli")
+COMMAND("mon debug list_features " \
+        "name=feature_type,type=CephString,req=false", \
+        "list available mon map features to be set/unset", \
+        "mon", "rw", "cli")
+COMMAND("mon debug unset_feature " \
+        "name=feature_type,type=CephChoices,strings=persistent|optional " \
+        "name=feature_name,type=CephString " \
+        "name=sure,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \
+        "unset provided feature from monmap", \
+        "mon", "rw", "cli")
 
 /*
  * OSD commands

src/mon/Monitor.cc

@@ -19,6 +19,7 @@
 #include <limits.h>
 #include <cstring>
 #include <boost/scope_exit.hpp>
+#include <boost/algorithm/string/predicate.hpp>
 
 #include "Monitor.h"
 #include "common/version.h"
@@ -294,7 +295,7 @@ void Monitor::do_admin_command(string command, cmdmap_t& cmdmap, string format,
     args += cmd_vartype_stringify(p->second);
   }
   args = "[" + args + "]";
- 
+
   bool read_only = (command == "mon_status" ||
                     command == "mon metadata" ||
                     command == "quorum_status" ||
@@ -336,6 +337,123 @@ void Monitor::do_admin_command(string command, cmdmap_t& cmdmap, string format,
     if (f) {
       f->flush(ss);
     }
+  } else if (boost::starts_with(command, "debug mon features")) {
+  
+    // check if unsupported feature is set
+    if (!cct->check_experimental_feature_enabled("mon_debug_features_commands")) {
+      ss << "error: this is an experimental feature and is not enabled.";
+      goto abort;
+    }
+
+    if (command == "debug mon features list") {
+
+      mon_feature_t supported = ceph::features::mon::get_supported();
+      mon_feature_t persistent = ceph::features::mon::get_persistent();
+
+      if (f) {
+
+        f->open_object_section("features");
+        f->open_object_section("ceph-mon");
+        supported.dump_with_value(f.get(), "supported");
+        persistent.dump_with_value(f.get(), "persistent");
+        f->close_section(); // ceph-mon
+        f->open_object_section("monmap");
+        monmap->persistent_features.dump_with_value(f.get(), "persistent");
+        monmap->optional_features.dump_with_value(f.get(), "optional");
+        mon_feature_t required = monmap->get_required_features();
+        required.dump_with_value(f.get(), "required");
+        f->close_section(); // monmap
+        f->close_section(); // features
+
+        f->flush(ss);
+      } else {
+        ss << "only structured formats allowed when listing";
+      }
+    } else if (command == "debug mon features set" ||
+               command == "debug mon features set_val" ||
+               command == "debug mon features unset" ||
+               command == "debug mon features unset_val") {
+
+      string n;
+      if (!cmd_getval(cct, cmdmap, "feature", n)) {
+        ss << "missing feature to set";
+        goto abort;
+      }
+
+      string f_type;
+      bool do_persistent = false, do_optional = false;
+
+      if (cmd_getval(cct, cmdmap, "feature_type", f_type)) {
+        if (f_type == "--persistent") {
+          do_persistent = true;
+        } else {
+          do_optional = true;
+        }
+      }
+
+      mon_feature_t feature;
+
+      if (command == "debug mon features set" ||
+          command == "debug mon features unset") {
+        feature = ceph::features::mon::get_feature_by_name(n);
+        if (feature == ceph::features::mon::FEATURE_NONE) {
+          ss << "no such feature '" << n << "'";
+          goto abort;
+        }
+      } else {
+        uint64_t feature_val;
+        string interr;
+        feature_val = strict_strtoll(n.c_str(), 10, &interr);
+        if (!interr.empty()) {
+          ss << "unable to parse feature value: " << interr;
+          goto abort;
+        }
+
+        feature = mon_feature_t(feature_val);
+      }
+
+      bool do_unset = false;
+      if (boost::ends_with(command, "unset") ||
+          boost::ends_with(command, "unset_val")) {
+        do_unset = true;
+      }
+
+      ss << (do_unset? "un" : "") << "setting feature '";
+      feature.print_with_value(ss);
+      ss << "' on current monmap\n";
+      ss << "please note this change is not persistent; "
+         << "changes to monmap will overwrite the changes\n";
+
+      if (!do_persistent && !do_optional) {
+        if (ceph::features::mon::get_persistent().contains_all(feature)) {
+          do_persistent = true;
+        } else {
+          do_optional = true;
+        }
+      }
+
+      ss << "\n" << (do_unset ? "un" : "") << "setting ";
+
+      mon_feature_t &target_feature = (do_persistent ?
+          monmap->persistent_features : monmap->optional_features);
+
+      if (do_persistent) {
+        ss << "persistent feature";
+      } else {
+        ss << "optional feature";
+      }
+
+      if (do_unset) {
+        target_feature.unset_feature(feature);
+      } else {
+        target_feature.set_feature(feature);
+      }
+
+    } else {
+
+      ss << "unrecognized command";
+    }
+
   } else {
     assert(0 == "bad AdminSocket command binding");
   }
@@ -770,6 +888,46 @@ int Monitor::preinit()
                                      admin_hook,
                                      "show the ops currently in flight");
   assert(r == 0);
+
+  // debugging api
+  r = admin_socket->register_command("debug mon features list",
+                                     "debug mon features list",
+                                     admin_hook,
+                                     "list monmap features");
+  assert(r == 0);
+  r = admin_socket->register_command("debug mon features set",
+                                     "debug mon features set "
+                                     "name=feature,type=CephString "
+                                     "name=feature_type,type=CephChoices,req=false,"
+                                     "strings=--persistent|--optional",
+                                     admin_hook,
+                                     "set a given feature, by name, in the monmap");
+  assert(r == 0);
+  r = admin_socket->register_command("debug mon features set_val",
+                                     "debug mon features set_val "
+                                     "name=feature,type=CephString "
+                                     "name=feature_type,type=CephChoices,req=false,"
+                                     "strings=--persistent|--optional",
+                                     admin_hook,
+                                     "set a given feature, by value, in the monmap");
+  assert(r == 0);
+  r = admin_socket->register_command("debug mon features unset",
+                                     "debug mon features unset "
+                                     "name=feature,type=CephString "
+                                     "name=feature_type,type=CephChoices,req=false,"
+                                     "strings=--persistent|--optional",
+                                     admin_hook,
+                                     "unset a given feature, by name, in the monmap");
+  assert(r == 0);
+  r = admin_socket->register_command("debug mon features unset_val",
+                                     "debug mon features unset_val "
+                                     "name=feature,type=CephString "
+                                     "name=feature_type,type=CephChoices,req=false,"
+                                     "strings=--persistent|--optional",
+                                     admin_hook,
+                                     "unset a given feature, by value, in the monmap");
+  assert(r == 0);
+
   lock.Lock();
 
   // add ourselves as a conf observer
@@ -891,12 +1049,18 @@ void Monitor::shutdown()
     admin_socket->unregister_command("quorum enter");
     admin_socket->unregister_command("quorum exit");
     admin_socket->unregister_command("ops");
+    // debugging api
+    admin_socket->unregister_command("debug mon features list");
+    admin_socket->unregister_command("debug mon features set");
+    admin_socket->unregister_command("debug mon features set_val");
+    admin_socket->unregister_command("debug mon features unset");
+    admin_socket->unregister_command("debug mon features unset_val");
     delete admin_hook;
     admin_hook = NULL;
   }
 
   elector.shutdown();
-  
+
   // clean up
   paxos->shutdown();
   for (vector<PaxosService*>::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p)

src/mon/mon_types.h

@@ -288,6 +288,20 @@ public:
     return (*this);
   }
 
+  /**
+   * Obtain raw features
+   *
+   * @remarks
+   *    Consumers should not assume this interface will never change.
+   * @remarks
+   *    As the number of features increase, so may the internal representation
+   *    of the raw features. When this happens, this interface will change
+   *    accordingly. So should consumers of this interface.
+   */
+  uint64_t get_raw() const {
+    return features;
+  }
+
   constexpr
   friend mon_feature_t operator&(const mon_feature_t a,
                                  const mon_feature_t b) {
@@ -379,18 +393,34 @@ public:
     features |= f.features;
   }
 
+  void unset_feature(const mon_feature_t f) {
+    features &= ~(f.features);
+  }
+
   void print(ostream& out) const {
     out << "[";
     print_bit_str(features, out, ceph::features::mon::get_feature_name);
     out << "]";
   }
 
+  void print_with_value(ostream& out) const {
+    out << "[";
+    print_bit_str(features, out, ceph::features::mon::get_feature_name, true);
+    out << "]";
+  }
+
   void dump(Formatter *f, const char *sec_name = NULL) const {
     f->open_array_section((sec_name ? sec_name : "features"));
     dump_bit_str(features, f, ceph::features::mon::get_feature_name);
     f->close_section();
   }
 
+  void dump_with_value(Formatter *f, const char *sec_name = NULL) const {
+    f->open_array_section((sec_name ? sec_name : "features"));
+    dump_bit_str(features, f, ceph::features::mon::get_feature_name, true);
+    f->close_section();
+  }
+
   void encode(bufferlist& bl) const {
     ENCODE_START(HEAD_VERSION, COMPAT_VERSION, bl);
     ::encode(features, bl);
@@ -438,6 +468,8 @@ namespace ceph {
             FEATURE_NONE
             );
       }
+
+      static inline mon_feature_t get_feature_by_name(std::string n);
     }
   }
 }
@@ -453,6 +485,17 @@ static inline const char *ceph::features::mon::get_feature_name(uint64_t b) {
   return "unknown";
 }
 
+static inline
+mon_feature_t ceph::features::mon::get_feature_by_name(std::string n) {
+
+  if (n == "kraken") {
+    return FEATURE_KRAKEN;
+  } else if (n == "reserved") {
+    return FEATURE_RESERVED;
+  }
+  return FEATURE_NONE;
+}
+
 static inline ostream& operator<<(ostream& out, const mon_feature_t& f) {
   out << "mon_feature_t(";
   f.print(out);