mds: big MDS refactor stage 2: encapsulation

Encapsulate MDSRank in MDS instead of inheriting from
it.

Signed-off-by: John Spray <john.spray@redhat.com>

src/ceph_mds.cc

@@ -205,7 +205,7 @@ int main(int argc, const char **argv)
   mds->orig_argc = argc;
   mds->orig_argv = argv;
 
-  if (shadow)
+  if (shadow != MDSMap::STATE_NULL)
     r = mds->init(shadow);
   else
     r = mds->init();
@@ -237,6 +237,7 @@ int main(int argc, const char **argv)
   pidfile_remove();
 
   delete mds;
+  delete msgr;
 
   g_ceph_context->put();
 

src/mds/Beacon.cc

@@ -57,7 +57,6 @@ void Beacon::init(MDSMap const *mdsmap, MDSMap::DaemonState want_state_,
   Mutex::Locker l(lock);
   assert(mdsmap != NULL);
 
-  // Initialize copies of MDS state
   want_state = want_state_;
   _notify_mdsmap(mdsmap);
   standby_for_rank = standby_rank_;
@@ -195,6 +194,8 @@ void Beacon::_send()
 	   << dendl;
 
   seq_stamp[last_seq] = ceph_clock_now(g_ceph_context);
+
+  assert(want_state != MDSMap::STATE_NULL);
   
   MMDSBeacon *beacon = new MMDSBeacon(
       monc->get_fsid(), mds_gid_t(monc->get_global_id()),
@@ -230,10 +231,13 @@ void Beacon::notify_mdsmap(MDSMap const *mdsmap)
 void Beacon::_notify_mdsmap(MDSMap const *mdsmap)
 {
   assert(mdsmap != NULL);
+  assert(mdsmap->get_epoch() >= epoch);
 
-  epoch = mdsmap->get_epoch();
-  compat = get_mdsmap_compat_set_default();
-  compat.merge(mdsmap->compat);
+  if (mdsmap->get_epoch() != epoch) {
+    epoch = mdsmap->get_epoch();
+    compat = get_mdsmap_compat_set_default();
+    compat.merge(mdsmap->compat);
+  }
 }
 
 
@@ -270,10 +274,17 @@ utime_t Beacon::get_laggy_until() const
   return laggy_until;
 }
 
-void Beacon::set_want_state(MDSMap::DaemonState const newstate)
+void Beacon::set_want_state(MDSMap const *mdsmap, MDSMap::DaemonState const newstate)
 {
   Mutex::Locker l(lock);
 
+  // Update mdsmap epoch atomically with updating want_state, so that when
+  // we send a beacon with the new want state it has the latest epoch, and
+  // once we have updated to the latest epoch, we are not sending out
+  // a stale want_state (i.e. one from before making it through MDSMap
+  // handling)
+  _notify_mdsmap(mdsmap);
+
   if (want_state != newstate) {
     dout(10) << __func__ << ": "
       << ceph_mds_state_name(want_state) << " -> "

src/mds/Beacon.h

@@ -101,7 +101,7 @@ public:
   void handle_mds_beacon(MMDSBeacon *m);
   void send();
 
-  void set_want_state(MDSMap::DaemonState const newstate);
+  void set_want_state(MDSMap const *mdsmap, MDSMap::DaemonState const newstate);
   MDSMap::DaemonState get_want_state() const;
 
   /**

src/mds/MDCache.cc

@@ -11804,3 +11804,8 @@ void MDCache::notify_mdsmap_changed()
   stray_manager.update_op_limit();
 }
 
+void MDCache::notify_osdmap_changed()
+{
+  stray_manager.update_op_limit();
+}
+

src/mds/MDCache.h

@@ -1093,6 +1093,7 @@ public:
   void discard_delayed_expire(CDir *dir);
 
   void notify_mdsmap_changed();
+  void notify_osdmap_changed();
 
 protected:
   void dump_cache(const char *fn, Formatter *f);

src/mds/MDS.h

@@ -72,15 +72,13 @@ class MDSTableClient;
 
 class AuthAuthorizeHandlerRegistry;
 
-class MDS : public MDSRank, public Dispatcher, public md_config_obs_t {
+class MDS : public Dispatcher, public md_config_obs_t {
  public:
-
   /* Global MDS lock: every time someone takes this, they must
    * also check the `stopping` flag.  If stopping is true, you
    * must either do nothing and immediately drop the lock, or
    * never drop the lock again (i.e. call respawn()) */
   Mutex        mds_lock;
-  bool         stopping;
 
   SafeTimer    timer;
 
@@ -98,7 +96,8 @@ class MDS : public MDSRank, public Dispatcher, public md_config_obs_t {
   Objecter     *objecter;
   LogClient    log_client;
   LogChannelRef clog;
-  Finisher finisher;
+
+  MDSRank mds_rank;
 
  public:
   MDS(const std::string &n, Messenger *m, MonClient *mc);
@@ -117,14 +116,12 @@ class MDS : public MDSRank, public Dispatcher, public md_config_obs_t {
   virtual void handle_conf_change(const struct md_config_t *conf,
 				  const std::set <std::string> &changed);
  protected:
-
-
   // tick and other timer fun
   class C_MDS_Tick : public MDSInternalContext {
     protected:
       MDS *mds_daemon;
   public:
-    C_MDS_Tick(MDS *m) : MDSInternalContext(m), mds_daemon(m) {}
+    C_MDS_Tick(MDS *m, MDSRank *rank) : MDSInternalContext(rank), mds_daemon(m) {}
     void finish(int r) {
       mds_daemon->tick_event = 0;
       mds_daemon->tick();
@@ -132,8 +129,11 @@ class MDS : public MDSRank, public Dispatcher, public md_config_obs_t {
   } *tick_event;
   void     reset_tick();
 
-  // -- client map --
-  epoch_t      last_client_mdsmap_bcast;
+  void wait_for_omap_osds();
+
+  mds_rank_t standby_for_rank;
+  string standby_for_name;
+  MDSMap::DaemonState standby_type;  // one of STANDBY_REPLAY, ONESHOT_REPLAY
 
  private:
   bool ms_dispatch(Message *m);
@@ -155,44 +155,14 @@ class MDS : public MDSRank, public Dispatcher, public md_config_obs_t {
   void set_up_admin_socket();
   void clean_up_admin_socket();
   void check_ops_in_flight(); // send off any slow ops to monitor
-  void command_scrub_path(Formatter *f, const string& path);
-  void command_flush_path(Formatter *f, const string& path);
-  void command_flush_journal(Formatter *f);
-  void command_get_subtrees(Formatter *f);
-  void command_export_dir(Formatter *f,
-      const std::string &path, mds_rank_t dest);
-  bool command_dirfrag_split(
-      cmdmap_t cmdmap,
-      std::ostream &ss);
-  bool command_dirfrag_merge(
-      cmdmap_t cmdmap,
-      std::ostream &ss);
-  bool command_dirfrag_ls(
-      cmdmap_t cmdmap,
-      std::ostream &ss,
-      Formatter *f);
- private:
-  int _command_export_dir(const std::string &path, mds_rank_t dest);
-  int _command_flush_journal(std::stringstream *ss);
-  CDir *_command_dirfrag_get(
-      const cmdmap_t &cmdmap,
-      std::ostream &ss);
- protected:
-  void create_logger();
-  void update_log_config();
 
-  void bcast_mds_map();  // to mounted clients
-public:
   /**
    * Terminate this daemon process.
    *
    * This function will return, but once it does so the calling thread
    * must do no more work as all subsystems will have been shut down.
-   *
-   * @param fast: if true, do not send a message to the mon before shutting
-   *              down
    */
-  void suicide(bool fast = false);
+  void suicide();
 
   /**
    * Start a new daemon process with the same command line parameters that

src/mds/MDSMap.cc

@@ -372,6 +372,9 @@ void MDSMap::get_health(list<pair<health_status_t,string> >& summary,
   set<string> laggy;
   for (; u != u_end; ++u) {
     map<mds_gid_t, mds_info_t>::const_iterator m = mds_info.find(u->second);
+    if (m == m_end) {
+      std::cerr << "Up rank " << u->first << " GID " << u->second << " not found!" << std::endl;
+    }
     assert(m != m_end);
     const mds_info_t &mds_info(m->second);
     if (mds_info.laggy()) {

src/mds/MDSRank.h

@@ -125,11 +125,12 @@ class MMDSMap;
  */
 class MDSRank {
   public:
+    // FIXME: this should be private in favour of getter
     mds_rank_t whoami;
 
-    // FIXME: incarnation is logically a daemon property, not a rank property,
-    // but used in log msgs and set on objecter (objecter *is* logically
-    // a rank thing rather than daemon thing)
+    // Incarnation as seen in MDSMap at the point where a rank is
+    // assigned.  FIXME this shoudl be a const once MDSRank
+    // is only constructed upon a rank being issued.
     int incarnation;
 
     mds_rank_t get_nodeid() const { return whoami; }
@@ -205,6 +206,10 @@ class MDSRank {
     void handle_write_error(int err);
 
   protected:
+    // Flag to indicate we entered shutdown: anyone seeing this to be true
+    // after taking mds_lock must drop out.
+    bool stopping;
+
     class ProgressThread : public Thread {
       MDSRank *mds;
       Cond cond;
@@ -246,11 +251,16 @@ class MDSRank {
     // when it's laggy.
     Beacon &beacon;
 
+    /**
+     * Emit clog warnings for any ops reported as warnings by optracker
+     */
+    void check_ops_in_flight();
 
-    // Call into me from MDS::ms_dispatch
-    // FIXME separate dispatchers for MDSRank vs MDSDaemon
-    bool handle_rank_message(Message *m);
-
+    /**
+     * Share MDSMap with clients
+     */
+    void bcast_mds_map();  // to mounted clients
+    epoch_t      last_client_mdsmap_bcast;
   public:
 
     void queue_waiter(MDSInternalContextBase *c) {
@@ -268,10 +278,11 @@ class MDSRank {
         SafeTimer &timer_,
         Beacon &beacon_,
         MDSMap *& mdsmap_,
-        Finisher *finisher_,
-        MDS *mds_daemon_,
         Messenger *msgr,
-        MonClient *monc_);
+        MonClient *monc_,
+        Objecter *objecter_,
+        Context *respawn_hook_,
+        Context *suicide_hook_);
     ~MDSRank();
 
     // Daemon lifetime functions: these guys break the abstraction
@@ -280,7 +291,7 @@ class MDSRank {
     // to be able to e.g. tear down the whole process, we have to
     // have a reference going all the way down.
     // >>>
-    void suicide(bool fast = false);
+    void suicide();
     void respawn();
     // <<<
 
@@ -365,12 +376,54 @@ class MDSRank {
 
     int get_req_rate() { return logger->get(l_mds_request); }
 
+    // FIXME: interface for MDSDaemon to call, don't really want to expose
+    // this to every subsystem that carries an MDSRank reference though
+    // >>>
+    void init(mds_rank_t rank, int incarnation);
+    void tick();
+    void shutdown();
+    void create_logger();
+    void update_log_config();
+    bool handle_asok_command(std::string command, cmdmap_t& cmdmap,
+                             Formatter *f, std::ostream& ss);
+    void handle_mds_map(MMDSMap *m, MDSMap *oldmap);
+    void handle_osd_map();
+    bool kill_session(int64_t session_id);
+
+    // Call into me from MDS::ms_dispatch
+    bool ms_dispatch(Message *m);
+
+  protected:
+    void command_scrub_path(Formatter *f, const string& path);
+    void command_flush_path(Formatter *f, const string& path);
+    void command_flush_journal(Formatter *f);
+    void command_get_subtrees(Formatter *f);
+    void command_export_dir(Formatter *f,
+        const std::string &path, mds_rank_t dest);
+    bool command_dirfrag_split(
+        cmdmap_t cmdmap,
+        std::ostream &ss);
+    bool command_dirfrag_merge(
+        cmdmap_t cmdmap,
+        std::ostream &ss);
+    bool command_dirfrag_ls(
+        cmdmap_t cmdmap,
+        std::ostream &ss,
+        Formatter *f);
+    int _command_export_dir(const std::string &path, mds_rank_t dest);
+    int _command_flush_journal(std::stringstream *ss);
+    CDir *_command_dirfrag_get(
+        const cmdmap_t &cmdmap,
+        std::ostream &ss);
+    // <<<
+
   protected:
-    // FIXME: reference cycle: MDSRank should not carry a reference up to MDSDaemon
-    MDS          *mds_daemon;
     Messenger    *messenger;
     MonClient    *monc;
 
+    Context *respawn_hook;
+    Context *suicide_hook;
+
     // Friended to access retry_dispatch
     friend class C_MDS_RetryMessage;
 
@@ -380,9 +433,6 @@ class MDSRank {
     void calc_recovery_set();
     void request_state(MDSMap::DaemonState s);
 
-    mds_rank_t standby_for_rank;
-    MDSMap::DaemonState standby_type;  // one of STANDBY_REPLAY, ONESHOT_REPLAY
-    string standby_for_name;
     bool standby_replaying;  // true if current replay pass is in standby-replay mode
 
     typedef enum {
@@ -426,12 +476,7 @@ class MDSRank {
     void stopping_done();
     // <<<
     
-    // FIXME: remove 'rank' from name once MDSRank is encapsulated
-    // in MDSDaemon instead of being a parent
     // >>>
-    void handle_mds_map_rank(
-        MMDSMap *m, MDSMap *oldmap,
-        int oldwhoami, MDSMap::DaemonState oldstate);
     void handle_mds_recovery(mds_rank_t who);
     void handle_mds_failure(mds_rank_t who);
     // <<<

src/mds/MDSTableClient.h

@@ -19,13 +19,13 @@
 #include "MDSContext.h"
 #include "mds_table_types.h"
 
-class MDS;
+class MDSRank;
 class LogSegment;
 class MMDSTableRequest;
 
 class MDSTableClient {
 protected:
-  MDS *mds;
+  MDSRank *mds;
   int table;
 
   uint64_t last_reqid;
@@ -57,7 +57,7 @@ protected:
   friend class C_LoggedAck;
 
 public:
-  MDSTableClient(MDS *m, int tab) :
+  MDSTableClient(MDSRank *m, int tab) :
     mds(m), table(tab), last_reqid(~0ULL), server_ready(false) {}
   virtual ~MDSTableClient() {}
 

src/mds/SnapClient.h

@@ -19,12 +19,12 @@
 #include "snap.h"
 
 class MDSInternalContextBase;
-class MDS;
+class MDSRank;
 class LogSegment;
 
 class SnapClient : public MDSTableClient {
 public:
-  SnapClient(MDS *m) : MDSTableClient(m, TABLE_SNAP) {}
+  SnapClient(MDSRank *m) : MDSTableClient(m, TABLE_SNAP) {}
 
   void resend_queries() {}
   void handle_query_result(MMDSTableRequest *m) {}