cephfs: Change behavior of cluster_down flag

Setting the cluster_down flag will now set all active MDS
to standby and clearing it will restore the previous max_mds.
Changing max_mds when the cluster_down flag is set will clear the
flag automatically.

Signed-off-by: Douglas Fuller <dfuller@redhat.com>

doc/cephfs/administration.rst

@@ -76,19 +76,20 @@ to enumerate the objects during operations like stats or deletes.
 
 Taking the cluster down
 -----------------------
+Taking a CephFS cluster down is done by setting the cluster_down flag:
+ 
+:: 
+ 
+    mds set <fs name> cluster_down true 
+ 
+To bring the cluster back online:
+ 
+:: 
 
-Taking a CephFS cluster down is done by reducing the number of ranks to 1,
-setting the cluster_down flag, and then failing the last rank. For example:
+   mds set <fs name> cluster_down false
 
-::
-    ceph fs set <fs_name> max_mds 1
-    ceph mds deactivate <fs_name>:1 # rank 2 of 2
-    ceph status # wait for rank 1 to finish stopping
-    ceph fs set <fs_name> cluster_down true
-    ceph mds fail <fs_name>:0
+This will restore the previous value of max_mds.
 
-Setting the ``cluster_down`` flag prevents standbys from taking over the failed
-rank.
 
 Daemons
 -------

src/mds/MDSMap.cc

@@ -651,7 +651,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
   encode(cas_pool, bl);
 
   // kclient ignores everything from here
-  __u16 ev = 12;
+  __u16 ev = 13;
   encode(ev, bl);
   encode(compat, bl);
   encode(metadata_pool, bl);
@@ -672,6 +672,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
   encode(damaged, bl);
   encode(balancer, bl);
   encode(standby_count_wanted, bl);
+  encode(old_max_mds, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -802,6 +803,10 @@ void MDSMap::decode(bufferlist::iterator& p)
     decode(standby_count_wanted, p);
   }
 
+  if (ev >= 13) {
+    decode(old_max_mds, p);
+  }
+
   DECODE_FINISH(p);
 }
 

src/mds/MDSMap.h

@@ -202,6 +202,7 @@ protected:
    */
 
   mds_rank_t max_mds; /* The maximum number of active MDSes. Also, the maximum rank. */
+  mds_rank_t old_max_mds; /* Value to restore when MDS cluster is marked up */
   mds_rank_t standby_count_wanted;
   string balancer;    /* The name/version of the mantle balancer (i.e. the rados obj name) */
 
@@ -238,6 +239,7 @@ public:
       cas_pool(-1),
       metadata_pool(-1),
       max_mds(1),
+      old_max_mds(0),
       standby_count_wanted(-1),
       ever_allowed_features(0),
       explicitly_allowed_features(0),
@@ -311,6 +313,8 @@ public:
 
   mds_rank_t get_max_mds() const { return max_mds; }
   void set_max_mds(mds_rank_t m) { max_mds = m; }
+  void set_old_max_mds() { old_max_mds = max_mds; }
+  mds_rank_t get_old_max_mds() const { return old_max_mds; }
 
   mds_rank_t get_standby_count_wanted(mds_rank_t standby_daemon_count) const {
     assert(standby_daemon_count >= 0);

src/mon/FSCommands.cc

@@ -279,6 +279,7 @@ public:
           fs->fscid,
           [n](std::shared_ptr<Filesystem> fs)
       {
+	fs->mds_map.clear_flag(CEPH_MDSMAP_DOWN);
         fs->mds_map.set_max_mds(n);
       });
     } else if (var == "inline_data") {
@@ -428,15 +429,21 @@ public:
         return r;
       }
 
+      ss << fs->mds_map.get_fs_name();
+
       fsmap.modify_filesystem(
           fs->fscid,
           [is_down](std::shared_ptr<Filesystem> fs)
       {
-        if (is_down) {
-          fs->mds_map.set_flag(CEPH_MDSMAP_DOWN);
-        } else {
-          fs->mds_map.clear_flag(CEPH_MDSMAP_DOWN);
-        }
+	if (is_down) {
+	  fs->mds_map.set_flag(CEPH_MDSMAP_DOWN);
+	  fs->mds_map.set_old_max_mds();
+	  fs->mds_map.set_max_mds(0);
+	} else {
+	  mds_rank_t oldmax = fs->mds_map.get_old_max_mds();
+	  fs->mds_map.clear_flag(CEPH_MDSMAP_DOWN);
+	  fs->mds_map.set_max_mds(oldmax ? oldmax : 1);
+	}
       });
 
       if (is_down) {