From f005e8af6b52bd5e2194e5aa483a17acc9a7c3cf Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Sat, 4 Feb 2017 11:58:56 -0500 Subject: [PATCH] qa: disable max_mds changes during thrashing While the trasher supports the behavior desired by issue 10792 [1], the bugs uncovered due to deactivating MDS (and sometimes killing deactivating MDS) are presently a distraction from addressing issues during normal failures. So now thrashing max_mds is turned off by default. I have added a TODO to deactivate ranks in order (configurably) as random deactivation causes a lot of other problems. This also fixes a bug: random.randrange(0.0, 1.0) always returns 0. Oops. [1] http://tracker.ceph.com/issues/10792 Signed-off-by: Patrick Donnelly --- qa/tasks/mds_thrash.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qa/tasks/mds_thrash.py b/qa/tasks/mds_thrash.py index 0f9dfcb288b..6e95aae80a5 100644 --- a/qa/tasks/mds_thrash.py +++ b/qa/tasks/mds_thrash.py @@ -47,7 +47,7 @@ class MDSThrasher(Greenlet): thrash_in_replay: [default: 0.0] likelihood that the MDS will be thrashed during replay. Value should be between 0.0 and 1.0. - thrash_max_mds: [default: 0.25] likelihood that the max_mds of the mds + thrash_max_mds: [default: 0.0] likelihood that the max_mds of the mds cluster will be modified to a value [1, current) or (current, starting max_mds]. When reduced, randomly selected MDSs other than rank 0 will be deactivated to reach the new max_mds. Value should be between 0.0 and 1.0. @@ -112,7 +112,7 @@ class MDSThrasher(Greenlet): self.stopping = Event() self.randomize = bool(self.config.get('randomize', True)) - self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.25)) + self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.0)) self.max_thrash = int(self.config.get('max_thrash', 1)) self.max_thrash_delay = float(self.config.get('thrash_delay', 120.0)) self.thrash_in_replay = float(self.config.get('thrash_in_replay', False)) @@ -231,7 +231,7 @@ class MDSThrasher(Greenlet): status = self.fs.status() - if random.randrange(0.0, 1.0) <= self.thrash_max_mds: + if random.random() <= self.thrash_max_mds: max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] options = range(1, max_mds)+range(max_mds+1, self.max_mds+1) if len(options) > 0: @@ -242,6 +242,7 @@ class MDSThrasher(Greenlet): stats['max_mds'] += 1 # Now randomly deactivate mds if we shrank + # TODO: it's desirable to deactivate in order. Make config to do random. targets = filter(lambda r: r['rank'] > 0, status.get_ranks(self.fs.id)) # can't deactivate 0 for target in random.sample(targets, max(0, max_mds-new_max_mds)): self.log("deactivating rank %d" % target['rank'])