RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-22 10:17:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

qa: disable max_mds changes during thrashing

Browse Source 
While the trasher supports the behavior desired by issue 10792 [1], the
bugs uncovered due to deactivating MDS (and sometimes killing
deactivating MDS) are presently a distraction from addressing issues
during normal failures. So now thrashing max_mds is turned off by
default. I have added a TODO to deactivate ranks in order (configurably)
as random deactivation causes a lot of other problems.

This also fixes a bug: random.randrange(0.0, 1.0) always returns 0.
Oops.

[1] http://tracker.ceph.com/issues/10792

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
This commit is contained in:
Patrick Donnelly 2017-02-04 11:58:56 -05:00
parent 82662edd7f
commit f005e8af6b
No known key found for this signature in database
GPG Key ID: 3A2A7E25BEA8AADB
1 changed files with 4 additions and 3 deletions

7
qa/tasks/mds_thrash.py
View File

@ -47,7 +47,7 @@ class MDSThrasher(Greenlet):
thrash_in_replay: [default: 0.0] likelihood that the MDS will be thrashed
during replay. Value should be between 0.0 and 1.0.
thrash_max_mds: [default: 0.25] likelihood that the max_mds of the mds
thrash_max_mds: [default: 0.0] likelihood that the max_mds of the mds
cluster will be modified to a value [1, current) or (current, starting
max_mds]. When reduced, randomly selected MDSs other than rank 0 will be
deactivated to reach the new max_mds. Value should be between 0.0 and 1.0.
@ -112,7 +112,7 @@ class MDSThrasher(Greenlet):
self.stopping = Event()
self.randomize = bool(self.config.get('randomize', True))
self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.25))
self.thrash_max_mds = float(self.config.get('thrash_max_mds', 0.0))
self.max_thrash = int(self.config.get('max_thrash', 1))
self.max_thrash_delay = float(self.config.get('thrash_delay', 120.0))
self.thrash_in_replay = float(self.config.get('thrash_in_replay', False))
@ -231,7 +231,7 @@ class MDSThrasher(Greenlet):
status = self.fs.status()
if random.randrange(0.0, 1.0) <= self.thrash_max_mds:
if random.random() <= self.thrash_max_mds:
max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds']
options = range(1, max_mds)+range(max_mds+1, self.max_mds+1)
if len(options) > 0:
@ -242,6 +242,7 @@ class MDSThrasher(Greenlet):
stats['max_mds'] += 1
# Now randomly deactivate mds if we shrank
# TODO: it's desirable to deactivate in order. Make config to do random.
targets = filter(lambda r: r['rank'] > 0, status.get_ranks(self.fs.id)) # can't deactivate 0
for target in random.sample(targets, max(0, max_mds-new_max_mds)):
self.log("deactivating rank %d" % target['rank'])