diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py index b4e26ae379d..c9969081400 100644 --- a/qa/tasks/ceph_manager.py +++ b/qa/tasks/ceph_manager.py @@ -153,7 +153,7 @@ class Thrasher: first_mon[1], opt) self.saved_options.append((service, opt, old_value)) - self._set_config(service, '*', opt, new_value) + manager.inject_args(service, '*', opt, new_value) # initialize ceph_objectstore_tool property - must be done before # do_thrash is spawned - http://tracker.ceph.com/issues/18799 if (self.config.get('powercycle') or @@ -180,13 +180,6 @@ class Thrasher: if self.noscrub_toggle_delay: self.noscrub_toggle_thread = gevent.spawn(self.do_noscrub_toggle) - def _set_config(self, service_type, service_id, name, value): - opt_arg = '--{name} {value}'.format(name=name, value=value) - whom = '.'.join([service_type, service_id]) - self.ceph_manager.raw_cluster_cmd('--', 'tell', whom, - 'injectargs', opt_arg) - - def cmd_exists_on_osds(self, cmd): allremotes = self.ceph_manager.ctx.cluster.only(\ teuthology.is_type('osd', self.cluster)).remotes.keys() @@ -392,10 +385,12 @@ class Thrasher: self.dead_osds.remove(osd) self.live_osds.append(osd) if self.random_eio > 0 and osd is self.rerrosd: - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--filestore_debug_random_read_err='+str(self.random_eio)) - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--bluestore_debug_random_read_err='+str(self.random_eio)) + self.ceph_manager.inject_args('osd', self.rerrosd, + 'filestore_debug_random_read_err', + self.random_eio) + self.ceph_manager.inject_args('osd', self.rerrosd, + 'bluestore_debug_random_read_err', + self.random_eio) def out_osd(self, osd=None): @@ -893,8 +888,9 @@ class Thrasher: osd_state = "false" else: osd_state = "true" - self.ceph_manager.raw_cluster_cmd_result('tell', 'osd.*', - 'injectargs', '--osd_enable_op_tracker=%s' % osd_state) + self.ceph_manager.inject_args('osd', '*', + 'osd_enable_op_tracker', + osd_state) gevent.sleep(delay) @log_exc @@ -952,10 +948,12 @@ class Thrasher: delay = self.config.get("op_delay", 5) self.rerrosd = self.live_osds[0] if self.random_eio > 0: - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--filestore_debug_random_read_err='+str(self.random_eio)) - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--bluestore_debug_random_read_err='+str(self.random_eio)) + self.ceph_manager.inject_args('osd', self.rerrosd, + 'filestore_debug_random_read_err', + self.random_eio) + self.ceph_manager.inject_args('osd', self.rerrosd, + 'bluestore_debug_random_read_err', + self.random_eio) self.log("starting do_thrash") while not self.stopping: to_log = [str(x) for x in ["in_osds: ", self.in_osds, @@ -985,16 +983,16 @@ class Thrasher: time.sleep(delay) self.all_up() if self.random_eio > 0: - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--filestore_debug_random_read_err=0.0') - self.ceph_manager.raw_cluster_cmd('tell', 'osd.'+str(self.rerrosd), - 'injectargs', '--', '--bluestore_debug_random_read_err=0.0') + self.ceph_manager.inject_args('osd', self.rerrosd, + 'filestore_debug_random_read_err', '0.0') + self.ceph_manager.inject_args('osd', self.rerrosd, + 'bluestore_debug_random_read_err', '0.0') for pool in list(self.pools_to_fix_pgp_num): if self.ceph_manager.get_pool_pg_num(pool) > 0: self.fix_pgp_num(pool) self.pools_to_fix_pgp_num.clear() for service, opt, saved_value in self.saved_options: - self._set_config(service, '*', opt, saved_value) + self.ceph_manager.inject_args(service, '*', opt, saved_value) self.saved_options = [] self.all_up_in() @@ -1481,6 +1479,13 @@ class CephManager: j = json.loads(proc.stdout.getvalue()) return j[name] + def inject_args(self, service_type, service_id, name, value): + whom = '{0}.{1}'.format(service_type, service_id) + if isinstance(value, bool): + value = 'true' if value else 'false' + opt_arg = '--{name}={value}'.format(name=name, value=value) + self.raw_cluster_cmd('--', 'tell', whom, 'injectargs', opt_arg) + def set_config(self, osdnum, **argdict): """ :param osdnum: osd number @@ -2314,11 +2319,9 @@ class CephManager: remote.console.power_off() elif self.config.get('bdev_inject_crash') and self.config.get('bdev_inject_crash_probability'): if random.uniform(0, 1) < self.config.get('bdev_inject_crash_probability', .5): - self.raw_cluster_cmd( - '--', 'tell', 'osd.%d' % osd, - 'injectargs', - '--bdev-inject-crash %d' % self.config.get('bdev_inject_crash'), - ) + self.inject_args( + 'osd', osd, + 'bdev-inject-crash', self.config.get('bdev_inject_crash')) try: self.ctx.daemons.get_daemon('osd', osd, self.cluster).wait() except: @@ -2340,9 +2343,8 @@ class CephManager: """ Stop osd if nothing else works. """ - self.raw_cluster_cmd('--', 'tell', 'osd.%d' % osd, - 'injectargs', - '--objectstore-blackhole') + self.inject_args('osd', osd, + 'objectstore-blackhole', True) time.sleep(2) self.ctx.daemons.get_daemon('osd', osd, self.cluster).stop()