ceph/teuthology/nuke.py
Greg Farnum 1ea825c643 teuthology-nuke: split the big main function
It was getting a bit big, but now all the functions fit on
one screen each.

Signed-off-by: Greg Farnum <gregory.farnum@dreamhost.com>
2011-08-10 16:06:45 -07:00

202 lines
6.2 KiB
Python

import argparse
import yaml
def parse_args():
from teuthology.run import config_file
from teuthology.run import MergeConfig
parser = argparse.ArgumentParser(description='Reset test machines')
parser.add_argument(
'-v', '--verbose',
action='store_true', default=None,
help='be more verbose'
)
parser.add_argument(
'config',
metavar='CONFFILE',
nargs='+',
type=config_file,
action=MergeConfig,
default={},
help='config file to read',
)
parser.add_argument(
'--archive',
metavar='DIR',
help='path to archive results in',
)
parser.add_argument(
'--owner',
help='job owner',
)
args = parser.parse_args()
return args
def shutdown_daemons(ctx, log):
from orchestra import run
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'if', 'grep', '-q', 'cfuse', '/etc/mtab', run.Raw(';'),
'then',
'grep', 'cfuse', '/etc/mtab', run.Raw('|'),
'grep', '-o', " /.* fuse", run.Raw('|'),
'grep', '-o', "/.* ", run.Raw('|'),
'xargs', 'sudo', 'fusermount', '-u', run.Raw(';'),
'fi',
run.Raw(';'),
'killall',
'--quiet',
'/tmp/cephtest/binary/usr/local/bin/cmon',
'/tmp/cephtest/binary/usr/local/bin/cosd',
'/tmp/cephtest/binary/usr/local/bin/cmds',
'/tmp/cephtest/binary/usr/local/bin/cfuse',
run.Raw(';'),
'if', 'test', '-e', '/etc/rsyslog.d/80-cephtest.conf',
run.Raw(';'),
'then',
'sudo', 'rm', '-f', '--', '/etc/rsyslog.d/80-cephtest.conf',
run.Raw('&&'),
'sudo', 'initctl', 'restart', 'rsyslog',
run.Raw(';'),
'fi',
run.Raw(';'),
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to finish shutdowns...', name)
proc.exitstatus.get()
def find_kernel_mounts(ctx, log):
from orchestra import run
nodes = {}
log.info('Looking for kernel mounts to handle...')
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'grep', '-q', " ceph " , '/etc/mtab'
],
wait=False,
)
nodes[remote] = proc
kernel_mounts = list()
for remote, proc in nodes.iteritems():
try:
proc.exitstatus.get()
log.debug('kernel mount exists on %s', remote.name)
kernel_mounts.append(remote)
except run.CommandFailedError: # no mounts!
log.debug('no kernel mount on %s', remote.name)
return kernel_mounts
def remove_kernel_mounts(ctx, kernel_mounts, log):
"""
properly we should be able to just do a forced unmount,
but that doesn't seem to be working, so you should reboot instead
"""
from orchestra import run
nodes = {}
for remote in kernel_mounts:
log.info('clearing kernel mount from %s', remote.name)
proc = remote.run(
args=[
'grep', 'ceph', '/etc/mtab', run.Raw('|'),
'grep', '-o', "on /.* type", run.Raw('|'),
'grep', '-o', "/.* ", run.Raw('|'),
'xargs', 'sudo', 'umount', '-f', run.Raw(';'),
'fi'
],
wait=False
)
nodes[remote] = proc
for remote, proc in nodes:
proc.exitstatus.get()
def reboot_kernel_mounts(ctx, kernel_mounts, log):
from orchestra import run
import time
nodes = {}
for remote in kernel_mounts:
log.info('rebooting %s', remote.name)
proc = remote.run( # note use of -n to force a no-sync reboot
args=['sudo', 'reboot', '-f', '-n'],
wait=False
)
nodes[remote] = proc
# we just ignore these procs because reboot -f doesn't actually
# send anything back to the ssh client!
#for remote, proc in nodes.iteritems():
#proc.exitstatus.get()
from teuthology.misc import reconnect
if kernel_mounts:
log.info('waiting for nodes to reboot')
time.sleep(5) #if we try and reconnect too quickly, it succeeds!
reconnect(ctx, 300) #allow 5 minutes for the reboots
def remove_testing_tree(ctx, log):
from orchestra import run
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'sudo', 'rm', '-rf', '/tmp/cephtest',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to clear filesystem...', name)
proc.exitstatus.get()
def main():
from gevent import monkey; monkey.patch_all()
from orchestra import monkey; monkey.patch_all()
import logging
log = logging.getLogger(__name__)
ctx = parse_args()
loglevel = logging.INFO
if ctx.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
level=loglevel,
)
from teuthology.misc import read_config
read_config(ctx)
log.info('\n '.join(['targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
if ctx.owner is None:
from teuthology.misc import get_user
ctx.owner = get_user()
from teuthology.task.internal import check_lock, connect
check_lock(ctx, None)
connect(ctx, None)
log.info('Unmount cfuse and killing daemons...')
shutdown_daemons(ctx, log)
log.info('All daemons killed.')
log.info('Dealing with any kernel mounts...')
kernel_mounts = find_kernel_mounts(ctx, log)
#remove_kernel_mounts(ctx, kernel_mounts, log)
reboot_kernel_mounts(ctx, kernel_mounts, log)
log.info('All kernel mounts gone.')
log.info('Clearing filesystem of test data...')
remove_testing_tree(ctx, log)
log.info('Filesystem Cleared.')