RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-16 07:17:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/wip-sandon-vm'

Browse Source 
Conflicts:
	teuthology/lock.py
	teuthology/misc.py
	teuthology/task/install.py
This commit is contained in:
Sandon Van Ness 2013-07-25 19:50:02 -07:00
commit 41cafb1a10
10 changed files with 108 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
schedule_suite.sh
View File

@ -8,15 +8,18 @@ flavor=$5
teuthology_branch=$6
mtype=$7
template=$8
distro=$9
if [ -z "$email" ]; then
echo "usage: $0 <suite> <ceph branch> <kernel branch> <email> [flavor] [teuthology-branch] [machinetype] [template]"
echo "usage: $0 <suite> <ceph branch> <kernel branch> <email> [flavor] [teuthology-branch] [machinetype] [template] [distro]"
echo " flavor can be 'basic', 'gcov', 'notcmalloc'."
exit 1
fi
[ -z "$flavor" ] && flavor='basic'
[ -z "$distro" ] && distro='ubuntu'
##
test ! -d ~/src/ceph-qa-suite && echo "error: expects to find ~/src/ceph-qa-suite" && exit 1
@ -62,6 +65,7 @@ kernel:
sha1: $KERNEL_SHA1
nuke-on-error: true
machine_type: $mtype
os_type: $distro
tasks:
- chef:
- clock.check:
@ -109,7 +113,7 @@ fi
##
stamp=`date +%Y-%m-%d_%H:%M:%S`
name=`whoami`"-$stamp-$suite-$ceph-$kernel-$flavor"
name=`whoami`"-$stamp-$suite-$ceph-$kernel-$flavor-$mtype"
echo "name $name"

29
teuthology/lock.py
View File

@ -35,11 +35,11 @@ def lock_many(ctx, num, machinetype, user=None, description=None):
log.error('Could not lock %d nodes, reason: unknown.', num)
return []
def lock(ctx, name, user=None):
def lock(ctx, name, user=None, description=None):
if user is None:
user = teuthology.get_user()
success, _, _ = ls.send_request('POST', ls._lock_url(ctx) + '/' + name,
urllib.urlencode(dict(user=user)))
urllib.urlencode(dict(user=user, desc=description)))
if success:
log.debug('locked %s as %s', name, user)
else:
@ -543,10 +543,17 @@ def create_if_vm(ctx, machine_name):
phys_host = status_info['vpshost']
if not phys_host:
return False
try:
vm_type = ctx.vm_type
except AttributeError:
vm_type = 'ubuntu'
from teuthology.misc import get_distro
os_type = get_distro(ctx)
os_version = dict(
ubuntu="12.04",
fedora="18",
centos="6.4",
opensuse="12.2",
sles="11-sp2",
rhel="6.3",
debian='6.0'
)
createMe = decanonicalize_hostname(machine_name)
with tempfile.NamedTemporaryFile() as tmp:
try:
@ -554,13 +561,15 @@ def create_if_vm(ctx, machine_name):
except KeyError:
lcnfg = {}
distro = lcnfg.get('distro', os_type.lower())
file_info = {}
file_info['disk-size'] = lcnfg.get('disk-size', '30G')
file_info['ram'] = lcnfg.get('ram', '1.9G')
file_info['cpus'] = lcnfg.get('cpus', 1)
file_info['networks'] = lcnfg.get('networks',
[{'source' : 'front', 'mac' : status_info['mac']}])
file_info['distro'] = lcnfg.get('distro', vm_type.lower())
file_info['distro'] = distro
file_info['distroversion'] = os_version[distro]
file_info['additional-disks'] = lcnfg.get(
'additional-disks', 3)
file_info['additional-disks-size'] = lcnfg.get(
@ -582,6 +591,12 @@ def create_if_vm(ctx, machine_name):
(machine_name,err))
else:
log.info("%s created: %s" % (machine_name,owt))
#If the guest already exists first destroy then re-create:
if 'exists' in err:
log.info("Guest files exist. Re-creating guest: %s" %
(machine_name))
destroy_if_vm(ctx, machine_name)
create_if_vm(ctx, machine_name)
return True
#
# Use downburst to destroy a virtual machine

10
teuthology/misc.py
View File

@ -885,3 +885,13 @@ def get_system_type(remote):
if system_value in ['CentOS','Fedora','RedHatEnterpriseServer']:
return "rpm"
return system_value
def get_distro(ctx):
try:
os_type = ctx.config.get('os_type', ctx.os_type)
except AttributeError:
os_type = 'ubuntu'
try:
return ctx.config['downburst'].get('distro', os_type)
except KeyError:
return os_type

15
teuthology/orchestra/run.py
View File

@ -81,12 +81,12 @@ def execute(client, args):
)
return r
def copy_to_log(f, logger, loglevel=logging.INFO):
def copy_to_log(f, logger, host, loglevel=logging.INFO):
# i can't seem to get fudge to fake an iterable, so using this old
# api for now
for line in f.xreadlines():
line = line.rstrip()
logger.log(loglevel, line)
logger.log(loglevel, '[' + host + ']: ' + line)
def copy_and_close(src, fdst):
if src is not None:
@ -95,14 +95,15 @@ def copy_and_close(src, fdst):
shutil.copyfileobj(src, fdst)
fdst.close()
def copy_file_to(f, dst):
def copy_file_to(f, dst, host):
if hasattr(dst, 'log'):
# looks like a Logger to me; not using isinstance to make life
# easier for unit tests
handler = copy_to_log
return handler(f, dst, host)
else:
handler = shutil.copyfileobj
return handler(f, dst)
return handler(f, dst)
class CommandFailedError(Exception):
@ -217,12 +218,12 @@ def run(
if logger is None:
logger = log
(host,port) = client.get_transport().getpeername()
g_err = None
if stderr is not PIPE:
if stderr is None:
stderr = logger.getChild('err')
g_err = gevent.spawn(copy_file_to, r.stderr, stderr)
g_err = gevent.spawn(copy_file_to, r.stderr, stderr, host)
r.stderr = stderr
else:
assert not wait, "Using PIPE for stderr without wait=False would deadlock."
@ -231,7 +232,7 @@ def run(
if stdout is not PIPE:
if stdout is None:
stdout = logger.getChild('out')
g_out = gevent.spawn(copy_file_to, r.stdout, stdout)
g_out = gevent.spawn(copy_file_to, r.stdout, stdout, host)
r.stdout = stdout
else:
assert not wait, "Using PIPE for stdout without wait=False would deadlock."

10
teuthology/run.py
View File

@ -62,6 +62,11 @@ def parse_args():
default=None,
help='Type of machine to lock/run tests on.',
)
parser.add_argument(
'--os-type',
default='ubuntu',
help='Distro/OS of machine to run test on.',
)
parser.add_argument(
'--block',
action='store_true',
@ -166,7 +171,10 @@ def main():
{'internal.vm_setup': None},
])
if 'kernel' in ctx.config:
init_tasks.append({'kernel': ctx.config['kernel']})
from teuthology.misc import get_distro
distro = get_distro(ctx)
if distro == 'ubuntu':
init_tasks.append({'kernel': ctx.config['kernel']})
init_tasks.extend([
{'internal.base': None},
{'internal.archive': None},

3
teuthology/task/ceph-deploy.py
View File

@ -405,6 +405,9 @@ def task(ctx, config):
assert isinstance(config, dict), \
"task ceph-deploy only supports a dictionary for configuration"
overrides = ctx.config.get('overrides', {})
teuthology.deep_merge(config, overrides.get('ceph-deploy', {}))
if config.get('branch') is not None:
assert isinstance(config['branch'], dict), 'branch must be a dictionary'

4
teuthology/task/ceph.py
View File

@ -550,7 +550,7 @@ def cluster(ctx, config):
mkfs_options = config.get('mkfs_options')
mount_options = config.get('mount_options')
if fs == 'btrfs':
package = 'btrfs-tools'
#package = 'btrfs-tools'
if mount_options is None:
mount_options = ['noatime','user_subvol_rm_allowed']
if mkfs_options is None:
@ -558,7 +558,7 @@ def cluster(ctx, config):
'-l', '32768',
'-n', '32768']
if fs == 'xfs':
package = 'xfsprogs'
#package = 'xfsprogs'
if mount_options is None:
mount_options = ['noatime']
if mkfs_options is None:

8
teuthology/task/install.py
View File

@ -172,7 +172,15 @@ def _update_rpm_package_list_and_install(ctx, remote, rpm, config):
pkg=ceph_release)
if cmp_msg != err_mess.getvalue().strip():
raise
remote.run(args=['rm', '-f', rpm_name])
#Fix Repo Priority
remote.run(
args=[
'sudo', 'sed', '-i', run.Raw('\':a;N;$!ba;s/enabled=1\\ngpg/enabled=1\\npriority=1\\ngpg/g\''), '/etc/yum.repos.d/ceph.repo',
])
remote.run(
args=[
'sudo', 'yum', 'clean', 'all',

12
teuthology/task/internal.py
View File

@ -103,10 +103,20 @@ def lock_machines(ctx, config):
if vmlist:
log.info('Waiting for virtual machines to come up')
keyscan_out = ''
loopcount=0
while len(keyscan_out.splitlines()) != len(vmlist):
loopcount += 1
time.sleep(10)
keyscan_out, current_locks = lock.keyscan_check(ctx, vmlist)
log.info('virtual machine is stil unavailable')
if loopcount == 40:
loopcount = 0
log.info('virtual machine(s) still not up, recreating unresponsive ones.')
for guest in vmlist:
if guest not in keyscan_out:
log.info('recreating: ' + guest)
lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
lock.create_if_vm(ctx, 'ubuntu@' + guest)
if lock.update_keys(ctx, keyscan_out, current_locks):
log.info("Error in virtual machine keys")
newscandict = {}
@ -523,7 +533,7 @@ def vm_setup(ctx, config):
def _handle_vm_init(remote):
log.info('Running ceph_qa_chef on ', remote)
remote.run(args=['wget','-q','-O-',
'https://raw.github.com/ceph/ceph-qa-chef/master/solo/solo-from-scratch',
'http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD',
run.Raw('|'),
'sh',
])

29
teuthology/test/test_get_distro.py Normal file
View File

@ -0,0 +1,29 @@
from .. import misc as teuthology
class Mock: pass
class TestGetDistro(object):
def setup(self):
self.fake_ctx = Mock()
self.fake_ctx.config = {}
self.fake_ctx.os_type = 'ubuntu'
def test_default_distro(self):
distro = teuthology.get_distro(self.fake_ctx)
assert distro == 'ubuntu'
def test_argument(self):
self.fake_ctx.os_type = 'centos'
distro = teuthology.get_distro(self.fake_ctx)
assert distro == 'centos'
def test_teuth_config(self):
self.fake_ctx.config = {'os_type': 'fedora'}
distro = teuthology.get_distro(self.fake_ctx)
assert distro == 'fedora'
def test_teuth_config_downburst(self):
self.fake_ctx.config = {'downburst' : {'distro': 'sles'}}
distro = teuthology.get_distro(self.fake_ctx)
assert distro == 'sles'