2011-06-15 19:10:27 +00:00
|
|
|
import os
|
2011-06-03 21:47:44 +00:00
|
|
|
import yaml
|
2013-02-21 22:51:54 +00:00
|
|
|
import StringIO
|
|
|
|
import contextlib
|
2013-08-29 19:01:04 +00:00
|
|
|
import sys
|
2013-10-11 16:38:35 +00:00
|
|
|
import logging
|
2013-08-29 19:01:04 +00:00
|
|
|
from traceback import format_tb
|
|
|
|
|
2013-10-11 16:38:35 +00:00
|
|
|
from . import report
|
|
|
|
from .misc import get_distro
|
|
|
|
from .misc import get_user
|
|
|
|
from .misc import read_config
|
|
|
|
from .nuke import nuke
|
|
|
|
from .run_tasks import run_tasks
|
2013-10-11 17:09:40 +00:00
|
|
|
from .results import email_results
|
2011-06-03 21:47:44 +00:00
|
|
|
|
2013-10-08 21:55:40 +00:00
|
|
|
|
2013-08-29 18:35:52 +00:00
|
|
|
def set_up_logging(ctx):
|
2011-06-03 21:47:44 +00:00
|
|
|
loglevel = logging.INFO
|
|
|
|
if ctx.verbose:
|
|
|
|
loglevel = logging.DEBUG
|
|
|
|
|
2013-08-29 18:35:52 +00:00
|
|
|
logging.basicConfig(level=loglevel)
|
2013-08-29 16:52:45 +00:00
|
|
|
if ctx.archive is not None:
|
|
|
|
os.mkdir(ctx.archive)
|
|
|
|
|
|
|
|
handler = logging.FileHandler(
|
|
|
|
filename=os.path.join(ctx.archive, 'teuthology.log'),
|
2013-08-29 18:35:52 +00:00
|
|
|
)
|
2013-08-29 16:52:45 +00:00
|
|
|
formatter = logging.Formatter(
|
|
|
|
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
|
|
|
|
datefmt='%Y-%m-%dT%H:%M:%S',
|
2013-08-29 18:35:52 +00:00
|
|
|
)
|
2013-08-29 16:52:45 +00:00
|
|
|
handler.setFormatter(formatter)
|
|
|
|
logging.getLogger().addHandler(handler)
|
|
|
|
|
2013-08-29 19:01:04 +00:00
|
|
|
install_except_hook()
|
|
|
|
|
|
|
|
|
|
|
|
def install_except_hook():
|
|
|
|
def log_exception(exception_class, exception, traceback):
|
|
|
|
logging.critical(''.join(format_tb(traceback)))
|
2013-08-29 19:56:15 +00:00
|
|
|
if not exception.message:
|
|
|
|
logging.critical(exception_class.__name__)
|
|
|
|
return
|
2013-10-08 21:55:40 +00:00
|
|
|
logging.critical('{0}: {1}'.format(
|
|
|
|
exception_class.__name__, exception))
|
2013-08-29 19:01:04 +00:00
|
|
|
|
|
|
|
sys.excepthook = log_exception
|
|
|
|
|
|
|
|
|
2013-08-29 18:35:52 +00:00
|
|
|
def write_initial_metadata(ctx):
|
|
|
|
if ctx.archive is not None:
|
2013-08-29 16:52:45 +00:00
|
|
|
with file(os.path.join(ctx.archive, 'pid'), 'w') as f:
|
|
|
|
f.write('%d' % os.getpid())
|
|
|
|
|
|
|
|
with file(os.path.join(ctx.archive, 'owner'), 'w') as f:
|
|
|
|
f.write(ctx.owner + '\n')
|
|
|
|
|
|
|
|
with file(os.path.join(ctx.archive, 'orig.config.yaml'), 'w') as f:
|
|
|
|
yaml.safe_dump(ctx.config, f, default_flow_style=False)
|
2012-04-03 21:53:17 +00:00
|
|
|
|
2013-08-29 19:07:13 +00:00
|
|
|
info = {
|
|
|
|
'name': ctx.name,
|
|
|
|
'description': ctx.description,
|
|
|
|
'owner': ctx.owner,
|
|
|
|
'pid': os.getpid(),
|
|
|
|
}
|
2013-09-11 18:56:48 +00:00
|
|
|
if 'job_id' in ctx.config:
|
|
|
|
info['job_id'] = ctx.config['job_id']
|
|
|
|
|
2013-08-29 19:07:13 +00:00
|
|
|
with file(os.path.join(ctx.archive, 'info.yaml'), 'w') as f:
|
|
|
|
yaml.safe_dump(info, f, default_flow_style=False)
|
|
|
|
|
2013-08-29 18:35:52 +00:00
|
|
|
|
2013-10-09 18:19:10 +00:00
|
|
|
def main(ctx):
|
2013-08-29 18:35:52 +00:00
|
|
|
set_up_logging(ctx)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
if ctx.owner is None:
|
|
|
|
ctx.owner = get_user()
|
|
|
|
|
2013-10-04 17:35:03 +00:00
|
|
|
# Older versions of teuthology stored job_id as an int. Convert it to a str
|
|
|
|
# if necessary.
|
|
|
|
job_id = ctx.config.get('job_id')
|
|
|
|
if job_id is not None:
|
|
|
|
job_id = str(job_id)
|
|
|
|
ctx.config['job_id'] = job_id
|
|
|
|
|
2013-08-29 18:35:52 +00:00
|
|
|
write_initial_metadata(ctx)
|
2013-10-04 15:25:07 +00:00
|
|
|
report.try_push_job_info(ctx.config)
|
2013-08-29 18:35:52 +00:00
|
|
|
|
2012-04-03 21:53:17 +00:00
|
|
|
if 'targets' in ctx.config and 'roles' in ctx.config:
|
|
|
|
targets = len(ctx.config['targets'])
|
|
|
|
roles = len(ctx.config['roles'])
|
|
|
|
assert targets >= roles, \
|
2013-10-08 21:55:40 +00:00
|
|
|
'%d targets are needed for all roles but found %d listed.' % (
|
|
|
|
roles, targets)
|
2013-02-05 20:53:08 +00:00
|
|
|
|
|
|
|
machine_type = ctx.machine_type
|
|
|
|
if machine_type is None:
|
2013-03-29 19:16:39 +00:00
|
|
|
fallback_default = ctx.config.get('machine_type', 'plana')
|
|
|
|
machine_type = ctx.config.get('machine-type', fallback_default)
|
2013-02-05 20:53:08 +00:00
|
|
|
|
2011-07-07 23:15:18 +00:00
|
|
|
if ctx.block:
|
|
|
|
assert ctx.lock, \
|
|
|
|
'the --block option is only supported with the --lock option'
|
|
|
|
|
2011-07-07 18:43:35 +00:00
|
|
|
read_config(ctx)
|
|
|
|
|
2013-10-08 21:55:40 +00:00
|
|
|
log.debug('\n '.join(['Config:', ] + yaml.safe_dump(
|
|
|
|
ctx.config, default_flow_style=False).splitlines()))
|
2012-06-20 18:35:43 +00:00
|
|
|
|
|
|
|
ctx.summary = dict(success=True)
|
|
|
|
|
|
|
|
ctx.summary['owner'] = ctx.owner
|
|
|
|
|
|
|
|
if ctx.description is not None:
|
|
|
|
ctx.summary['description'] = ctx.description
|
|
|
|
|
2011-06-30 22:53:42 +00:00
|
|
|
for task in ctx.config['tasks']:
|
2013-10-08 21:55:40 +00:00
|
|
|
msg = ('kernel installation shouldn be a base-level item, not part ' +
|
|
|
|
'of the tasks list')
|
|
|
|
assert 'kernel' not in task, msg
|
2011-06-30 22:53:42 +00:00
|
|
|
|
2011-07-06 21:22:43 +00:00
|
|
|
init_tasks = []
|
|
|
|
if ctx.lock:
|
2013-10-08 21:55:40 +00:00
|
|
|
msg = ('You cannot specify targets in a config file when using the ' +
|
|
|
|
'--lock option')
|
|
|
|
assert 'targets' not in ctx.config, msg
|
|
|
|
init_tasks.append({'internal.lock_machines': (
|
|
|
|
len(ctx.config['roles']), machine_type)})
|
2011-07-06 21:22:43 +00:00
|
|
|
|
|
|
|
init_tasks.extend([
|
2013-10-08 21:55:40 +00:00
|
|
|
{'internal.save_config': None},
|
|
|
|
{'internal.check_lock': None},
|
|
|
|
{'internal.connect': None},
|
|
|
|
{'internal.check_conflict': None},
|
|
|
|
{'internal.check_ceph_data': None},
|
|
|
|
{'internal.vm_setup': None},
|
|
|
|
])
|
2011-06-30 22:53:42 +00:00
|
|
|
if 'kernel' in ctx.config:
|
2013-07-25 21:45:02 +00:00
|
|
|
distro = get_distro(ctx)
|
2013-07-13 03:20:45 +00:00
|
|
|
if distro == 'ubuntu':
|
|
|
|
init_tasks.append({'kernel': ctx.config['kernel']})
|
2011-06-30 22:53:42 +00:00
|
|
|
init_tasks.extend([
|
2013-10-08 21:55:40 +00:00
|
|
|
{'internal.base': None},
|
|
|
|
{'internal.archive': None},
|
|
|
|
{'internal.coredump': None},
|
|
|
|
{'internal.sudo': None},
|
|
|
|
{'internal.syslog': None},
|
|
|
|
{'internal.timer': None},
|
|
|
|
])
|
2011-06-30 22:53:42 +00:00
|
|
|
|
|
|
|
ctx.config['tasks'][:0] = init_tasks
|
2011-06-16 21:17:14 +00:00
|
|
|
|
2011-06-15 22:52:30 +00:00
|
|
|
try:
|
2011-06-16 20:01:09 +00:00
|
|
|
run_tasks(tasks=ctx.config['tasks'], ctx=ctx)
|
2011-06-15 22:52:30 +00:00
|
|
|
finally:
|
2011-11-09 00:01:39 +00:00
|
|
|
if not ctx.summary.get('success') and ctx.config.get('nuke-on-error'):
|
2012-04-25 00:51:16 +00:00
|
|
|
# only unlock if we locked them in the first place
|
|
|
|
nuke(ctx, log, ctx.lock)
|
2011-11-18 21:53:51 +00:00
|
|
|
if ctx.archive is not None:
|
|
|
|
with file(os.path.join(ctx.archive, 'summary.yaml'), 'w') as f:
|
|
|
|
yaml.safe_dump(ctx.summary, f, default_flow_style=False)
|
2013-02-21 22:51:54 +00:00
|
|
|
with contextlib.closing(StringIO.StringIO()) as f:
|
|
|
|
yaml.safe_dump(ctx.summary, f)
|
|
|
|
log.info('Summary data:\n%s' % f.getvalue())
|
2013-02-27 19:32:37 +00:00
|
|
|
with contextlib.closing(StringIO.StringIO()) as f:
|
2013-10-08 21:55:40 +00:00
|
|
|
if ('email-on-error' in ctx.config
|
|
|
|
and not ctx.summary.get('success', False)):
|
2013-02-27 19:32:37 +00:00
|
|
|
yaml.safe_dump(ctx.summary, f)
|
|
|
|
yaml.safe_dump(ctx.config, f)
|
|
|
|
emsg = f.getvalue()
|
2013-10-08 21:55:40 +00:00
|
|
|
subject = "Teuthology error -- %s" % ctx.summary[
|
|
|
|
'failure_reason']
|
|
|
|
email_results(subject, "Teuthology", ctx.config[
|
|
|
|
'email-on-error'], emsg)
|
2013-10-04 15:25:07 +00:00
|
|
|
|
|
|
|
report.try_push_job_info(ctx.config, ctx.summary)
|
2013-10-02 23:27:09 +00:00
|
|
|
|
2013-02-27 19:35:55 +00:00
|
|
|
if ctx.summary.get('success', True):
|
|
|
|
log.info('pass')
|
|
|
|
else:
|
|
|
|
log.info('FAIL')
|
|
|
|
sys.exit(1)
|