ceph/teuthology/run_tasks.py

113 lines
4.1 KiB
Python
Raw Normal View History

2011-06-02 22:04:01 +00:00
import sys
import logging
2013-09-11 15:17:02 +00:00
from .sentry import get_client as get_sentry_client
from .misc import get_http_log_path
from .config import config as teuth_config
2013-09-11 20:17:22 +00:00
from copy import deepcopy
2011-06-02 22:04:01 +00:00
log = logging.getLogger(__name__)
def run_one_task(taskname, **kwargs):
submod = taskname
subtask = 'task'
if '.' in taskname:
(submod, subtask) = taskname.rsplit('.', 1)
parent = __import__('teuthology.task', globals(), locals(), [submod], 0)
mod = getattr(parent, submod)
fn = getattr(mod, subtask)
2011-06-02 22:04:01 +00:00
return fn(**kwargs)
def run_tasks(tasks, ctx):
2011-06-02 22:04:01 +00:00
stack = []
try:
for taskdict in tasks:
try:
((taskname, config),) = taskdict.iteritems()
except ValueError:
raise RuntimeError('Invalid task definition: %s' % taskdict)
log.info('Running task %s...', taskname)
manager = run_one_task(taskname, ctx=ctx, config=config)
2011-06-02 22:04:01 +00:00
if hasattr(manager, '__enter__'):
manager.__enter__()
stack.append((taskname, manager))
except Exception as e:
ctx.summary['success'] = False
if 'failure_reason' not in ctx.summary:
ctx.summary['failure_reason'] = str(e)
2013-08-23 19:53:38 +00:00
log.exception('Saw exception from tasks.')
2013-08-23 16:40:44 +00:00
sentry = get_sentry_client()
2013-08-21 15:07:12 +00:00
if sentry:
config = deepcopy(ctx.config)
tags = {
'task': taskname,
'owner': ctx.owner,
}
if 'teuthology_branch' in config:
tags['teuthology_branch'] = config['teuthology_branch']
2013-09-11 20:17:22 +00:00
# Remove ssh keys from reported config
if 'targets' in config:
targets = config['targets']
for host in targets.keys():
targets[host] = '<redacted>'
2013-09-18 15:05:14 +00:00
job_id = ctx.config.get('job_id')
archive_path = ctx.config.get('archive_path')
2013-09-11 15:17:02 +00:00
extra = {
2013-09-11 20:17:22 +00:00
'config': config,
2013-09-18 15:05:14 +00:00
'logs': get_http_log_path(archive_path, job_id),
2013-09-11 15:17:02 +00:00
}
2013-09-11 19:00:58 +00:00
exc_id = sentry.get_ident(sentry.captureException(
2013-09-11 15:17:02 +00:00
tags=tags,
extra=extra,
2013-09-11 19:00:58 +00:00
))
event_url = "{server}/search?q={id}".format(
server=teuth_config.sentry_server.strip('/'), id=exc_id)
log.exception(" Sentry event: %s" % event_url)
ctx.summary['sentry_event'] = event_url
if ctx.config.get('interactive-on-error'):
from .task import interactive
log.warning('Saw failure, going into interactive mode...')
interactive.task(ctx=ctx, config=None)
2011-06-02 22:04:01 +00:00
finally:
try:
exc_info = sys.exc_info()
while stack:
taskname, manager = stack.pop()
log.debug('Unwinding manager %s', taskname)
2011-06-02 22:04:01 +00:00
try:
suppress = manager.__exit__(*exc_info)
except Exception as e:
ctx.summary['success'] = False
if 'failure_reason' not in ctx.summary:
ctx.summary['failure_reason'] = str(e)
log.exception('Manager failed: %s', taskname)
if exc_info == (None, None, None):
# if first failure is in an __exit__, we don't
# have exc_info set yet
exc_info = sys.exc_info()
if ctx.config.get('interactive-on-error'):
from .task import interactive
log.warning(
'Saw failure, going into interactive mode...')
interactive.task(ctx=ctx, config=None)
2011-06-02 22:04:01 +00:00
else:
if suppress:
sys.exc_clear()
exc_info = (None, None, None)
if exc_info != (None, None, None):
log.debug('Exception was not quenched, exiting: %s: %s',
exc_info[0].__name__, exc_info[1])
raise SystemExit(1)
2011-06-02 22:04:01 +00:00
finally:
# be careful about cyclic references
del exc_info