mirror of
https://github.com/ceph/ceph
synced 2024-12-26 13:33:57 +00:00
6fbf98bb0c
Many errors in yaml configurations cause ValueError to get thrown with the message 'too many values to unpack.' A previously reverted change tried to handle all these situations and print an appropriate message. The current behavior of throwing the ValueError exception and exiting is probably what we still want in these cases. So instead of handling the error, the code now checks for the exception at the top-most call and displays appropriate messages in log.error and in the ctx.summary data itself. Fixes: 7510 Signed-off-by: Warren Usui <warren.usui@inktank.com>
130 lines
4.8 KiB
Python
130 lines
4.8 KiB
Python
import sys
|
|
import logging
|
|
from .sentry import get_client as get_sentry_client
|
|
from .misc import get_http_log_path
|
|
from .config import config as teuth_config
|
|
from copy import deepcopy
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def run_one_task(taskname, **kwargs):
|
|
submod = taskname
|
|
subtask = 'task'
|
|
if '.' in taskname:
|
|
(submod, subtask) = taskname.rsplit('.', 1)
|
|
parent = __import__('teuthology.task', globals(), locals(), [submod], 0)
|
|
try:
|
|
mod = getattr(parent, submod)
|
|
except AttributeError:
|
|
log.error("No task named %s was found", submod)
|
|
raise
|
|
try:
|
|
fn = getattr(mod, subtask)
|
|
except AttributeError:
|
|
log.error("No subtask of %s named %s was found", mod, subtask)
|
|
raise
|
|
return fn(**kwargs)
|
|
|
|
|
|
def run_tasks(tasks, ctx):
|
|
stack = []
|
|
try:
|
|
for taskdict in tasks:
|
|
try:
|
|
((taskname, config),) = taskdict.iteritems()
|
|
except ValueError:
|
|
raise RuntimeError('Invalid task definition: %s' % taskdict)
|
|
log.info('Running task %s...', taskname)
|
|
manager = run_one_task(taskname, ctx=ctx, config=config)
|
|
if hasattr(manager, '__enter__'):
|
|
manager.__enter__()
|
|
stack.append((taskname, manager))
|
|
except Exception as e:
|
|
ctx.summary['success'] = False
|
|
if 'failure_reason' not in ctx.summary:
|
|
ctx.summary['failure_reason'] = str(e)
|
|
log.exception('Saw exception from tasks.')
|
|
|
|
sentry = get_sentry_client()
|
|
if sentry:
|
|
config = deepcopy(ctx.config)
|
|
|
|
tags = {
|
|
'task': taskname,
|
|
'owner': ctx.owner,
|
|
}
|
|
if 'teuthology_branch' in config:
|
|
tags['teuthology_branch'] = config['teuthology_branch']
|
|
|
|
# Remove ssh keys from reported config
|
|
if 'targets' in config:
|
|
targets = config['targets']
|
|
for host in targets.keys():
|
|
targets[host] = '<redacted>'
|
|
|
|
job_id = ctx.config.get('job_id')
|
|
archive_path = ctx.config.get('archive_path')
|
|
extra = {
|
|
'config': config,
|
|
'logs': get_http_log_path(archive_path, job_id),
|
|
}
|
|
exc_id = sentry.get_ident(sentry.captureException(
|
|
tags=tags,
|
|
extra=extra,
|
|
))
|
|
event_url = "{server}/search?q={id}".format(
|
|
server=teuth_config.sentry_server.strip('/'), id=exc_id)
|
|
log.exception(" Sentry event: %s" % event_url)
|
|
ctx.summary['sentry_event'] = event_url
|
|
|
|
if ctx.config.get('interactive-on-error'):
|
|
from .task import interactive
|
|
log.warning('Saw failure, going into interactive mode...')
|
|
interactive.task(ctx=ctx, config=None)
|
|
# Throughout teuthology, (x,) = y has been used to assign values
|
|
# from yaml files where only one entry of type y is correct. This
|
|
# causes failures with 'too many values to unpack.' We want to
|
|
# fail as before, but with easier to understand error indicators.
|
|
if type(e) == ValueError:
|
|
if e.message == 'too many values to unpack':
|
|
emsg = 'Possible configuration error in yaml file'
|
|
log.error(emsg)
|
|
ctx.summary['failure_info'] = emsg
|
|
finally:
|
|
try:
|
|
exc_info = sys.exc_info()
|
|
while stack:
|
|
taskname, manager = stack.pop()
|
|
log.debug('Unwinding manager %s', taskname)
|
|
try:
|
|
suppress = manager.__exit__(*exc_info)
|
|
except Exception as e:
|
|
ctx.summary['success'] = False
|
|
if 'failure_reason' not in ctx.summary:
|
|
ctx.summary['failure_reason'] = str(e)
|
|
log.exception('Manager failed: %s', taskname)
|
|
|
|
if exc_info == (None, None, None):
|
|
# if first failure is in an __exit__, we don't
|
|
# have exc_info set yet
|
|
exc_info = sys.exc_info()
|
|
|
|
if ctx.config.get('interactive-on-error'):
|
|
from .task import interactive
|
|
log.warning(
|
|
'Saw failure, going into interactive mode...')
|
|
interactive.task(ctx=ctx, config=None)
|
|
else:
|
|
if suppress:
|
|
sys.exc_clear()
|
|
exc_info = (None, None, None)
|
|
|
|
if exc_info != (None, None, None):
|
|
log.debug('Exception was not quenched, exiting: %s: %s',
|
|
exc_info[0].__name__, exc_info[1])
|
|
raise SystemExit(1)
|
|
finally:
|
|
# be careful about cyclic references
|
|
del exc_info
|