mirror of
https://github.com/ceph/ceph
synced 2025-03-11 02:39:05 +00:00
Move teuthology-results' arg parsing to scripts/
Signed-off-by: Zack Cerza <zack.cerza@inktank.com>
This commit is contained in:
Zack Cerza
parent
477e4ae2ce
commit
193b3112a3
40
scripts/results.py
Normal file
40
scripts/results.py
Normal file
@ -0,0 +1,40 @@
|
||||
import argparse
|
||||
|
||||
import teuthology.results
|
||||
|
||||
|
||||
def main():
|
||||
teuthology.results.main(parse_args())
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Email teuthology suite results')
|
||||
parser.add_argument(
|
||||
'--email',
|
||||
help='address to email test failures to',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--timeout',
|
||||
help='how many seconds to wait for all tests to finish (default no ' +
|
||||
'wait)',
|
||||
type=int,
|
||||
default=0,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--archive-dir',
|
||||
metavar='DIR',
|
||||
help='path under which results for the suite are stored',
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--name',
|
||||
help='name of the suite',
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
'-v', '--verbose',
|
||||
action='store_true', default=False,
|
||||
help='be more verbose',
|
||||
)
|
||||
return parser.parse_args()
|
||||
2
setup.py
2
setup.py
@ -25,7 +25,7 @@ setup(
|
||||
'teuthology-schedule = scripts.schedule:main',
|
||||
'teuthology-updatekeys = scripts.updatekeys:main',
|
||||
'teuthology-coverage = teuthology.coverage:analyze',
|
||||
'teuthology-results = teuthology.suite:results',
|
||||
'teuthology-results = scripts.results:main',
|
||||
'teuthology-report = scripts.report:main',
|
||||
],
|
||||
},
|
||||
|
||||
259
teuthology/results.py
Normal file
259
teuthology/results.py
Normal file
@ -0,0 +1,259 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import yaml
|
||||
import logging
|
||||
import subprocess
|
||||
from textwrap import dedent
|
||||
from textwrap import fill
|
||||
|
||||
from teuthology import misc
|
||||
from teuthology import suite
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
loglevel = logging.INFO
|
||||
if args.verbose:
|
||||
loglevel = logging.DEBUG
|
||||
|
||||
logging.basicConfig(
|
||||
level=loglevel,
|
||||
)
|
||||
|
||||
misc.read_config(args)
|
||||
|
||||
handler = logging.FileHandler(
|
||||
filename=os.path.join(args.archive_dir, 'results.log'),
|
||||
)
|
||||
formatter = logging.Formatter(
|
||||
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
|
||||
datefmt='%Y-%m-%dT%H:%M:%S',
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
try:
|
||||
results(args)
|
||||
except Exception:
|
||||
log.exception('error generating results')
|
||||
raise
|
||||
|
||||
|
||||
def results(args):
|
||||
running_tests = [
|
||||
f for f in sorted(os.listdir(args.archive_dir))
|
||||
if not f.startswith('.')
|
||||
and os.path.isdir(os.path.join(args.archive_dir, f))
|
||||
and not os.path.exists(os.path.join(
|
||||
args.archive_dir, f, 'summary.yaml'))
|
||||
]
|
||||
starttime = time.time()
|
||||
log.info('Waiting up to %d seconds for tests to finish...', args.timeout)
|
||||
while running_tests and args.timeout > 0:
|
||||
if os.path.exists(os.path.join(
|
||||
args.archive_dir,
|
||||
running_tests[-1], 'summary.yaml')):
|
||||
running_tests.pop()
|
||||
else:
|
||||
if time.time() - starttime > args.timeout:
|
||||
log.warn('test(s) did not finish before timeout of %d seconds',
|
||||
args.timeout)
|
||||
break
|
||||
time.sleep(10)
|
||||
log.info('Tests finished! gathering results...')
|
||||
|
||||
(subject, body) = build_email_body(args.name, args.archive_dir,
|
||||
args.timeout)
|
||||
|
||||
try:
|
||||
if args.email:
|
||||
email_results(
|
||||
subject=subject,
|
||||
from_=args.teuthology_config['results_sending_email'],
|
||||
to=args.email,
|
||||
body=body,
|
||||
)
|
||||
finally:
|
||||
generate_coverage(args)
|
||||
|
||||
|
||||
def generate_coverage(args):
|
||||
log.info('starting coverage generation')
|
||||
subprocess.Popen(
|
||||
args=[
|
||||
os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'),
|
||||
'-v',
|
||||
'-o',
|
||||
os.path.join(args.teuthology_config[
|
||||
'coverage_output_dir'], args.name),
|
||||
'--html-output',
|
||||
os.path.join(args.teuthology_config[
|
||||
'coverage_html_dir'], args.name),
|
||||
'--cov-tools-dir',
|
||||
args.teuthology_config['coverage_tools_dir'],
|
||||
args.archive_dir,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def email_results(subject, from_, to, body):
|
||||
log.info('Sending results to {to}: {body}'.format(to=to, body=body))
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
msg = MIMEText(body)
|
||||
msg['Subject'] = subject
|
||||
msg['From'] = from_
|
||||
msg['To'] = to
|
||||
log.debug('sending email %s', msg.as_string())
|
||||
smtp = smtplib.SMTP('localhost')
|
||||
smtp.sendmail(msg['From'], [msg['To']], msg.as_string())
|
||||
smtp.quit()
|
||||
|
||||
|
||||
def build_email_body(name, archive_dir, timeout):
|
||||
failed = {}
|
||||
hung = {}
|
||||
passed = {}
|
||||
|
||||
for job in suite.get_jobs(archive_dir):
|
||||
job_dir = os.path.join(archive_dir, job)
|
||||
summary_file = os.path.join(job_dir, 'summary.yaml')
|
||||
|
||||
# Unfinished jobs will have no summary.yaml
|
||||
if not os.path.exists(summary_file):
|
||||
info_file = os.path.join(job_dir, 'info.yaml')
|
||||
|
||||
desc = ''
|
||||
if os.path.exists(info_file):
|
||||
with file(info_file) as f:
|
||||
info = yaml.safe_load(f)
|
||||
desc = info['description']
|
||||
|
||||
hung[job] = email_templates['hung_templ'].format(
|
||||
job_id=job,
|
||||
desc=desc,
|
||||
)
|
||||
continue
|
||||
|
||||
with file(summary_file) as f:
|
||||
summary = yaml.safe_load(f)
|
||||
|
||||
if summary['success']:
|
||||
passed[job] = email_templates['pass_templ'].format(
|
||||
job_id=job,
|
||||
desc=summary.get('description'),
|
||||
time=int(summary.get('duration', 0)),
|
||||
)
|
||||
else:
|
||||
log = misc.get_http_log_path(archive_dir, job)
|
||||
if log:
|
||||
log_line = email_templates['fail_log_templ'].format(log=log)
|
||||
else:
|
||||
log_line = ''
|
||||
sentry_events = summary.get('sentry_events')
|
||||
if sentry_events:
|
||||
sentry_line = email_templates['fail_sentry_templ'].format(
|
||||
sentries='\n '.join(sentry_events))
|
||||
else:
|
||||
sentry_line = ''
|
||||
|
||||
# 'fill' is from the textwrap module and it collapses a given
|
||||
# string into multiple lines of a maximum width as specified. We
|
||||
# want 75 characters here so that when we indent by 4 on the next
|
||||
# line, we have 79-character exception paragraphs.
|
||||
reason = fill(summary.get('failure_reason'), 75)
|
||||
reason = '\n'.join((' ') + line for line in reason.splitlines())
|
||||
|
||||
failed[job] = email_templates['fail_templ'].format(
|
||||
job_id=job,
|
||||
desc=summary.get('description'),
|
||||
time=int(summary.get('duration', 0)),
|
||||
reason=reason,
|
||||
log_line=log_line,
|
||||
sentry_line=sentry_line,
|
||||
)
|
||||
|
||||
maybe_comma = lambda s: ', ' if s else ' '
|
||||
|
||||
subject = ''
|
||||
fail_sect = ''
|
||||
hung_sect = ''
|
||||
pass_sect = ''
|
||||
if failed:
|
||||
subject += '{num_failed} failed{sep}'.format(
|
||||
num_failed=len(failed),
|
||||
sep=maybe_comma(hung or passed)
|
||||
)
|
||||
fail_sect = email_templates['sect_templ'].format(
|
||||
title='Failed',
|
||||
jobs=''.join(failed.values())
|
||||
)
|
||||
if hung:
|
||||
subject += '{num_hung} hung{sep}'.format(
|
||||
num_hung=len(hung),
|
||||
sep=maybe_comma(passed),
|
||||
)
|
||||
hung_sect = email_templates['sect_templ'].format(
|
||||
title='Hung',
|
||||
jobs=''.join(hung.values()),
|
||||
)
|
||||
if passed:
|
||||
subject += '%s passed ' % len(passed)
|
||||
pass_sect = email_templates['sect_templ'].format(
|
||||
title='Passed',
|
||||
jobs=''.join(passed.values()),
|
||||
)
|
||||
|
||||
body = email_templates['body_templ'].format(
|
||||
name=name,
|
||||
log_root=misc.get_http_log_path(archive_dir),
|
||||
fail_count=len(failed),
|
||||
hung_count=len(hung),
|
||||
pass_count=len(passed),
|
||||
fail_sect=fail_sect,
|
||||
hung_sect=hung_sect,
|
||||
pass_sect=pass_sect,
|
||||
)
|
||||
|
||||
subject += 'in {suite}'.format(suite=name)
|
||||
return (subject.strip(), body.strip())
|
||||
|
||||
email_templates = {
|
||||
'body_templ': dedent("""\
|
||||
Test Run: {name}
|
||||
=================================================================
|
||||
logs: {log_root}
|
||||
failed: {fail_count}
|
||||
hung: {hung_count}
|
||||
passed: {pass_count}
|
||||
|
||||
{fail_sect}{hung_sect}{pass_sect}
|
||||
"""),
|
||||
'sect_templ': dedent("""\
|
||||
{title}
|
||||
=================================================================
|
||||
{jobs}
|
||||
"""),
|
||||
'fail_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
-----------------------------------------------------------------
|
||||
time: {time}s{log_line}{sentry_line}
|
||||
|
||||
{reason}
|
||||
|
||||
"""),
|
||||
'fail_log_templ': "\nlog: {log}",
|
||||
'fail_sentry_templ': "\nsentry: {sentries}",
|
||||
'hung_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
"""),
|
||||
'pass_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
time: {time}s
|
||||
|
||||
"""),
|
||||
}
|
||||
@ -2,7 +2,6 @@
|
||||
# by generating combinations of facets found in
|
||||
# https://github.com/ceph/ceph-qa-suite.git
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import errno
|
||||
import itertools
|
||||
@ -11,11 +10,8 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from textwrap import dedent, fill
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from teuthology import misc
|
||||
from teuthology import lock as lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@ -245,136 +241,6 @@ def ls(archive_dir, verbose):
|
||||
print ' {reason}'.format(reason=summary['failure_reason'])
|
||||
|
||||
|
||||
def generate_coverage(args):
|
||||
log.info('starting coverage generation')
|
||||
subprocess.Popen(
|
||||
args=[
|
||||
os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'),
|
||||
'-v',
|
||||
'-o',
|
||||
os.path.join(args.teuthology_config[
|
||||
'coverage_output_dir'], args.name),
|
||||
'--html-output',
|
||||
os.path.join(args.teuthology_config[
|
||||
'coverage_html_dir'], args.name),
|
||||
'--cov-tools-dir',
|
||||
args.teuthology_config['coverage_tools_dir'],
|
||||
args.archive_dir,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def email_results(subject, from_, to, body):
|
||||
log.info('Sending results to {to}: {body}'.format(to=to, body=body))
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
msg = MIMEText(body)
|
||||
msg['Subject'] = subject
|
||||
msg['From'] = from_
|
||||
msg['To'] = to
|
||||
log.debug('sending email %s', msg.as_string())
|
||||
smtp = smtplib.SMTP('localhost')
|
||||
smtp.sendmail(msg['From'], [msg['To']], msg.as_string())
|
||||
smtp.quit()
|
||||
|
||||
|
||||
def results():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Email teuthology suite results')
|
||||
parser.add_argument(
|
||||
'--email',
|
||||
help='address to email test failures to',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--timeout',
|
||||
help='how many seconds to wait for all tests to finish (default no ' +
|
||||
'wait)',
|
||||
type=int,
|
||||
default=0,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--archive-dir',
|
||||
metavar='DIR',
|
||||
help='path under which results for the suite are stored',
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--name',
|
||||
help='name of the suite',
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
'-v', '--verbose',
|
||||
action='store_true', default=False,
|
||||
help='be more verbose',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
loglevel = logging.INFO
|
||||
if args.verbose:
|
||||
loglevel = logging.DEBUG
|
||||
|
||||
logging.basicConfig(
|
||||
level=loglevel,
|
||||
)
|
||||
|
||||
misc.read_config(args)
|
||||
|
||||
handler = logging.FileHandler(
|
||||
filename=os.path.join(args.archive_dir, 'results.log'),
|
||||
)
|
||||
formatter = logging.Formatter(
|
||||
fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
|
||||
datefmt='%Y-%m-%dT%H:%M:%S',
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
try:
|
||||
_results(args)
|
||||
except Exception:
|
||||
log.exception('error generating results')
|
||||
raise
|
||||
|
||||
|
||||
def _results(args):
|
||||
running_tests = [
|
||||
f for f in sorted(os.listdir(args.archive_dir))
|
||||
if not f.startswith('.')
|
||||
and os.path.isdir(os.path.join(args.archive_dir, f))
|
||||
and not os.path.exists(os.path.join(
|
||||
args.archive_dir, f, 'summary.yaml'))
|
||||
]
|
||||
starttime = time.time()
|
||||
log.info('Waiting up to %d seconds for tests to finish...', args.timeout)
|
||||
while running_tests and args.timeout > 0:
|
||||
if os.path.exists(os.path.join(
|
||||
args.archive_dir,
|
||||
running_tests[-1], 'summary.yaml')):
|
||||
running_tests.pop()
|
||||
else:
|
||||
if time.time() - starttime > args.timeout:
|
||||
log.warn('test(s) did not finish before timeout of %d seconds',
|
||||
args.timeout)
|
||||
break
|
||||
time.sleep(10)
|
||||
log.info('Tests finished! gathering results...')
|
||||
|
||||
(subject, body) = build_email_body(args.name, args.archive_dir,
|
||||
args.timeout)
|
||||
|
||||
try:
|
||||
if args.email:
|
||||
email_results(
|
||||
subject=subject,
|
||||
from_=args.teuthology_config['results_sending_email'],
|
||||
to=args.email,
|
||||
body=body,
|
||||
)
|
||||
finally:
|
||||
generate_coverage(args)
|
||||
|
||||
|
||||
def get_jobs(archive_dir):
|
||||
dir_contents = os.listdir(archive_dir)
|
||||
|
||||
@ -388,152 +254,6 @@ def get_jobs(archive_dir):
|
||||
return sorted(jobs)
|
||||
|
||||
|
||||
email_templates = {
|
||||
'body_templ': dedent("""\
|
||||
Test Run: {name}
|
||||
=================================================================
|
||||
logs: {log_root}
|
||||
failed: {fail_count}
|
||||
hung: {hung_count}
|
||||
passed: {pass_count}
|
||||
|
||||
{fail_sect}{hung_sect}{pass_sect}
|
||||
"""),
|
||||
'sect_templ': dedent("""\
|
||||
{title}
|
||||
=================================================================
|
||||
{jobs}
|
||||
"""),
|
||||
'fail_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
-----------------------------------------------------------------
|
||||
time: {time}s{log_line}{sentry_line}
|
||||
|
||||
{reason}
|
||||
|
||||
"""),
|
||||
'fail_log_templ': "\nlog: {log}",
|
||||
'fail_sentry_templ': "\nsentry: {sentries}",
|
||||
'hung_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
"""),
|
||||
'pass_templ': dedent("""\
|
||||
[{job_id}] {desc}
|
||||
time: {time}s
|
||||
|
||||
"""),
|
||||
}
|
||||
|
||||
|
||||
def build_email_body(name, archive_dir, timeout):
|
||||
failed = {}
|
||||
hung = {}
|
||||
passed = {}
|
||||
|
||||
for job in get_jobs(archive_dir):
|
||||
job_dir = os.path.join(archive_dir, job)
|
||||
summary_file = os.path.join(job_dir, 'summary.yaml')
|
||||
|
||||
# Unfinished jobs will have no summary.yaml
|
||||
if not os.path.exists(summary_file):
|
||||
info_file = os.path.join(job_dir, 'info.yaml')
|
||||
|
||||
desc = ''
|
||||
if os.path.exists(info_file):
|
||||
with file(info_file) as f:
|
||||
info = yaml.safe_load(f)
|
||||
desc = info['description']
|
||||
|
||||
hung[job] = email_templates['hung_templ'].format(
|
||||
job_id=job,
|
||||
desc=desc,
|
||||
)
|
||||
continue
|
||||
|
||||
with file(summary_file) as f:
|
||||
summary = yaml.safe_load(f)
|
||||
|
||||
if summary['success']:
|
||||
passed[job] = email_templates['pass_templ'].format(
|
||||
job_id=job,
|
||||
desc=summary.get('description'),
|
||||
time=int(summary.get('duration', 0)),
|
||||
)
|
||||
else:
|
||||
log = misc.get_http_log_path(archive_dir, job)
|
||||
if log:
|
||||
log_line = email_templates['fail_log_templ'].format(log=log)
|
||||
else:
|
||||
log_line = ''
|
||||
sentry_events = summary.get('sentry_events')
|
||||
if sentry_events:
|
||||
sentry_line = email_templates['fail_sentry_templ'].format(
|
||||
sentries='\n '.join(sentry_events))
|
||||
else:
|
||||
sentry_line = ''
|
||||
|
||||
# 'fill' is from the textwrap module and it collapses a given
|
||||
# string into multiple lines of a maximum width as specified. We
|
||||
# want 75 characters here so that when we indent by 4 on the next
|
||||
# line, we have 79-character exception paragraphs.
|
||||
reason = fill(summary.get('failure_reason'), 75)
|
||||
reason = '\n'.join((' ') + line for line in reason.splitlines())
|
||||
|
||||
failed[job] = email_templates['fail_templ'].format(
|
||||
job_id=job,
|
||||
desc=summary.get('description'),
|
||||
time=int(summary.get('duration', 0)),
|
||||
reason=reason,
|
||||
log_line=log_line,
|
||||
sentry_line=sentry_line,
|
||||
)
|
||||
|
||||
maybe_comma = lambda s: ', ' if s else ' '
|
||||
|
||||
subject = ''
|
||||
fail_sect = ''
|
||||
hung_sect = ''
|
||||
pass_sect = ''
|
||||
if failed:
|
||||
subject += '{num_failed} failed{sep}'.format(
|
||||
num_failed=len(failed),
|
||||
sep=maybe_comma(hung or passed)
|
||||
)
|
||||
fail_sect = email_templates['sect_templ'].format(
|
||||
title='Failed',
|
||||
jobs=''.join(failed.values())
|
||||
)
|
||||
if hung:
|
||||
subject += '{num_hung} hung{sep}'.format(
|
||||
num_hung=len(hung),
|
||||
sep=maybe_comma(passed),
|
||||
)
|
||||
hung_sect = email_templates['sect_templ'].format(
|
||||
title='Hung',
|
||||
jobs=''.join(hung.values()),
|
||||
)
|
||||
if passed:
|
||||
subject += '%s passed ' % len(passed)
|
||||
pass_sect = email_templates['sect_templ'].format(
|
||||
title='Passed',
|
||||
jobs=''.join(passed.values()),
|
||||
)
|
||||
|
||||
body = email_templates['body_templ'].format(
|
||||
name=name,
|
||||
log_root=misc.get_http_log_path(archive_dir),
|
||||
fail_count=len(failed),
|
||||
hung_count=len(hung),
|
||||
pass_count=len(passed),
|
||||
fail_sect=fail_sect,
|
||||
hung_sect=hung_sect,
|
||||
pass_sect=pass_sect,
|
||||
)
|
||||
|
||||
subject += 'in {suite}'.format(suite=name)
|
||||
return (subject.strip(), body.strip())
|
||||
|
||||
|
||||
def get_arch(config):
|
||||
for yamlfile in config:
|
||||
y = yaml.safe_load(file(yamlfile))
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import os
|
||||
import textwrap
|
||||
from .. import suite
|
||||
from .. import results
|
||||
from .fake_archive import FakeArchive
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ class TestResultsEmail(object):
|
||||
run_name = self.reference['run_name']
|
||||
run_dir = os.path.join(self.archive_base, run_name)
|
||||
self.archive.populate_archive(run_name, self.reference['jobs'])
|
||||
(subject, body) = suite.build_email_body(
|
||||
(subject, body) = results.build_email_body(
|
||||
run_name,
|
||||
run_dir,
|
||||
36000)
|
||||
Loading…
Reference in New Issue
Block a user