ceph/teuthology/coverage.py

import argparse
from contextlib import closing
import logging
import os
import shutil
import subprocess
import MySQLdb
import yaml

from teuthology import misc as teuthology

log = logging.getLogger(__name__)

"""
The coverage database can be created in mysql with:

CREATE TABLE `coverage` (
  `run_id` bigint(20) NOT NULL AUTO_INCREMENT,
  `rev` char(40) NOT NULL,
  `test` varchar(255) NOT NULL,
  `suite` varchar(255) NOT NULL,
  `lines` int(10) unsigned NOT NULL,
  `line_cov` float unsigned NOT NULL,
  `functions` int(10) unsigned NOT NULL,
  `function_cov` float unsigned NOT NULL,
  `branches` int(10) unsigned NOT NULL,
  `branch_cov` float unsigned NOT NULL,
  `run_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (`run_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8

"""

def connect_to_db(ctx):
    db = MySQLdb.connect(
        host=ctx.teuthology_config['coverage_db_host'],
        user=ctx.teuthology_config['coverage_db_user'],
        db=ctx.teuthology_config['coverage_db_name'],
        passwd=ctx.teuthology_config['coverage_db_password'],
        )
    db.autocommit(False)
    return db

def store_coverage(ctx, test_coverage, rev, suite):
    with closing(connect_to_db(ctx)) as db:
        rows = []
        for test, coverage in test_coverage.iteritems():
            flattened_cov = [item for sublist in coverage for item in sublist]
            rows.append([rev, test, suite] + flattened_cov)
        log.debug('inserting rows into db: %s', str(rows))
        try:
            cursor = db.cursor()
            cursor.executemany(
                'INSERT INTO `coverage`'
                ' (`rev`, `test`, `suite`, `lines`, `line_cov`, `functions`,'
                ' `function_cov`, `branches`, `branch_cov`)'
                ' VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)',
                rows)
        except Exception:
            log.exception('error updating database')
            db.rollback()
            raise
        else:
            db.commit()
            log.info('added coverage to database')
        finally:
            cursor.close()

def read_coverage(output):
    log.debug('reading coverage from output: %s', output)
    coverage = [None, None, None]
    prefixes = ['  lines......: ', '  functions..: ', '  branches...: ']
    for line in reversed(output.splitlines()):
        for i, prefix in enumerate(prefixes):
            if line.startswith(prefix):
                if '%' in line:
                    cov_num = int(line[line.find('(') + 1:line.find(' of')])
                    cov_percent = float(line[len(prefix):line.find('%')])
                    coverage[i] = (cov_num, cov_percent)
                else:
                    # may have no data for e.g. branches on the initial run
                    coverage[i] = (None, None)
                break
        if None not in coverage:
            break
    return coverage

def analyze():
    parser = argparse.ArgumentParser(description="""
Analyze the coverage of a suite of test runs, generating html output with lcov.
""")
    parser.add_argument(
        '-o', '--lcov-output',
        help='the directory in which to store results',
        required=True,
        )
    parser.add_argument(
        '--html-output',
        help='the directory in which to store html output',
        )
    parser.add_argument(
        '--cov-tools-dir',
        help='the location of coverage scripts (cov-init and cov-analyze)',
        default='../../coverage',
        )
    parser.add_argument(
        '--skip-init',
        help='skip initialization (useful if a run stopped partway through)',
        action='store_true',
        default=False,
        )
    parser.add_argument(
        '-v', '--verbose',
        help='be more verbose',
        action='store_true',
        default=False,
        )
    parser.add_argument(
        'test_dir',
        help='the location of the test results',
        )
    args = parser.parse_args()

    loglevel = logging.INFO
    if args.verbose:
        loglevel = logging.DEBUG

    logging.basicConfig(
        level=loglevel,
        )

    teuthology.read_config(args)

    handler = logging.FileHandler(
        filename=os.path.join(args.test_dir, 'coverage.log'),
        )
    formatter = logging.Formatter(
        fmt='%(asctime)s.%(msecs)03d %(levelname)s:%(message)s',
        datefmt='%Y-%m-%dT%H:%M:%S',
        )
    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)

    try:
        _analyze(args)
    except Exception:
        log.exception('error generating coverage')
        raise

def _analyze(args):
    tests = [
        f for f in sorted(os.listdir(args.test_dir))
        if not f.startswith('.')
        and os.path.isdir(os.path.join(args.test_dir, f))
        and os.path.exists(os.path.join(args.test_dir, f, 'summary.yaml'))
        and os.path.exists(os.path.join(args.test_dir, f, 'ceph-sha1'))]

    test_summaries = {}
    for test in tests:
        summary = {}
        with file(os.path.join(args.test_dir, test, 'summary.yaml')) as f:
            g = yaml.safe_load_all(f)
            for new in g:
                summary.update(new)

        if summary['flavor'] != 'gcov':
            log.debug('Skipping %s, since it does not include coverage', test)
            continue
        test_summaries[test] = summary

    assert len(test_summaries) > 0

    suite = os.path.basename(args.test_dir)

    # only run cov-init once.
    # this only works if all tests were run against the same version.
    if not args.skip_init:
        log.info('initializing coverage data...')
        subprocess.check_call(
            args=[
                os.path.join(args.cov_tools_dir, 'cov-init.sh'),
                os.path.join(args.test_dir, tests[0]),
                args.lcov_output,
                os.path.join(
                    args.teuthology_config['ceph_build_output_dir'],
                    '{suite}.tgz'.format(suite=suite),
                    ),
                ])
        shutil.copy(
            os.path.join(args.lcov_output, 'base.lcov'),
            os.path.join(args.lcov_output, 'total.lcov')
            )

    test_coverage = {}
    for test, summary in test_summaries.iteritems():
        lcov_file = '{name}.lcov'.format(name=test)

        log.info('analyzing coverage for %s', test)
        proc = subprocess.Popen(
            args=[
                os.path.join(args.cov_tools_dir, 'cov-analyze.sh'),
                '-t', os.path.join(args.test_dir, test),
                '-d', args.lcov_output,
                '-o', test,
                ],
            stdout=subprocess.PIPE,
            )
        output, _ = proc.communicate()
        desc = summary.get('description', test)
        test_coverage[desc] = read_coverage(output)

        log.info('adding %s data to total', test)
        proc = subprocess.Popen(
            args=[
                'lcov',
                '-a', os.path.join(args.lcov_output, lcov_file),
                '-a', os.path.join(args.lcov_output, 'total.lcov'),
                '-o', os.path.join(args.lcov_output, 'total_tmp.lcov'),
                ],
            stdout=subprocess.PIPE,
            )
        output, _ = proc.communicate()

        os.rename(
            os.path.join(args.lcov_output, 'total_tmp.lcov'),
            os.path.join(args.lcov_output, 'total.lcov')
            )

    coverage = read_coverage(output)
    test_coverage['total for {suite}'.format(suite=suite)] = coverage
    log.debug('total coverage is %s', str(coverage))

    if args.html_output:
        subprocess.check_call(
            args=[
                'genhtml',
                '-s',
                '-o', os.path.join(args.html_output, 'total'),
                '-t', 'Total for {suite}'.format(suite=suite),
                '--',
                os.path.join(args.lcov_output, 'total.lcov'),
                ])

    store_coverage(args, test_coverage, summary['ceph-sha1'], suite)