ceph/qa/tasks/buildpackages.py

"""
Build ceph packages

Unit tests:

py.test -v -s tests/test_buildpackages.py

Integration tests:

teuthology-openstack --verbose --key-name myself --key-filename ~/Downloads/myself --ceph infernalis --suite teuthology/buildpackages

"""
import copy
import logging
import os
import types
from teuthology import packaging
from teuthology import misc
from teuthology.config import config as teuth_config
from teuthology.openstack import OpenStack

log = logging.getLogger(__name__)

class LocalGitbuilderProject(packaging.GitbuilderProject):

    def __init__(self):
        pass


def get_pkg_type(os_type):
    if os_type in ('centos', 'fedora', 'opensuse', 'rhel', 'sles'):
        return 'rpm'
    else:
        return 'deb'

def apply_overrides(ctx, config):
    if config is None:
        config = {}
    else:
        config = copy.deepcopy(config)

    assert isinstance(config, dict), \
        "task install only supports a dictionary for configuration"

    project, = config.get('project', 'ceph'),
    log.debug('project %s' % project)
    overrides = ctx.config.get('overrides')
    if overrides:
        install_overrides = overrides.get('install', {})
        misc.deep_merge(config, install_overrides.get(project, {}))
    return config

def get_config_install(ctx, config):
    config = apply_overrides(ctx, config)
    log.debug('install config %s' % config)
    return [(config.get('flavor', 'basic'),
             config.get('tag', ''),
             config.get('branch', ''),
             config.get('sha1'))]

def get_config_install_upgrade(ctx, config):
    log.debug('install.upgrade config before override %s' % config)
    configs = []
    for (role, role_config) in config.iteritems():
        if role_config is None:
            role_config = {}
        o = apply_overrides(ctx, role_config)

        log.debug('install.upgrade config ' + str(role_config) +
                  ' and with overrides ' + str(o))
        # for install.upgrade overrides are actually defaults
        configs.append((o.get('flavor', 'basic'),
                        role_config.get('tag', o.get('tag', '')),
                        role_config.get('branch', o.get('branch', '')),
                        role_config.get('sha1', o.get('sha1'))))
    return configs

GET_CONFIG_FUNCTIONS = {
    'install': get_config_install,
    'install.upgrade': get_config_install_upgrade,
}

def lookup_configs(ctx, node):
    configs = []
    if type(node) is types.ListType:
        for leaf in node:
            configs.extend(lookup_configs(ctx, leaf))
    elif type(node) is types.DictType:
        for (key, value) in node.iteritems():
            if key in ('install', 'install.upgrade'):
                configs.extend(GET_CONFIG_FUNCTIONS[key](ctx, value))
            elif key in ('overrides',):
                pass
            else:
                configs.extend(lookup_configs(ctx, value))
    return configs

def get_sha1(ref):
    url = teuth_config.get_ceph_git_url()
    ls_remote = misc.sh("git ls-remote " + url + " " + ref)
    return ls_remote.split()[0]

def task(ctx, config):
    """
    Build Ceph packages. This task will automagically be run
    before the task that need to install packages (this is taken
    care of by the internal teuthology task).

    The config should be as follows:

    buildpackages:
      good_machine:
        disk: 40 # GB
        ram: 48000 # MB
        cpus: 16
      min_machine:
        disk: 40 # GB
        ram: 8000 # MB
        cpus: 1

    example:

    tasks:
    - buildpackages:
        good_machine:
          disk: 40 # GB
          ram: 15000 # MB
          cpus: 16
        min_machine:
          disk: 40 # GB
          ram: 8000 # MB
          cpus: 1
    - install:

    When a buildpackages task is already included, the values it contains can be
    overriden with:

    overrides:
      buildpackages:
        good_machine:
          disk: 20 # GB
          ram: 2000 # MB
          cpus: 2
        min_machine:
          disk: 10 # GB
          ram: 1000 # MB
          cpus: 1

    """
    log.info('Beginning buildpackages...')
    if config is None:
        config = {}
    assert isinstance(config, dict), \
        'task only accepts a dict for config not ' + str(config)
    overrides = ctx.config.get('overrides', {})
    misc.deep_merge(config, overrides.get('buildpackages', {}))
    d = os.path.join(os.path.dirname(__file__), 'buildpackages')
    os_type = misc.get_distro(ctx)
    os_version = misc.get_distro_version(ctx)
    arch = ctx.config.get('arch', OpenStack().get_default_arch())
    dist = LocalGitbuilderProject()._get_distro(distro=os_type,
                                                version=os_version)
    pkg_type = get_pkg_type(os_type)
    misc.sh(
        "flock --close /tmp/buildpackages " +
        "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
    for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
        if tag:
            sha1 = get_sha1(tag)
        elif branch:
            sha1 = get_sha1(branch)
        log.info("building flavor = " + flavor + "," +
                 " tag = " + tag + "," +
                 " branch = " + branch + "," +
                 " sha1 = " + sha1)
        target = ('ceph-' +
                  pkg_type + '-' +
                  dist + '-' +
                  arch + '-' +
                  flavor + '-' +
                  sha1)
        openstack = OpenStack()
        openstack.set_provider()
        if openstack.provider == 'ovh':
            select = '^(vps|hg)-.*ssd'
        else:
            select = ''
        network = openstack.net()
        if network != "":
            network = " OPENSTACK_NETWORK='" + network + "' "
        openstack.image(os_type, os_version, arch) # create if it does not exist
        build_flavor = openstack.flavor_range(
            config['min_machine'], config['good_machine'], arch, select)
        default_arch = openstack.get_default_arch()
        http_flavor = openstack.flavor({
            'disk': 30, # GB
            'ram': 1024, # MB
            'cpus': 1,
        }, default_arch, select)
        lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
        cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
               " flock --close " + lock +
               " make -C " + d +
               network +
               " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
               " CEPH_PKG_TYPE=" + pkg_type +
               " CEPH_OS_TYPE=" + os_type +
               " CEPH_OS_VERSION=" + os_version +
               " CEPH_DIST=" + dist +
               " CEPH_ARCH=" + arch +
               " CEPH_SHA1=" + sha1 +
               " CEPH_TAG=" + tag +
               " CEPH_BRANCH=" + branch +
               " CEPH_FLAVOR=" + flavor +
               " BUILD_FLAVOR=" + build_flavor +
               " HTTP_FLAVOR=" + http_flavor +
               " HTTP_ARCH=" + default_arch +
               " " + target +
               " ")
        log.info("buildpackages: " + cmd)
        misc.sh(cmd)
    teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
    log.info('Finished buildpackages')