ceph/qa/tasks/lua_tests.py

"""
Run a set of lua tests on rgw.
"""
from io import BytesIO
from configobj import ConfigObj
import base64
import contextlib
import logging
import os
import random
import string

from teuthology import misc as teuthology
from teuthology import contextutil
from teuthology.orchestra import run

log = logging.getLogger(__name__)

@contextlib.contextmanager
def download(ctx, config):
    assert isinstance(config, dict)
    log.info('Downloading lua-tests...')
    testdir = teuthology.get_testdir(ctx)
    branch = ctx.config.get('suite_branch')
    repo = ctx.config.get('suite_repo')
    log.info('Using branch %s from %s for lua tests', branch, repo)
    for (client, client_config) in config.items():

        ctx.cluster.only(client).run(
            args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)],
            )

        sha1 = client_config.get('sha1')

        if sha1 is not None:
            ctx.cluster.only(client).run(
                args=[
                    'cd', '{tdir}/ceph'.format(tdir=testdir),
                    run.Raw('&&'),
                    'git', 'reset', '--hard', sha1,
                    ],
                )

    try:
        yield
    finally:
        log.info('Removing lua-tests...')
        testdir = teuthology.get_testdir(ctx)
        for client in config:
            ctx.cluster.only(client).run(
                args=[
                    'rm',
                    '-rf',
                    '{tdir}/ceph'.format(tdir=testdir),
                    ],
                )


def _config_user(luatests_conf, section, user):
    """
    Configure users for this section by stashing away keys, ids, and
    email addresses.
    """
    luatests_conf[section].setdefault('user_id', user)
    luatests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
    luatests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
    luatests_conf[section].setdefault('access_key',
        ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
    luatests_conf[section].setdefault('secret_key',
        base64.b64encode(os.urandom(40)).decode())


@contextlib.contextmanager
def create_users(ctx, config):
    """
    Create a main and an alternate s3 user.
    """
    assert isinstance(config, dict)
    log.info('Creating rgw user...')
    testdir = teuthology.get_testdir(ctx)

    users = {'s3 main': 'foo'}
    for client in config['clients']:
        luatests_conf = config['luatests_conf'][client]
        for section, user in users.items():
            _config_user(luatests_conf, section, '{user}.{client}'.format(user=user, client=client))
            log.debug('Creating user {user} on {host}'.format(user=luatests_conf[section]['user_id'], host=client))
            cluster_name, daemon_type, client_id = teuthology.split_role(client)
            client_with_id = daemon_type + '.' + client_id
            ctx.cluster.only(client).run(
                args=[
                    'adjust-ulimits',
                    'ceph-coverage',
                    '{tdir}/archive/coverage'.format(tdir=testdir),
                    'radosgw-admin',
                    '-n', client_with_id,
                    'user', 'create',
                    '--uid', luatests_conf[section]['user_id'],
                    '--display-name', luatests_conf[section]['display_name'],
                    '--access-key', luatests_conf[section]['access_key'],
                    '--secret', luatests_conf[section]['secret_key'],
                    '--cluster', cluster_name,
                    ],
                )

    try:
        yield
    finally:
        for client in config['clients']:
            for user in users.values():
                uid = '{user}.{client}'.format(user=user, client=client)
                cluster_name, daemon_type, client_id = teuthology.split_role(client)
                client_with_id = daemon_type + '.' + client_id
                ctx.cluster.only(client).run(
                    args=[
                        'adjust-ulimits',
                        'ceph-coverage',
                        '{tdir}/archive/coverage'.format(tdir=testdir),
                        'radosgw-admin',
                        '-n', client_with_id,
                        'user', 'rm',
                        '--uid', uid,
                        '--purge-data',
                        '--cluster', cluster_name,
                        ],
                    )


@contextlib.contextmanager
def configure(ctx, config):
    assert isinstance(config, dict)
    log.info('Configuring lua-tests...')
    testdir = teuthology.get_testdir(ctx)
    for client, properties in config['clients'].items():
        (remote,) = ctx.cluster.only(client).remotes.keys()
        luatests_conf = config['luatests_conf'][client]

        conf_fp = BytesIO()
        luatests_conf.write(conf_fp)
        remote.write_file(
            path='{tdir}/ceph/src/test/rgw/lua/lua-tests.{client}.conf'.format(tdir=testdir, client=client),
            data=conf_fp.getvalue(),
            )

    try:
        yield
    finally:
        log.info('Removing lua-tests.conf file...')
        testdir = teuthology.get_testdir(ctx)
        for client, properties in config['clients'].items():
            (remote,) = ctx.cluster.only(client).remotes.keys()
            remote.run(
                 args=['rm', '-f',
                       '{tdir}/ceph/src/test/rgw/lua/lua-tests.{client}.conf'.format(tdir=testdir,client=client),
                 ],
                 )


def get_toxvenv_dir(ctx):
    return ctx.tox.venv_path


def toxvenv_sh(ctx, remote, args, **kwargs):
    activate = get_toxvenv_dir(ctx) + '/bin/activate'
    return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)


@contextlib.contextmanager
def run_tests(ctx, config):
    """
    Run the lua tests after everything is set up.
    :param ctx: Context passed to task
    :param config: specific configuration information
    """
    assert isinstance(config, dict)
    log.info('Running lua-tests...')
    testdir = teuthology.get_testdir(ctx)
    for client, client_config in config.items():
        (remote,) = ctx.cluster.only(client).remotes.keys()

        # test marks to use by default
        attr = ['basic_test', 'request_test', 'example_test']

        if 'extra_attr' in client_config:
            attr = client_config.get('extra_attr')

        args = ['cd', '{tdir}/ceph/src/test/rgw/lua/'.format(tdir=testdir), run.Raw('&&'),
            'LUATESTS_CONF=./lua-tests.{client}.conf'.format(client=client),
            'tox', '--', '-v', '-m', ' or '.join(attr)]

        toxvenv_sh(ctx, remote, args, label="lua tests against rgw")

    yield


@contextlib.contextmanager
def task(ctx,config):
    """

    If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo'
    parameters in your teuthology-suite command. Example command for this is as follows::

    teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:lua --ceph your_ceph_branch_name -m smithi --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name
    
    """
    assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
    assert hasattr(ctx, 'tox'), 's3tests must run after the tox task'
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task only supports a list or dictionary for configuration"

    all_clients = ['client.{id}'.format(id=id_)
                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients=config.keys()

    log.debug('config is %s', config)

    luatests_conf = {}

    for client in clients:
        endpoint = ctx.rgw.role_endpoints.get(client)
        assert endpoint, 'luatests: no rgw endpoint for {}'.format(client)

        luatests_conf[client] = ConfigObj(
            indent_type='',
            infile={
                'DEFAULT':
                    {
                    'port':endpoint.port,
                    'host':endpoint.dns_name,
                    },
                's3 main':{}
            }
        )

    with contextutil.nested(
        lambda: download(ctx=ctx, config=config),
        lambda: create_users(ctx=ctx, config=dict(
                clients=clients,
                luatests_conf=luatests_conf,
                )),
        lambda: configure(ctx=ctx, config=dict(
                clients=config,
                luatests_conf=luatests_conf,
                )),
        lambda: run_tests(ctx=ctx, config=config),
        ):
        pass
    yield