2014-02-15 01:08:28 +00:00
|
|
|
"""
|
|
|
|
rgw routines
|
|
|
|
"""
|
2013-08-17 17:14:37 +00:00
|
|
|
import argparse
|
2011-06-22 23:36:58 +00:00
|
|
|
import contextlib
|
2013-07-11 00:45:35 +00:00
|
|
|
import json
|
2011-06-22 23:36:58 +00:00
|
|
|
import logging
|
|
|
|
import os
|
2016-04-13 08:19:06 +00:00
|
|
|
import errno
|
2016-01-19 14:56:18 +00:00
|
|
|
import util.rgw as rgw_utils
|
2011-06-22 23:36:58 +00:00
|
|
|
|
2014-08-07 14:24:59 +00:00
|
|
|
from teuthology.orchestra import run
|
2011-06-22 23:36:58 +00:00
|
|
|
from teuthology import misc as teuthology
|
|
|
|
from teuthology import contextutil
|
2014-10-22 21:40:07 +00:00
|
|
|
from teuthology.orchestra.run import CommandFailedError
|
2017-04-26 01:38:52 +00:00
|
|
|
from util.rgw import rgwadmin, wait_for_radosgw
|
2014-08-07 14:24:59 +00:00
|
|
|
from util.rados import (rados, create_ec_pool,
|
2014-05-01 21:36:08 +00:00
|
|
|
create_replicated_pool,
|
|
|
|
create_cache_pool)
|
2011-06-22 23:36:58 +00:00
|
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
2011-09-03 00:58:19 +00:00
|
|
|
@contextlib.contextmanager
|
2017-06-26 08:42:08 +00:00
|
|
|
def start_rgw(ctx, config, clients):
|
2014-02-15 01:08:28 +00:00
|
|
|
"""
|
|
|
|
Start rgw on remote sites.
|
|
|
|
"""
|
2011-09-03 00:58:19 +00:00
|
|
|
log.info('Starting rgw...')
|
2013-01-23 20:37:39 +00:00
|
|
|
testdir = teuthology.get_testdir(ctx)
|
2017-06-26 08:42:08 +00:00
|
|
|
for client in clients:
|
2014-03-27 16:35:28 +00:00
|
|
|
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
|
2016-12-16 18:23:09 +00:00
|
|
|
cluster_name, daemon_type, client_id = teuthology.split_role(client)
|
|
|
|
client_with_id = daemon_type + '.' + client_id
|
|
|
|
client_with_cluster = cluster_name + '.' + client_with_id
|
|
|
|
|
2012-02-22 00:08:21 +00:00
|
|
|
client_config = config.get(client)
|
|
|
|
if client_config is None:
|
|
|
|
client_config = {}
|
2012-02-24 20:04:58 +00:00
|
|
|
log.info("rgw %s config is %s", client, client_config)
|
2014-04-15 18:12:00 +00:00
|
|
|
cmd_prefix = [
|
2013-02-18 20:14:14 +00:00
|
|
|
'sudo',
|
2013-09-06 20:08:01 +00:00
|
|
|
'adjust-ulimits',
|
2013-07-22 17:38:28 +00:00
|
|
|
'ceph-coverage',
|
|
|
|
'{tdir}/archive/coverage'.format(tdir=testdir),
|
2013-09-06 20:08:01 +00:00
|
|
|
'daemon-helper',
|
2013-07-22 17:38:28 +00:00
|
|
|
'term',
|
2012-02-22 00:08:21 +00:00
|
|
|
]
|
2014-04-15 19:49:03 +00:00
|
|
|
|
|
|
|
rgw_cmd = ['radosgw']
|
|
|
|
|
2017-04-26 01:38:52 +00:00
|
|
|
log.info("Using %s as radosgw frontend", ctx.rgw.frontend)
|
2014-04-15 19:49:03 +00:00
|
|
|
|
2017-05-19 20:05:36 +00:00
|
|
|
host, port = ctx.rgw.role_endpoints[client]
|
2017-07-19 14:23:41 +00:00
|
|
|
frontends = \
|
|
|
|
'{frontend} port={port}'.format(frontend=ctx.rgw.frontend,
|
|
|
|
port=port)
|
2017-08-13 23:36:38 +00:00
|
|
|
frontend_prefix = client_config.get('frontend_prefix', None)
|
|
|
|
if frontend_prefix:
|
|
|
|
frontends += ' prefix={pfx}'.format(pfx=frontend_prefix)
|
2014-04-15 19:49:03 +00:00
|
|
|
rgw_cmd.extend([
|
2017-07-19 14:23:41 +00:00
|
|
|
'--rgw-frontends', frontends,
|
2016-12-16 18:23:09 +00:00
|
|
|
'-n', client_with_id,
|
|
|
|
'--cluster', cluster_name,
|
|
|
|
'-k', '/etc/ceph/{client_with_cluster}.keyring'.format(client_with_cluster=client_with_cluster),
|
2013-07-22 17:38:28 +00:00
|
|
|
'--log-file',
|
2016-12-16 18:23:09 +00:00
|
|
|
'/var/log/ceph/rgw.{client_with_cluster}.log'.format(client_with_cluster=client_with_cluster),
|
2013-07-22 17:38:28 +00:00
|
|
|
'--rgw_ops_log_socket_path',
|
2016-12-16 18:23:09 +00:00
|
|
|
'{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(tdir=testdir,
|
2017-07-19 14:20:05 +00:00
|
|
|
client_with_cluster=client_with_cluster)
|
|
|
|
])
|
|
|
|
|
|
|
|
keystone_role = client_config.get('use-keystone-role', None)
|
|
|
|
if keystone_role is not None:
|
|
|
|
if not ctx.keystone:
|
|
|
|
raise ConfigError('rgw must run after the keystone task')
|
|
|
|
url = 'http://{host}:{port}/v1/KEY_$(tenant_id)s'.format(host=host,
|
|
|
|
port=port)
|
|
|
|
ctx.keystone.create_endpoint(ctx, keystone_role, 'swift', url)
|
|
|
|
|
|
|
|
keystone_host, keystone_port = \
|
|
|
|
ctx.keystone.public_endpoints[keystone_role]
|
|
|
|
rgw_cmd.extend([
|
|
|
|
'--rgw_keystone_url',
|
|
|
|
'http://{khost}:{kport}'.format(khost=keystone_host,
|
|
|
|
kport=keystone_port),
|
|
|
|
])
|
|
|
|
|
|
|
|
rgw_cmd.extend([
|
2013-07-22 17:38:28 +00:00
|
|
|
'--foreground',
|
|
|
|
run.Raw('|'),
|
|
|
|
'sudo',
|
|
|
|
'tee',
|
2016-12-16 18:23:09 +00:00
|
|
|
'/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(tdir=testdir,
|
|
|
|
client_with_cluster=client_with_cluster),
|
2013-07-22 17:38:28 +00:00
|
|
|
run.Raw('2>&1'),
|
2014-04-15 19:49:03 +00:00
|
|
|
])
|
2012-02-22 17:18:17 +00:00
|
|
|
|
2013-09-07 00:38:29 +00:00
|
|
|
if client_config.get('valgrind'):
|
2014-04-15 18:12:00 +00:00
|
|
|
cmd_prefix = teuthology.get_valgrind_args(
|
2013-09-07 00:38:29 +00:00
|
|
|
testdir,
|
2017-04-18 19:34:59 +00:00
|
|
|
client_with_cluster,
|
2014-04-15 18:12:00 +00:00
|
|
|
cmd_prefix,
|
2013-09-07 00:38:29 +00:00
|
|
|
client_config.get('valgrind')
|
|
|
|
)
|
2012-02-22 00:08:21 +00:00
|
|
|
|
2014-04-15 18:12:00 +00:00
|
|
|
run_cmd = list(cmd_prefix)
|
|
|
|
run_cmd.extend(rgw_cmd)
|
2012-02-22 00:08:21 +00:00
|
|
|
|
2013-04-30 23:37:48 +00:00
|
|
|
ctx.daemons.add_daemon(
|
2017-04-18 19:34:59 +00:00
|
|
|
remote, 'rgw', client_with_id,
|
2016-12-16 18:23:09 +00:00
|
|
|
cluster=cluster_name,
|
2012-02-22 00:08:21 +00:00
|
|
|
args=run_cmd,
|
2011-09-03 00:58:19 +00:00
|
|
|
logger=log.getChild(client),
|
|
|
|
stdin=run.PIPE,
|
|
|
|
wait=False,
|
|
|
|
)
|
|
|
|
|
2016-11-15 18:44:27 +00:00
|
|
|
# XXX: add_daemon() doesn't let us wait until radosgw finishes startup
|
2017-05-05 23:59:05 +00:00
|
|
|
for client in config.keys():
|
2016-11-15 18:44:27 +00:00
|
|
|
host, port = ctx.rgw.role_endpoints[client]
|
|
|
|
endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
|
|
|
|
log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint))
|
2017-04-17 19:39:37 +00:00
|
|
|
wait_for_radosgw(endpoint)
|
2016-11-15 18:44:27 +00:00
|
|
|
|
2011-09-03 00:58:19 +00:00
|
|
|
try:
|
|
|
|
yield
|
|
|
|
finally:
|
2013-04-30 23:37:48 +00:00
|
|
|
for client in config.iterkeys():
|
2017-04-22 15:33:44 +00:00
|
|
|
cluster_name, daemon_type, client_id = teuthology.split_role(client)
|
|
|
|
client_with_id = daemon_type + '.' + client_id
|
|
|
|
client_with_cluster = cluster_name + '.' + client_with_id
|
|
|
|
ctx.daemons.get_daemon('rgw', client_with_id, cluster_name).stop()
|
2012-11-20 00:19:06 +00:00
|
|
|
ctx.cluster.only(client).run(
|
|
|
|
args=[
|
|
|
|
'rm',
|
2013-04-30 23:37:48 +00:00
|
|
|
'-f',
|
2017-04-22 15:33:44 +00:00
|
|
|
'{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
|
|
|
|
client=client_with_cluster),
|
2013-04-30 23:37:48 +00:00
|
|
|
],
|
|
|
|
)
|
2011-09-03 00:58:19 +00:00
|
|
|
|
2013-07-22 17:38:28 +00:00
|
|
|
def assign_ports(ctx, config):
|
2014-02-15 01:08:28 +00:00
|
|
|
"""
|
|
|
|
Assign port numberst starting with port 7280.
|
|
|
|
"""
|
2013-07-22 17:38:28 +00:00
|
|
|
port = 7280
|
|
|
|
role_endpoints = {}
|
|
|
|
for remote, roles_for_host in ctx.cluster.remotes.iteritems():
|
|
|
|
for role in roles_for_host:
|
|
|
|
if role in config:
|
|
|
|
role_endpoints[role] = (remote.name.split('@')[1], port)
|
|
|
|
port += 1
|
|
|
|
|
|
|
|
return role_endpoints
|
|
|
|
|
2014-03-25 01:19:41 +00:00
|
|
|
@contextlib.contextmanager
|
2017-06-26 08:42:08 +00:00
|
|
|
def create_pools(ctx, clients):
|
2014-03-25 01:19:41 +00:00
|
|
|
"""Create replicated or erasure coded data pools for rgw."""
|
|
|
|
|
2017-04-26 01:38:52 +00:00
|
|
|
log.info('Creating data pools')
|
2017-06-26 08:42:08 +00:00
|
|
|
for client in clients:
|
|
|
|
log.debug("Obtaining remote for client {}".format(client))
|
2014-03-27 16:35:28 +00:00
|
|
|
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
|
2014-03-25 01:19:41 +00:00
|
|
|
data_pool = '.rgw.buckets'
|
2016-12-16 18:23:09 +00:00
|
|
|
cluster_name, daemon_type, client_id = teuthology.split_role(client)
|
|
|
|
|
2014-03-25 01:19:41 +00:00
|
|
|
if ctx.rgw.ec_data_pool:
|
2014-08-21 23:27:17 +00:00
|
|
|
create_ec_pool(remote, data_pool, client, 64,
|
2017-08-22 17:56:11 +00:00
|
|
|
ctx.rgw.erasure_code_profile, cluster_name, 'rgw')
|
2014-03-25 01:19:41 +00:00
|
|
|
else:
|
2017-08-22 17:56:11 +00:00
|
|
|
create_replicated_pool(remote, data_pool, 64, cluster_name, 'rgw')
|
2014-05-01 21:36:08 +00:00
|
|
|
if ctx.rgw.cache_pools:
|
|
|
|
create_cache_pool(remote, data_pool, data_pool + '.cache', 64,
|
2017-08-22 17:56:11 +00:00
|
|
|
64*1024*1024, cluster_name, 'rgw')
|
2017-04-26 01:38:52 +00:00
|
|
|
log.debug('Pools created')
|
2014-03-25 01:19:41 +00:00
|
|
|
yield
|
|
|
|
|
2013-07-11 00:45:35 +00:00
|
|
|
@contextlib.contextmanager
|
2017-06-26 08:42:08 +00:00
|
|
|
def configure_compression(ctx, clients, compression):
|
2017-04-26 01:38:52 +00:00
|
|
|
""" set a compression type in the default zone placement """
|
|
|
|
log.info('Configuring compression type = %s', compression)
|
2017-06-26 08:42:08 +00:00
|
|
|
for client in clients:
|
2016-12-02 19:48:44 +00:00
|
|
|
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
|
|
|
|
# issue a 'radosgw-admin user list' command to trigger this
|
|
|
|
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
|
|
|
|
|
|
|
|
rgwadmin(ctx, client,
|
|
|
|
cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
|
2017-04-18 19:34:59 +00:00
|
|
|
'--placement-id', 'default-placement',
|
|
|
|
'--compression', compression],
|
2016-12-02 19:48:44 +00:00
|
|
|
check_status=True)
|
2013-07-22 17:38:28 +00:00
|
|
|
yield
|
2011-06-22 23:36:58 +00:00
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
def task(ctx, config):
|
|
|
|
"""
|
|
|
|
For example, to run rgw on all clients::
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
- ceph:
|
|
|
|
- rgw:
|
|
|
|
|
|
|
|
To only run on certain clients::
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
- ceph:
|
|
|
|
- rgw: [client.0, client.3]
|
2012-02-22 00:08:21 +00:00
|
|
|
|
|
|
|
or
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
- ceph:
|
|
|
|
- rgw:
|
|
|
|
client.0:
|
|
|
|
client.3:
|
|
|
|
|
|
|
|
To run radosgw through valgrind:
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
- ceph:
|
|
|
|
- rgw:
|
|
|
|
client.0:
|
|
|
|
valgrind: [--tool=memcheck]
|
|
|
|
client.3:
|
|
|
|
valgrind: [--tool=memcheck]
|
2011-06-22 23:36:58 +00:00
|
|
|
"""
|
|
|
|
if config is None:
|
2012-02-21 23:47:32 +00:00
|
|
|
config = dict(('client.{id}'.format(id=id_), None)
|
2014-04-15 15:50:07 +00:00
|
|
|
for id_ in teuthology.all_roles_of_type(
|
|
|
|
ctx.cluster, 'client'))
|
2012-02-21 23:47:32 +00:00
|
|
|
elif isinstance(config, list):
|
|
|
|
config = dict((name, None) for name in config)
|
2011-06-22 23:36:58 +00:00
|
|
|
|
2017-06-26 08:42:08 +00:00
|
|
|
clients = config.keys() # http://tracker.ceph.com/issues/20417
|
|
|
|
|
2014-03-26 01:04:35 +00:00
|
|
|
overrides = ctx.config.get('overrides', {})
|
|
|
|
teuthology.deep_merge(config, overrides.get('rgw', {}))
|
|
|
|
|
2013-07-22 17:38:28 +00:00
|
|
|
role_endpoints = assign_ports(ctx, config)
|
2013-08-17 17:14:37 +00:00
|
|
|
ctx.rgw = argparse.Namespace()
|
|
|
|
ctx.rgw.role_endpoints = role_endpoints
|
2015-11-23 16:04:12 +00:00
|
|
|
|
2017-04-18 19:34:59 +00:00
|
|
|
ctx.rgw.ec_data_pool = bool(config.pop('ec-data-pool', False))
|
|
|
|
ctx.rgw.erasure_code_profile = config.pop('erasure_code_profile', {})
|
|
|
|
ctx.rgw.cache_pools = bool(config.pop('cache-pools', False))
|
|
|
|
ctx.rgw.frontend = config.pop('frontend', 'civetweb')
|
|
|
|
ctx.rgw.compression_type = config.pop('compression type', None)
|
2016-12-16 18:23:09 +00:00
|
|
|
ctx.rgw.config = config
|
2016-01-19 14:56:18 +00:00
|
|
|
|
2017-06-26 08:42:08 +00:00
|
|
|
log.debug("config is {}".format(config))
|
|
|
|
log.debug("client list is {}".format(clients))
|
2017-05-19 20:05:36 +00:00
|
|
|
subtasks = [
|
2017-06-26 08:42:08 +00:00
|
|
|
lambda: create_pools(ctx=ctx, clients=clients),
|
2017-05-19 20:05:36 +00:00
|
|
|
]
|
2017-05-05 23:59:05 +00:00
|
|
|
if ctx.rgw.compression_type:
|
|
|
|
subtasks.extend([
|
2017-06-26 08:42:08 +00:00
|
|
|
lambda: configure_compression(ctx=ctx, clients=clients,
|
2017-05-05 23:59:05 +00:00
|
|
|
compression=ctx.rgw.compression_type),
|
|
|
|
])
|
|
|
|
subtasks.extend([
|
2017-06-26 08:42:08 +00:00
|
|
|
lambda: start_rgw(ctx=ctx, config=config, clients=clients),
|
2017-04-26 01:38:52 +00:00
|
|
|
])
|
2014-04-16 14:35:04 +00:00
|
|
|
|
|
|
|
with contextutil.nested(*subtasks):
|
|
|
|
yield
|