mirror of
https://github.com/ceph/ceph
synced 2025-03-04 23:40:07 +00:00
rountrip: add task
This commit is contained in:
Yehuda Sadeh
parent
97cc6c2990
commit
7eec30946d
251
teuthology/task/s3roundtrip.py
Normal file
251
teuthology/task/s3roundtrip.py
Normal file
@ -0,0 +1,251 @@
|
||||
from cStringIO import StringIO
|
||||
import base64
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import yaml
|
||||
|
||||
from teuthology import misc as teuthology
|
||||
from teuthology import contextutil
|
||||
from ..orchestra import run
|
||||
from ..orchestra.connection import split_user
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def download(ctx, config):
|
||||
assert isinstance(config, list)
|
||||
log.info('Downloading s3-tests...')
|
||||
for client in config:
|
||||
ctx.cluster.only(client).run(
|
||||
args=[
|
||||
'git', 'clone',
|
||||
'https://github.com/NewDreamNetwork/s3-tests.git',
|
||||
'/tmp/cephtest/s3-tests',
|
||||
],
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
log.info('Removing s3-tests...')
|
||||
for client in config:
|
||||
ctx.cluster.only(client).run(
|
||||
args=[
|
||||
'rm',
|
||||
'-rf',
|
||||
'/tmp/cephtest/s3-tests',
|
||||
],
|
||||
)
|
||||
|
||||
def _config_user(s3tests_conf, section, user):
|
||||
s3tests_conf[section].setdefault('user_id', user)
|
||||
s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
|
||||
s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
|
||||
s3tests_conf[section].setdefault('access_key', base64.b64encode(os.urandom(20)))
|
||||
s3tests_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40)))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def create_users(ctx, config):
|
||||
assert isinstance(config, dict)
|
||||
log.info('Creating rgw users...')
|
||||
for client in config['clients']:
|
||||
s3tests_conf = config['s3tests_conf'][client]
|
||||
s3tests_conf.setdefault('roundtrip', {})
|
||||
s3tests_conf['roundtrip'].setdefault('bucket', 'rttest-' + client + '-{random}-')
|
||||
s3tests_conf['roundtrip'].setdefault('readers', 10)
|
||||
s3tests_conf['roundtrip'].setdefault('writers', 3)
|
||||
s3tests_conf['roundtrip'].setdefault('duration', 300)
|
||||
s3tests_conf['roundtrip'].setdefault('files', {})
|
||||
rtconf = s3tests_conf['roundtrip']
|
||||
rtconf['files'].setdefault('num', 10)
|
||||
rtconf['files'].setdefault('size', 2000)
|
||||
rtconf['files'].setdefault('stddev', 500)
|
||||
for section, user in [('s3', 'foo')]:
|
||||
_config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
|
||||
ctx.cluster.only(client).run(
|
||||
args=[
|
||||
'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib',
|
||||
'/tmp/cephtest/enable-coredump',
|
||||
'/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
|
||||
'/tmp/cephtest/archive/coverage',
|
||||
'/tmp/cephtest/binary/usr/local/bin/radosgw-admin',
|
||||
'-c', '/tmp/cephtest/ceph.conf',
|
||||
'user', 'create',
|
||||
'--uid', s3tests_conf[section]['user_id'],
|
||||
'--display-name', s3tests_conf[section]['display_name'],
|
||||
'--access-key', s3tests_conf[section]['access_key'],
|
||||
'--secret', s3tests_conf[section]['secret_key'],
|
||||
'--email', s3tests_conf[section]['email'],
|
||||
],
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def configure(ctx, config):
|
||||
assert isinstance(config, dict)
|
||||
log.info('Configuring s3-roundtrip-tests...')
|
||||
for client, properties in config['clients'].iteritems():
|
||||
s3tests_conf = config['s3tests_conf'][client]
|
||||
if properties is not None and 'rgw_server' in properties:
|
||||
host = None
|
||||
for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
|
||||
log.info('roles: ' + str(roles))
|
||||
log.info('target: ' + str(target))
|
||||
if properties['rgw_server'] in roles:
|
||||
_, host = split_user(target)
|
||||
assert host is not None, "Invalid client specified as the rgw_server"
|
||||
s3tests_conf['s3']['host'] = host
|
||||
else:
|
||||
s3tests_conf['s3']['host'] = 'localhost'
|
||||
|
||||
def_conf = s3tests_conf['DEFAULT']
|
||||
s3tests_conf['s3'].setdefault('port', def_conf['port'])
|
||||
s3tests_conf['s3'].setdefault('is_secure', def_conf['is_secure'])
|
||||
|
||||
(remote,) = ctx.cluster.only(client).remotes.keys()
|
||||
remote.run(
|
||||
args=[
|
||||
'cd',
|
||||
'/tmp/cephtest/s3-tests',
|
||||
run.Raw('&&'),
|
||||
'./bootstrap',
|
||||
],
|
||||
)
|
||||
conf_fp = StringIO()
|
||||
conf = dict(
|
||||
s3=s3tests_conf['s3'],
|
||||
roundtrip=s3tests_conf['roundtrip'],
|
||||
)
|
||||
yaml.dump(conf, conf_fp, default_flow_style=False)
|
||||
teuthology.write_file(
|
||||
remote=remote,
|
||||
path='/tmp/cephtest/archive/s3roundtrip.{client}.config.yaml'.format(client=client),
|
||||
data=conf_fp.getvalue(),
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def run_tests(ctx, config):
|
||||
assert isinstance(config, dict)
|
||||
for client, client_config in config.iteritems():
|
||||
(remote,) = ctx.cluster.only(client).remotes.keys()
|
||||
conf = teuthology.get_file(remote, '/tmp/cephtest/archive/s3roundtrip.{client}.config.yaml'.format(client=client))
|
||||
args = [
|
||||
'/tmp/cephtest/s3-tests/virtualenv/bin/s3tests-test-roundtrip',
|
||||
]
|
||||
if client_config is not None and 'extra_args' in client_config:
|
||||
args.extend(client_config['extra_args'])
|
||||
|
||||
ctx.cluster.only(client).run(
|
||||
args=args,
|
||||
stdin=conf,
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def task(ctx, config):
|
||||
"""
|
||||
Run the s3tests-test-roundtrip suite against rgw.
|
||||
|
||||
To run all tests on all clients::
|
||||
|
||||
tasks:
|
||||
- ceph:
|
||||
- rgw:
|
||||
- s3roundtrip:
|
||||
|
||||
To restrict testing to particular clients::
|
||||
|
||||
tasks:
|
||||
- ceph:
|
||||
- rgw: [client.0]
|
||||
- s3roundtrip: [client.0]
|
||||
|
||||
To run against a server on client.1::
|
||||
|
||||
tasks:
|
||||
- ceph:
|
||||
- rgw: [client.1]
|
||||
- s3roundtrip:
|
||||
client.0:
|
||||
rgw_server: client.1
|
||||
|
||||
To pass extra test arguments
|
||||
|
||||
tasks:
|
||||
- ceph:
|
||||
- rgw: [client.0]
|
||||
- s3roundtrip:
|
||||
client.0:
|
||||
roundtrip:
|
||||
bucket: mybucket
|
||||
readers: 10
|
||||
writers: 3
|
||||
duration: 600
|
||||
files:
|
||||
num: 10
|
||||
size: 2000
|
||||
stddev: 500
|
||||
client.1:
|
||||
...
|
||||
|
||||
To override s3 configuration
|
||||
|
||||
tasks:
|
||||
- ceph:
|
||||
- rgw: [client.0]
|
||||
- s3roundtrip:
|
||||
client.0:
|
||||
s3:
|
||||
user_id: myuserid
|
||||
display_name: myname
|
||||
email: my@email
|
||||
access_key: myaccesskey
|
||||
secret_key: mysecretkey
|
||||
|
||||
"""
|
||||
assert config is None or isinstance(config, list) \
|
||||
or isinstance(config, dict), \
|
||||
"task s3tests only supports a list or dictionary for configuration"
|
||||
all_clients = ['client.{id}'.format(id=id_)
|
||||
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
|
||||
if config is None:
|
||||
config = all_clients
|
||||
if isinstance(config, list):
|
||||
config = dict.fromkeys(config)
|
||||
clients = config.keys()
|
||||
|
||||
s3tests_conf = {}
|
||||
for client in clients:
|
||||
if config[client] is None:
|
||||
config[client] = {}
|
||||
config[client].setdefault('s3', {})
|
||||
config[client].setdefault('roundtrip', {})
|
||||
|
||||
s3tests_conf[client] = ({
|
||||
'DEFAULT':
|
||||
{
|
||||
'port' : 7280,
|
||||
'is_secure' : False,
|
||||
},
|
||||
'roundtrip' : config[client]['roundtrip'],
|
||||
's3' : config[client]['s3'],
|
||||
})
|
||||
|
||||
with contextutil.nested(
|
||||
lambda: download(ctx=ctx, config=clients),
|
||||
lambda: create_users(ctx=ctx, config=dict(
|
||||
clients=clients,
|
||||
s3tests_conf=s3tests_conf,
|
||||
)),
|
||||
lambda: configure(ctx=ctx, config=dict(
|
||||
clients=config,
|
||||
s3tests_conf=s3tests_conf,
|
||||
)),
|
||||
lambda: run_tests(ctx=ctx, config=config),
|
||||
):
|
||||
yield
|
||||
Loading…
Reference in New Issue
Block a user