"""
Run a series of rgw admin commands through the rest interface.
The test cases in this file have been annotated for inventory.
To extract the inventory (in csv format) use the command:
grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
"""
from cStringIO import StringIO
import logging
import json
import boto.exception
import boto.s3.connection
import boto.s3.acl
import requests
import time
from boto.connection import AWSAuthConnection
from teuthology import misc as teuthology
from util.rgw import get_user_summary, get_user_successful_ops
log = logging.getLogger(__name__)
def rgwadmin(ctx, client, cmd):
"""
Perform rgw admin command
:param client: client
:param cmd: command to execute.
:return: command exit status, json result.
"""
log.info('radosgw-admin: %s' % cmd)
testdir = teuthology.get_testdir(ctx)
pre = [
'adjust-ulimits',
'ceph-coverage',
'{tdir}/archive/coverage'.format(tdir=testdir),
'radosgw-admin',
'--log-to-stderr',
'--format', 'json',
]
pre.extend(cmd)
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
proc = remote.run(
args=pre,
check_status=False,
stdout=StringIO(),
stderr=StringIO(),
)
r = proc.exitstatus
out = proc.stdout.getvalue()
j = None
if not r and out != '':
try:
j = json.loads(out)
log.info(' json result: %s' % j)
except ValueError:
j = out
log.info(' raw result: %s' % j)
return (r, j)
def rgwadmin_rest(connection, cmd, params=None, headers=None, raw=False):
"""
perform a rest command
"""
log.info('radosgw-admin-rest: %s %s' % (cmd, params))
put_cmds = ['create', 'link', 'add']
post_cmds = ['unlink', 'modify']
delete_cmds = ['trim', 'rm', 'process']
get_cmds = ['check', 'info', 'show', 'list']
bucket_sub_resources = ['object', 'policy', 'index']
user_sub_resources = ['subuser', 'key', 'caps']
zone_sub_resources = ['pool', 'log', 'garbage']
def get_cmd_method_and_handler(cmd):
"""
Get the rest command and handler from information in cmd and
from the imported requests object.
"""
if cmd[1] in put_cmds:
return 'PUT', requests.put
elif cmd[1] in delete_cmds:
return 'DELETE', requests.delete
elif cmd[1] in post_cmds:
return 'POST', requests.post
elif cmd[1] in get_cmds:
return 'GET', requests.get
def get_resource(cmd):
"""
Get the name of the resource from information in cmd.
"""
if cmd[0] == 'bucket' or cmd[0] in bucket_sub_resources:
if cmd[0] == 'bucket':
return 'bucket', ''
else:
return 'bucket', cmd[0]
elif cmd[0] == 'user' or cmd[0] in user_sub_resources:
if cmd[0] == 'user':
return 'user', ''
else:
return 'user', cmd[0]
elif cmd[0] == 'usage':
return 'usage', ''
elif cmd[0] == 'zone' or cmd[0] in zone_sub_resources:
if cmd[0] == 'zone':
return 'zone', ''
else:
return 'zone', cmd[0]
def build_admin_request(conn, method, resource = '', headers=None, data='',
query_args=None, params=None):
"""
Build an administative request adapted from the build_request()
method of boto.connection
"""
path = conn.calling_format.build_path_base('admin', resource)
auth_path = conn.calling_format.build_auth_path('admin', resource)
host = conn.calling_format.build_host(conn.server_name(), 'admin')
if query_args:
path += '?' + query_args
boto.log.debug('path=%s' % path)
auth_path += '?' + query_args
boto.log.debug('auth_path=%s' % auth_path)
return AWSAuthConnection.build_base_http_request(conn, method, path,
auth_path, params, headers, data, host)
method, handler = get_cmd_method_and_handler(cmd)
resource, query_args = get_resource(cmd)
request = build_admin_request(connection, method, resource,
query_args=query_args, headers=headers)
url = '{protocol}://{host}{path}'.format(protocol=request.protocol,
host=request.host, path=request.path)
request.authorize(connection=connection)
result = handler(url, params=params, headers=request.headers)
if raw:
log.info(' text result: %s' % result.txt)
return result.status_code, result.txt
else:
log.info(' json result: %s' % result.json())
return result.status_code, result.json()
def task(ctx, config):
"""
Test radosgw-admin functionality through the RESTful interface
"""
assert config is None or isinstance(config, list) \
or isinstance(config, dict), \
"task s3tests only supports a list or dictionary for configuration"
all_clients = ['client.{id}'.format(id=id_)
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
# just use the first client...
client = clients[0]
##
admin_user = 'ada'
admin_display_name = 'Ms. Admin User'
admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
user1 = 'foo'
user2 = 'fud'
subuser1 = 'foo:foo1'
subuser2 = 'foo:foo2'
display_name1 = 'Foo'
display_name2 = 'Fud'
email = 'foo@foo.com'
access_key = '9te6NH5mcdcq0Tc5i8i1'
secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2 = 'p5YnriCv1nAtykxBrupQ'
secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name = 'myfoo'
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', admin_user,
'--display-name', admin_display_name,
'--access-key', admin_access_key,
'--secret', admin_secret_key,
'--max-buckets', '0',
'--caps', admin_caps
])
logging.error(out)
logging.error(err)
assert not err
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
remote_host = remote.name.split('@')[1]
admin_conn = boto.s3.connection.S3Connection(
aws_access_key_id=admin_access_key,
aws_secret_access_key=admin_secret_key,
is_secure=False,
port=7280,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
assert ret == 404
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['user', 'create'],
{'uid' : user1,
'display-name' : display_name1,
'email' : email,
'access-key' : access_key,
'secret-key' : secret_key,
'max-buckets' : '4'
})
assert ret == 200
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
assert ret == 200
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert out['suspended']
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
assert not err
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'create'],
{'uid' : user1,
'access-key' : access_key2,
'secret-key' : secret_key2
})
assert ret == 200
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'rm'],
{'uid' : user1,
'access-key' : access_key2
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'create'],
{'subuser' : subuser1,
'secret-key' : swift_secret1,
'key-type' : 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'create'],
{'subuser' : subuser2,
'secret-key' : swift_secret2,
'key-type' : 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'rm'],
{'subuser' : subuser1,
'key-type' :'swift'
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'rm'],
{'subuser' : subuser1
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'rm'],
{'subuser' : subuser2,
'key-type' : 'swift',
'{purge-keys' :True
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
assert ret == 200
assert len(out) == 0
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=7280,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
assert ret == 200
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
assert ret == 200
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn,
['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['owner'] == user1
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
assert ret == 200
assert len(out) == 1
assert out[0]['id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name})
assert ret == 200
# create a second user to link the bucket to
(ret, out) = rgwadmin_rest(admin_conn,
['user', 'create'],
{'uid' : user2,
'display-name' : display_name2,
'access-key' : access_key2,
'secret-key' : secret_key2,
'max-buckets' : '1',
})
assert ret == 200
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user2, 'bucket' : bucket_name})
assert ret == 200
# try creating an object with the first user which should cause an error
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('three')
except boto.exception.S3ResponseError:
denied = True
assert denied
# relink the bucket to the first user and delete the second user
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user1, 'bucket' : bucket_name})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2})
assert ret == 200
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
assert ret == 200
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# create a bucket for deletion stats
useless_bucket = connection.create_bucket('useless_bucket')
useless_key = useless_bucket.new_key('useless_key')
useless_key.set_contents_from_string('useless string')
# delete it
useless_key.delete()
useless_bucket.delete()
# wait for the statistics to flush
time.sleep(60)
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'}) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
for cat in test_categories:
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat})
assert ret == 200
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
assert ret == 200
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
assert ret == 200
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
assert ret == 200
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo'*10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
assert ret == 409
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
assert ret == 200
acl = key.get_xml_acl()
assert acl == out.strip('\n')
# add another grantee by making the object public read
key.set_acl('public-read')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
assert ret == 200
acl = key.get_xml_acl()
assert acl == out.strip('\n')
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True})
assert ret == 200
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = 'usage=read'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' : user1, 'user-caps' : caps})
assert ret == 200
assert out[0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' : user1, 'user-caps' : caps})
assert ret == 200
assert not out
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
assert ret == 409
# TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True})
assert ret == 200
# TESTCASE 'rm-user3','user','info','deleted user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 404