mirror of
https://github.com/ceph/ceph
synced 2024-12-25 04:43:17 +00:00
2678db532a
This script uses the python bindings to libcephfs and rados to create files and check the correctness of the backtrace written to the 'parent' xattr on the first object (if its a file) or inode (if its a dir). The script includes test cases that kill the mds at specific kill points and restart it through teuthology using the teuthology restart task. Signed-off-by: Sam Lang <sam.lang@inktank.com>
253 lines
6.8 KiB
Python
Executable File
253 lines
6.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import subprocess as sub
|
|
from cStringIO import StringIO
|
|
import json
|
|
import os
|
|
import time
|
|
import sys
|
|
import thread
|
|
|
|
import rados as rados
|
|
import cephfs as cephfs
|
|
|
|
prefix='testbt'
|
|
|
|
def get_name(b, i, j):
|
|
c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j)
|
|
return c, b + '/' + c
|
|
|
|
def mkdir(ceph, d):
|
|
print >>sys.stderr, "mkdir {d}".format(d=d)
|
|
ceph.mkdir(d, 0755)
|
|
return ceph.stat(d)['st_ino']
|
|
|
|
def create(ceph, f):
|
|
print >>sys.stderr, "creating {f}".format(f=f)
|
|
fd = ceph.open(f, os.O_CREAT|os.O_RDWR, 0644)
|
|
ceph.close(fd)
|
|
return ceph.stat(f)['st_ino']
|
|
|
|
def set_mds_config_param(ceph, param):
|
|
with file('/dev/null', 'rb') as devnull:
|
|
confarg = ''
|
|
if conf != '':
|
|
confarg = '-c {c}'.format(c=conf)
|
|
r = sub.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull)
|
|
if (r != 0):
|
|
raise
|
|
|
|
import ConfigParser
|
|
import contextlib
|
|
|
|
class _TrimIndentFile(object):
|
|
def __init__(self, fp):
|
|
self.fp = fp
|
|
|
|
def readline(self):
|
|
line = self.fp.readline()
|
|
return line.lstrip(' \t')
|
|
|
|
def _optionxform(s):
|
|
s = s.replace('_', ' ')
|
|
s = '_'.join(s.split())
|
|
return s
|
|
|
|
def conf_set_kill_mds(location, killnum):
|
|
print >>sys.stderr, 'setting mds kill config option for {l}.{k}'.format(l=location, k=killnum)
|
|
print "restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum)
|
|
sys.stdout.flush()
|
|
for l in sys.stdin.readline():
|
|
if l == 'restarted':
|
|
break
|
|
|
|
def flush(ceph, testnum):
|
|
print >>sys.stderr, 'flushing {t}'.format(t=testnum)
|
|
set_mds_config_param(ceph, '--mds_log_max_segments 1')
|
|
|
|
for i in range(1, 500):
|
|
f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i)
|
|
print >>sys.stderr, 'flushing with create {f}'.format(f=f)
|
|
fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0644)
|
|
ceph.close(fd)
|
|
ceph.unlink(f)
|
|
|
|
print >> sys.stderr, 'flush doing shutdown'
|
|
ceph.shutdown()
|
|
print >> sys.stderr, 'flush reinitializing ceph'
|
|
ceph = cephfs.LibCephFS(conffile=conf)
|
|
print >> sys.stderr, 'flush doing mount'
|
|
ceph.mount()
|
|
return ceph
|
|
|
|
def kill_mds(ceph, location, killnum):
|
|
print >>sys.stderr, 'killing mds: {l}.{k}'.format(l=location, k=killnum)
|
|
set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum))
|
|
|
|
def wait_for_mds(ceph):
|
|
# wait for restart
|
|
while True:
|
|
confarg = ''
|
|
if conf != '':
|
|
confarg = '-c {c}'.format(c=conf)
|
|
r = sub.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True)
|
|
if r.find('a=up:active'):
|
|
break
|
|
time.sleep(1)
|
|
|
|
def decode(value):
|
|
|
|
tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid())
|
|
with open(tmpfile, 'w+') as f:
|
|
f.write(value)
|
|
|
|
p = sub.Popen(
|
|
[
|
|
'ceph-dencoder',
|
|
'import',
|
|
tmpfile,
|
|
'type',
|
|
'inode_backtrace_t',
|
|
'decode',
|
|
'dump_json',
|
|
],
|
|
stdin=sub.PIPE,
|
|
stdout=sub.PIPE,
|
|
)
|
|
(stdout, _) = p.communicate(input=value)
|
|
p.stdin.close()
|
|
if (p.returncode != 0):
|
|
raise
|
|
os.remove(tmpfile)
|
|
return json.loads(stdout)
|
|
|
|
class VerifyFailure(Exception):
|
|
pass
|
|
|
|
def verify(rados_ioctx, ino, values, pool):
|
|
print >>sys.stderr, 'getting parent attr for ino: %lx.00000000' % ino
|
|
savede = None
|
|
for i in range(1, 20):
|
|
try:
|
|
savede = None
|
|
binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent')
|
|
except rados.ObjectNotFound as e:
|
|
# wait for a bit to let segments get flushed out
|
|
savede = e
|
|
time.sleep(10)
|
|
if savede:
|
|
raise savede
|
|
|
|
bt = decode(binbt)
|
|
|
|
if bt['ino'] != ino:
|
|
raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
|
|
bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values))
|
|
ind = 0
|
|
for (n, i) in values:
|
|
if bt['ancestors'][ind]['dirino'] != i:
|
|
raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
|
|
b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values))
|
|
if bt['ancestors'][ind]['dname'] != n:
|
|
raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
|
|
b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values))
|
|
ind += 1
|
|
|
|
if bt['pool'] != pool:
|
|
raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
|
|
btp=bt['pool'], p=pool, bt=bt, i=ino, v=values))
|
|
|
|
def make_abc(ceph, rooti, i):
|
|
expected_bt = []
|
|
c, d = get_name("/", i, 0)
|
|
expected_bt = [(c, rooti)] + expected_bt
|
|
di = mkdir(ceph, d)
|
|
c, d = get_name(d, i, 1)
|
|
expected_bt = [(c, di)] + expected_bt
|
|
di = mkdir(ceph, d)
|
|
c, f = get_name(d, i, 2)
|
|
fi = create(ceph, f)
|
|
expected_bt = [(c, di)] + expected_bt
|
|
return fi, expected_bt
|
|
|
|
test = -1
|
|
if len(sys.argv) > 1:
|
|
test = int(sys.argv[1])
|
|
|
|
conf = ''
|
|
if len(sys.argv) > 2:
|
|
conf = sys.argv[2]
|
|
|
|
radosobj = rados.Rados(conffile=conf)
|
|
radosobj.connect()
|
|
ioctx = radosobj.open_ioctx('data')
|
|
|
|
ceph = cephfs.LibCephFS(conffile=conf)
|
|
ceph.mount()
|
|
|
|
rooti = ceph.stat('/')['st_ino']
|
|
|
|
test = -1
|
|
if len(sys.argv) > 1:
|
|
test = int(sys.argv[1])
|
|
|
|
conf = '/etc/ceph/ceph.conf'
|
|
if len(sys.argv) > 2:
|
|
conf = sys.argv[2]
|
|
|
|
# create /a/b/c
|
|
# flush
|
|
# verify
|
|
|
|
i = 0
|
|
if test < 0 or test == i:
|
|
print >>sys.stderr, 'Running test %d: basic verify' % i
|
|
ino, expected_bt = make_abc(ceph, rooti, i)
|
|
ceph = flush(ceph, i)
|
|
verify(ioctx, ino, expected_bt, 0)
|
|
|
|
i += 1
|
|
|
|
# kill-mds-at-openc-1
|
|
# create /a/b/c
|
|
# restart-mds
|
|
# flush
|
|
# verify
|
|
|
|
if test < 0 or test == i:
|
|
print >>sys.stderr, 'Running test %d: kill openc' % i
|
|
print "restart mds a"
|
|
sys.stdout.flush()
|
|
kill_mds(ceph, 'openc', 1)
|
|
ino, expected_bt = make_abc(ceph, rooti, i)
|
|
ceph = flush(ceph, i)
|
|
verify(ioctx, ino, expected_bt, 0)
|
|
|
|
i += 1
|
|
|
|
# kill-mds-at-openc-1
|
|
# create /a/b/c
|
|
# restart-mds with kill-mds-at-replay-1
|
|
# restart-mds
|
|
# flush
|
|
# verify
|
|
if test < 0 or test == i:
|
|
print >>sys.stderr, 'Running test %d: kill openc/replay' % i
|
|
# these are reversed because we want to prepare the config
|
|
conf_set_kill_mds('journal_replay', 1)
|
|
kill_mds(ceph, 'openc', 1)
|
|
print "restart mds a"
|
|
sys.stdout.flush()
|
|
ino, expected_bt = make_abc(ceph, rooti, i)
|
|
ceph = flush(ceph, i)
|
|
verify(ioctx, ino, expected_bt, 0)
|
|
|
|
i += 1
|
|
|
|
ioctx.close()
|
|
radosobj.shutdown()
|
|
ceph.shutdown()
|
|
|
|
print "done"
|
|
sys.stdout.flush()
|