mirror of
https://github.com/ceph/ceph
synced 2024-12-23 20:03:56 +00:00
149 lines
4.6 KiB
Python
149 lines
4.6 KiB
Python
|
|
"""
|
|
Exercise the MDS's auto repair functions
|
|
"""
|
|
|
|
import contextlib
|
|
import logging
|
|
import time
|
|
|
|
from teuthology.orchestra.run import CommandFailedError
|
|
|
|
from tasks.cephfs.filesystem import Filesystem
|
|
from tasks.cephfs.fuse_mount import FuseMount
|
|
from tasks.cephfs.cephfs_test_case import CephFSTestCase, run_tests
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# Arbitrary timeouts for operations involving restarting
|
|
# an MDS or waiting for it to come up
|
|
MDS_RESTART_GRACE = 60
|
|
|
|
class TestMDSAutoRepair(CephFSTestCase):
|
|
# Environment references
|
|
mount_a = None
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(TestMDSAutoRepair, self).__init__(*args, **kwargs)
|
|
|
|
self.configs_set = set()
|
|
|
|
def set_conf(self, subsys, key, value):
|
|
self.configs_set.add((subsys, key))
|
|
self.fs.set_ceph_conf(subsys, key, value)
|
|
|
|
def setUp(self):
|
|
self.fs.mds_restart()
|
|
self.fs.wait_for_daemons()
|
|
self.mount_a.mount()
|
|
self.mount_a.wait_until_mounted()
|
|
|
|
def tearDown(self):
|
|
self.fs.clear_firewall()
|
|
self.mount_a.teardown()
|
|
|
|
for subsys, key in self.configs_set:
|
|
self.fs.clear_ceph_conf(subsys, key)
|
|
|
|
def test_backtrace_repair(self):
|
|
"""
|
|
MDS should verify/fix backtrace on fetch dirfrag
|
|
"""
|
|
|
|
# trim log segment as fast as possible
|
|
self.fs.set_ceph_conf('mds', 'mds cache size', 100)
|
|
self.fs.set_ceph_conf('mds', 'mds log max segments', 2)
|
|
self.fs.set_ceph_conf('mds', 'mds log events per segment', 1)
|
|
self.fs.set_ceph_conf('mds', 'mds verify backtrace', 1)
|
|
self.fs.mds_restart()
|
|
self.fs.wait_for_daemons()
|
|
|
|
create_script = "mkdir {0}; for i in `seq 0 500`; do touch {0}/file$i; done"
|
|
# create main test directory
|
|
self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir1")])
|
|
|
|
# create more files in another directory. make sure MDS trim dentries in testdir1
|
|
self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir2")])
|
|
|
|
# flush journal entries to dirfrag objects
|
|
self.fs.mds_asok(['flush', 'journal'])
|
|
|
|
# drop inodes caps
|
|
self.mount_a.umount_wait()
|
|
self.mount_a.mount()
|
|
self.mount_a.wait_until_mounted()
|
|
|
|
# wait MDS to trim dentries in testdir1. 60 seconds should be long enough.
|
|
time.sleep(60)
|
|
|
|
# remove testdir1's backtrace
|
|
proc = self.mount_a.run_shell(["sudo", "ls", "-id", "testdir1"])
|
|
self.assertEqual(proc.exitstatus, 0)
|
|
objname = "{:x}.00000000".format(long(proc.stdout.getvalue().split()[0]))
|
|
proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "rmxattr", objname, "parent"])
|
|
self.assertEqual(proc.exitstatus, 0);
|
|
|
|
# readdir (fetch dirfrag) should fix testdir1's backtrace
|
|
self.mount_a.run_shell(["sudo", "ls", "testdir1"])
|
|
|
|
# add more entries to journal
|
|
self.mount_a.run_shell(["sudo", "rm", "-rf", " testdir2"])
|
|
|
|
# flush journal entries to dirfrag objects
|
|
self.fs.mds_asok(['flush', 'journal'])
|
|
|
|
# check if backtrace exists
|
|
proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "getxattr", objname, "parent"])
|
|
self.assertEqual(proc.exitstatus, 0)
|
|
|
|
def test_mds_readonly(self):
|
|
"""
|
|
test if MDS behave correct when it's readonly
|
|
"""
|
|
# operation should successd when MDS is not readonly
|
|
self.mount_a.run_shell(["sudo", "touch", "test_file1"])
|
|
writer = self.mount_a.write_background(loop=True)
|
|
|
|
time.sleep(10)
|
|
self.assertFalse(writer.finished)
|
|
|
|
# force MDS to read-only mode
|
|
self.fs.mds_asok(['force_readonly'])
|
|
time.sleep(10)
|
|
|
|
# touching test file should fail
|
|
try:
|
|
self.mount_a.run_shell(["sudo", "touch", "test_file1"])
|
|
except CommandFailedError, e:
|
|
pass
|
|
else:
|
|
self.assertTrue(False)
|
|
|
|
# background writer also should fail
|
|
self.assertTrue(writer.finished)
|
|
|
|
# restart mds to make it writable
|
|
self.fs.mds_restart()
|
|
self.fs.wait_for_daemons()
|
|
|
|
@contextlib.contextmanager
|
|
def task(ctx, config):
|
|
fs = Filesystem(ctx, config)
|
|
mount_a = ctx.mounts.values()[0]
|
|
|
|
# Stash references on ctx so that we can easily debug in interactive mode
|
|
# =======================================================================
|
|
ctx.filesystem = fs
|
|
ctx.mount_a = mount_a
|
|
|
|
run_tests(ctx, config, TestMDSAutoRepair, {
|
|
'fs': fs,
|
|
'mount_a': mount_a,
|
|
})
|
|
|
|
# Continue to any downstream tasks
|
|
# ================================
|
|
yield
|