ceph/tasks/cephfs/test_auto_repair.py

"""
Exercise the MDS's auto repair functions
"""

import logging
import time

from teuthology.orchestra.run import CommandFailedError
from tasks.cephfs.cephfs_test_case import CephFSTestCase


log = logging.getLogger(__name__)


# Arbitrary timeouts for operations involving restarting
# an MDS or waiting for it to come up
MDS_RESTART_GRACE = 60


class TestMDSAutoRepair(CephFSTestCase):
    def test_backtrace_repair(self):
        """
        MDS should verify/fix backtrace on fetch dirfrag
        """

        # trim log segment as fast as possible
        self.set_conf('mds', 'mds cache size', 100)
        self.set_conf('mds', 'mds verify backtrace', 1)
        self.fs.mds_fail_restart()
        self.fs.wait_for_daemons()

        create_script = "mkdir {0}; for i in `seq 0 500`; do touch {0}/file$i; done"
        # create main test directory
        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir1")])

        # create more files in another directory. make sure MDS trim dentries in testdir1
        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir2")])

        # drop inodes caps
        self.mount_a.umount_wait()

        # flush journal entries to dirfrag objects
        self.fs.mds_asok(['flush', 'journal'])

        self.mount_a.mount()
        self.mount_a.wait_until_mounted()

        # wait MDS to trim dentries in testdir1. 60 seconds should be long enough.
        time.sleep(60)

        # remove testdir1's backtrace
        proc = self.mount_a.run_shell(["sudo", "ls", "-id", "testdir1"])
        self.assertEqual(proc.exitstatus, 0)
        objname = "{:x}.00000000".format(long(proc.stdout.getvalue().split()[0]))
        self.fs.rados(["rmxattr", objname, "parent"])

        # readdir (fetch dirfrag) should fix testdir1's backtrace
        self.mount_a.run_shell(["sudo", "ls", "testdir1"])

        # add more entries to journal
        self.mount_a.run_shell(["sudo", "rm", "-rf", " testdir2"])

        # flush journal entries to dirfrag objects
        self.fs.mds_asok(['flush', 'journal'])

        # check if backtrace exists
        self.fs.rados(["getxattr", objname, "parent"])

    def test_mds_readonly(self):
        """
        test if MDS behave correct when it's readonly
        """
        # operation should successd when MDS is not readonly
        self.mount_a.run_shell(["sudo", "touch", "test_file1"])
        writer = self.mount_a.write_background(loop=True)

        time.sleep(10)
        self.assertFalse(writer.finished)

        # force MDS to read-only mode
        self.fs.mds_asok(['force_readonly'])
        time.sleep(10)

        # touching test file should fail
        try:
            self.mount_a.run_shell(["sudo", "touch", "test_file1"])
        except CommandFailedError:
            pass
        else:
            self.assertTrue(False)

        # background writer also should fail
        self.assertTrue(writer.finished)

        # restart mds to make it writable
        self.fs.mds_fail_restart()
        self.fs.wait_for_daemons()