ceph/tasks/mds_auto_repair.py

"""
Exercise the MDS's auto repair functions
"""

import contextlib
import logging
import time

from teuthology.orchestra.run import CommandFailedError

from tasks.cephfs.filesystem import Filesystem
from tasks.cephfs.cephfs_test_case import CephFSTestCase, run_tests


log = logging.getLogger(__name__)


# Arbitrary timeouts for operations involving restarting
# an MDS or waiting for it to come up
MDS_RESTART_GRACE = 60


class TestMDSAutoRepair(CephFSTestCase):
    def test_backtrace_repair(self):
        """
        MDS should verify/fix backtrace on fetch dirfrag
        """

        # trim log segment as fast as possible
        self.fs.set_ceph_conf('mds', 'mds cache size', 100)
        self.fs.set_ceph_conf('mds', 'mds log max segments', 2)
        self.fs.set_ceph_conf('mds', 'mds log events per segment', 1)
        self.fs.set_ceph_conf('mds', 'mds verify backtrace', 1)
        self.fs.mds_restart()
        self.fs.wait_for_daemons()

        create_script = "mkdir {0}; for i in `seq 0 500`; do touch {0}/file$i; done"
        # create main test directory
        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir1")])

        # create more files in another directory. make sure MDS trim dentries in testdir1
        self.mount_a.run_shell(["sudo", "bash", "-c", create_script.format("testdir2")])

        # flush journal entries to dirfrag objects
        self.fs.mds_asok(['flush', 'journal'])

        # drop inodes caps
        self.mount_a.umount_wait()
        self.mount_a.mount()
        self.mount_a.wait_until_mounted()

        # wait MDS to trim dentries in testdir1. 60 seconds should be long enough.
        time.sleep(60)

        # remove testdir1's backtrace
        proc = self.mount_a.run_shell(["sudo", "ls", "-id", "testdir1"])
        self.assertEqual(proc.exitstatus, 0)
        objname = "{:x}.00000000".format(long(proc.stdout.getvalue().split()[0]))
        proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "rmxattr", objname, "parent"])
        self.assertEqual(proc.exitstatus, 0)

        # readdir (fetch dirfrag) should fix testdir1's backtrace
        self.mount_a.run_shell(["sudo", "ls", "testdir1"])

        # add more entries to journal
        self.mount_a.run_shell(["sudo", "rm", "-rf", " testdir2"])

        # flush journal entries to dirfrag objects
        self.fs.mds_asok(['flush', 'journal'])

        # check if backtrace exists
        proc = self.mount_a.run_shell(["sudo", "rados", "-p", "metadata", "getxattr", objname, "parent"])
        self.assertEqual(proc.exitstatus, 0)

    def test_mds_readonly(self):
        """
        test if MDS behave correct when it's readonly
        """
        # operation should successd when MDS is not readonly
        self.mount_a.run_shell(["sudo", "touch", "test_file1"])
        writer = self.mount_a.write_background(loop=True)

        time.sleep(10)
        self.assertFalse(writer.finished)

        # force MDS to read-only mode
        self.fs.mds_asok(['force_readonly'])
        time.sleep(10)

        # touching test file should fail
        try:
            self.mount_a.run_shell(["sudo", "touch", "test_file1"])
        except CommandFailedError:
            pass
        else:
            self.assertTrue(False)

        # background writer also should fail
        self.assertTrue(writer.finished)

        # restart mds to make it writable
        self.fs.mds_restart()
        self.fs.wait_for_daemons()


@contextlib.contextmanager
def task(ctx, config):
    fs = Filesystem(ctx, config)
    mount_a = ctx.mounts.values()[0]

    # Stash references on ctx so that we can easily debug in interactive mode
    # =======================================================================
    ctx.filesystem = fs
    ctx.mount_a = mount_a

    run_tests(ctx, config, TestMDSAutoRepair, {
        'fs': fs,
        'mount_a': mount_a,
    })

    # Continue to any downstream tasks
    # ================================
    yield