From 9552efde4a0fcf72d03eb2e7fcbb5a2dd96fc89b Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 5 May 2017 19:00:31 -0400 Subject: [PATCH] qa: improve time handling for test_exports test Also catches corner-case found by Zheng where an unjournaled directory will cause export pinning to fail because it cannot be made a subtree until its parent is stable. Signed-off-by: Patrick Donnelly --- qa/tasks/cephfs/filesystem.py | 8 ++- qa/tasks/cephfs/test_exports.py | 106 ++++++++++---------------------- 2 files changed, 39 insertions(+), 75 deletions(-) diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py index 273bb39ebe4..4a860c86d64 100644 --- a/qa/tasks/cephfs/filesystem.py +++ b/qa/tasks/cephfs/filesystem.py @@ -788,7 +788,7 @@ class Filesystem(MDSCluster): return result - def wait_for_state(self, goal_state, reject=None, timeout=None, mds_id=None): + def wait_for_state(self, goal_state, reject=None, timeout=None, mds_id=None, rank=None): """ Block until the MDS reaches a particular state, or a failure condition is met. @@ -805,7 +805,11 @@ class Filesystem(MDSCluster): started_at = time.time() while True: status = self.status() - if mds_id is not None: + if rank is not None: + mds_info = status.get_rank(self.id, rank) + current_state = mds_info['state'] if mds_info else None + log.info("Looked up MDS state for mds.{0}: {1}".format(rank, current_state)) + elif mds_id is not None: # mds_info is None if no daemon with this ID exists in the map mds_info = status.get_mds(mds_id) current_state = mds_info['state'] if mds_info else None diff --git a/qa/tasks/cephfs/test_exports.py b/qa/tasks/cephfs/test_exports.py index 92dfe3f8f20..7cbf464a953 100644 --- a/qa/tasks/cephfs/test_exports.py +++ b/qa/tasks/cephfs/test_exports.py @@ -5,8 +5,22 @@ from tasks.cephfs.cephfs_test_case import CephFSTestCase log = logging.getLogger(__name__) - class TestExports(CephFSTestCase): + def _wait_subtrees(self, status, rank, test): + timeout = 30 + pause = 2 + test = sorted(test) + for i in range(timeout/pause): + subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, rank)['name']) + subtrees = filter(lambda s: s['dir']['path'].startswith('/'), subtrees) + log.info(subtrees) + filtered = sorted([(s['dir']['path'], s['auth_first']) for s in subtrees]) + log.info(filtered) + if filtered == test: + return subtrees + time.sleep(pause) + raise RuntimeError("rank {0} failed to reach desired subtree state", rank) + def test_export_pin(self): self.fs.set_allow_multimds(True) self.fs.set_max_mds(2) @@ -14,33 +28,19 @@ class TestExports(CephFSTestCase): status = self.fs.status() self.mount_a.run_shell(["mkdir", "-p", "1/2/3"]) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 0) + self._wait_subtrees(status, 0, []) # NOP self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "-1", "1"]) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 0) + self._wait_subtrees(status, 0, []) # NOP (rank < -1) self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "-2341", "1"]) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 0) + self._wait_subtrees(status, 0, []) # pin /1 to rank 1 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", "1"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 1)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - log.info(subtrees) - self.assertTrue(len(subtrees) == 1 and subtrees[0]['auth_first'] == 1) + self._wait_subtrees(status, 1, [('/1', 1)]) # Check export_targets is set properly status = self.fs.status() @@ -50,72 +50,39 @@ class TestExports(CephFSTestCase): # redundant pin /1/2 to rank 1 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", "1/2"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 1)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 2 and subtrees[0]['auth_first'] == 1 and subtrees[1]['auth_first'] == 1) + self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1)]) # change pin /1/2 to rank 0 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "0", "1/2"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 2) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1/2'), subtrees) - self.assertTrue(len(subtrees) == 1 and subtrees[0]['auth_first'] == 0) + self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 0)]) + self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) # change pin /1/2/3 to (presently) non-existent rank 2 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "2", "1/2/3"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 2) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1/2'), subtrees) - self.assertTrue(len(subtrees) == 1 and subtrees[0]['auth_first'] == 0) + self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0)]) + self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 0)]) # change pin /1/2 back to rank 1 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", "1/2"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 1)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 2 and subtrees[0]['auth_first'] == 1 and subtrees[1]['auth_first'] == 1) + self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1)]) # add another directory pinned to 1 self.mount_a.run_shell(["mkdir", "-p", "1/4/5"]) self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", "1/4/5"]) + self._wait_subtrees(status, 1, [('/1', 1), ('/1/2', 1), ('/1/4/5', 1)]) # change pin /1 to 0 self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "0", "1"]) - time.sleep(20) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'] == '/1', subtrees) - self.assertTrue(len(subtrees) == 1) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 1)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1/'), subtrees) - self.assertTrue(len(subtrees) == 2 and subtrees[0]['auth_first'] == 1 and subtrees[1]['auth_first'] == 1) # /1/2 and /1/4/5 + self._wait_subtrees(status, 0, [('/1', 0), ('/1/2', 1), ('/1/4/5', 1)]) # change pin /1/2 to default (-1); does the subtree root properly respect it's parent pin? self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "-1", "1/2"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 1)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1'), subtrees) - self.assertTrue(len(subtrees) == 1) # /1/4 still here! + self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1)]) if len(list(status.get_standbys())): self.fs.set_max_mds(3) - time.sleep(10) - status = self.fs.status() - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 2)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/1/2/3'), subtrees) - self.assertTrue(len(subtrees) == 1) + self.fs.wait_for_state('up:active', rank=2) + self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2)]) # Check export_targets is set properly status = self.fs.status() @@ -130,14 +97,7 @@ class TestExports(CephFSTestCase): # Test rename self.mount_a.run_shell(["mkdir", "-p", "a/b", "aa/bb"]) self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "1", "a"]) - time.sleep(10); - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/a'), subtrees) - self.assertTrue(len(subtrees) == 1 and subtrees[0]['auth_first'] == 1) + self.mount_a.run_shell(["setfattr", "-n", "ceph.dir.pin", "-v", "0", "aa/bb"]) + self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/aa/bb', 0)]) self.mount_a.run_shell(["mv", "aa", "a/b/"]) - time.sleep(10) - subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, 0)['name']) - log.info(subtrees) - subtrees = filter(lambda s: s['dir']['path'].startswith('/a'), subtrees) - self.assertTrue(len(subtrees) == 1 and subtrees[0]['auth_first'] == 1) + self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/a/b/aa/bb', 0)])