Merge pull request #13099 from jcsp/wip-18663

qa/tasks: force umount during kclient teardown

qa/tasks/cephfs/kernel_mount.py

@@ -7,11 +7,15 @@ from teuthology import misc
 
 from teuthology.orchestra import remote as orchestra_remote
 from teuthology.orchestra import run
+from teuthology.contextutil import MaxWhileTries
 from .mount import CephFSMount
 
 log = logging.getLogger(__name__)
 
 
+UMOUNT_TIMEOUT = 300
+
+
 class KernelMount(CephFSMount):
     def __init__(self, mons, test_dir, client_id, client_remote,
                  ipmi_user, ipmi_password, ipmi_domain):
@@ -96,13 +100,15 @@ class KernelMount(CephFSMount):
 
         self.client_remote.run(args=cmd)
 
-        self.client_remote.run(
+        rproc = self.client_remote.run(
             args=[
                 'rmdir',
                 '--',
                 self.mountpoint,
             ],
+            wait=False
         )
+        run.wait([rproc], UMOUNT_TIMEOUT)
         self.mounted = False
 
     def cleanup(self):
@@ -117,7 +123,7 @@ class KernelMount(CephFSMount):
 
         try:
             self.umount(force)
-        except CommandFailedError:
+        except (CommandFailedError, MaxWhileTries):
             if not force:
                 raise
 

qa/tasks/kclient.py

@@ -5,7 +5,9 @@ import contextlib
 import logging
 
 from teuthology.misc import deep_merge
+from teuthology.orchestra.run import CommandFailedError
 from teuthology import misc
+from teuthology.contextutil import MaxWhileTries
 from cephfs.kernel_mount import KernelMount
 
 log = logging.getLogger(__name__)
@@ -104,11 +106,32 @@ def task(ctx, config):
 
         kernel_mount.mount()
 
+
+    def umount_all():
+        log.info('Unmounting kernel clients...')
+
+        forced = False
+        for mount in mounts.values():
+            if mount.is_mounted():
+                try:
+                    mount.umount()
+                except (CommandFailedError, MaxWhileTries):
+                    log.warn("Ordinary umount failed, forcing...")
+                    forced = True
+                    mount.umount_wait(force=True)
+
+        return forced
+
     ctx.mounts = mounts
     try:
         yield mounts
+    except:
+        umount_all()  # ignore forced retval, we are already in error handling
     finally:
-        log.info('Unmounting kernel clients...')
-        for mount in mounts.values():
-            if mount.is_mounted():
-                mount.umount()
+
+        forced = umount_all()
+        if forced:
+            # The context managers within the kclient manager worked (i.e.
+            # the test workload passed) but for some reason we couldn't
+            # umount, so turn this into a test failure.
+            raise RuntimeError("Kernel mounts did not umount cleanly")