cephfs/test_client_recovery: check FUSE version before invoking flock

flock only works properly on FUSE versions >=2.9, which is newer
than eg Ubuntu Precise. So check the version on our client mounts and
only test flock if it's at least that new.

Fixes: #9995

Signed-off-by: Greg Farnum <gfarnum@redhat.com>

tasks/cephfs/mount.py

@@ -167,7 +167,7 @@ class CephFSMount(object):
         raise RuntimeError("Timed out after {0}s waiting for {1} to become visible from {2}".format(
             i, basename, self.client_id))
 
-    def lock_background(self, basename="background_file"):
+    def lock_background(self, basename="background_file", do_flock=True):
         """
         Open and lock a files for writing, hold the lock in a background process
         """
@@ -175,36 +175,40 @@ class CephFSMount(object):
 
         path = os.path.join(self.mountpoint, basename)
 
-        pyscript = dedent("""
+        script_builder = """
             import time
             import fcntl
-            import struct
-
+            import struct"""
+        if do_flock:
+            script_builder += """
             f1 = open("{path}-1", 'w')
-            fcntl.flock(f1, fcntl.LOCK_EX | fcntl.LOCK_NB)
-
+            fcntl.flock(f1, fcntl.LOCK_EX | fcntl.LOCK_NB)"""
+        script_builder += """
             f2 = open("{path}-2", 'w')
             lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
             fcntl.fcntl(f2, fcntl.F_SETLK, lockdata)
             while True:
                 time.sleep(1)
-            """).format(path=path)
+            """
+
+        pyscript = dedent(script_builder).format(path=path)
 
         log.info("lock file {0}".format(basename))
         rproc = self._run_python(pyscript)
         self.background_procs.append(rproc)
         return rproc
 
-    def check_filelock(self, basename="background_file"):
+    def check_filelock(self, basename="background_file", do_flock=True):
         assert(self.is_mounted())
 
         path = os.path.join(self.mountpoint, basename)
 
-        pyscript = dedent("""
+        script_builder = """
             import fcntl
             import errno
-            import struct
-
+            import struct"""
+        if do_flock:
+            script_builder += """
             f1 = open("{path}-1", 'r')
             try:
                 fcntl.flock(f1, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -212,8 +216,8 @@ class CephFSMount(object):
                 if e.errno == errno.EAGAIN:
                     pass
             else:
-                raise RuntimeError("flock on file {path}-1 not found")
-
+                raise RuntimeError("flock on file {path}-1 not found")"""
+        script_builder += """
             f2 = open("{path}-2", 'r')
             try:
                 lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
@@ -223,7 +227,8 @@ class CephFSMount(object):
                     pass
             else:
                 raise RuntimeError("posix lock on file {path}-2 not found")
-            """).format(path=path)
+            """
+        pyscript = dedent(script_builder).format(path=path)
 
         log.info("check lock on file {0}".format(basename))
         self.client_remote.run(args=[

tasks/cephfs/test_client_recovery.py

@@ -5,9 +5,12 @@ Teuthology task for exercising CephFS client recovery
 
 import logging
 import time
+import distutils.version as version
+import re
 
 from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
 from tasks.cephfs.cephfs_test_case import CephFSTestCase
+from teuthology.packaging import get_package_version
 
 
 log = logging.getLogger(__name__)
@@ -307,19 +310,41 @@ class TestClientRecovery(CephFSTestCase):
         self.assertLess(recovery_time, self.ms_max_backoff * 2)
         self.assert_session_state(client_id, "open")
 
+
     def test_filelock(self):
         """
         Check that file lock doesn't get lost after an MDS restart
         """
-        lock_holder = self.mount_a.lock_background()
+        a_version_str = get_package_version(self.mount_a.client_remote, "fuse")
+        b_version_str = get_package_version(self.mount_b.client_remote, "fuse")
+        flock_version_str = "2.9"
+
+        version_regex = re.compile(r"[0-9\.]+")
+        a_result = version_regex.match(a_version_str)
+        self.assertTrue(a_result)
+        b_result = version_regex.match(b_version_str)
+        self.assertTrue(b_result)
+        a_version = version.StrictVersion(a_result.group())
+        b_version = version.StrictVersion(b_result.group())
+        flock_version=version.StrictVersion(flock_version_str)
+
+        flockable = False
+        if (a_version >= flock_version and b_version >= flock_version):
+            log.info("testing flock locks")
+            flockable = True
+        else:
+            log.info("not testing flock locks, machines have versions {av} and {bv}".format(
+                av=a_version_str,bv=b_version_str))
+
+        lock_holder = self.mount_a.lock_background(do_flock=flockable)
 
         self.mount_b.wait_for_visible("background_file-2")
-        self.mount_b.check_filelock()
+        self.mount_b.check_filelock(do_flock=flockable)
 
         self.fs.mds_fail_restart()
         self.fs.wait_for_state('up:active', timeout=MDS_RESTART_GRACE)
 
-        self.mount_b.check_filelock()
+        self.mount_b.check_filelock(do_flock=flockable)
 
         # Tear down the background process
         lock_holder.stdin.close()