tasks/cephfs: implement TestDataScan.test_stashed_layout

This is for testing how cephfs-data-scan handles
the new 'layout' xattr when found during recovery.

Signed-off-by: John Spray <john.spray@redhat.com>

tasks/cephfs/mount.py

@@ -278,6 +278,42 @@ class CephFSMount(object):
                         "seek={0}".format(seek)
         ])
 
+    def write_test_pattern(self, filename, size):
+        log.info("Writing {0} bytes to {1}".format(size, filename))
+        return self.run_python(dedent("""
+            import zlib
+            path = "{path}"
+            f = open(path, 'w')
+            for i in range(0, {size}):
+                val = zlib.crc32("%s" % i) & 7
+                f.write(chr(val))
+            f.close()
+        """.format(
+            path=os.path.join(self.mountpoint, filename),
+            size=size
+        )))
+
+    def validate_test_pattern(self, filename, size):
+        log.info("Validating {0} bytes from {1}".format(size, filename))
+        return self.run_python(dedent("""
+            import zlib
+            path = "{path}"
+            f = open(path, 'r')
+            bytes = f.read()
+            f.close()
+            if len(bytes) != {size}:
+                raise RuntimeError("Bad length {{0}} vs. expected {{1}}".format(
+                    len(bytes), {size}
+                ))
+            for i, b in enumerate(bytes):
+                val = zlib.crc32("%s" % i) & 7
+                if b != chr(val):
+                    raise RuntimeError("Bad data at offset {{0}}".format(i))
+        """.format(
+            path=os.path.join(self.mountpoint, filename),
+            size=size
+        )))
+
     def open_n_background(self, fs_path, count):
         """
         Open N files for writing, hold them open in a background process

tasks/cephfs/test_data_scan.py

@@ -4,6 +4,7 @@ Test our tools for recovering metadata from the data pool
 """
 
 import logging
+import os
 from textwrap import dedent
 import traceback
 from collections import namedtuple
@@ -131,6 +132,86 @@ class BacktracelessFile(Workload):
         return self._errors
 
 
+class StripedStashedLayout(Workload):
+    def __init__(self, fs, m):
+        super(StripedStashedLayout, self).__init__(fs, m)
+
+        # Nice small stripes so we can quickly do our writes+validates
+        self.sc = 4
+        self.ss = 65536
+        self.os = 262144
+
+        self.interesting_sizes = [
+            # Exactly stripe_count objects will exist
+            self.os * self.sc,
+            # Fewer than stripe_count objects will exist
+            self.os * self.sc / 2,
+            self.os * (self.sc - 1) + self.os / 2,
+            self.os * (self.sc - 1) + self.os / 2 - 1,
+            self.os * (self.sc + 1) + self.os / 2,
+            self.os * (self.sc + 1) + self.os / 2 + 1,
+            # More than stripe_count objects will exist
+            self.os * self.sc + self.os * self.sc / 2
+        ]
+
+    def write(self):
+        # Create a dir with a striped layout set on it
+        self._mount.run_shell(["mkdir", "stripey"])
+
+        self._mount.run_shell([
+            "setfattr", "-n", "ceph.dir.layout", "-v",
+            "stripe_unit={ss} stripe_count={sc} object_size={os} pool=data".format(
+                ss=self.ss, os=self.os, sc=self.sc
+            ),
+            "./stripey"])
+
+        # Write files, then flush metadata so that its layout gets written into an xattr
+        for i, n_bytes in enumerate(self.interesting_sizes):
+            self._mount.write_test_pattern("stripey/flushed_file_{0}".format(i), n_bytes)
+            # This is really just validating the validator
+            self._mount.validate_test_pattern("stripey/flushed_file_{0}".format(i), n_bytes)
+        self._filesystem.mds_asok(["flush", "journal"])
+
+        # Write another file in the same way, but this time don't flush the metadata,
+        # so that it won't have the layout xattr
+        self._mount.write_test_pattern("stripey/unflushed_file", 1024 * 512)
+        self._mount.validate_test_pattern("stripey/unflushed_file", 1024 * 512)
+
+        self._initial_state = {
+            "unflushed_ino": self._mount.path_to_ino("stripey/unflushed_file")
+        }
+
+    def flush(self):
+        # Pass because we already selectively flushed during write
+        pass
+
+    def validate(self):
+        # The first files should have been recovered into its original location
+        # with the correct layout: read back correct data
+        for i, n_bytes in enumerate(self.interesting_sizes):
+            try:
+                self._mount.validate_test_pattern("stripey/flushed_file_{0}".format(i), n_bytes)
+            except CommandFailedError as e:
+                self._errors.append(
+                    ValidationError("File {0} (size {1}): {2}".format(i, n_bytes, e), traceback.format_exc(3))
+                )
+
+        # The unflushed file should have been recovered into lost+found without
+        # the correct layout: read back junk
+        ino_name = "%x" % self._initial_state["unflushed_ino"]
+        self.assert_equal(self._mount.ls("lost+found"), [ino_name])
+        try:
+            self._mount.validate_test_pattern(os.path.join("lost+found", ino_name), 1024 * 512)
+        except CommandFailedError:
+            pass
+        else:
+            self._errors.append(
+                ValidationError("Unexpectedly valid data in unflushed striped file", "")
+            )
+
+        return self._errors
+
+
 class MovedDir(Workload):
     def write(self):
         # Create a nested dir that we will then move.  Two files with two different
@@ -271,6 +352,7 @@ class TestDataScan(CephFSTestCase):
         # See that the files are present and correct
         errors = workload.validate()
         if errors:
+            log.error("Validation errors found: {0}".format(len(errors)))
             for e in errors:
                 log.error(e.exception)
                 log.error(e.backtrace)
@@ -296,6 +378,9 @@ class TestDataScan(CephFSTestCase):
     def test_rebuild_nondefault_layout(self):
         self._rebuild_metadata(NonDefaultLayout(self.fs, self.mount_a))
 
+    def test_stashed_layout(self):
+        self._rebuild_metadata(StripedStashedLayout(self.fs, self.mount_a))
+
     def _dirfrag_keys(self, object_id):
         keys_str = self.fs.rados(["listomapkeys", object_id])
         if keys_str: