import os import json import errno import random import logging from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) class TestVolumes(CephFSTestCase): TEST_VOLUME_NAME = "fs_test_vol" TEST_SUBVOLUME_PREFIX="subvolume" TEST_GROUP_PREFIX="group" TEST_SNAPSHOT_PREFIX="snapshot" TEST_FILE_NAME_PREFIX="subvolume_file" # for filling subvolume with data CLIENTS_REQUIRED = 1 # io defaults DEFAULT_FILE_SIZE = 1 # MB DEFAULT_NUMBER_OF_FILES = 1024 def _fs_cmd(self, *args): return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args) def _generate_random_subvolume_name(self): return "{0}_{1}".format(TestVolumes.TEST_SUBVOLUME_PREFIX, random.randint(0, 10000)) def _generate_random_group_name(self): return "{0}_{1}".format(TestVolumes.TEST_GROUP_PREFIX, random.randint(0, 100)) def _generate_random_snapshot_name(self): return "{0}_{1}".format(TestVolumes.TEST_SNAPSHOT_PREFIX, random.randint(0, 100)) def _enable_multi_fs(self): self._fs_cmd("flag", "set", "enable_multiple", "true", "--yes-i-really-mean-it") def _create_or_reuse_test_volume(self): result = json.loads(self._fs_cmd("volume", "ls")) if len(result) == 0: self.vol_created = True self.volname = TestVolumes.TEST_VOLUME_NAME self._fs_cmd("volume", "create", self.volname) else: self.volname = result[0]['name'] def _get_subvolume_group_path(self, vol_name, group_name): args = ("subvolumegroup", "getpath", vol_name, group_name) path = self._fs_cmd(*args) # remove the leading '/', and trailing whitespaces return path[1:].rstrip() def _get_subvolume_path(self, vol_name, subvol_name, group_name=None): args = ["subvolume", "getpath", vol_name, subvol_name] if group_name: args.append(group_name) args = tuple(args) path = self._fs_cmd(*args) # remove the leading '/', and trailing whitespaces return path[1:].rstrip() def _delete_test_volume(self): self._fs_cmd("volume", "rm", self.volname) def _do_subvolume_io(self, subvolume, number_of_files=DEFAULT_NUMBER_OF_FILES, file_size=DEFAULT_FILE_SIZE): # get subvolume path for IO subvolpath = self._fs_cmd("subvolume", "getpath", self.volname, subvolume) self.assertNotEqual(subvolpath, None) subvolpath = subvolpath[1:].rstrip() # remove "/" prefix and any trailing newline log.debug("filling subvolume {0} with {1} files each {2}MB size".format(subvolume, number_of_files, file_size)) for i in range(number_of_files): filename = "{0}.{1}".format(TestVolumes.TEST_FILE_NAME_PREFIX, i) self.mount_a.write_n_mb(os.path.join(subvolpath, filename), file_size) def _wait_for_trash_empty(self, timeout=30): # XXX: construct the trash dir path (note that there is no mgr # [sub]volume interface for this). trashdir = os.path.join("./", "volumes", "_deleting") self.mount_a.wait_for_dir_empty(trashdir) def setUp(self): super(TestVolumes, self).setUp() self.volname = None self.vol_created = False self._enable_multi_fs() self._create_or_reuse_test_volume() def tearDown(self): if self.vol_created: self._delete_test_volume() super(TestVolumes, self).tearDown() ### basic subvolume operations def test_subvolume_create_and_rm(self): # create subvolume subvolume = self._generate_random_subvolume_name() self._fs_cmd("subvolume", "create", self.volname, subvolume) # make sure it exists subvolpath = self._fs_cmd("subvolume", "getpath", self.volname, subvolume) self.assertNotEqual(subvolpath, None) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) # make sure its gone try: self._fs_cmd("subvolume", "getpath", self.volname, subvolume) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise # verify trash dir is clean self._wait_for_trash_empty() def test_subvolume_create_idempotence(self): # create subvolume subvolume = self._generate_random_subvolume_name() self._fs_cmd("subvolume", "create", self.volname, subvolume) # try creating w/ same subvolume name -- should be idempotent self._fs_cmd("subvolume", "create", self.volname, subvolume) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) # verify trash dir is clean self._wait_for_trash_empty() def test_subvolume_create_with_invalid_data_pool_layout(self): subvolume = self._generate_random_subvolume_name() data_pool = "invalid_pool" # create subvolume with invalid data pool layout try: self._fs_cmd("subvolume", "create", self.volname, subvolume, "--pool_layout", data_pool) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise else: raise # clean up self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--force") def test_subvolume_create_with_auto_cleanup_on_fail(self): subvolume = self._generate_random_subvolume_name() data_pool = "invalid_pool" # create subvolume with invalid data pool layout fails with self.assertRaises(CommandFailedError): self._fs_cmd("subvolume", "create", self.volname, subvolume, "--pool_layout", data_pool) # check whether subvol path is cleaned up try: self._fs_cmd("subvolume", "getpath", self.volname, subvolume) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise else: raise def test_nonexistent_subvolume_rm(self): # remove non-existing subvolume subvolume = "non_existent_subvolume" # try, remove subvolume try: self._fs_cmd("subvolume", "rm", self.volname, subvolume) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise # force remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--force") def test_nonexistent_subvolume_group_create(self): subvolume = self._generate_random_subvolume_name() group = "non_existent_group" # try, creating subvolume in a nonexistent group try: self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise def test_default_uid_gid_subvolume(self): subvolume = self._generate_random_subvolume_name() expected_uid = 0 expected_gid = 0 # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) subvol_path = self._get_subvolume_path(self.volname, subvolume) # check subvolume's uid and gid stat = self.mount_a.stat(subvol_path) self.assertEqual(stat['st_uid'], expected_uid) self.assertEqual(stat['st_gid'], expected_gid) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) ### subvolume group operations def test_subvolume_create_and_rm_in_group(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) # verify trash dir is clean self._wait_for_trash_empty() # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_create_with_desired_data_pool_layout(self): group1 = self._generate_random_group_name() group2 = self._generate_random_group_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group1) group1_path = self._get_subvolume_group_path(self.volname, group1) default_pool = self.mount_a.getfattr(group1_path, "ceph.dir.layout.pool") new_pool = "new_pool" self.assertNotEqual(default_pool, new_pool) # add data pool self.fs.add_data_pool(new_pool) # create group specifying the new data pool as its pool layout self._fs_cmd("subvolumegroup", "create", self.volname, group2, "--pool_layout", new_pool) group2_path = self._get_subvolume_group_path(self.volname, group2) desired_pool = self.mount_a.getfattr(group2_path, "ceph.dir.layout.pool") self.assertEqual(desired_pool, new_pool) self._fs_cmd("subvolumegroup", "rm", self.volname, group1) self._fs_cmd("subvolumegroup", "rm", self.volname, group2) def test_subvolume_group_create_with_invalid_data_pool_layout(self): group = self._generate_random_group_name() data_pool = "invalid_pool" # create group with invalid data pool layout try: self._fs_cmd("subvolumegroup", "create", self.volname, group, "--pool_layout", data_pool) except CommandFailedError as ce: if ce.exitstatus != errno.EINVAL: raise else: raise # clean up self._fs_cmd("subvolumegroup", "rm", self.volname, group, "--force") def test_subvolume_group_create_with_auto_cleanup_on_fail(self): group = self._generate_random_group_name() data_pool = "invalid_pool" # create group with invalid data pool layout with self.assertRaises(CommandFailedError): self._fs_cmd("subvolumegroup", "create", self.volname, group, "--pool_layout", data_pool) # check whether group path is cleaned up try: self._fs_cmd("subvolumegroup", "getpath", self.volname, group) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise else: raise def test_subvolume_create_with_desired_data_pool_layout_in_group(self): subvol1 = self._generate_random_subvolume_name() subvol2 = self._generate_random_subvolume_name() group = self._generate_random_group_name() # create group. this also helps set default pool layout for subvolumes # created within the group. self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group. self._fs_cmd("subvolume", "create", self.volname, subvol1, "--group_name", group) subvol1_path = self._get_subvolume_path(self.volname, subvol1, group_name=group) default_pool = self.mount_a.getfattr(subvol1_path, "ceph.dir.layout.pool") new_pool = "new_pool" self.assertNotEqual(default_pool, new_pool) # add data pool self.fs.add_data_pool(new_pool) # create subvolume specifying the new data pool as its pool layout self._fs_cmd("subvolume", "create", self.volname, subvol2, "--group_name", group, "--pool_layout", new_pool) subvol2_path = self._get_subvolume_path(self.volname, subvol2, group_name=group) desired_pool = self.mount_a.getfattr(subvol2_path, "ceph.dir.layout.pool") self.assertEqual(desired_pool, new_pool) self._fs_cmd("subvolume", "rm", self.volname, subvol2, group) self._fs_cmd("subvolume", "rm", self.volname, subvol1, group) self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_create_with_desired_mode(self): group1 = self._generate_random_group_name() group2 = self._generate_random_group_name() # default mode expected_mode1 = "755" # desired mode expected_mode2 = "777" # create group self._fs_cmd("subvolumegroup", "create", self.volname, group1) self._fs_cmd("subvolumegroup", "create", self.volname, group2, "--mode", "777") group1_path = self._get_subvolume_group_path(self.volname, group1) group2_path = self._get_subvolume_group_path(self.volname, group2) # check group's mode actual_mode1 = self.mount_a.run_shell(['stat', '-c' '%a', group1_path]).stdout.getvalue().strip() actual_mode2 = self.mount_a.run_shell(['stat', '-c' '%a', group2_path]).stdout.getvalue().strip() self.assertEqual(actual_mode1, expected_mode1) self.assertEqual(actual_mode2, expected_mode2) self._fs_cmd("subvolumegroup", "rm", self.volname, group1) self._fs_cmd("subvolumegroup", "rm", self.volname, group2) def test_subvolume_create_with_desired_mode_in_group(self): subvol1 = self._generate_random_subvolume_name() subvol2 = self._generate_random_subvolume_name() subvol3 = self._generate_random_subvolume_name() group = self._generate_random_group_name() # default mode expected_mode1 = "755" # desired mode expected_mode2 = "777" # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvol1, "--group_name", group) self._fs_cmd("subvolume", "create", self.volname, subvol2, "--group_name", group, "--mode", "777") # check whether mode 0777 also works self._fs_cmd("subvolume", "create", self.volname, subvol3, "--group_name", group, "--mode", "0777") subvol1_path = self._get_subvolume_path(self.volname, subvol1, group_name=group) subvol2_path = self._get_subvolume_path(self.volname, subvol2, group_name=group) subvol3_path = self._get_subvolume_path(self.volname, subvol3, group_name=group) # check subvolume's mode actual_mode1 = self.mount_a.run_shell(['stat', '-c' '%a', subvol1_path]).stdout.getvalue().strip() actual_mode2 = self.mount_a.run_shell(['stat', '-c' '%a', subvol2_path]).stdout.getvalue().strip() actual_mode3 = self.mount_a.run_shell(['stat', '-c' '%a', subvol3_path]).stdout.getvalue().strip() self.assertEqual(actual_mode1, expected_mode1) self.assertEqual(actual_mode2, expected_mode2) self.assertEqual(actual_mode3, expected_mode2) self._fs_cmd("subvolume", "rm", self.volname, subvol1, group) self._fs_cmd("subvolume", "rm", self.volname, subvol2, group) self._fs_cmd("subvolume", "rm", self.volname, subvol3, group) self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_nonexistent_subvolme_group_rm(self): group = "non_existent_group" # try, remove subvolume group try: self._fs_cmd("subvolumegroup", "rm", self.volname, group) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise # force remove subvolume self._fs_cmd("subvolumegroup", "rm", self.volname, group, "--force") def test_default_uid_gid_subvolume_group(self): group = self._generate_random_group_name() expected_uid = 0 expected_gid = 0 # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) group_path = self._get_subvolume_group_path(self.volname, group) # check group's uid and gid stat = self.mount_a.stat(group_path) self.assertEqual(stat['st_uid'], expected_uid) self.assertEqual(stat['st_gid'], expected_gid) # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) ### snapshot operations def test_subvolume_snapshot_create_and_rm(self): subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) # verify trash dir is clean self._wait_for_trash_empty() def test_subvolume_snapshot_create_idempotence(self): subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) # try creating w/ same subvolume snapshot name -- should be idempotent self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) # verify trash dir is clean self._wait_for_trash_empty() def test_nonexistent_subvolume_snapshot_rm(self): subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) # remove snapshot again try: self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise # force remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot, "--force") # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) # verify trash dir is clean self._wait_for_trash_empty() def test_subvolume_snapshot_in_group(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() snapshot = self._generate_random_snapshot_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) # snapshot subvolume in group self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot, group) # remove snapshot self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot, group) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) # verify trash dir is clean self._wait_for_trash_empty() # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_snapshot_create_and_rm(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() snapshot = self._generate_random_snapshot_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) # snapshot group self._fs_cmd("subvolumegroup", "snapshot", "create", self.volname, group, snapshot) # remove snapshot self._fs_cmd("subvolumegroup", "snapshot", "rm", self.volname, group, snapshot) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) # verify trash dir is clean self._wait_for_trash_empty() # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_subvolume_group_snapshot_idempotence(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() snapshot = self._generate_random_snapshot_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) # snapshot group self._fs_cmd("subvolumegroup", "snapshot", "create", self.volname, group, snapshot) # try creating snapshot w/ same snapshot name -- shoule be idempotent self._fs_cmd("subvolumegroup", "snapshot", "create", self.volname, group, snapshot) # remove snapshot self._fs_cmd("subvolumegroup", "snapshot", "rm", self.volname, group, snapshot) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) # verify trash dir is clean self._wait_for_trash_empty() # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_nonexistent_subvolume_group_snapshot_rm(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() snapshot = self._generate_random_snapshot_name() # create group self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) # snapshot group self._fs_cmd("subvolumegroup", "snapshot", "create", self.volname, group, snapshot) # remove snapshot self._fs_cmd("subvolumegroup", "snapshot", "rm", self.volname, group, snapshot) # remove snapshot try: self._fs_cmd("subvolumegroup", "snapshot", "rm", self.volname, group, snapshot) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) # verify trash dir is clean self._wait_for_trash_empty() # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) def test_async_subvolume_rm(self): subvolume = self._generate_random_subvolume_name() # create subvolume self._fs_cmd("subvolume", "create", self.volname, subvolume) # fill subvolume w/ some data self._do_subvolume_io(subvolume) self.mount_a.umount_wait() # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) self.mount_a.mount() # verify trash dir is clean self._wait_for_trash_empty()