diff --git a/qa/workunits/ceph-disk/ceph-disk-test.py b/qa/workunits/ceph-disk/ceph-disk-test.py index 0e4a50b2df2..137869b6713 100644 --- a/qa/workunits/ceph-disk/ceph-disk-test.py +++ b/qa/workunits/ceph-disk/ceph-disk-test.py @@ -143,6 +143,15 @@ class CephDisk: def get_journal_partition(self, uuid): return self.get_space_partition('journal', uuid) + def get_block_partition(self, uuid): + return self.get_space_partition('block', uuid) + + def get_blockdb_partition(self, uuid): + return self.get_space_partition('block.db', uuid) + + def get_blockwal_partition(self, uuid): + return self.get_space_partition('block.wal', uuid) + def get_space_partition(self, name, uuid): data_partition = self.get_osd_partition(uuid) space_dev = data_partition[name + '_dev'] @@ -454,10 +463,84 @@ class TestCephDisk(object): " " + disk) c.wait_for_osd_up(osd_uuid) device = json.loads(c.sh("ceph-disk list --format json " + disk))[0] - assert len(device['partitions']) == 2 + assert len(device['partitions']) == 4 c.check_osd_status(osd_uuid, 'block') + c.check_osd_status(osd_uuid, 'block.wal') + c.check_osd_status(osd_uuid, 'block.db') c.helper("pool_read_write") c.destroy_osd(osd_uuid) + c.sh("ceph-disk --verbose zap " + disk) + + def test_activate_bluestore_seperated_block_db_wal(self): + c = CephDisk() + disk1 = c.unused_disks()[0] + disk2 = c.unused_disks()[1] + osd_uuid = str(uuid.uuid1()) + c.sh("ceph-disk --verbose zap " + disk1 + " " + disk2) + c.conf['global']['osd objectstore'] = 'bluestore' + c.save_conf() + c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " + osd_uuid + + " " + disk1 + " --block.db " + disk2 + " --block.wal " + disk2) + c.wait_for_osd_up(osd_uuid) + device = json.loads(c.sh("ceph-disk list --format json " + disk1))[0] + assert len(device['partitions']) == 2 + device = json.loads(c.sh("ceph-disk list --format json " + disk2))[0] + assert len(device['partitions']) == 2 + c.check_osd_status(osd_uuid, 'block') + c.check_osd_status(osd_uuid, 'block.wal') + c.check_osd_status(osd_uuid, 'block.db') + c.helper("pool_read_write") + c.destroy_osd(osd_uuid) + c.sh("ceph-disk --verbose zap " + disk1 + " " + disk2) + + def test_activate_bluestore_reuse_db_wal_partition(self): + c = CephDisk() + disks = c.unused_disks() + block_disk = disks[0] + db_wal_disk = disks[1] + # + # Create an OSD with two disks (one for block, + # the other for block.db and block.wal ) and then destroy osd. + # + osd_uuid1 = str(uuid.uuid1()) + c.sh("ceph-disk --verbose zap " + block_disk + " " + db_wal_disk) + c.conf['global']['osd objectstore'] = 'bluestore' + c.save_conf() + c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " + + osd_uuid1 + " " + block_disk + " --block.db " + db_wal_disk + + " --block.wal " + db_wal_disk) + c.wait_for_osd_up(osd_uuid1) + blockdb_partition = c.get_blockdb_partition(osd_uuid1) + blockdb_path = blockdb_partition['path'] + blockwal_partition = c.get_blockwal_partition(osd_uuid1) + blockwal_path = blockwal_partition['path'] + c.destroy_osd(osd_uuid1) + c.sh("ceph-disk --verbose zap " + block_disk) + # + # Create another OSD with the block.db and block.wal partition + # of the previous OSD + # + osd_uuid2 = str(uuid.uuid1()) + c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " + + osd_uuid2 + " " + block_disk + " --block.db " + blockdb_path + + " --block.wal " + blockwal_path) + c.wait_for_osd_up(osd_uuid2) + device = json.loads(c.sh("ceph-disk list --format json " + block_disk))[0] + assert len(device['partitions']) == 2 + device = json.loads(c.sh("ceph-disk list --format json " + db_wal_disk))[0] + assert len(device['partitions']) == 2 + c.check_osd_status(osd_uuid2, 'block') + c.check_osd_status(osd_uuid2, 'block.wal') + c.check_osd_status(osd_uuid2, 'block.db') + blockdb_partition = c.get_blockdb_partition(osd_uuid2) + blockwal_partition = c.get_blockwal_partition(osd_uuid2) + # + # Verify the previous OSD partition has been reused + # + assert blockdb_partition['path'] == blockdb_path + assert blockwal_partition['path'] == blockwal_path + c.destroy_osd(osd_uuid2) + c.sh("ceph-disk --verbose zap " + block_disk + " " + db_wal_disk) def test_activate_with_journal_dev_is_symlink(self): c = CephDisk() diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py index e2922ef66e5..2328c969850 100755 --- a/src/ceph-disk/ceph_disk/main.py +++ b/src/ceph-disk/ceph_disk/main.py @@ -57,6 +57,16 @@ PTYPE = { 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106', 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106', }, + 'block.db': { + # identical because creating a block is atomic + 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876', + 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be', + }, + 'block.wal': { + # identical because creating a block is atomic + 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9', + 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be', + }, 'osd': { 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d', 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be', @@ -75,6 +85,14 @@ PTYPE = { 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106', 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be', }, + 'block.db': { + 'ready': '166418da-c469-4022-adf4-b30afd37f176', + 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be', + }, + 'block.wal': { + 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86', + 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be', + }, 'osd': { 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d', 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be', @@ -89,6 +107,14 @@ PTYPE = { 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106', 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be', }, + 'block.db': { + 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3', + 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be', + }, + 'block.wal': { + 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966', + 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be', + }, 'osd': { 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d', 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be', @@ -103,6 +129,14 @@ PTYPE = { 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560', 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560', }, + 'block.db': { + 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261', + 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be', + }, + 'block.wal': { + 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f', + 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be', + }, 'osd': { 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560', 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560', @@ -1805,6 +1839,8 @@ class PrepareBluestore(Prepare): self.lockbox = Lockbox(args) self.data = PrepareBluestoreData(args) self.block = PrepareBluestoreBlock(args) + self.blockdb = PrepareBluestoreBlockDB(args) + self.blockwal = PrepareBluestoreBlockWAL(args) @staticmethod def parser(): @@ -1821,17 +1857,19 @@ class PrepareBluestore(Prepare): return [ PrepareBluestore.parser(), PrepareBluestoreBlock.parser(), + PrepareBluestoreBlockDB.parser(), + PrepareBluestoreBlockWAL.parser(), ] def prepare_locked(self): if self.data.args.dmcrypt: self.lockbox.prepare() - self.data.prepare(self.block) + self.data.prepare(self.blockdb, self.blockwal, self.block) class Space(object): - NAMES = ('block', 'journal') + NAMES = ('block', 'journal', 'block.db', 'block.wal') class PrepareSpace(object): @@ -1903,7 +1941,7 @@ class PrepareSpace(object): return self.type == self.DEVICE @staticmethod - def parser(name): + def parser(name, positional=True): parser = argparse.ArgumentParser(add_help=False) parser.add_argument( '--%s-uuid' % name, @@ -1920,13 +1958,15 @@ class PrepareSpace(object): action='store_true', default=None, help='verify that %s is a block device' % name.upper(), ) - parser.add_argument( - name, - metavar=name.upper(), - nargs='?', - help=('path to OSD %s disk block device;' % name + - ' leave out to store %s in file' % name), - ) + + if positional: + parser.add_argument( + name, + metavar=name.upper(), + nargs='?', + help=('path to OSD %s disk block device;' % name + + ' leave out to store %s in file' % name), + ) return parser def wants_space(self): @@ -2068,17 +2108,18 @@ class PrepareSpace(object): partition.format() partition.map() - command_check_call( - [ - 'sgdisk', - '--typecode={num}:{uuid}'.format( - num=num, - uuid=partition.ptype_for_name(self.name), - ), - '--', - getattr(self.args, self.name), - ], - ) + command_check_call( + [ + 'sgdisk', + '--typecode={num}:{uuid}'.format( + num=num, + uuid=partition.ptype_for_name(self.name), + ), + '--', + getattr(self.args, self.name), + ], + ) + update_partition(getattr(self.args, self.name), 'prepared') LOG.debug('%s is GPT partition %s', self.name.capitalize(), @@ -2128,7 +2169,15 @@ class PrepareBluestoreBlock(PrepareSpace): super(PrepareBluestoreBlock, self).__init__(args) def get_space_size(self): - return 0 # get as much space as possible + block_size = get_conf( + cluster=self.args.cluster, + variable='bluestore_block_size', + ) + + if block_size is None: + return 0 # get as much space as possible + else: + return int(block_size) / 1048576 # MB def desired_partition_number(self): if self.args.block == self.args.data: @@ -2142,6 +2191,76 @@ class PrepareBluestoreBlock(PrepareSpace): return PrepareSpace.parser('block') +class PrepareBluestoreBlockDB(PrepareSpace): + + def __init__(self, args): + self.name = 'block.db' + super(PrepareBluestoreBlockDB, self).__init__(args) + + def get_space_size(self): + block_size = get_conf( + cluster=self.args.cluster, + variable='bluestore_block_db_size', + ) + + if block_size is None: + return 64 # MB, default value + else: + return int(block_size) / 1048576 # MB + + def desired_partition_number(self): + if getattr(self.args, 'block.db') == self.args.data: + num = 3 + else: + num = 0 + return num + + @staticmethod + def parser(): + parser = PrepareSpace.parser('block.db', positional=False) + parser.add_argument( + '--block.db', + metavar='BLOCKDB', + help='path to the device or file for bluestore block.db', + ) + return parser + + +class PrepareBluestoreBlockWAL(PrepareSpace): + + def __init__(self, args): + self.name = 'block.wal' + super(PrepareBluestoreBlockWAL, self).__init__(args) + + def get_space_size(self): + block_size = get_conf( + cluster=self.args.cluster, + variable='bluestore_block_wal_size', + ) + + if block_size is None: + return 128 # MB, default value + else: + return int(block_size) / 1048576 # MB + + def desired_partition_number(self): + if getattr(self.args, 'block.wal') == self.args.data: + num = 4 + else: + num = 0 + return num + + @staticmethod + def parser(): + parser = PrepareSpace.parser('block.wal', positional=False) + parser.add_argument( + '--block.wal', + metavar='BLOCKWAL', + help='path to the device or file for bluestore block.wal', + ) + return parser + + class CryptHelpers(object): @staticmethod @@ -4362,7 +4481,11 @@ def main_trigger(args): ) elif parttype in (PTYPE['regular']['block']['ready'], - PTYPE['mpath']['block']['ready']): + PTYPE['regular']['block.db']['ready'], + PTYPE['regular']['block.wal']['ready'], + PTYPE['mpath']['block']['ready'], + PTYPE['mpath']['block.db']['ready'], + PTYPE['mpath']['block.wal']['ready']): out, err, ret = command( ceph_disk + [ @@ -4372,7 +4495,11 @@ def main_trigger(args): ) elif parttype in (PTYPE['plain']['block']['ready'], - PTYPE['luks']['block']['ready']): + PTYPE['plain']['block.db']['ready'], + PTYPE['plain']['block.wal']['ready'], + PTYPE['luks']['block']['ready'], + PTYPE['luks']['block.db']['ready'], + PTYPE['luks']['block.wal']['ready']): out, err, ret = command( ceph_disk + [ diff --git a/udev/95-ceph-osd.rules b/udev/95-ceph-osd.rules index 5349d0f13a5..d49461bb0ac 100644 --- a/udev/95-ceph-osd.rules +++ b/udev/95-ceph-osd.rules @@ -28,6 +28,26 @@ ACTION=="change", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="cafecafe-9b03-4f30-b4c6-b4b80ceff106", \ OWNER="ceph", GROUP="ceph", MODE="660" +# BLOCK.DB_UUID +ACTION=="add", SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="30cd0809-c2b2-499c-8879-2d6b78529876", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="30cd0809-c2b2-499c-8879-2d6b78529876", \ + OWNER="ceph", GROUP="ceph", MODE="660" + +# BLOCK.WAL_UUID +ACTION=="add", SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="5ce17fce-4087-4169-b7ff-056cc58473f9", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="5ce17fce-4087-4169-b7ff-056cc58473f9", \ + OWNER="ceph", GROUP="ceph", MODE="660" + # LOCKBOX_UUID ACTION=="add", SUBSYSTEM=="block", \ ENV{DEVTYPE}=="partition", \ @@ -65,6 +85,24 @@ ACTION=="change", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="cafecafe-8ae0-4982-bf9d-5a8d867af560", \ OWNER="ceph", GROUP="ceph", MODE="660" +# MPATH_BLOCK.DB_UUID +ACTION=="add", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="ec6d6385-e346-45dc-be91-da2a7c8b3261", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="ec6d6385-e346-45dc-be91-da2a7c8b3261", \ + OWNER="ceph", GROUP="ceph", MODE="660" + +# MPATH_BLOCK.WAL_UUID +ACTION=="add", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="01b41e1b-002a-453c-9f17-88793989ff8f", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="01b41e1b-002a-453c-9f17-88793989ff8f", \ + OWNER="ceph", GROUP="ceph", MODE="660" + # MPATH_BLOCK_UUID ACTION=="add", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="7f4a666a-16f3-47a2-8445-152ef4d03f6c", \ @@ -94,6 +132,26 @@ ACTION=="change", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="cafecafe-9b03-4f30-b4c6-5ec00ceff106", \ OWNER="ceph", GROUP="ceph", MODE="660" +# DMCRYPT_BLOCK.DB_UUID +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3", \ + OWNER="ceph", GROUP="ceph", MODE="660" + +# DMCRYPT_BLOCK.WAL_UUID +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="306e8683-4fe2-4330-b7c0-00a917c16966", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="306e8683-4fe2-4330-b7c0-00a917c16966", \ + OWNER="ceph", GROUP="ceph", MODE="660" + # DMCRYPT_LUKS_JOURNAL_UUID ACTION=="add" SUBSYSTEM=="block", \ ENV{DEVTYPE}=="partition", \ @@ -114,6 +172,26 @@ ACTION=="change", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="cafecafe-9b03-4f30-b4c6-35865ceff106", \ OWNER="ceph", GROUP="ceph", MODE="660" +# DMCRYPT_LUKS_BLOCK.DB_UUID +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="166418da-c469-4022-adf4-b30afd37f176", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="166418da-c469-4022-adf4-b30afd37f176", \ + OWNER="ceph", GROUP="ceph", MODE="660" + +# DMCRYPT_LUKS_BLOCK.WAL_UUID +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="86a32090-3647-40b9-bbbd-38d8c573aa86", \ + OWNER:="ceph", GROUP:="ceph", MODE:="660", \ + RUN+="/usr/sbin/ceph-disk --log-stdout -v trigger /dev/$name" +ACTION=="change", SUBSYSTEM=="block", \ + ENV{ID_PART_ENTRY_TYPE}=="86a32090-3647-40b9-bbbd-38d8c573aa86", \ + OWNER="ceph", GROUP="ceph", MODE="660" + # DMCRYPT_OID_UUID ACTION=="add" SUBSYSTEM=="block", \ ENV{DEVTYPE}=="partition", \