Merge pull request #35728 from jan--f/c-v-add-subcommand-parse-drive-groups

ceph-volume: add drive-group subcommand

doc/ceph-volume/drive-group.rst

@@ -0,0 +1,12 @@
+.. _ceph-volume-drive-group:
+
+``drive-group``
+===============
+The drive-group subcommand allows for passing :ref:'drivegroups' specifications
+straight to ceph-volume as json. ceph-volume will then attempt to deploy this
+drive groups via the batch subcommand.
+
+The specification can be passed via a file, string argument or on stdin.
+See the subcommand help for further details::
+
+    # ceph-volume drive-group --help

doc/ceph-volume/index.rst

@@ -65,6 +65,7 @@ and ``ceph-disk`` is fully disabled. Encryption is fully supported.
    intro
    systemd
    inventory
+   drive-group
    lvm/index
    lvm/activate
    lvm/batch

src/ceph-volume/ceph_volume/drive_group/__init__.py

@@ -0,0 +1 @@
+from .main import Deploy # noqa

src/ceph-volume/ceph_volume/drive_group/main.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+import json
+import logging
+import sys
+
+from ceph.deployment.drive_group import DriveGroupSpec
+from ceph.deployment.drive_selection.selector import DriveSelection
+from ceph.deployment.translate import to_ceph_volume
+from ceph.deployment.inventory import Device
+from ceph_volume.inventory import Inventory
+from ceph_volume.devices.lvm.batch import Batch
+
+logger = logging.getLogger(__name__)
+
+class Deploy(object):
+
+    help = '''
+    Deploy OSDs according to a drive groups specification.
+
+    The DriveGroup specification must be passed in json.
+    It can either be (preference in this order)
+      - in a file, path passed as a positional argument
+      - read from stdin, pass "-" as a positional argument
+      - a json string passed via the --spec argument
+
+    Either the path postional argument or --spec must be specifed.
+    '''
+
+    def __init__(self, argv):
+        logger.error(f'argv: {argv}')
+        self.argv = argv
+
+    def main(self):
+        parser = argparse.ArgumentParser(
+            prog='ceph-volume drive-group',
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description=self.help,
+        )
+        parser.add_argument(
+            'path',
+            nargs='?',
+            default=None,
+            help=('Path to file containing drive group spec or "-" to read from stdin'),
+        )
+        parser.add_argument(
+            '--spec',
+            default='',
+            nargs='?',
+            help=('drive-group json string')
+        )
+        parser.add_argument(
+            '--dry-run',
+            default=False,
+            action='store_true',
+            help=('dry run, only print the batch command that would be run'),
+        )
+        self.args = parser.parse_args(self.argv)
+        if self.args.path:
+            if self.args.path == "-":
+                commands = self.from_json(sys.stdin)
+            else:
+                with open(self.args.path, 'r') as f:
+                    commands = self.from_json(f)
+        elif self.args.spec:
+            dg = json.loads(self.args.spec)
+            commands = self.get_dg_spec(dg)
+        else:
+            # either --spec or path arg must be specified
+            parser.print_help(sys.stderr)
+            sys.exit(0)
+        cmd = commands.run()
+        if not cmd:
+            logger.error('DriveGroup didn\'t produce any commands')
+            return
+        if self.args.dry_run:
+            logger.info('Returning ceph-volume command (--dry-run was passed): {}'.format(cmd))
+            print(cmd)
+        else:
+            logger.info('Running ceph-volume command: {}'.format(cmd))
+            batch_args = cmd.split(' ')[2:]
+            b = Batch(batch_args)
+            b.main()
+
+    def from_json(self, file_):
+        dg = {}
+        dg = json.load(file_)
+        return self.get_dg_spec(dg)
+
+    def get_dg_spec(self, dg):
+        dg_spec = DriveGroupSpec._from_json_impl(dg)
+        dg_spec.validate()
+        i = Inventory([])
+        i.main()
+        inventory = i.get_report()
+        devices = [Device.from_json(i) for i in inventory]
+        selection = DriveSelection(dg_spec, devices)
+        return to_ceph_volume(selection)

src/ceph-volume/ceph_volume/inventory/main.py

@@ -37,6 +37,12 @@ class Inventory(object):
         else:
             self.format_report(Devices())
 
+    def get_report(self):
+        if self.args.path:
+            return Device(self.args.path).json_report()
+        else:
+            return Devices().json_report()
+
     def format_report(self, inventory):
         if self.args.format == 'json':
             print(json.dumps(inventory.json_report()))

src/ceph-volume/ceph_volume/main.py

@@ -6,7 +6,7 @@ import sys
 import logging
 
 from ceph_volume.decorators import catches
-from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory
+from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory, drive_group
 
 
 class Volume(object):
@@ -29,6 +29,7 @@ Ceph Conf: {ceph_path}
             'simple': devices.simple.Simple,
             'raw': devices.raw.Raw,
             'inventory': inventory.Inventory,
+            'drive-group': drive_group.Deploy,
         }
         self.plugin_help = "No plugins found/loaded"
         if argv is None:

src/ceph-volume/setup.py

@@ -1,4 +1,5 @@
 from setuptools import setup, find_packages
+import os
 
 
 setup(
@@ -13,6 +14,9 @@ setup(
     keywords='ceph volume disk devices lvm',
     url="https://github.com/ceph/ceph",
     zip_safe = False,
+    install_requires='ceph',
+    dependency_links=[''.join(['file://', os.path.join(os.getcwd(), '../',
+                                                       'python-common#egg=ceph-1.0.0')])],
     tests_require=[
         'pytest >=2.1.3',
         'tox',

src/python-common/.gitignore

@@ -1,2 +1,3 @@
 ceph.egg-info
 build
+setup.cfg

src/python-common/ceph/deployment/drive_group.py

@@ -191,7 +191,7 @@ class DriveGroupSpec(ServiceSpec):
         #: Set (or override) the "bluestore_block_db_size" value, in bytes
         self.block_db_size = block_db_size
 
-        #: set journal_size is bytes
+        #: set journal_size in bytes
         self.journal_size = journal_size
 
         #: Number of osd daemons per "DATA" device.

src/python-common/ceph/deployment/translate.py

@@ -10,6 +10,7 @@ from ceph.deployment.drive_selection.selector import DriveSelection
 logger = logging.getLogger(__name__)
 
 
+# TODO refactor this to a DriveSelection method
 class to_ceph_volume(object):
 
     def __init__(self,