Merge pull request #39648 from liewegas/cephadm-spec-config

mgr/cephadm: add config section to ServiceSpec

Reviewed-by: Sebastian Wagner <sebastian.wagner@suse.com>

src/pybind/mgr/cephadm/serve.py

@@ -25,6 +25,9 @@ from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
 from cephadm.schedule import HostAssignment
 from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest, \
     CephadmNoImage, CEPH_UPGRADE_ORDER, ContainerInspectInfo
+from mgr_module import MonCommandFailed
+
+from . import utils
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -448,6 +451,34 @@ class CephadmServe:
 
         return r
 
+    def _apply_service_config(self, spec: ServiceSpec) -> None:
+        if spec.config:
+            section = utils.name_to_config_section(spec.service_name())
+            for k, v in spec.config.items():
+                try:
+                    current = self.mgr.get_foreign_ceph_option(section, k)
+                except KeyError:
+                    self.log.warning(
+                        f'Ignoring invalid {spec.service_name()} config option {k}'
+                    )
+                    self.mgr.events.for_service(
+                        spec, OrchestratorEvent.ERROR, f'Invalid config option {k}'
+                    )
+                    continue
+                if current != v:
+                    self.log.debug(f'setting [{section}] {k} = {v}')
+                    try:
+                        self.mgr.check_mon_command({
+                            'prefix': 'config set',
+                            'name': k,
+                            'value': str(v),
+                            'who': section,
+                        })
+                    except MonCommandFailed as e:
+                        self.log.warning(
+                            f'Failed to set {spec.service_name()} option {k}: {e}'
+                        )
+
     def _apply_service(self, spec: ServiceSpec) -> bool:
         """
         Schedule a service.  Deploy new daemons or remove old ones, depending
@@ -465,6 +496,8 @@ class CephadmServe:
             return False
         self.log.debug('Applying service %s spec' % service_name)
 
+        self._apply_service_config(spec)
+
         if service_type == 'osd':
             self.mgr.osd_service.create_from_spec(cast(DriveGroupSpec, spec))
             # TODO: return True would result in a busy loop

src/pybind/mgr/orchestrator/module.py

@@ -10,7 +10,8 @@ from prettytable import PrettyTable
 
 from ceph.deployment.inventory import Device
 from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection
-from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
+from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, \
+    ServiceSpecValidationError
 from ceph.utils import datetime_now
 
 from mgr_util import to_pretty_timedelta, format_dimless
@@ -1062,6 +1063,15 @@ Usage:
             specs: List[Union[ServiceSpec, HostSpec]] = []
             for s in content:
                 spec = json_to_generic_spec(s)
+
+                # validate the config (we need MgrModule for that)
+                if isinstance(spec, ServiceSpec) and spec.config:
+                    for k, v in spec.config.items():
+                        try:
+                            self.get_foreign_ceph_option('mon', k)
+                        except KeyError:
+                            raise ServiceSpecValidationError(f'Invalid config option {k} in spec')
+
                 if dry_run and not isinstance(spec, HostSpec):
                     spec.preview_only = dry_run
                 specs.append(spec)

src/python-common/ceph/deployment/service_spec.py

@@ -391,6 +391,9 @@ class ServiceSpec(object):
                           'node-exporter osd prometheus rbd-mirror rgw ' \
                           'container cephadm-exporter ha-rgw'.split()
     REQUIRES_SERVICE_ID = 'iscsi mds nfs osd rgw container ha-rgw '.split()
+    MANAGED_CONFIG_OPTIONS = [
+        'mds_join_fs',
+    ]
 
     @classmethod
     def _cls(cls: Type[ServiceSpecT], service_type: str) -> Type[ServiceSpecT]:
@@ -429,6 +432,7 @@ class ServiceSpec(object):
                  service_id: Optional[str] = None,
                  placement: Optional[PlacementSpec] = None,
                  count: Optional[int] = None,
+                 config: Optional[Dict[str, str]] = None,
                  unmanaged: bool = False,
                  preview_only: bool = False,
                  ):
@@ -442,6 +446,10 @@ class ServiceSpec(object):
         self.unmanaged = unmanaged
         self.preview_only = preview_only
 
+        self.config: Optional[Dict[str, str]] = None
+        if config:
+            self.config = {k.replace(' ', '_'): v for k, v in config.items()}
+
     @classmethod
     @handle_type_error
     def from_json(cls: Type[ServiceSpecT], json_spec: Dict) -> ServiceSpecT:
@@ -465,6 +473,8 @@ class ServiceSpec(object):
 
             service_type: nfs
             service_id: foo
+            config:
+              some_option: the_value
             spec:
               pool: mypool
               namespace: myns
@@ -556,6 +566,12 @@ class ServiceSpec(object):
 
         if self.placement is not None:
             self.placement.validate()
+        if self.config:
+            for k, v in self.config.items():
+                if k in self.MANAGED_CONFIG_OPTIONS:
+                    raise ServiceSpecValidationError(
+                        f'Cannot set config option {k} in spec: it is managed by cephadm'
+                    )
 
     def __repr__(self) -> str:
         return "{}({!r})".format(self.__class__.__name__, self.__dict__)
@@ -584,12 +600,14 @@ class NFSServiceSpec(ServiceSpec):
                  namespace: Optional[str] = None,
                  placement: Optional[PlacementSpec] = None,
                  unmanaged: bool = False,
-                 preview_only: bool = False
+                 preview_only: bool = False,
+                 config: Optional[Dict[str, str]] = None,
                  ):
         assert service_type == 'nfs'
         super(NFSServiceSpec, self).__init__(
             'nfs', service_id=service_id,
-            placement=placement, unmanaged=unmanaged, preview_only=preview_only)
+            placement=placement, unmanaged=unmanaged, preview_only=preview_only,
+            config=config)
 
         #: RADOS pool where NFS client recovery data is stored.
         self.pool = pool
@@ -627,6 +645,12 @@ class RGWSpec(ServiceSpec):
     Settings to configure a (multisite) Ceph RGW
 
     """
+    MANAGED_CONFIG_OPTIONS = ServiceSpec.MANAGED_CONFIG_OPTIONS + [
+        'rgw_zone',
+        'rgw_realm',
+        'rgw_frontends',
+    ]
+
     def __init__(self,
                  service_type: str = 'rgw',
                  service_id: Optional[str] = None,
@@ -640,6 +664,7 @@ class RGWSpec(ServiceSpec):
                  unmanaged: bool = False,
                  ssl: bool = False,
                  preview_only: bool = False,
+                 config: Optional[Dict[str, str]] = None,
                  ):
         assert service_type == 'rgw', service_type
         if service_id:
@@ -657,7 +682,7 @@ class RGWSpec(ServiceSpec):
         super(RGWSpec, self).__init__(
             'rgw', service_id=service_id,
             placement=placement, unmanaged=unmanaged,
-            preview_only=preview_only)
+            preview_only=preview_only, config=config)
 
         self.rgw_realm = rgw_realm
         self.rgw_zone = rgw_zone
@@ -713,12 +738,14 @@ class IscsiServiceSpec(ServiceSpec):
                  ssl_key: Optional[str] = None,
                  placement: Optional[PlacementSpec] = None,
                  unmanaged: bool = False,
-                 preview_only: bool = False
+                 preview_only: bool = False,
+                 config: Optional[Dict[str, str]] = None,
                  ):
         assert service_type == 'iscsi'
         super(IscsiServiceSpec, self).__init__('iscsi', service_id=service_id,
                                                placement=placement, unmanaged=unmanaged,
-                                               preview_only=preview_only)
+                                               preview_only=preview_only,
+                                               config=config)
 
         #: RADOS pool where ceph-iscsi config data is stored.
         self.pool = pool
@@ -758,12 +785,13 @@ class AlertManagerSpec(ServiceSpec):
                  unmanaged: bool = False,
                  preview_only: bool = False,
                  user_data: Optional[Dict[str, Any]] = None,
+                 config: Optional[Dict[str, str]] = None,
                  ):
         assert service_type == 'alertmanager'
         super(AlertManagerSpec, self).__init__(
             'alertmanager', service_id=service_id,
             placement=placement, unmanaged=unmanaged,
-            preview_only=preview_only)
+            preview_only=preview_only, config=config)
 
         # Custom configuration.
         #
@@ -789,6 +817,7 @@ class HA_RGWSpec(ServiceSpec):
     def __init__(self,
                  service_type: str = 'ha-rgw',
                  service_id: Optional[str] = None,
+                 config: Optional[Dict[str, str]] = None,
                  placement: Optional[PlacementSpec] = None,
                  virtual_ip_interface: Optional[str] = None,
                  virtual_ip_address: Optional[str] = None,
@@ -810,7 +839,8 @@ class HA_RGWSpec(ServiceSpec):
                  definitive_host_list: Optional[List[HostPlacementSpec]] = None
                  ):
         assert service_type == 'ha-rgw'
-        super(HA_RGWSpec, self).__init__('ha-rgw', service_id=service_id, placement=placement)
+        super(HA_RGWSpec, self).__init__('ha-rgw', service_id=service_id,
+                                         placement=placement, config=config)
 
         self.virtual_ip_interface = virtual_ip_interface
         self.virtual_ip_address = virtual_ip_address
@@ -878,6 +908,7 @@ class CustomContainerSpec(ServiceSpec):
     def __init__(self,
                  service_type: str = 'container',
                  service_id: Optional[str] = None,
+                 config: Optional[Dict[str, str]] = None,
                  placement: Optional[PlacementSpec] = None,
                  unmanaged: bool = False,
                  preview_only: bool = False,
@@ -901,7 +932,7 @@ class CustomContainerSpec(ServiceSpec):
         super(CustomContainerSpec, self).__init__(
             service_type, service_id,
             placement=placement, unmanaged=unmanaged,
-            preview_only=preview_only)
+            preview_only=preview_only, config=config)
 
         self.image = image
         self.entrypoint = entrypoint