mirror of
https://github.com/ceph/ceph
synced 2024-12-29 06:52:35 +00:00
7210d7f951
if mgr is not active, monitor will refuse to set any option consumed by mgr modules. the reason the tests pass somtimes is that, we have a racing here: 1. stop all mgr daemons 2. MgrMonitor gets updated and updates its mgr_module_options accordingly. 3. in TestDashboard.setUp(), we reset the port number for dashboard using "ceph config set mgr mgr/dashboard/y/ssl_server_port 7789" 4. restart all mgr daemons but the 2nd step and 3rd step could race with each other, if the 2nd step happens after 3rd step, the test passes. otherwise it fails. in this change, "--force" is passed to the "ceph config set" command, so ConfigMonitor can bypass the sanity test for the option, and just set this option. Signed-off-by: Kefu Chai <kchai@redhat.com>
219 lines
7.6 KiB
Python
219 lines
7.6 KiB
Python
import json
|
|
import logging
|
|
|
|
from teuthology import misc
|
|
from tasks.ceph_test_case import CephTestCase
|
|
|
|
# TODO move definition of CephCluster away from the CephFS stuff
|
|
from tasks.cephfs.filesystem import CephCluster
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class MgrCluster(CephCluster):
|
|
def __init__(self, ctx):
|
|
super(MgrCluster, self).__init__(ctx)
|
|
self.mgr_ids = list(misc.all_roles_of_type(ctx.cluster, 'mgr'))
|
|
|
|
if len(self.mgr_ids) == 0:
|
|
raise RuntimeError(
|
|
"This task requires at least one manager daemon")
|
|
|
|
self.mgr_daemons = dict(
|
|
[(mgr_id, self._ctx.daemons.get_daemon('mgr', mgr_id)) for mgr_id
|
|
in self.mgr_ids])
|
|
|
|
def mgr_stop(self, mgr_id):
|
|
self.mgr_daemons[mgr_id].stop()
|
|
|
|
def mgr_fail(self, mgr_id):
|
|
self.mon_manager.raw_cluster_cmd("mgr", "fail", mgr_id)
|
|
|
|
def mgr_restart(self, mgr_id):
|
|
self.mgr_daemons[mgr_id].restart()
|
|
|
|
def get_mgr_map(self):
|
|
return json.loads(
|
|
self.mon_manager.raw_cluster_cmd("mgr", "dump", "--format=json-pretty"))
|
|
|
|
def get_active_id(self):
|
|
return self.get_mgr_map()["active_name"]
|
|
|
|
def get_standby_ids(self):
|
|
return [s['name'] for s in self.get_mgr_map()["standbys"]]
|
|
|
|
def set_module_conf(self, module, key, val):
|
|
self.mon_manager.raw_cluster_cmd("config", "set", "mgr",
|
|
"mgr/{0}/{1}".format(
|
|
module, key
|
|
), val)
|
|
|
|
def set_module_localized_conf(self, module, mgr_id, key, val, force):
|
|
cmd = ["config", "set", "mgr",
|
|
"/".join(["mgr", module, mgr_id, key]),
|
|
val]
|
|
if force:
|
|
cmd.append("--force")
|
|
self.mon_manager.raw_cluster_cmd(*cmd)
|
|
|
|
|
|
class MgrTestCase(CephTestCase):
|
|
MGRS_REQUIRED = 1
|
|
|
|
@classmethod
|
|
def setup_mgrs(cls):
|
|
# Stop all the daemons
|
|
for daemon in cls.mgr_cluster.mgr_daemons.values():
|
|
daemon.stop()
|
|
|
|
for mgr_id in cls.mgr_cluster.mgr_ids:
|
|
cls.mgr_cluster.mgr_fail(mgr_id)
|
|
|
|
# Unload all non-default plugins
|
|
loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
"mgr", "module", "ls"))['enabled_modules']
|
|
unload_modules = set(loaded) - {"restful"}
|
|
|
|
for m in unload_modules:
|
|
cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
"mgr", "module", "disable", m)
|
|
|
|
# Start all the daemons
|
|
for daemon in cls.mgr_cluster.mgr_daemons.values():
|
|
daemon.restart()
|
|
|
|
# Wait for an active to come up
|
|
cls.wait_until_true(lambda: cls.mgr_cluster.get_active_id() != "",
|
|
timeout=20)
|
|
|
|
expect_standbys = set(cls.mgr_cluster.mgr_ids) \
|
|
- {cls.mgr_cluster.get_active_id()}
|
|
cls.wait_until_true(
|
|
lambda: set(cls.mgr_cluster.get_standby_ids()) == expect_standbys,
|
|
timeout=20)
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
# The test runner should have populated this
|
|
assert cls.mgr_cluster is not None
|
|
|
|
if len(cls.mgr_cluster.mgr_ids) < cls.MGRS_REQUIRED:
|
|
cls.skipTest(
|
|
"Only have {0} manager daemons, {1} are required".format(
|
|
len(cls.mgr_cluster.mgr_ids), cls.MGRS_REQUIRED))
|
|
|
|
cls.setup_mgrs()
|
|
|
|
@classmethod
|
|
def _unload_module(cls, module_name):
|
|
def is_disabled():
|
|
enabled_modules = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
'mgr', 'module', 'ls'))['enabled_modules']
|
|
return module_name not in enabled_modules
|
|
|
|
if is_disabled():
|
|
return
|
|
|
|
log.info("Unloading Mgr module %s ...", module_name)
|
|
cls.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'module', 'disable', module_name)
|
|
cls.wait_until_true(is_disabled, timeout=30)
|
|
|
|
@classmethod
|
|
def _load_module(cls, module_name):
|
|
loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
"mgr", "module", "ls"))['enabled_modules']
|
|
if module_name in loaded:
|
|
# The enable command is idempotent, but our wait for a restart
|
|
# isn't, so let's return now if it's already loaded
|
|
return
|
|
|
|
initial_mgr_map = cls.mgr_cluster.get_mgr_map()
|
|
|
|
# check if the the module is configured as an always on module
|
|
mgr_daemons = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
"mgr", "metadata"))
|
|
|
|
for daemon in mgr_daemons:
|
|
if daemon["name"] == initial_mgr_map["active_name"]:
|
|
ceph_version = daemon["ceph_release"]
|
|
always_on = initial_mgr_map["always_on_modules"].get(ceph_version, [])
|
|
if module_name in always_on:
|
|
return
|
|
|
|
log.info("Loading Mgr module %s ...", module_name)
|
|
initial_gid = initial_mgr_map['active_gid']
|
|
cls.mgr_cluster.mon_manager.raw_cluster_cmd(
|
|
"mgr", "module", "enable", module_name, "--force")
|
|
|
|
# Wait for the module to load
|
|
def has_restarted():
|
|
mgr_map = cls.mgr_cluster.get_mgr_map()
|
|
done = mgr_map['active_gid'] != initial_gid and mgr_map['available']
|
|
if done:
|
|
log.info("Restarted after module load (new active {0}/{1})".format(
|
|
mgr_map['active_name'], mgr_map['active_gid']))
|
|
return done
|
|
cls.wait_until_true(has_restarted, timeout=30)
|
|
|
|
|
|
@classmethod
|
|
def _get_uri(cls, service_name):
|
|
# Little dict hack so that I can assign into this from
|
|
# the get_or_none function
|
|
mgr_map = {'x': None}
|
|
|
|
def _get_or_none():
|
|
mgr_map['x'] = cls.mgr_cluster.get_mgr_map()
|
|
result = mgr_map['x']['services'].get(service_name, None)
|
|
return result
|
|
|
|
cls.wait_until_true(lambda: _get_or_none() is not None, 30)
|
|
|
|
uri = mgr_map['x']['services'][service_name]
|
|
|
|
log.info("Found {0} at {1} (daemon {2}/{3})".format(
|
|
service_name, uri, mgr_map['x']['active_name'],
|
|
mgr_map['x']['active_gid']))
|
|
|
|
return uri
|
|
|
|
@classmethod
|
|
def _assign_ports(cls, module_name, config_name, min_port=7789):
|
|
"""
|
|
To avoid the need to run lots of hosts in teuthology tests to
|
|
get different URLs per mgr, we will hand out different ports
|
|
to each mgr here.
|
|
|
|
This is already taken care of for us when running in a vstart
|
|
environment.
|
|
"""
|
|
# Start handing out ports well above Ceph's range.
|
|
assign_port = min_port
|
|
|
|
for mgr_id in cls.mgr_cluster.mgr_ids:
|
|
cls.mgr_cluster.mgr_stop(mgr_id)
|
|
cls.mgr_cluster.mgr_fail(mgr_id)
|
|
|
|
for mgr_id in cls.mgr_cluster.mgr_ids:
|
|
log.info("Using port {0} for {1} on mgr.{2}".format(
|
|
assign_port, module_name, mgr_id
|
|
))
|
|
cls.mgr_cluster.set_module_localized_conf(module_name, mgr_id,
|
|
config_name,
|
|
str(assign_port),
|
|
force=True)
|
|
assign_port += 1
|
|
|
|
for mgr_id in cls.mgr_cluster.mgr_ids:
|
|
cls.mgr_cluster.mgr_restart(mgr_id)
|
|
|
|
def is_available():
|
|
mgr_map = cls.mgr_cluster.get_mgr_map()
|
|
done = mgr_map['available']
|
|
if done:
|
|
log.info("Available after assign ports (new active {0}/{1})".format(
|
|
mgr_map['active_name'], mgr_map['active_gid']))
|
|
return done
|
|
cls.wait_until_true(is_available, timeout=30)
|