mirror of
https://github.com/ceph/ceph
synced 2024-12-30 07:23:11 +00:00
474 lines
14 KiB
Python
474 lines
14 KiB
Python
import io
|
|
import os
|
|
import sys
|
|
import contextlib
|
|
|
|
from docutils.parsers.rst import directives
|
|
from docutils.parsers.rst import Directive
|
|
from jinja2 import Template
|
|
from pcpp.preprocessor import Preprocessor
|
|
from sphinx.util import logging
|
|
from sphinx.util.console import bold
|
|
from importlib import reload
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Flags:
|
|
NOFORWARD = (1 << 0)
|
|
OBSOLETE = (1 << 1)
|
|
DEPRECATED = (1 << 2)
|
|
MGR = (1 << 3)
|
|
POLL = (1 << 4)
|
|
HIDDEN = (1 << 5)
|
|
|
|
VALS = {
|
|
NOFORWARD: 'no_forward',
|
|
OBSOLETE: 'obsolete',
|
|
DEPRECATED: 'deprecated',
|
|
MGR: 'mgr',
|
|
POLL: 'poll',
|
|
HIDDEN: 'hidden',
|
|
}
|
|
|
|
def __init__(self, fs):
|
|
self.fs = fs
|
|
|
|
def __contains__(self, other):
|
|
return other in str(self)
|
|
|
|
def __str__(self):
|
|
keys = Flags.VALS.keys()
|
|
es = {Flags.VALS[k] for k in keys if self.fs & k == k}
|
|
return ', '.join(sorted(es))
|
|
|
|
def __bool__(self):
|
|
return bool(str(self))
|
|
|
|
|
|
class CmdParam(object):
|
|
t = {
|
|
'CephInt': 'int',
|
|
'CephString': 'str',
|
|
'CephChoices': 'str',
|
|
'CephPgid': 'str',
|
|
'CephOsdName': 'str',
|
|
'CephPoolname': 'str',
|
|
'CephObjectname': 'str',
|
|
'CephUUID': 'str',
|
|
'CephEntityAddr': 'str',
|
|
'CephIPAddr': 'str',
|
|
'CephName': 'str',
|
|
'CephBool': 'bool',
|
|
'CephFloat': 'float',
|
|
'CephFilepath': 'str',
|
|
}
|
|
|
|
bash_example = {
|
|
'CephInt': '1',
|
|
'CephString': 'string',
|
|
'CephChoices': 'choice',
|
|
'CephPgid': '0',
|
|
'CephOsdName': 'osd.0',
|
|
'CephPoolname': 'poolname',
|
|
'CephObjectname': 'objectname',
|
|
'CephUUID': 'uuid',
|
|
'CephEntityAddr': 'entityaddr',
|
|
'CephIPAddr': '0.0.0.0',
|
|
'CephName': 'name',
|
|
'CephBool': 'true',
|
|
'CephFloat': '0.0',
|
|
'CephFilepath': '/path/to/file',
|
|
}
|
|
|
|
def __init__(self, type, name,
|
|
who=None, n=None, req=True, range=None, strings=None,
|
|
goodchars=None, positional=True):
|
|
self.type = type
|
|
self.name = name
|
|
self.who = who
|
|
self.n = n == 'N'
|
|
self.req = req != 'false'
|
|
self.range = range.split('|') if range else []
|
|
self.strings = strings.split('|') if strings else []
|
|
self.goodchars = goodchars
|
|
self.positional = positional != 'false'
|
|
|
|
assert who == None
|
|
|
|
def help(self):
|
|
advanced = []
|
|
if self.type != 'CephString':
|
|
advanced.append(self.type + ' ')
|
|
if self.range:
|
|
advanced.append('range= ``{}`` '.format('..'.join(self.range)))
|
|
if self.strings:
|
|
advanced.append('strings=({}) '.format(' '.join(self.strings)))
|
|
if self.goodchars:
|
|
advanced.append('goodchars= ``{}`` '.format(self.goodchars))
|
|
if self.n:
|
|
advanced.append('(can be repeated)')
|
|
|
|
advanced = advanced or ["(string)"]
|
|
return ' '.join(advanced)
|
|
|
|
def mk_example_value(self):
|
|
if self.type == 'CephChoices' and self.strings:
|
|
return self.strings[0]
|
|
if self.range:
|
|
return self.range[0]
|
|
return CmdParam.bash_example[self.type]
|
|
|
|
def mk_bash_example(self, simple):
|
|
val = self.mk_example_value()
|
|
|
|
if self.type == 'CephBool':
|
|
return '--' + self.name
|
|
if simple:
|
|
if self.type == "CephChoices" and self.strings:
|
|
return val
|
|
elif self.type == "CephString" and self.name != 'who':
|
|
return 'my_' + self.name
|
|
else:
|
|
return CmdParam.bash_example[self.type]
|
|
else:
|
|
return '--{}={}'.format(self.name, val)
|
|
|
|
|
|
class CmdCommand(object):
|
|
def __init__(self, prefix, args, desc,
|
|
module=None, perm=None, flags=0, poll=None):
|
|
self.prefix = prefix
|
|
self.params = sorted([CmdParam(**arg) for arg in args],
|
|
key=lambda p: p.req, reverse=True)
|
|
self.help = desc
|
|
self.module = module
|
|
self.perm = perm
|
|
self.flags = Flags(flags)
|
|
self.needs_overload = False
|
|
|
|
def is_reasonably_simple(self):
|
|
if len(self.params) > 3:
|
|
return False
|
|
if any(p.n for p in self.params):
|
|
return False
|
|
return True
|
|
|
|
def mk_bash_example(self):
|
|
simple = self.is_reasonably_simple()
|
|
line = ' '.join(['ceph', self.prefix] + [p.mk_bash_example(simple) for p in self.params])
|
|
return line
|
|
|
|
|
|
class Sig:
|
|
@staticmethod
|
|
def _parse_arg_desc(desc):
|
|
try:
|
|
return dict(kv.split('=', 1) for kv in desc.split(',') if kv)
|
|
except ValueError:
|
|
return desc
|
|
|
|
@staticmethod
|
|
def parse_cmd(cmd):
|
|
parsed = [Sig._parse_arg_desc(s) or s for s in cmd.split()]
|
|
prefix = [s for s in parsed if isinstance(s, str)]
|
|
params = [s for s in parsed if not isinstance(s, str)]
|
|
return ' '.join(prefix), params
|
|
|
|
@staticmethod
|
|
def parse_args(args):
|
|
return [Sig._parse_arg_desc(arg) for arg in args]
|
|
|
|
|
|
TEMPLATE = '''
|
|
{%- set punct_char = '-' -%}
|
|
{# add a header if we have multiple commands in this section #}
|
|
{% if commands | length > 1 %}
|
|
{{ section }}
|
|
{{ section | length * '-' }}
|
|
{# and demote the subsection #}
|
|
{% set punct_char = '^' %}
|
|
{% endif %}
|
|
{% for command in commands %}
|
|
{{ command.prefix }}
|
|
{{ command.prefix | length * punct_char }}
|
|
|
|
{{ command.help | wordwrap(70) }}
|
|
|
|
:Example command:
|
|
.. code-block:: bash
|
|
|
|
{{ command.mk_bash_example() | wordwrap(70) | indent(9) }}
|
|
|
|
{%- if command.params %}
|
|
:Parameters:{% for param in command.params -%}
|
|
{{" -" | indent(12, not loop.first) }} **{% if param.positional %}{{param.name}}{% else %}--{{param.name}}{% endif %}**: {{ param.help() }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
:Ceph Module: {{ command.module }}
|
|
:Required Permissions: ``{{ command.perm }}``
|
|
{%- if command.flags %}
|
|
:Command Flags: ``{{ command.flags }}``
|
|
{% endif %}
|
|
{% endfor %}
|
|
'''
|
|
|
|
|
|
def group_by_prefix(commands):
|
|
last_prefix = None
|
|
grouped = []
|
|
for cmd in commands:
|
|
prefix = cmd.prefix.split(' ', 1)[0]
|
|
if prefix == last_prefix:
|
|
grouped.append(cmd)
|
|
elif last_prefix is None:
|
|
last_prefix = prefix
|
|
grouped = [cmd]
|
|
else:
|
|
yield last_prefix, grouped
|
|
last_prefix = prefix
|
|
grouped = [cmd]
|
|
assert grouped
|
|
yield last_prefix, grouped
|
|
|
|
|
|
def render_commands(commands):
|
|
rendered = io.StringIO()
|
|
for section, grouped in group_by_prefix(commands):
|
|
logger.debug('rendering commands: %s: %d', section, len(grouped))
|
|
rendered.write(Template(TEMPLATE).render(
|
|
section=section,
|
|
commands=grouped))
|
|
return rendered.getvalue().split('\n')
|
|
|
|
|
|
class CephMgrCommands(Directive):
|
|
"""
|
|
extracts commands from specified mgr modules
|
|
"""
|
|
has_content = True
|
|
required_arguments = 1
|
|
optional_arguments = 0
|
|
final_argument_whitespace = False
|
|
option_spec = {'python_path': directives.unchanged}
|
|
|
|
def _normalize_path(self, dirname):
|
|
my_dir = os.path.dirname(os.path.realpath(__file__))
|
|
src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
|
|
return os.path.join(src_dir, dirname)
|
|
|
|
def _is_mgr_module(self, dirname, name):
|
|
if not os.path.isdir(os.path.join(dirname, name)):
|
|
return False
|
|
if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
|
|
return False
|
|
return name not in ['tests']
|
|
|
|
@contextlib.contextmanager
|
|
def mocked_modules(self):
|
|
# src/pybind/mgr/tests
|
|
from tests import mock
|
|
mock_imports = ['rados',
|
|
'rbd',
|
|
'cephfs',
|
|
'dateutil',
|
|
'dateutil.parser']
|
|
# make dashboard happy
|
|
mock_imports += ['OpenSSL',
|
|
'jwt',
|
|
'bcrypt',
|
|
'jsonpatch',
|
|
'rook.rook_client',
|
|
'rook.rook_client.ceph',
|
|
'rook.rook_client._helper',
|
|
'cherrypy=3.2.3']
|
|
# make diskprediction_local happy
|
|
mock_imports += ['numpy',
|
|
'scipy']
|
|
# make restful happy
|
|
mock_imports += ['pecan',
|
|
'pecan.rest',
|
|
'pecan.hooks',
|
|
'werkzeug',
|
|
'werkzeug.serving']
|
|
|
|
for m in mock_imports:
|
|
args = {}
|
|
parts = m.split('=', 1)
|
|
mocked = parts[0]
|
|
if len(parts) > 1:
|
|
args['__version__'] = parts[1]
|
|
sys.modules[mocked] = mock.Mock(**args)
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
for m in mock_imports:
|
|
mocked = m.split('=', 1)[0]
|
|
sys.modules.pop(mocked)
|
|
|
|
def _collect_module_commands(self, name):
|
|
with self.mocked_modules():
|
|
logger.info(bold(f"loading mgr module '{name}'..."))
|
|
mgr_mod = __import__(name, globals(), locals(), [], 0)
|
|
reload(mgr_mod)
|
|
from tests import M
|
|
|
|
def subclass(x):
|
|
try:
|
|
return issubclass(x, M)
|
|
except TypeError:
|
|
return False
|
|
ms = [c for c in mgr_mod.__dict__.values()
|
|
if subclass(c) and 'Standby' not in c.__name__]
|
|
[m] = ms
|
|
assert isinstance(m.COMMANDS, list)
|
|
return m.COMMANDS
|
|
|
|
def _normalize_command(self, command):
|
|
if 'handler' in command:
|
|
del command['handler']
|
|
if 'cmd' in command:
|
|
command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
|
|
del command['cmd']
|
|
else:
|
|
command['args'] = Sig.parse_args(command['args'])
|
|
command['flags'] = (1 << 3)
|
|
command['module'] = 'mgr'
|
|
return command
|
|
|
|
def _render_cmds(self, commands):
|
|
lines = render_commands(commands)
|
|
assert lines
|
|
lineno = self.lineno - self.state_machine.input_offset - 1
|
|
source = self.state_machine.input_lines.source(lineno)
|
|
self.state_machine.insert_input(lines, source)
|
|
|
|
def run(self):
|
|
module_path = self._normalize_path(self.arguments[0])
|
|
sys.path.insert(0, module_path)
|
|
for path in self.options.get('python_path', '').split(':'):
|
|
sys.path.insert(0, self._normalize_path(path))
|
|
os.environ['UNITTEST'] = 'true'
|
|
modules = [name for name in os.listdir(module_path)
|
|
if self._is_mgr_module(module_path, name)]
|
|
commands = sum([self._collect_module_commands(name) for name in modules], [])
|
|
cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
|
|
cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
|
|
cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
|
|
self._render_cmds(cmds)
|
|
|
|
if 'pybind_rgw_mod' in sys.modules:
|
|
orig_rgw_mod = sys.modules['pybind_rgw_mod']
|
|
sys.modules['rgw'] = orig_rgw_mod
|
|
|
|
return []
|
|
|
|
|
|
class MyProcessor(Preprocessor):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.cmds = []
|
|
self.undef('__DATE__')
|
|
self.undef('__TIME__')
|
|
self.expand_linemacro = False
|
|
self.expand_filemacro = False
|
|
self.expand_countermacro = False
|
|
self.line_directive = '#line'
|
|
self.define("__PCPP_VERSION__ " + '')
|
|
self.define("__PCPP_ALWAYS_FALSE__ 0")
|
|
self.define("__PCPP_ALWAYS_TRUE__ 1")
|
|
|
|
def eval(self, src):
|
|
_cmds = []
|
|
|
|
NONE = 0
|
|
NOFORWARD = (1 << 0)
|
|
OBSOLETE = (1 << 1)
|
|
DEPRECATED = (1 << 2)
|
|
MGR = (1 << 3)
|
|
POLL = (1 << 4)
|
|
HIDDEN = (1 << 5)
|
|
TELL = (1 << 6)
|
|
|
|
def FLAG(a):
|
|
return a
|
|
|
|
def COMMAND(cmd, desc, module, perm):
|
|
_cmds.append({
|
|
'cmd': cmd,
|
|
'desc': desc,
|
|
'module': module,
|
|
'perm': perm
|
|
})
|
|
|
|
def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
|
|
_cmds.append({
|
|
'cmd': cmd,
|
|
'desc': desc,
|
|
'module': module,
|
|
'perm': perm,
|
|
'flags': flag
|
|
})
|
|
|
|
self.parse(src)
|
|
out = io.StringIO()
|
|
self.write(out)
|
|
out.seek(0)
|
|
s = out.read()
|
|
exec(s, globals(), locals())
|
|
return _cmds
|
|
|
|
|
|
class CephMonCommands(Directive):
|
|
"""
|
|
extracts commands from specified header file
|
|
"""
|
|
has_content = True
|
|
required_arguments = 1
|
|
optional_arguments = 0
|
|
final_argument_whitespace = True
|
|
|
|
def _src_dir(self):
|
|
my_dir = os.path.dirname(os.path.realpath(__file__))
|
|
return os.path.abspath(os.path.join(my_dir, '../..'))
|
|
|
|
def _parse_headers(self, headers):
|
|
src_dir = self._src_dir()
|
|
src = '\n'.join(f'#include "{src_dir}/{header}"' for header in headers)
|
|
return MyProcessor().eval(src)
|
|
|
|
def _normalize_command(self, command):
|
|
if 'handler' in command:
|
|
del command['handler']
|
|
command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
|
|
del command['cmd']
|
|
return command
|
|
|
|
def _render_cmds(self, commands):
|
|
lines = render_commands(commands)
|
|
assert lines
|
|
lineno = self.lineno - self.state_machine.input_offset - 1
|
|
source = self.state_machine.input_lines.source(lineno)
|
|
self.state_machine.insert_input(lines, source)
|
|
|
|
def run(self):
|
|
headers = self.arguments[0].split()
|
|
commands = self._parse_headers(headers)
|
|
cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
|
|
cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
|
|
cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
|
|
self._render_cmds(cmds)
|
|
return []
|
|
|
|
|
|
def setup(app):
|
|
app.add_directive("ceph-mgr-commands", CephMgrCommands)
|
|
app.add_directive("ceph-mon-commands", CephMonCommands)
|
|
|
|
return {
|
|
'version': '0.1',
|
|
'parallel_read_safe': True,
|
|
'parallel_write_safe': True,
|
|
}
|