ceph/doc/_ext/ceph_commands.py

import io
import os
import sys
import contextlib

from docutils.parsers.rst import directives
from docutils.parsers.rst import Directive
from jinja2 import Template
from pcpp.preprocessor import Preprocessor
from sphinx.util import logging
from sphinx.util.console import bold
from importlib import reload

logger = logging.getLogger(__name__)


class Flags:
    NOFORWARD = (1 << 0)
    OBSOLETE = (1 << 1)
    DEPRECATED = (1 << 2)
    MGR = (1 << 3)
    POLL = (1 << 4)
    HIDDEN = (1 << 5)

    VALS = {
        NOFORWARD: 'no_forward',
        OBSOLETE: 'obsolete',
        DEPRECATED: 'deprecated',
        MGR: 'mgr',
        POLL: 'poll',
        HIDDEN: 'hidden',
    }

    def __init__(self, fs):
        self.fs = fs

    def __contains__(self, other):
        return other in str(self)

    def __str__(self):
        keys = Flags.VALS.keys()
        es = {Flags.VALS[k] for k in keys if self.fs & k == k}
        return ', '.join(sorted(es))

    def __bool__(self):
        return bool(str(self))


class CmdParam(object):
    t = {
        'CephInt': 'int',
        'CephString': 'str',
        'CephChoices': 'str',
        'CephPgid': 'str',
        'CephOsdName': 'str',
        'CephPoolname': 'str',
        'CephObjectname': 'str',
        'CephUUID': 'str',
        'CephEntityAddr': 'str',
        'CephIPAddr': 'str',
        'CephName': 'str',
        'CephBool': 'bool',
        'CephFloat': 'float',
        'CephFilepath': 'str',
    }

    bash_example = {
        'CephInt': '1',
        'CephString': 'string',
        'CephChoices': 'choice',
        'CephPgid': '0',
        'CephOsdName': 'osd.0',
        'CephPoolname': 'poolname',
        'CephObjectname': 'objectname',
        'CephUUID': 'uuid',
        'CephEntityAddr': 'entityaddr',
        'CephIPAddr': '0.0.0.0',
        'CephName': 'name',
        'CephBool': 'true',
        'CephFloat': '0.0',
        'CephFilepath': '/path/to/file',
    }

    def __init__(self, type, name,
                 who=None, n=None, req=True, range=None, strings=None,
                 goodchars=None, positional=True):
        self.type = type
        self.name = name
        self.who = who
        self.n = n == 'N'
        self.req = req != 'false'
        self.range = range.split('|') if range else []
        self.strings = strings.split('|') if strings else []
        self.goodchars = goodchars
        self.positional = positional != 'false'

        assert who == None

    def help(self):
        advanced = []
        if self.type != 'CephString':
            advanced.append(self.type + ' ')
        if self.range:
            advanced.append('range= ``{}`` '.format('..'.join(self.range)))
        if self.strings:
            advanced.append('strings=({}) '.format(' '.join(self.strings)))
        if self.goodchars:
            advanced.append('goodchars= ``{}`` '.format(self.goodchars))
        if self.n:
            advanced.append('(can be repeated)')

        advanced = advanced or ["(string)"]
        return ' '.join(advanced)

    def mk_example_value(self):
        if self.type == 'CephChoices' and self.strings:
            return self.strings[0]
        if self.range:
            return self.range[0]
        return CmdParam.bash_example[self.type]

    def mk_bash_example(self, simple):
        val = self.mk_example_value()

        if self.type == 'CephBool':
            return '--' + self.name
        if simple:
            if self.type == "CephChoices" and self.strings:
                return val
            elif self.type == "CephString" and self.name != 'who':
                return 'my_' + self.name
            else:
                return CmdParam.bash_example[self.type]
        else:
            return '--{}={}'.format(self.name, val)


class CmdCommand(object):
    def __init__(self, prefix, args, desc,
                 module=None, perm=None, flags=0, poll=None):
        self.prefix = prefix
        self.params = sorted([CmdParam(**arg) for arg in args],
                             key=lambda p: p.req, reverse=True)
        self.help = desc
        self.module = module
        self.perm = perm
        self.flags = Flags(flags)
        self.needs_overload = False

    def is_reasonably_simple(self):
        if len(self.params) > 3:
            return False
        if any(p.n for p in self.params):
            return False
        return True

    def mk_bash_example(self):
        simple = self.is_reasonably_simple()
        line = ' '.join(['ceph', self.prefix] + [p.mk_bash_example(simple) for p in self.params])
        return line


class Sig:
    @staticmethod
    def _parse_arg_desc(desc):
        try:
            return dict(kv.split('=', 1) for kv in desc.split(',') if kv)
        except ValueError:
            return desc

    @staticmethod
    def parse_cmd(cmd):
        parsed = [Sig._parse_arg_desc(s) or s for s in cmd.split()]
        prefix = [s for s in parsed if isinstance(s, str)]
        params = [s for s in parsed if not isinstance(s, str)]
        return ' '.join(prefix), params

    @staticmethod
    def parse_args(args):
        return [Sig._parse_arg_desc(arg) for arg in args.split()]


TEMPLATE = '''
{%- set punct_char = '-' -%}
{# add a header if we have multiple commands in this section #}
{% if commands | length > 1 %}
{{ section }}
{{ section | length * '-' }}
{# and demote the subsection #}
{% set punct_char = '^' %}
{% endif %}
{% for command in commands %}
{{ command.prefix }}
{{ command.prefix | length * punct_char }}

{{ command.help | wordwrap(70) }}

:Example command:
    .. code-block:: bash

       {{ command.mk_bash_example() | wordwrap(70) | indent(9) }}

{%- if command.params %}
:Parameters:{% for param in command.params -%}
{{" -" | indent(12, not loop.first) }} **{% if param.positional %}{{param.name}}{% else %}--{{param.name}}{% endif %}**: {{ param.help() }}
{% endfor %}
{% endif %}
:Ceph Module: {{ command.module }}
:Required Permissions: ``{{ command.perm }}``
{%- if command.flags %}
:Command Flags: ``{{ command.flags }}``
{% endif %}
{% endfor %}
'''


def group_by_prefix(commands):
    last_prefix = None
    grouped = []
    for cmd in commands:
        prefix = cmd.prefix.split(' ', 1)[0]
        if prefix == last_prefix:
            grouped.append(cmd)
        elif last_prefix is None:
            last_prefix = prefix
            grouped = [cmd]
        else:
            yield last_prefix, grouped
            last_prefix = prefix
            grouped = [cmd]
    assert grouped
    yield last_prefix, grouped


def render_commands(commands):
    rendered = io.StringIO()
    for section, grouped in group_by_prefix(commands):
        logger.debug('rendering commands: %s: %d', section, len(grouped))
        rendered.write(Template(TEMPLATE).render(
            section=section,
            commands=grouped))
    return rendered.getvalue().split('\n')


class CephMgrCommands(Directive):
    """
    extracts commands from specified mgr modules
    """
    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec = {'python_path': directives.unchanged}

    def _normalize_path(self, dirname):
        my_dir = os.path.dirname(os.path.realpath(__file__))
        src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
        return os.path.join(src_dir, dirname)

    def _is_mgr_module(self, dirname, name):
        if not os.path.isdir(os.path.join(dirname, name)):
            return False
        if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
            return False
        return name not in ['tests']

    @contextlib.contextmanager
    def mocked_modules(self):
        # src/pybind/mgr/tests
        from tests import mock
        mock_imports = ['rados',
                        'rbd',
                        'cephfs',
                        'dateutil',
                        'dateutil.parser']
        # make dashboard happy
        mock_imports += ['OpenSSL',
                         'jwt',
                         'bcrypt',
                         'jsonpatch',
                         'rook.rook_client',
                         'rook.rook_client.ceph',
                         'rook.rook_client._helper',
                         'cherrypy=3.2.3']
        # make diskprediction_local happy
        mock_imports += ['numpy',
                         'scipy']
        # make restful happy
        mock_imports += ['pecan',
                         'pecan.rest',
                         'pecan.hooks',
                         'werkzeug',
                         'werkzeug.serving']

        for m in mock_imports:
            args = {}
            parts = m.split('=', 1)
            mocked = parts[0]
            if len(parts) > 1:
                args['__version__'] = parts[1]
            sys.modules[mocked] = mock.Mock(**args)

        try:
            yield
        finally:
            for m in mock_imports:
                mocked = m.split('=', 1)[0]
                sys.modules.pop(mocked)

    def _collect_module_commands(self, name):
        with self.mocked_modules():
            logger.info(bold(f"loading mgr module '{name}'..."))
            mgr_mod = __import__(name, globals(), locals(), [], 0)
            reload(mgr_mod)
            from tests import M

            def subclass(x):
                try:
                    return issubclass(x, M)
                except TypeError:
                    return False
            ms = [c for c in mgr_mod.__dict__.values()
                  if subclass(c) and 'Standby' not in c.__name__]
            [m] = ms
            assert isinstance(m.COMMANDS, list)
            return m.COMMANDS

    def _normalize_command(self, command):
        if 'handler' in command:
            del command['handler']
        if 'cmd' in command:
            command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
            del command['cmd']
        else:
            command['args'] = Sig.parse_args(command['args'])
        command['flags'] = (1 << 3)
        command['module'] = 'mgr'
        return command

    def _render_cmds(self, commands):
        lines = render_commands(commands)
        assert lines
        lineno = self.lineno - self.state_machine.input_offset - 1
        source = self.state_machine.input_lines.source(lineno)
        self.state_machine.insert_input(lines, source)

    def run(self):
        module_path = self._normalize_path(self.arguments[0])
        sys.path.insert(0, module_path)
        for path in self.options.get('python_path', '').split(':'):
            sys.path.insert(0, self._normalize_path(path))
        os.environ['UNITTEST'] = 'true'
        modules = [name for name in os.listdir(module_path)
                   if self._is_mgr_module(module_path, name)]
        commands = sum([self._collect_module_commands(name) for name in modules], [])
        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
        self._render_cmds(cmds)

        orig_rgw_mod = sys.modules['pybind_rgw_mod']
        sys.modules['rgw'] = orig_rgw_mod

        return []


class MyProcessor(Preprocessor):
    def __init__(self):
        super().__init__()
        self.cmds = []
        self.undef('__DATE__')
        self.undef('__TIME__')
        self.expand_linemacro = False
        self.expand_filemacro = False
        self.expand_countermacro = False
        self.line_directive = '#line'
        self.define("__PCPP_VERSION__ " + '')
        self.define("__PCPP_ALWAYS_FALSE__ 0")
        self.define("__PCPP_ALWAYS_TRUE__ 1")

    def eval(self, src):
        _cmds = []

        NONE = 0
        NOFORWARD = (1 << 0)
        OBSOLETE = (1 << 1)
        DEPRECATED = (1 << 2)
        MGR = (1 << 3)
        POLL = (1 << 4)
        HIDDEN = (1 << 5)
        TELL = (1 << 6)

        def FLAG(a):
            return a

        def COMMAND(cmd, desc, module, perm):
            _cmds.append({
                'cmd': cmd,
                'desc': desc,
                'module': module,
                'perm': perm
            })

        def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
            _cmds.append({
                'cmd': cmd,
                'desc': desc,
                'module': module,
                'perm': perm,
                'flags': flag
            })

        self.parse(src)
        out = io.StringIO()
        self.write(out)
        out.seek(0)
        s = out.read()
        exec(s, globals(), locals())
        return _cmds


class CephMonCommands(Directive):
    """
    extracts commands from specified header file
    """
    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True

    def _src_dir(self):
        my_dir = os.path.dirname(os.path.realpath(__file__))
        return os.path.abspath(os.path.join(my_dir, '../..'))

    def _parse_headers(self, headers):
        src_dir = self._src_dir()
        src = '\n'.join(f'#include "{src_dir}/{header}"' for header in headers)
        return MyProcessor().eval(src)

    def _normalize_command(self, command):
        if 'handler' in command:
            del command['handler']
        command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
        del command['cmd']
        return command

    def _render_cmds(self, commands):
        lines = render_commands(commands)
        assert lines
        lineno = self.lineno - self.state_machine.input_offset - 1
        source = self.state_machine.input_lines.source(lineno)
        self.state_machine.insert_input(lines, source)

    def run(self):
        headers = self.arguments[0].split()
        commands = self._parse_headers(headers)
        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
        self._render_cmds(cmds)
        return []


def setup(app):
    app.add_directive("ceph-mgr-commands", CephMgrCommands)
    app.add_directive("ceph-mon-commands", CephMonCommands)

    return {
        'version': '0.1',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }