Merge pull request #33886 from sebastian-philipp/doc-mon-command-api

doc: Add a generated reference of all mon commands.

Reviewed-by: Ernesto Puerta <epuertat@redhat.com>
Reviewed-by: Zac Dover <zac.dover@gmail.com>

admin/build-doc

@@ -70,6 +70,10 @@ mkdir -p $vdir/lib
 export LD_LIBRARY_PATH="$vdir/lib"
 export PYTHONPATH=$TOPDIR/src/pybind
 
+
+$vdir/bin/python $TOPDIR/doc/scripts/gen_mon_command_api.py > $TOPDIR/doc/api/mon_command_api.rst
+
+
 # FIXME(sileht): I dunno how to pass the include-dirs correctly with pip
 # for build_ext step, it should be:
 # --global-option=build_ext --global-option="--cython-include-dirs $TOPDIR/src/pybind/rados/"

admin/doc-requirements.txt

@@ -6,3 +6,6 @@ Cython
 prettytable
 sphinx-autodoc-typehints
 typed-ast
+pcpp
+-e../src/python-common
+Jinja2

doc/.gitignore

@@ -1,2 +1,3 @@
 /overview.png
 /object_store.png
+/api/mon_command_api.rst

doc/api/index.rst

@@ -39,6 +39,18 @@ Ceph Object Store APIs
 - See `Swift-compatible API`_.
 - See `Admin Ops API`_.
 
+Ceph MON Command API
+====================
+
+- See `Mon command API`_.
+
 .. _S3-compatible API: ../radosgw/s3/
 .. _Swift-compatible API: ../radosgw/swift/
 .. _Admin Ops API: ../radosgw/adminops
+.. _Mon command API: mon_command_api
+
+
+.. toctree::
+    :hidden:
+    
+    mon_command_api

doc/scripts/gen_mon_command_api.py

@@ -0,0 +1,206 @@
+import os
+import json
+import sys
+from subprocess import check_output
+
+from jinja2 import Template
+
+
+class Flags:
+    NOFORWARD = (1 << 0)
+    OBSOLETE = (1 << 1)
+    DEPRECATED = (1 << 2)
+    MGR = (1 << 3)
+    POLL = (1 << 4)
+    HIDDEN = (1 << 5)
+
+    VALS = {
+        NOFORWARD: 'no_forward',
+        OBSOLETE: 'obsolete',
+        DEPRECATED: 'deprecated',
+        MGR: 'mgr',
+        POLL: 'poll',
+        HIDDEN: 'hidden',
+    }
+
+    def __init__(self, fs):
+        self.fs = fs
+
+    def __contains__(self, other):
+        return other in str(self)
+
+    def __str__(self):
+        keys = Flags.VALS.keys()
+        es = {Flags.VALS[k] for k in keys if self.fs & k == k}
+        return ', '.join(sorted(es))
+
+    def __bool__(self):
+        return bool(str(self))
+
+
+class CmdParam(object):
+    t = {
+        'CephInt': 'int',
+        'CephString': 'str',
+        'CephChoices': 'str',
+        'CephPgid': 'str',
+        'CephOsdName': 'str',
+        'CephPoolname': 'str',
+        'CephObjectname': 'str',
+        'CephUUID': 'str',
+        'CephEntityAddr': 'str',
+        'CephIPAddr': 'str',
+        'CephName': 'str',
+        'CephBool': 'bool',
+        'CephFloat': 'float',
+        'CephFilepath': 'str',
+    }
+
+    bash_example = {
+        'CephInt': '1',
+        'CephString': 'string',
+        'CephChoices': 'choice',
+        'CephPgid': '0',
+        'CephOsdName': 'osd.0',
+        'CephPoolname': 'poolname',
+        'CephObjectname': 'objectname',
+        'CephUUID': 'uuid',
+        'CephEntityAddr': 'entityaddr',
+        'CephIPAddr': '0.0.0.0',
+        'CephName': 'name',
+        'CephBool': 'true',
+        'CephFloat': '0.0',
+        'CephFilepath': '/path/to/file',
+    }
+
+    def __init__(self, type, name, who=None, n=None, req=True, range=None, strings=None,
+                 goodchars=None):
+        self.type = type
+        self.name = name
+        self.who = who
+        self.n = n == 'N'
+        self.req = req != 'false'
+        self.range = range.split('|') if range else []
+        self.strings = strings.split('|') if strings else []
+        self.goodchars = goodchars
+
+        assert who == None
+
+    def help(self):
+        advanced = []
+        if self.type != 'CephString':
+            advanced.append(self.type + ' ')
+        if self.range:
+            advanced.append('range= ``{}`` '.format('..'.join(self.range)))
+        if self.strings:
+            advanced.append('strings=({}) '.format(' '.join(self.strings)))
+        if self.goodchars:
+            advanced.append('goodchars= ``{}`` '.format(self.goodchars))
+        if self.n:
+            advanced.append('(can be repeated)')
+
+        advanced = advanced or ["(string)"]
+        return ' '.join(advanced)
+
+    def mk_example_value(self):
+        if self.type == 'CephChoices' and self.strings:
+            return self.strings[0]
+        if self.range:
+            return self.range[0]
+        return CmdParam.bash_example[self.type]
+
+    def mk_bash_example(self, simple):
+        val = self.mk_example_value()
+
+        if self.type == 'CephBool':
+            return '--' + self.name
+        if simple:
+            if self.type == "CephChoices" and self.strings:
+                return val
+            elif self.type == "CephString" and self.name != 'who':
+                return 'my_' + self.name
+            else:
+                return CmdParam.bash_example[self.type]
+        else:
+            return '--{}={}'.format(self.name, val)
+
+
+class CmdCommand(object):
+    def __init__(self, sig, desc, module=None, perm=None, flags=0, poll=None):
+        self.sig = [s for s in sig if isinstance(s, str)]
+        self.params = sorted([CmdParam(**s) for s in sig if not isinstance(s, str)],
+                             key=lambda p: p.req, reverse=True)
+        self.help = desc
+        self.module = module
+        self.perm = perm
+        self.flags = Flags(flags)
+        self.needs_overload = False
+
+    def prefix(self):
+        return ' '.join(self.sig)
+
+    def is_reasonably_simple(self):
+        if len(self.params) > 3:
+            return False
+        if any(p.n for p in self.params):
+            return False
+        return True
+
+    def mk_bash_example(self):
+        simple = self.is_reasonably_simple()
+        line = ' '.join(['ceph', self.prefix()] + [p.mk_bash_example(simple) for p in self.params])
+        return line
+
+
+tpl = '''
+.. This file is automatically generated. do not modify
+
+{% for command in commands %}
+
+{{ command.prefix() }}
+{{ command.prefix() | length * '^' }}
+
+{{ command.help | wordwrap(70)}}
+
+Example command:
+
+.. code-block:: bash
+
+    {{ command.mk_bash_example() }}
+{% if command.params %}
+Parameters:
+
+{% for param in command.params %}* **{{param.name}}**: {{ param.help() | wordwrap(70) | indent(2) }}
+{% endfor %}{% endif %}
+Ceph Module:
+
+* *{{ command.module }}*
+
+Required Permissions:
+
+* *{{ command.perm }}*
+
+{% if command.flags %}Command Flags:
+
+* *{{ command.flags }}*
+{% endif %}
+{% endfor %}
+
+'''
+
+def mk_sigs(all):
+    sigs = [CmdCommand(**e) for e in all]
+    sigs = [s for s in sigs if 'hidden' not in s.flags]
+    sigs = sorted(sigs, key=lambda f: f.sig)
+
+
+    tm = Template(tpl)
+    msg = tm.render(commands=list(sigs))
+
+    print(msg)
+
+
+if __name__ == '__main__':
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    commands = json.loads(check_output([sys.executable, script_dir + '/../../src/script/gen_static_command_descriptions.py']))
+    mk_sigs(commands)

src/pybind/mgr/dashboard/__init__.py

@@ -23,8 +23,6 @@ if 'UNITTEST' not in os.environ:
 
     mgr = _ModuleProxy()
 
-    # DO NOT REMOVE: required for ceph-mgr to load a module
-    from .module import Module, StandbyModule  # noqa: F401
 else:
     import logging
     logging.basicConfig(level=logging.DEBUG)
@@ -32,7 +30,12 @@ else:
     os.environ['PATH'] = '{}:{}'.format(os.path.abspath('../../../../build/bin'),
                                         os.environ['PATH'])
 
-    from tests import mock  # type: ignore
+    from tests import mock, mock_ceph_modules  # type: ignore
 
     mgr = mock.Mock()
     mgr.get_frontend_path.side_effect = lambda: os.path.abspath("./frontend/dist")
+
+    mock_ceph_modules()
+
+# DO NOT REMOVE: required for ceph-mgr to load a module
+from .module import Module, StandbyModule  # noqa: F401

src/pybind/mgr/dashboard/conftest.py

@@ -1,39 +0,0 @@
-import sys
-
-try:
-    from mock import Mock, patch
-except ImportError:
-    from unittest.mock import Mock, patch
-
-
-class MockRadosError(Exception):
-    def __init__(self, message, errno=None):
-        super(MockRadosError, self).__init__(message)
-        self.errno = errno
-
-    def __str__(self):
-        msg = super(MockRadosError, self).__str__()
-        if self.errno is None:
-            return msg
-        return '[errno {0}] {1}'.format(self.errno, msg)
-
-
-def pytest_configure(config):
-    sys.modules.update({
-        'rados': Mock(Error=MockRadosError, OSError=MockRadosError),
-        'rbd': Mock(),
-        'cephfs': Mock(),
-    })
-
-    # we need the following patches to fix the issue of multiple inheritance when
-    # one of the base classes is being mocked.
-    # Error example:
-    # TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) \
-    # subclass of the metaclasses of all its bases
-    class _BaseMgrModule:
-        pass
-
-    patcher = patch("ceph_module.BaseMgrStandbyModule", new=_BaseMgrModule)
-    patcher.start()
-    patcher = patch("ceph_module.BaseMgrModule", new=_BaseMgrModule)
-    patcher.start()

src/pybind/mgr/progress/__init__.py

@@ -1,7 +1,7 @@
 import os
-if 'UNITTEST' not in os.environ:
-    from .module import *
-else:
+
+if 'UNITTEST' in os.environ:
     import tests
+from .module import Module
 
 

src/pybind/mgr/progress/module.py

@@ -21,9 +21,6 @@ ENCODING_VERSION = 2
 # keep a global reference to the module so we can use it from Event methods
 _module = None  # type: Optional["Module"]
 
-# if unit test we want MgrModule to be blank
-if 'UNITTEST' in os.environ:
-    MgrModule = object  # type: ignore
 
 class Event(object):
     """

src/pybind/mgr/progress/test_progress.py

@@ -126,8 +126,9 @@ class TestModule(object):
         # bunch of attributes for testing
 
         module.PgRecoveryEvent.pg_update = mock.Mock()
-        self.test_module = module.Module() # so we can see if an event gets created
-        self.test_module.log = mock.Mock() # we don't need to log anything
+        module.Module._ceph_get_option = mock.Mock()  # .__init__
+        module.Module._configure_logging = lambda *args: ...  # .__init__
+        self.test_module = module.Module('module_name', 0, 0)  # so we can see if an event gets created
         self.test_module.get = mock.Mock() # so we can call pg_update
         self.test_module._complete = mock.Mock() # we want just to see if this event gets called
         self.test_module.get_osdmap = mock.Mock() # so that self.get_osdmap().get_epoch() works

src/pybind/mgr/tests/__init__.py

@@ -64,3 +64,22 @@ if 'UNITTEST' in os.environ:
     cm.BaseMgrModule = M
     cm.BaseMgrStandbyModule = M
     sys.modules['ceph_module'] = cm
+
+    def mock_ceph_modules():
+        class MockRadosError(Exception):
+            def __init__(self, message, errno=None):
+                super(MockRadosError, self).__init__(message)
+                self.errno = errno
+
+            def __str__(self):
+                msg = super(MockRadosError, self).__str__()
+                if self.errno is None:
+                    return msg
+                return '[errno {0}] {1}'.format(self.errno, msg)
+
+
+        sys.modules.update({
+            'rados': mock.Mock(Error=MockRadosError, OSError=MockRadosError),
+            'rbd': mock.Mock(),
+            'cephfs': mock.Mock(),
+        })

src/script/gen_static_command_descriptions.py

@@ -0,0 +1,168 @@
+"""
+Prints a statically compiled list of all commands.
+
+See
+
+* /admin/doc-requirements.txt
+* /doc/scripts/gen_mon_command_api.py
+
+Rational for putting this file here is to allow others to make use of this output.
+"""
+import json
+import os
+import io
+import sys
+
+from pcpp.preprocessor import Preprocessor, OutputDirective, Action
+
+os.environ['UNITTEST'] = 'true'
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+mgr_dir = os.path.abspath(script_dir + '/../../src/pybind/mgr')
+
+sys.path.insert(0, mgr_dir)
+
+from tests import mock, M
+
+
+def param_to_sig(p):
+    try:
+        return {kv.split('=')[0]: kv.split('=')[1] for kv in p.split(',')}
+    except IndexError:
+        return p
+
+def cmd_to_sig(cmd):
+    sig = cmd.split()
+    return [param_to_sig(s) or s for s in sig]
+
+
+def list_mgr_module(m_name):
+    sys.modules['rbd'] = mock.Mock()
+    sys.modules['cephfs'] = mock.Mock()
+    sys.modules['dateutil'] = mock.Mock()
+    sys.modules['dateutil.parser'] = mock.Mock()
+
+    # make dashboard happy
+    sys.modules['OpenSSL'] = mock.Mock()
+    sys.modules['jwt'] = mock.Mock()
+    sys.modules['bcrypt'] = mock.Mock()
+
+    sys.modules['scipy'] = mock.Mock()
+    sys.modules['jsonpatch'] = mock.Mock()
+    sys.modules['rook.rook_client'] = mock.Mock()
+    sys.modules['rook.rook_client.ceph'] = mock.Mock()
+
+    sys.modules['cherrypy'] = mock.Mock(__version__="3.2.3")
+
+    # make restful happy:
+    sys.modules['pecan'] = mock.Mock()
+    sys.modules['pecan.rest'] = mock.Mock()
+    sys.modules['pecan.hooks'] = mock.Mock()
+    sys.modules['werkzeug'] = mock.Mock()
+    sys.modules['werkzeug.serving'] = mock.Mock()
+
+    mgr_mod = __import__(m_name, globals(), locals(), [], 0)
+
+    def subclass(x):
+        try:
+            return issubclass(x, M)
+        except TypeError:
+            return False
+
+    ms = [c for c in mgr_mod.__dict__.values() if subclass(c) and 'Standby' not in c.__name__]
+    [m] = ms
+    assert isinstance(m.COMMANDS, list)
+    return m.COMMANDS
+
+
+def from_mgr_modules():
+    names = [name for name in os.listdir(mgr_dir)
+             if os.path.isdir(os.path.join(mgr_dir, name)) and
+             os.path.isfile(os.path.join(mgr_dir, name, '__init__.py')) and
+             name not in ['tests']]
+
+    comms = sum([list_mgr_module(name) for name in names], [])
+    for c in comms:
+        if 'handler' in c:
+            del c['handler']
+        c['sig'] = cmd_to_sig(c['cmd'])
+        del c['cmd']
+        c['flags'] = (1 << 3)
+        c['module'] = 'mgr'
+    return comms
+
+
+def from_mon_commands_h():
+    input_str = """
+    #include "{script_dir}/../mon/MonCommands.h"
+    #include "{script_dir}/../mgr/MgrCommands.h"
+    """.format(script_dir=script_dir)
+
+    cmds = []
+
+    class MyProcessor(Preprocessor):
+        def __init__(self):
+            super(MyProcessor, self).__init__()
+            self.undef('__DATE__')
+            self.undef('__TIME__')
+            self.expand_linemacro = False
+            self.expand_filemacro = False
+            self.expand_countermacro = False
+            self.line_directive = '#line'
+            self.define("__PCPP_VERSION__ " + '')
+            self.define("__PCPP_ALWAYS_FALSE__ 0")
+            self.define("__PCPP_ALWAYS_TRUE__ 1")
+            self.parse(input_str)
+            out = io.StringIO()
+            self.write(out)
+            out.seek(0)
+            s = out.read()
+
+            NONE = 0
+            NOFORWARD = (1 << 0)
+            OBSOLETE = (1 << 1)
+            DEPRECATED = (1 << 2)
+            MGR = (1 << 3)
+            POLL = (1 << 4)
+            HIDDEN = (1 << 5)
+            TELL = (1 << 6)
+
+            def FLAG(a):
+                return a
+
+            def COMMAND(cmd, desc, module, perm):
+                cmds.append({
+                    'cmd': cmd,
+                    'desc': desc,
+                    'module': module,
+                    'perm': perm
+                })
+
+            def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
+                cmds.append({
+                    'cmd': cmd,
+                    'desc': desc,
+                    'module': module,
+                    'perm': perm,
+                    'flags': flag
+                })
+
+            exec(s, globals(), locals())
+
+    MyProcessor()
+    for c in cmds:
+        if 'handler' in c:
+            del c['handler']
+        c['sig'] = cmd_to_sig(c['cmd'])
+        del c['cmd']
+    return cmds
+
+
+def gen_commands_dicts():
+    comms = from_mon_commands_h() + from_mgr_modules()
+    comms = sorted(comms, key=lambda c: [e for e in c['sig'] if isinstance(e, str)])
+    return comms
+
+if __name__ == '__main__':
+    print(json.dumps(gen_commands_dicts(), indent=2, sort_keys=True))