mirror of
https://github.com/ceph/ceph
synced 2024-12-26 21:43:10 +00:00
460 lines
15 KiB
Python
460 lines
15 KiB
Python
import io
|
|
import contextlib
|
|
import os
|
|
import sys
|
|
from typing import Any, Dict, List, Union
|
|
|
|
from docutils.nodes import Node
|
|
from docutils.parsers.rst import directives
|
|
from docutils.statemachine import StringList
|
|
|
|
from sphinx import addnodes
|
|
from sphinx.directives import ObjectDescription
|
|
from sphinx.domains.python import PyField
|
|
from sphinx.environment import BuildEnvironment
|
|
from sphinx.locale import _
|
|
from sphinx.util import logging, status_iterator, ws_re
|
|
from sphinx.util.docutils import switch_source_input, SphinxDirective
|
|
from sphinx.util.docfields import Field
|
|
from sphinx.util.nodes import make_id
|
|
import jinja2
|
|
import jinja2.filters
|
|
import yaml
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
TEMPLATE = '''
|
|
{% if desc %}
|
|
{{ desc | wordwrap(70) | indent(3) }}
|
|
{% endif %}
|
|
:type: ``{{opt.type}}``
|
|
{%- if default is not none %}
|
|
{%- if opt.type == 'size' %}
|
|
:default: ``{{ default | eval_size | iec_size }}``
|
|
{%- elif opt.type == 'secs' %}
|
|
:default: ``{{ default | readable_duration(opt.type) }}``
|
|
{%- elif opt.type in ('uint', 'int', 'float') %}
|
|
:default: ``{{ default | readable_num(opt.type) }}``
|
|
{%- elif opt.type == 'millisecs' %}
|
|
:default: ``{{ default }}`` milliseconds
|
|
{%- elif opt.type == 'bool' %}
|
|
:default: ``{{ default | string | lower }}``
|
|
{%- else %}
|
|
:default: {{ default | literal }}
|
|
{%- endif -%}
|
|
{%- endif %}
|
|
{%- if opt.enum_values %}
|
|
:valid choices:{% for enum_value in opt.enum_values -%}
|
|
{{" -" | indent(18, not loop.first) }} {{ enum_value | literal }}
|
|
{% endfor %}
|
|
{%- endif %}
|
|
{%- if opt.min is defined and opt.max is defined %}
|
|
:allowed range: ``[{{ opt.min }}, {{ opt.max }}]``
|
|
{%- elif opt.min is defined %}
|
|
:min: ``{{ opt.min }}``
|
|
{%- elif opt.max is defined %}
|
|
:max: ``{{ opt.max }}``
|
|
{%- endif %}
|
|
{%- if opt.constraint %}
|
|
:constraint: {{ opt.constraint }}
|
|
{% endif %}
|
|
{%- if opt.policies %}
|
|
:policies: {{ opt.policies }}
|
|
{% endif %}
|
|
{%- if opt.example %}
|
|
:example: {{ opt.example }}
|
|
{%- endif %}
|
|
{%- if opt.see_also %}
|
|
:see also: {{ opt.see_also | map('ref_confval') | join(', ') }}
|
|
{%- endif %}
|
|
{% if opt.note %}
|
|
.. note::
|
|
{{ opt.note }}
|
|
{%- endif -%}
|
|
{%- if opt.warning %}
|
|
.. warning::
|
|
{{ opt.warning }}
|
|
{%- endif %}
|
|
'''
|
|
|
|
|
|
def eval_size(value) -> int:
|
|
try:
|
|
return int(value)
|
|
except ValueError:
|
|
times = dict(_K=1 << 10,
|
|
_M=1 << 20,
|
|
_G=1 << 30,
|
|
_T=1 << 40)
|
|
for unit, m in times.items():
|
|
if value.endswith(unit):
|
|
return int(value[:-len(unit)]) * m
|
|
raise ValueError(f'unknown value: {value}')
|
|
|
|
|
|
def readable_duration(value: str, typ: str) -> str:
|
|
try:
|
|
if typ == 'sec':
|
|
v = int(value)
|
|
postfix = 'second' if v == 1 else 'seconds'
|
|
return f'{v} {postfix}'
|
|
elif typ == 'float':
|
|
return str(float(value))
|
|
else:
|
|
return str(int(value))
|
|
except ValueError:
|
|
times = dict(_min=['minute', 'minutes'],
|
|
_hr=['hour', 'hours'],
|
|
_day=['day', 'days'])
|
|
for unit, readables in times.items():
|
|
if value.endswith(unit):
|
|
v = int(value[:-len(unit)])
|
|
postfix = readables[0 if v == 1 else 1]
|
|
return f'{v} {postfix}'
|
|
raise ValueError(f'unknown value: {value}')
|
|
|
|
|
|
def do_plain_num(value: str, typ: str) -> str:
|
|
if typ == 'float':
|
|
return str(float(value))
|
|
else:
|
|
return str(int(value))
|
|
|
|
|
|
def iec_size(value: int) -> str:
|
|
if value == 0:
|
|
return '0B'
|
|
units = dict(Ei=60,
|
|
Pi=50,
|
|
Ti=40,
|
|
Gi=30,
|
|
Mi=20,
|
|
Ki=10,
|
|
B=0)
|
|
for unit, bits in units.items():
|
|
m = 1 << bits
|
|
if value % m == 0:
|
|
value //= m
|
|
return f'{value}{unit}'
|
|
raise Exception(f'iec_size() failed to convert {value}')
|
|
|
|
|
|
def do_fileize_num(value: str, typ: str) -> str:
|
|
v = eval_size(value)
|
|
return iec_size(v)
|
|
|
|
|
|
def readable_num(value: str, typ: str) -> str:
|
|
e = ValueError()
|
|
for eval_func in [do_plain_num,
|
|
readable_duration,
|
|
do_fileize_num]:
|
|
try:
|
|
return eval_func(value, typ)
|
|
except ValueError as ex:
|
|
e = ex
|
|
raise e
|
|
|
|
|
|
def literal(name) -> str:
|
|
if name:
|
|
return f'``{name}``'
|
|
else:
|
|
return f'<empty string>'
|
|
|
|
|
|
def ref_confval(name) -> str:
|
|
return f':confval:`{name}`'
|
|
|
|
|
|
def jinja_template() -> jinja2.Template:
|
|
env = jinja2.Environment()
|
|
env.filters['eval_size'] = eval_size
|
|
env.filters['iec_size'] = iec_size
|
|
env.filters['readable_duration'] = readable_duration
|
|
env.filters['readable_num'] = readable_num
|
|
env.filters['literal'] = literal
|
|
env.filters['ref_confval'] = ref_confval
|
|
return env.from_string(TEMPLATE)
|
|
|
|
|
|
FieldValueT = Union[bool, float, int, str]
|
|
|
|
|
|
class CephModule(SphinxDirective):
|
|
"""
|
|
Directive to name the mgr module for which options are documented.
|
|
"""
|
|
has_content = False
|
|
required_arguments = 1
|
|
optional_arguments = 0
|
|
final_argument_whitespace = False
|
|
|
|
def run(self) -> List[Node]:
|
|
module = self.arguments[0].strip()
|
|
if module == 'None':
|
|
self.env.ref_context.pop('ceph:module', None)
|
|
else:
|
|
self.env.ref_context['ceph:module'] = module
|
|
return []
|
|
|
|
|
|
class CephOption(ObjectDescription):
|
|
"""
|
|
emit option loaded from given command/options/<name>.yaml.in file
|
|
"""
|
|
has_content = True
|
|
required_arguments = 1
|
|
optional_arguments = 0
|
|
final_argument_whitespace = False
|
|
option_spec = {
|
|
'module': directives.unchanged,
|
|
'default': directives.unchanged
|
|
}
|
|
|
|
|
|
doc_field_types = [
|
|
Field('default',
|
|
label=_('Default'),
|
|
has_arg=False,
|
|
names=('default',)),
|
|
Field('type',
|
|
label=_('Type'),
|
|
has_arg=False,
|
|
names=('type',),
|
|
bodyrolename='class'),
|
|
]
|
|
|
|
template = jinja_template()
|
|
opts: Dict[str, Dict[str, FieldValueT]] = {}
|
|
mgr_opts: Dict[str, # module name
|
|
Dict[str, # option name
|
|
Dict[str, # field_name
|
|
FieldValueT]]] = {}
|
|
|
|
def _load_yaml(self) -> Dict[str, Dict[str, FieldValueT]]:
|
|
if CephOption.opts:
|
|
return CephOption.opts
|
|
opts = []
|
|
for fn in status_iterator(self.config.ceph_confval_imports,
|
|
'loading options...', 'red',
|
|
len(self.config.ceph_confval_imports),
|
|
self.env.app.verbosity):
|
|
self.env.note_dependency(fn)
|
|
try:
|
|
with open(fn, 'r') as f:
|
|
yaml_in = io.StringIO()
|
|
for line in f:
|
|
if '@' not in line:
|
|
yaml_in.write(line)
|
|
yaml_in.seek(0)
|
|
opts += yaml.safe_load(yaml_in)['options']
|
|
except OSError as e:
|
|
message = f'Unable to open option file "{fn}": {e}'
|
|
raise self.error(message)
|
|
CephOption.opts = dict((opt['name'], opt) for opt in opts)
|
|
return CephOption.opts
|
|
|
|
def _normalize_path(self, dirname):
|
|
my_dir = os.path.dirname(os.path.realpath(__file__))
|
|
src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
|
|
return os.path.join(src_dir, dirname)
|
|
|
|
def _is_mgr_module(self, dirname, name):
|
|
if not os.path.isdir(os.path.join(dirname, name)):
|
|
return False
|
|
if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
|
|
return False
|
|
return name not in ['tests']
|
|
|
|
@contextlib.contextmanager
|
|
def mocked_modules(self):
|
|
# src/pybind/mgr/tests
|
|
from tests import mock
|
|
mock_imports = ['rados',
|
|
'rbd',
|
|
'cephfs',
|
|
'dateutil',
|
|
'dateutil.parser']
|
|
# make dashboard happy
|
|
mock_imports += ['OpenSSL',
|
|
'jwt',
|
|
'bcrypt',
|
|
'jsonpatch',
|
|
'rook.rook_client',
|
|
'rook.rook_client.ceph',
|
|
'rook.rook_client._helper',
|
|
'cherrypy=3.2.3']
|
|
# make diskprediction_local happy
|
|
mock_imports += ['numpy',
|
|
'scipy']
|
|
# make restful happy
|
|
mock_imports += ['pecan',
|
|
'pecan.rest',
|
|
'pecan.hooks',
|
|
'werkzeug',
|
|
'werkzeug.serving']
|
|
|
|
for m in mock_imports:
|
|
args = {}
|
|
parts = m.split('=', 1)
|
|
mocked = parts[0]
|
|
if len(parts) > 1:
|
|
args['__version__'] = parts[1]
|
|
sys.modules[mocked] = mock.Mock(**args)
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
for m in mock_imports:
|
|
mocked = m.split('=', 1)[0]
|
|
sys.modules.pop(mocked)
|
|
|
|
def _collect_options_from_module(self, name):
|
|
with self.mocked_modules():
|
|
mgr_mod = __import__(name, globals(), locals(), [], 0)
|
|
# import 'M' from src/pybind/mgr/tests
|
|
from tests import M
|
|
|
|
def subclass(x):
|
|
try:
|
|
return issubclass(x, M)
|
|
except TypeError:
|
|
return False
|
|
ms = [c for c in mgr_mod.__dict__.values()
|
|
if subclass(c) and 'Standby' not in c.__name__]
|
|
[m] = ms
|
|
assert isinstance(m.MODULE_OPTIONS, list)
|
|
return m.MODULE_OPTIONS
|
|
|
|
def _load_module(self, module) -> Dict[str, Dict[str, FieldValueT]]:
|
|
mgr_opts = CephOption.mgr_opts.get(module)
|
|
if mgr_opts is not None:
|
|
return mgr_opts
|
|
python_path = self.config.ceph_confval_mgr_python_path
|
|
for path in python_path.split(':'):
|
|
sys.path.insert(0, self._normalize_path(path))
|
|
module_path = self.env.config.ceph_confval_mgr_module_path
|
|
module_path = self._normalize_path(module_path)
|
|
sys.path.insert(0, module_path)
|
|
if not self._is_mgr_module(module_path, module):
|
|
raise self.error(f'module "{module}" not found under {module_path}')
|
|
fn = os.path.join(module_path, module, 'module.py')
|
|
if os.path.exists(fn):
|
|
self.env.note_dependency(fn)
|
|
os.environ['UNITTEST'] = 'true'
|
|
opts = self._collect_options_from_module(module)
|
|
CephOption.mgr_opts[module] = dict((opt['name'], opt) for opt in opts)
|
|
return CephOption.mgr_opts[module]
|
|
|
|
def _current_module(self) -> str:
|
|
return self.options.get('module',
|
|
self.env.ref_context.get('ceph:module'))
|
|
|
|
def _render_option(self, name) -> str:
|
|
cur_module = self._current_module()
|
|
if cur_module:
|
|
try:
|
|
opt = self._load_module(cur_module).get(name)
|
|
except Exception as e:
|
|
message = f'Unable to load module "{cur_module}": {e}'
|
|
raise self.error(message)
|
|
else:
|
|
opt = self._load_yaml().get(name)
|
|
if opt is None:
|
|
raise self.error(f'Option "{name}" not found!')
|
|
if cur_module and 'type' not in opt:
|
|
# the type of module option defaults to 'str'
|
|
opt['type'] = 'str'
|
|
desc = opt.get('fmt_desc') or opt.get('long_desc') or opt.get('desc')
|
|
opt_default = opt.get('default')
|
|
default = self.options.get('default', opt_default)
|
|
try:
|
|
return self.template.render(opt=opt,
|
|
desc=desc,
|
|
default=default)
|
|
except Exception as e:
|
|
message = (f'Unable to render option "{name}": {e}. ',
|
|
f'opt={opt}, desc={desc}, default={default}')
|
|
raise self.error(message)
|
|
|
|
def handle_signature(self,
|
|
sig: str,
|
|
signode: addnodes.desc_signature) -> str:
|
|
signode.clear()
|
|
signode += addnodes.desc_name(sig, sig)
|
|
# normalize whitespace like XRefRole does
|
|
name = ws_re.sub(' ', sig)
|
|
cur_module = self._current_module()
|
|
if cur_module:
|
|
return '/'.join(['mgr', cur_module, name])
|
|
else:
|
|
return name
|
|
|
|
def transform_content(self, contentnode: addnodes.desc_content) -> None:
|
|
name = self.arguments[0]
|
|
source, lineno = self.get_source_info()
|
|
source = f'{source}:{lineno}:<confval>'
|
|
fields = StringList(self._render_option(name).splitlines() + [''],
|
|
source=source, parent_offset=lineno)
|
|
with switch_source_input(self.state, fields):
|
|
self.state.nested_parse(fields, 0, contentnode)
|
|
|
|
def add_target_and_index(self,
|
|
name: str,
|
|
sig: str,
|
|
signode: addnodes.desc_signature) -> None:
|
|
node_id = make_id(self.env, self.state.document, self.objtype, name)
|
|
signode['ids'].append(node_id)
|
|
self.state.document.note_explicit_target(signode)
|
|
entry = f'{name}; configuration option'
|
|
self.indexnode['entries'].append(('pair', entry, node_id, '', None))
|
|
std = self.env.get_domain('std')
|
|
std.note_object(self.objtype, name, node_id, location=signode)
|
|
|
|
|
|
def _reset_ref_context(app, env, docname):
|
|
env.ref_context.pop('ceph:module', None)
|
|
|
|
|
|
def setup(app) -> Dict[str, Any]:
|
|
app.add_config_value('ceph_confval_imports',
|
|
default=[],
|
|
rebuild='html',
|
|
types=[str])
|
|
app.add_config_value('ceph_confval_mgr_module_path',
|
|
default=[],
|
|
rebuild='html',
|
|
types=[str])
|
|
app.add_config_value('ceph_confval_mgr_python_path',
|
|
default=[],
|
|
rebuild='',
|
|
types=[str])
|
|
app.add_object_type(
|
|
'confsec',
|
|
'confsec',
|
|
objname='configuration section',
|
|
indextemplate='pair: %s; configuration section',
|
|
doc_field_types=[
|
|
Field(
|
|
'example',
|
|
label=_('Example'),
|
|
has_arg=False,
|
|
)]
|
|
)
|
|
app.add_object_type(
|
|
'confval',
|
|
'confval',
|
|
objname='configuration option',
|
|
)
|
|
app.add_directive_to_domain('std', 'mgr_module', CephModule)
|
|
app.add_directive_to_domain('std', 'confval', CephOption, override=True)
|
|
app.connect('env-purge-doc', _reset_ref_context)
|
|
|
|
return {
|
|
'version': 'builtin',
|
|
'parallel_read_safe': True,
|
|
'parallel_write_safe': True,
|
|
}
|