mirror of
https://github.com/ceph/ceph
synced 2025-04-01 23:02:17 +00:00
Merge pull request #21022 from sebastian-philipp/dashboard_get_range
mgr/dashboard: Refactor multiple duplicates of `get_rate()` Reviewed-by: Ricardo Dias <rdias@suse.com> Reviewed-by: Stephan Müller <smueller@suse.com>
This commit is contained in:
Lenz Grimmer
GitHub
commit
34232b347b
@ -7,8 +7,11 @@ import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import namedtuple
|
||||
|
||||
import requests
|
||||
import six
|
||||
|
||||
from ..mgr_test_case import MgrTestCase
|
||||
|
||||
|
||||
@ -117,6 +120,12 @@ class DashboardTestCase(MgrTestCase):
|
||||
body = self._resp.json()
|
||||
self.assertEqual(body, data)
|
||||
|
||||
def assertSchema(self, data, schema):
|
||||
try:
|
||||
return _validate_json(data, schema)
|
||||
except _ValError as e:
|
||||
self.assertEqual(data, str(e))
|
||||
|
||||
def assertBody(self, body):
|
||||
self.assertEqual(self._resp.text, body)
|
||||
|
||||
@ -148,3 +157,74 @@ class DashboardTestCase(MgrTestCase):
|
||||
out = cls.ceph_cluster.mon_manager.raw_cluster_cmd('mon_status')
|
||||
j = json.loads(out)
|
||||
return [mon['name'] for mon in j['monmap']['mons']]
|
||||
|
||||
|
||||
class JLeaf(namedtuple('JLeaf', ['typ', 'none'])):
|
||||
def __new__(cls, typ, none=False):
|
||||
if typ == str:
|
||||
typ = six.string_types
|
||||
return super(JLeaf, cls).__new__(cls, typ, none)
|
||||
|
||||
|
||||
JList = namedtuple('JList', ['elem_typ'])
|
||||
|
||||
JTuple = namedtuple('JList', ['elem_typs'])
|
||||
|
||||
|
||||
class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown'])):
|
||||
def __new__(cls, sub_elems, allow_unknown=False):
|
||||
"""
|
||||
:type sub_elems: dict[str, JAny | JLeaf | JList | JObj]
|
||||
:type allow_unknown: bool
|
||||
:return:
|
||||
"""
|
||||
return super(JObj, cls).__new__(cls, sub_elems, allow_unknown)
|
||||
|
||||
|
||||
JAny = namedtuple('JAny', ['none'])
|
||||
|
||||
|
||||
class _ValError(Exception):
|
||||
def __init__(self, msg, path):
|
||||
path_str = ''.join('[{}]'.format(repr(p)) for p in path)
|
||||
super(_ValError, self).__init__('In `input{}`: {}'.format(path_str, msg))
|
||||
|
||||
|
||||
def _validate_json(val, schema, path=[]):
|
||||
"""
|
||||
>>> d = {'a': 1, 'b': 'x', 'c': range(10)}
|
||||
... ds = JObj({'a': JLeaf(int), 'b': JLeaf(str), 'c': JList(JLeaf(int))})
|
||||
... _validate_json(d, ds)
|
||||
True
|
||||
"""
|
||||
if isinstance(schema, JAny):
|
||||
if not schema.none and val is None:
|
||||
raise _ValError('val is None', path)
|
||||
return True
|
||||
if isinstance(schema, JLeaf):
|
||||
if schema.none and val is None:
|
||||
return True
|
||||
if not isinstance(val, schema.typ):
|
||||
raise _ValError('val not of type {}'.format(schema.typ), path)
|
||||
return True
|
||||
if isinstance(schema, JList):
|
||||
return all(_validate_json(e, schema.elem_typ, path + [i]) for i, e in enumerate(val))
|
||||
if isinstance(schema, JTuple):
|
||||
return all(_validate_json(val[i], typ, path + [i])
|
||||
for i, typ in enumerate(schema.elem_typs))
|
||||
if isinstance(schema, JObj):
|
||||
missing_keys = set(schema.sub_elems.keys()).difference(set(val.keys()))
|
||||
if missing_keys:
|
||||
raise _ValError('missing keys: {}'.format(missing_keys), path)
|
||||
unknown_keys = set(val.keys()).difference(set(schema.sub_elems.keys()))
|
||||
if not schema.allow_unknown and unknown_keys:
|
||||
raise _ValError('unknown keys: {}'.format(unknown_keys), path)
|
||||
return all(
|
||||
_validate_json(val[sub_elem_name], sub_elem, path + [sub_elem_name])
|
||||
for sub_elem_name, sub_elem in schema.sub_elems.items()
|
||||
)
|
||||
|
||||
assert False, str(path)
|
||||
|
||||
|
||||
|
||||
|
||||
@ -2,15 +2,13 @@
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from .helper import DashboardTestCase, authenticate
|
||||
from .helper import DashboardTestCase, authenticate, JObj, JAny, JList, JLeaf, JTuple
|
||||
|
||||
|
||||
class OsdTest(DashboardTestCase):
|
||||
|
||||
def assert_in_and_not_none(self, data, properties):
|
||||
for prop in properties:
|
||||
self.assertIn(prop, data)
|
||||
self.assertIsNotNone(data[prop])
|
||||
self.assertSchema(data, JObj({p: JAny(none=False) for p in properties}, allow_unknown=True))
|
||||
|
||||
@authenticate
|
||||
def test_list(self):
|
||||
@ -25,6 +23,8 @@ class OsdTest(DashboardTestCase):
|
||||
self.assert_in_and_not_none(data['stats'], ['numpg', 'stat_bytes_used', 'stat_bytes',
|
||||
'op_r', 'op_w'])
|
||||
self.assert_in_and_not_none(data['stats_history'], ['op_out_bytes', 'op_in_bytes'])
|
||||
self.assertSchema(data['stats_history']['op_out_bytes'],
|
||||
JList(JTuple([JLeaf(int), JLeaf(float)])))
|
||||
|
||||
@authenticate
|
||||
def test_details(self):
|
||||
|
||||
@ -5,10 +5,9 @@ from collections import defaultdict
|
||||
|
||||
import cherrypy
|
||||
|
||||
from ..services.ceph_service import CephService
|
||||
|
||||
from . import ApiController, AuthRequired, BaseController
|
||||
from .. import mgr
|
||||
from ..services.ceph_service import CephService
|
||||
from ..tools import ViewCache
|
||||
|
||||
|
||||
@ -98,14 +97,6 @@ class CephFS(BaseController):
|
||||
|
||||
return names
|
||||
|
||||
def get_rate(self, daemon_type, daemon_name, stat):
|
||||
data = mgr.get_counter(daemon_type, daemon_name, stat)[stat]
|
||||
|
||||
if data and len(data) > 1:
|
||||
return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
|
||||
|
||||
return 0
|
||||
|
||||
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
|
||||
def fs_status(self, fs_id):
|
||||
mds_versions = defaultdict(list)
|
||||
@ -132,17 +123,17 @@ class CephFS(BaseController):
|
||||
if up:
|
||||
gid = mdsmap['up']["mds_{0}".format(rank)]
|
||||
info = mdsmap['info']['gid_{0}'.format(gid)]
|
||||
dns = self.get_latest("mds", info['name'], "mds.inodes")
|
||||
inos = self.get_latest("mds", info['name'], "mds_mem.ino")
|
||||
dns = mgr.get_latest("mds", info['name'], "mds.inodes")
|
||||
inos = mgr.get_latest("mds", info['name'], "mds_mem.ino")
|
||||
|
||||
if rank == 0:
|
||||
client_count = self.get_latest("mds", info['name'],
|
||||
"mds_sessions.session_count")
|
||||
client_count = mgr.get_latest("mds", info['name'],
|
||||
"mds_sessions.session_count")
|
||||
elif client_count == 0:
|
||||
# In case rank 0 was down, look at another rank's
|
||||
# sessionmap to get an indication of clients.
|
||||
client_count = self.get_latest("mds", info['name'],
|
||||
"mds_sessions.session_count")
|
||||
client_count = mgr.get_latest("mds", info['name'],
|
||||
"mds_sessions.session_count")
|
||||
|
||||
laggy = "laggy_since" in info
|
||||
|
||||
@ -150,20 +141,15 @@ class CephFS(BaseController):
|
||||
if laggy:
|
||||
state += "(laggy)"
|
||||
|
||||
# if state == "active" and not laggy:
|
||||
# c_state = self.colorize(state, self.GREEN)
|
||||
# else:
|
||||
# c_state = self.colorize(state, self.YELLOW)
|
||||
|
||||
# Populate based on context of state, e.g. client
|
||||
# ops for an active daemon, replay progress, reconnect
|
||||
# progress
|
||||
activity = ""
|
||||
|
||||
if state == "active":
|
||||
activity = self.get_rate("mds",
|
||||
info['name'],
|
||||
"mds_server.handle_client_request")
|
||||
activity = CephService.get_rate("mds",
|
||||
info['name'],
|
||||
"mds_server.handle_client_request")
|
||||
else:
|
||||
activity = 0.0
|
||||
|
||||
metadata = mgr.get_metadata('mds', info['name'])
|
||||
mds_versions[metadata.get('ceph_version', 'unknown')].append(
|
||||
@ -185,7 +171,7 @@ class CephFS(BaseController):
|
||||
"rank": rank,
|
||||
"state": "failed",
|
||||
"mds": "",
|
||||
"activity": "",
|
||||
"activity": 0.0,
|
||||
"dns": 0,
|
||||
"inos": 0
|
||||
}
|
||||
@ -197,10 +183,10 @@ class CephFS(BaseController):
|
||||
if daemon_info['state'] != "up:standby-replay":
|
||||
continue
|
||||
|
||||
inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
|
||||
dns = self.get_latest("mds", daemon_info['name'], "mds.inodes")
|
||||
inos = mgr.get_latest("mds", daemon_info['name'], "mds_mem.ino")
|
||||
dns = mgr.get_latest("mds", daemon_info['name'], "mds.inodes")
|
||||
|
||||
activity = self.get_rate(
|
||||
activity = CephService.get_rate(
|
||||
"mds", daemon_info['name'], "mds_log.replay")
|
||||
|
||||
rank_table.append(
|
||||
@ -292,12 +278,6 @@ class CephFS(BaseController):
|
||||
'data': clients
|
||||
}
|
||||
|
||||
def get_latest(self, daemon_type, daemon_name, stat):
|
||||
data = mgr.get_counter(daemon_type, daemon_name, stat)[stat]
|
||||
if data:
|
||||
return data[-1][1]
|
||||
return 0
|
||||
|
||||
|
||||
class CephFSClients(object):
|
||||
def __init__(self, module_inst, fscid):
|
||||
|
||||
@ -7,28 +7,12 @@ from mgr_module import CommandResult
|
||||
|
||||
from . import ApiController, AuthRequired, RESTController
|
||||
from .. import logger, mgr
|
||||
from ..services.ceph_service import CephService
|
||||
|
||||
|
||||
@ApiController('osd')
|
||||
@AuthRequired()
|
||||
class Osd(RESTController):
|
||||
def get_counter(self, daemon_name, stat):
|
||||
return mgr.get_counter('osd', daemon_name, stat)[stat]
|
||||
|
||||
def get_rate(self, daemon_name, stat):
|
||||
data = self.get_counter(daemon_name, stat)
|
||||
rate = 0
|
||||
if data and len(data) > 1:
|
||||
rate = (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
|
||||
return rate
|
||||
|
||||
def get_latest(self, daemon_name, stat):
|
||||
data = self.get_counter(daemon_name, stat)
|
||||
latest = 0
|
||||
if data and data[-1] and len(data[-1]) == 2:
|
||||
latest = data[-1][1]
|
||||
return latest
|
||||
|
||||
def list(self):
|
||||
osds = self.get_osd_map()
|
||||
# Extending by osd stats information
|
||||
@ -53,11 +37,11 @@ class Osd(RESTController):
|
||||
osd_spec = str(o['osd'])
|
||||
for s in ['osd.op_w', 'osd.op_in_bytes', 'osd.op_r', 'osd.op_out_bytes']:
|
||||
prop = s.split('.')[1]
|
||||
o['stats'][prop] = self.get_rate(osd_spec, s)
|
||||
o['stats_history'][prop] = self.get_counter(osd_spec, s)
|
||||
o['stats'][prop] = CephService.get_rate('osd', osd_spec, s)
|
||||
o['stats_history'][prop] = CephService.get_rates('osd', osd_spec, s)
|
||||
# Gauge stats
|
||||
for s in ['osd.numpg', 'osd.stat_bytes', 'osd.stat_bytes_used']:
|
||||
o['stats'][s.split('.')[1]] = self.get_latest(osd_spec, s)
|
||||
o['stats'][s.split('.')[1]] = mgr.get_latest('osd', osd_spec, s)
|
||||
return list(osds.values())
|
||||
|
||||
def get_osd_map(self):
|
||||
|
||||
@ -3,23 +3,12 @@ from __future__ import absolute_import
|
||||
|
||||
from . import ApiController, AuthRequired, RESTController
|
||||
from .. import mgr
|
||||
from ..services.ceph_service import CephService
|
||||
|
||||
|
||||
class PerfCounter(RESTController):
|
||||
service_type = None # type: str
|
||||
|
||||
def _get_rate(self, daemon_type, daemon_name, stat):
|
||||
data = mgr.get_counter(daemon_type, daemon_name, stat)[stat]
|
||||
if data and len(data) > 1:
|
||||
return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
|
||||
return 0
|
||||
|
||||
def _get_latest(self, daemon_type, daemon_name, stat):
|
||||
data = mgr.get_counter(daemon_type, daemon_name, stat)[stat]
|
||||
if data:
|
||||
return data[-1][1]
|
||||
return 0
|
||||
|
||||
def get(self, service_id):
|
||||
schema_dict = mgr.get_perf_schema(self.service_type, str(service_id))
|
||||
schema = schema_dict["{}.{}".format(self.service_type, service_id)]
|
||||
@ -31,11 +20,11 @@ class PerfCounter(RESTController):
|
||||
counter['description'] = value['description']
|
||||
# pylint: disable=W0212
|
||||
if mgr._stattype_to_str(value['type']) == 'counter':
|
||||
counter['value'] = self._get_rate(
|
||||
counter['value'] = CephService.get_rate(
|
||||
self.service_type, service_id, key)
|
||||
counter['unit'] = mgr._unit_to_str(value['units'])
|
||||
else:
|
||||
counter['value'] = self._get_latest(
|
||||
counter['value'] = mgr.get_latest(
|
||||
self.service_type, service_id, key)
|
||||
counter['unit'] = ''
|
||||
counters.append(counter)
|
||||
|
||||
@ -11,21 +11,6 @@ SERVICE_TYPE = 'tcmu-runner'
|
||||
@ApiController('tcmuiscsi')
|
||||
@AuthRequired()
|
||||
class TcmuIscsi(RESTController):
|
||||
def _get_rate(self, daemon_type, daemon_name, stat):
|
||||
data = mgr.get_counter(daemon_type, daemon_name, stat)[stat]
|
||||
|
||||
if data and len(data) > 1:
|
||||
last_value = data[0][1]
|
||||
last_time = data[0][0]
|
||||
rates = []
|
||||
for datum in data[1:]:
|
||||
rates.append([datum[0], ((datum[1] - last_value) /
|
||||
float(datum[0] - last_time))])
|
||||
last_value = datum[1]
|
||||
last_time = datum[0]
|
||||
return rates
|
||||
return [[0, 0]]
|
||||
|
||||
# pylint: disable=too-many-locals,too-many-nested-blocks
|
||||
def list(self): # pylint: disable=unused-argument
|
||||
daemons = {}
|
||||
@ -78,9 +63,9 @@ class TcmuIscsi(RESTController):
|
||||
image['stats_history'] = {}
|
||||
for s in ['rd', 'wr', 'rd_bytes', 'wr_bytes']:
|
||||
perf_key = "{}{}".format(perf_key_prefix, s)
|
||||
image['stats'][s] = self._get_rate(
|
||||
'tcmu-runner', service_id, perf_key)[-1][1]
|
||||
image['stats_history'][s] = self._get_rate(
|
||||
image['stats'][s] = CephService.get_rate(
|
||||
'tcmu-runner', service_id, perf_key)
|
||||
image['stats_history'][s] = CephService.get_rates(
|
||||
'tcmu-runner', service_id, perf_key)
|
||||
else:
|
||||
daemon['non_optimized_paths'] += 1
|
||||
|
||||
@ -7,6 +7,16 @@ from collections import defaultdict
|
||||
import json
|
||||
|
||||
from mgr_module import CommandResult
|
||||
|
||||
try:
|
||||
from more_itertools import pairwise
|
||||
except ImportError:
|
||||
def pairwise(iterable):
|
||||
from itertools import tee
|
||||
a, b = tee(iterable)
|
||||
next(b, None)
|
||||
return zip(a, b)
|
||||
|
||||
from .. import logger, mgr
|
||||
|
||||
|
||||
@ -90,8 +100,7 @@ class CephService(object):
|
||||
|
||||
def get_rate(series):
|
||||
if len(series) >= 2:
|
||||
return (float(series[0][1]) - float(series[1][1])) / \
|
||||
(float(series[0][0]) - float(series[1][0]))
|
||||
return differentiate(*series[0:1])
|
||||
return 0
|
||||
|
||||
for stat_name, stat_series in stats.items():
|
||||
@ -142,3 +151,34 @@ class CephService(object):
|
||||
return json.loads(outb)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return outb
|
||||
|
||||
@classmethod
|
||||
def get_rates(cls, svc_type, svc_name, path):
|
||||
"""
|
||||
:return: the derivative of mgr.get_counter()
|
||||
:rtype: list[tuple[int, float]]"""
|
||||
data = mgr.get_counter(svc_type, svc_name, path)[path]
|
||||
if not data:
|
||||
return [(0, 0)]
|
||||
elif len(data) == 1:
|
||||
return [(data[0][0], 0)]
|
||||
return [(data2[0], differentiate(data1, data2)) for data1, data2 in pairwise(data)]
|
||||
|
||||
@classmethod
|
||||
def get_rate(cls, svc_type, svc_name, path):
|
||||
"""returns most recent rate"""
|
||||
data = mgr.get_counter(svc_type, svc_name, path)[path]
|
||||
|
||||
if data and len(data) > 1:
|
||||
return differentiate(*data[-2:])
|
||||
return 0.0
|
||||
|
||||
|
||||
def differentiate(data1, data2):
|
||||
"""
|
||||
>>> times = [0, 2]
|
||||
>>> values = [100, 101]
|
||||
>>> differentiate(*zip(times, values))
|
||||
0.5
|
||||
"""
|
||||
return (data2[1] - data1[1]) / float(data2[0] - data1[0])
|
||||
|
||||
@ -12,7 +12,7 @@ setenv=
|
||||
LD_LIBRARY_PATH = {toxinidir}/../../../../build/lib
|
||||
PATH = {toxinidir}/../../../../build/bin:$PATH
|
||||
commands=
|
||||
{envbindir}/py.test --cov=. --cov-report= --junitxml=junit.{envname}.xml --doctest-modules controllers/rbd.py tests/
|
||||
{envbindir}/py.test --cov=. --cov-report= --junitxml=junit.{envname}.xml --doctest-modules controllers/rbd.py services/ tests/
|
||||
|
||||
[testenv:cov-init]
|
||||
setenv =
|
||||
|
||||
@ -578,6 +578,15 @@ class MgrModule(ceph_module.BaseMgrModule):
|
||||
"""
|
||||
return self._ceph_get_osdmap()
|
||||
|
||||
# TODO: improve C++->Python interface to return just
|
||||
# the latest if that's all we want.
|
||||
def get_latest(self, daemon_type, daemon_name, counter):
|
||||
data = self.get_counter(daemon_type, daemon_name, counter)[counter]
|
||||
if data:
|
||||
return data[-1][1]
|
||||
else:
|
||||
return 0
|
||||
|
||||
def get_all_perf_counters(self, prio_limit=PRIO_USEFUL):
|
||||
"""
|
||||
Return the perf counters currently known to this ceph-mgr
|
||||
@ -593,14 +602,6 @@ class MgrModule(ceph_module.BaseMgrModule):
|
||||
|
||||
result = defaultdict(dict)
|
||||
|
||||
# TODO: improve C++->Python interface to return just
|
||||
# the latest if that's all we want.
|
||||
def get_latest(daemon_type, daemon_name, counter):
|
||||
data = self.get_counter(daemon_type, daemon_name, counter)[counter]
|
||||
if data:
|
||||
return data[-1][1]
|
||||
else:
|
||||
return 0
|
||||
|
||||
for server in self.list_servers():
|
||||
for service in server['services']:
|
||||
@ -628,7 +629,8 @@ class MgrModule(ceph_module.BaseMgrModule):
|
||||
continue
|
||||
|
||||
counter_info = counter_schema
|
||||
counter_info['value'] = get_latest(service['type'], service['id'], counter_path)
|
||||
counter_info['value'] = self.get_latest(service['type'], service['id'],
|
||||
counter_path)
|
||||
result[svc_full_name][counter_path] = counter_info
|
||||
|
||||
self.log.debug("returning {0} counter".format(len(result)))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user