RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-06 10:23:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #21066 from sebastian-philipp/dashboard_error_handling

Browse Source 
mgr/dashboard: Improve exception handling

Reviewed-by: Tiago Melo <tmelo@suse.com>
Reviewed-by: Tatjana Dehler <tdehler@suse.com>
Reviewed-by: Ricardo Marques <rimarques@suse.com>
Reviewed-by: Ricardo Dias <rdias@suse.com>
This commit is contained in:
Volker Theile 2018-05-09 11:11:24 +02:00 committed by GitHub
commit a3cf914ded
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 699 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
qa/tasks/mgr/dashboard/helper.py
View File

@ -155,7 +155,7 @@ class DashboardTestCase(MgrTestCase):
@classmethod
def _task_request(cls, method, url, data, timeout):
res = cls._request(url, method, data)
cls._assertIn(cls._resp.status_code, [200, 201, 202, 204, 409])
cls._assertIn(cls._resp.status_code, [200, 201, 202, 204, 400])
if cls._resp.status_code != 202:
log.info("task finished immediately")
@ -166,53 +166,41 @@ class DashboardTestCase(MgrTestCase):
task_name = res['name']
task_metadata = res['metadata']
class Waiter(threading.Thread):
def __init__(self, task_name, task_metadata):
super(Waiter, self).__init__()
self.task_name = task_name
self.task_metadata = task_metadata
self.ev = threading.Event()
self.abort = False
self.res_task = None
retries = int(timeout)
res_task = None
while retries > 0 and not res_task:
retries -= 1
log.info("task (%s, %s) is still executing", task_name,
task_metadata)
time.sleep(1)
_res = cls._get('/api/task?name={}'.format(task_name))
cls._assertEq(cls._resp.status_code, 200)
executing_tasks = [task for task in _res['executing_tasks'] if
task['metadata'] == task_metadata]
finished_tasks = [task for task in _res['finished_tasks'] if
task['metadata'] == task_metadata]
if not executing_tasks and finished_tasks:
res_task = finished_tasks[0]
def run(self):
running = True
while running and not self.abort:
log.info("task (%s, %s) is still executing", self.task_name,
self.task_metadata)
time.sleep(1)
res = cls._get('/api/task?name={}'.format(self.task_name))
for task in res['finished_tasks']:
if task['metadata'] == self.task_metadata:
# task finished
running = False
self.res_task = task
self.ev.set()
if retries <= 0:
raise Exception("Waiting for task ({}, {}) to finish timed out. {}"
.format(task_name, task_metadata, _res))
thread = Waiter(task_name, task_metadata)
thread.start()
status = thread.ev.wait(timeout)
if not status:
# timeout expired
thread.abort = True
thread.join()
raise Exception("Waiting for task ({}, {}) to finish timed out"
.format(task_name, task_metadata))
log.info("task (%s, %s) finished", task_name, task_metadata)
if thread.res_task['success']:
if res_task['success']:
if method == 'POST':
cls._resp.status_code = 201
elif method == 'PUT':
cls._resp.status_code = 200
elif method == 'DELETE':
cls._resp.status_code = 204
return thread.res_task['ret_value']
return res_task['ret_value']
else:
if 'status' in thread.res_task['exception']:
cls._resp.status_code = thread.res_task['exception']['status']
if 'status' in res_task['exception']:
cls._resp.status_code = res_task['exception']['status']
else:
cls._resp.status_code = 500
return thread.res_task['exception']
return res_task['exception']
@classmethod
def _task_post(cls, url, data=None, timeout=60):

11
qa/tasks/mgr/dashboard/test_cephfs.py
View File

@ -37,3 +37,14 @@ class CephfsTest(DashboardTestCase):
self.assertIsInstance(data, dict)
self.assertIsNotNone(data)
@authenticate
def test_cephfs_mds_counters_wrong(self):
data = self._get("/api/cephfs/mds_counters/baadbaad")
self.assertStatus(400)
self.assertJsonBody({
"component": 'cephfs',
"code": "invalid_cephfs_id",
"detail": "Invalid cephfs id baadbaad"
})

14
qa/tasks/mgr/dashboard/test_pool.py
View File

@ -18,9 +18,6 @@ class PoolTest(DashboardTestCase):
cls._ceph_cmd(['osd', 'pool', 'delete', name, name, '--yes-i-really-really-mean-it'])
cls._ceph_cmd(['osd', 'erasure-code-profile', 'rm', 'ecprofile'])
@authenticate
def test_pool_list(self):
data = self._get("/api/pool")
@ -145,6 +142,17 @@ class PoolTest(DashboardTestCase):
for data in pools:
self._pool_create(data)
@authenticate
def test_pool_create_fail(self):
data = {'pool_type': u'replicated', 'rule_name': u'dnf', 'pg_num': u'8', 'pool': u'sadfs'}
self._post('/api/pool/', data)
self.assertStatus(400)
self.assertJsonBody({
'component': 'pool',
'code': "2",
'detail': "specified rule dnf doesn't exist"
})
@authenticate
def test_pool_info(self):
info_data = self._get("/api/pool/_info")

26
qa/tasks/mgr/dashboard/test_rbd.py
View File

@ -260,9 +260,12 @@ class RbdTest(DashboardTestCase):
res = self.create_image('rbd', 'test_rbd_twice', 10240)
res = self.create_image('rbd', 'test_rbd_twice', 10240)
self.assertStatus(409)
self.assertEqual(res, {"errno": 17, "status": 409, "component": "rbd",
"detail": "[errno 17] error creating image"})
self.assertStatus(400)
self.assertEqual(res, {"code": '17', 'status': 400, "component": "rbd",
"detail": "[errno 17] error creating image",
'task': {'name': 'rbd/create',
'metadata': {'pool_name': 'rbd',
'image_name': 'test_rbd_twice'}}})
self.remove_image('rbd', 'test_rbd_twice')
self.assertStatus(204)
@ -316,9 +319,12 @@ class RbdTest(DashboardTestCase):
def test_delete_non_existent_image(self):
res = self.remove_image('rbd', 'i_dont_exist')
self.assertStatus(409)
self.assertEqual(res, {"errno": 2, "status": 409, "component": "rbd",
"detail": "[errno 2] error removing image"})
self.assertStatus(400)
self.assertEqual(res, {u'code': u'2', "status": 400, "component": "rbd",
"detail": "[errno 2] error removing image",
'task': {'name': 'rbd/delete',
'metadata': {'pool_name': 'rbd',
'image_name': 'i_dont_exist'}}})
def test_image_delete(self):
self.create_image('rbd', 'delete_me', 2**30)
@ -472,9 +478,9 @@ class RbdTest(DashboardTestCase):
'snap_name': 'snap1'})
res = self.remove_image('rbd', 'cimg')
self.assertStatus(409)
self.assertIn('errno', res)
self.assertEqual(res['errno'], 39)
self.assertStatus(400)
self.assertIn('code', res)
self.assertEqual(res['code'], '39')
self.remove_image('rbd', 'cimg-clone')
self.assertStatus(204)
@ -525,7 +531,7 @@ class RbdTest(DashboardTestCase):
self.assertIsNotNone(img['parent'])
self.flatten_image('rbd_iscsi', 'img1_snapf_clone')
self.assertStatus(200)
self.assertStatus([200, 201])
img = self._get('/api/block/image/rbd_iscsi/img1_snapf_clone')
self.assertStatus(200)

79
src/pybind/mgr/dashboard/HACKING.rst
View File

@ -905,3 +905,82 @@ Usage example:
})
}
}
Error Handling in Python
~~~~~~~~~~~~~~~~~~~~~~~~
Good error handling is a key requirement in creating a good user experience
and providing a good API.
Dashboard code should not duplicate C++ code. Thus, if error handling in C++
is sufficient to provide good feedback, a new wrapper to catch these errors
is not necessary. On the other hand, input validation is the best place to
catch errors and generate the best error messages. If required, generate
errors as soon as possible.
The backend provides few standard ways of returning errors.
First, there is a generic Internal Server Error::
Status Code: 500
{
"version": <cherrypy version, e.g. 13.1.0>,
"detail": "The server encountered an unexpected condition which prevented it from fulfilling the request.",
}
For errors generated by the backend, we provide a standard error
format::
Status Code: 400
{
"detail": str(e), # E.g. "[errno -42] <some error message>"
"component": "rbd", # this can be null to represent a global error code
"code": "3", # Or a error name, e.g. "code": "some_error_key"
}
In case, the API Endpoints uses @ViewCache to temporarily cache results,
the error looks like so::
Status Code 400
{
"detail": str(e), # E.g. "[errno -42] <some error message>"
"component": "rbd", # this can be null to represent a global error code
"code": "3", # Or a error name, e.g. "code": "some_error_key"
'status': 3, # Indicating the @ViewCache error status
}
In case, the API Endpoints uses a task the error looks like so::
Status Code 400
{
"detail": str(e), # E.g. "[errno -42] <some error message>"
"component": "rbd", # this can be null to represent a global error code
"code": "3", # Or a error name, e.g. "code": "some_error_key"
"task": { # Information about the task itself
"name": "taskname",
"metadata": {...}
}
}
Our WebUI should show errors generated by the API to the user. Especially
field-related errors in wizards and dialogs or show non-intrusive notifications.
Handling exceptions in Python should be an exception. In general, we
should have few exception handlers in our project. Per default, propagate
errors to the API, as it will take care of all exceptions anyway. In general,
log the exception by adding ``logger.exception()`` with a description to the
handler.
We need to distinguish between user errors from internal errors and
programming errors. Using different exception types will ease the
task for the API layer and for the user interface:
Standard Python errors, like ``SystemError``, ``ValueError`` or ``KeyError``
will end up as internal server errors in the API.
In general, do not ``return`` error responses in the REST API. They will be
returned by the error handler. Instead, raise the appropriate exception.

105
src/pybind/mgr/dashboard/controllers/__init__.py
View File

@ -5,7 +5,6 @@ from __future__ import absolute_import
import collections
from datetime import datetime, timedelta
import fnmatch
from functools import wraps
import importlib
import inspect
import json
@ -21,7 +20,9 @@ from six import add_metaclass
from .. import logger
from ..settings import Settings
from ..tools import Session, TaskManager
from ..tools import Session, wraps, getargspec, TaskManager
from ..exceptions import ViewCacheNoDataException, DashboardException
from ..services.exception import serialize_dashboard_exception
def ApiController(path):
@ -31,7 +32,8 @@ def ApiController(path):
config = {
'tools.sessions.on': True,
'tools.sessions.name': Session.NAME,
'tools.session_expire_at_browser_close.on': True
'tools.session_expire_at_browser_close.on': True,
'tools.dashboard_exception_handler.on': True,
}
if not hasattr(cls, '_cp_config'):
cls._cp_config = {}
@ -155,34 +157,47 @@ class ApiRoot(object):
def browsable_api_view(meth):
@wraps(meth)
def wrapper(self, *vpath, **kwargs):
assert isinstance(self, BaseController)
if not Settings.ENABLE_BROWSABLE_API:
return meth(self, *vpath, **kwargs)
if 'text/html' not in cherrypy.request.headers.get('Accept', ''):
return meth(self, *vpath, **kwargs)
if '_method' in kwargs:
cherrypy.request.method = kwargs.pop('_method').upper()
# Form typically use None as default, but HTML defaults to empty-string.
for k in kwargs:
if not kwargs[k]:
del kwargs[k]
if '_raw' in kwargs:
kwargs.pop('_raw')
return meth(self, *vpath, **kwargs)
sub_path = cherrypy.request.path_info.split(self._cp_path_, 1)[-1].strip('/').split('/')
template = """
<html>
<h1>Browsable API</h1>
{docstring}
<h2>Request</h2>
<p>{method} {breadcrump}</p>
{params}
<h2>Response</h2>
<p>Status: {status_code}<p>
<pre>{reponse_headers}</pre>
<form action="/api/{path}/{vpath}" method="get">
<form action="/api/{path}/{sub_path}" method="get">
<input type="hidden" name="_raw" value="true" />
<button type="submit">GET raw data</button>
</form>
<h2>Data</h2>
<pre>{data}</pre>
{exception}
{create_form}
{delete_form}
<h2>Note</h2>
<p>Please note that this API is not an official Ceph REST API to be
used by third-party applications. It's primary purpose is to serve
@ -192,33 +207,51 @@ def browsable_api_view(meth):
create_form_template = """
<h2>Create Form</h2>
<form action="/api/{path}/{vpath}" method="post">
<form action="/api/{path}/{sub_path}" method="post">
{fields}<br>
<input type="hidden" name="_method" value="post" />
<button type="submit">Create</button>
</form>
"""
try:
data = meth(self, *vpath, **kwargs)
except Exception as e: # pylint: disable=broad-except
delete_form_template = """
<h2>Create Form</h2>
<form action="/api/{path}/{sub_path}" method="post">
<input type="hidden" name="_method" value="delete" />
<button type="submit">Delete</button>
</form>
"""
def mk_exception(e):
except_template = """
<h2>Exception: {etype}: {tostr}</h2>
<pre>{trace}</pre>
Params: {kwargs}
"""
import traceback
tb = sys.exc_info()[2]
cherrypy.response.headers['Content-Type'] = 'text/html'
data = except_template.format(
return except_template.format(
etype=e.__class__.__name__,
tostr=str(e),
trace='\n'.join(traceback.format_tb(tb)),
kwargs=kwargs
)
if cherrypy.response.headers['Content-Type'] == 'application/json':
data = json.dumps(json.loads(data), indent=2, sort_keys=True)
try:
data = meth(self, *vpath, **kwargs)
exception = ''
if cherrypy.response.headers['Content-Type'] == 'application/json':
try:
data = json.dumps(json.loads(data), indent=2, sort_keys=True)
except Exception: # pylint: disable=broad-except
pass
except (ViewCacheNoDataException, DashboardException) as e:
cherrypy.response.status = getattr(e, 'status', 400)
data = str(serialize_dashboard_exception(e))
exception = mk_exception(e)
except Exception as e: # pylint: disable=broad-except
data = ''
exception = mk_exception(e)
try:
create = getattr(self, 'create')
@ -228,7 +261,7 @@ def browsable_api_view(meth):
create_form = create_form_template.format(
fields='<br>'.join(input_fields),
path=self._cp_path_,
vpath='/'.join(vpath)
sub_path='/'.join(sub_path)
)
except AttributeError:
create_form = ''
@ -244,13 +277,18 @@ def browsable_api_view(meth):
docstring='<pre>{}</pre>'.format(self.__doc__) if self.__doc__ else '',
method=cherrypy.request.method,
path=self._cp_path_,
vpath='/'.join(vpath),
breadcrump=mk_breadcrump(['api', self._cp_path_] + list(vpath)),
sub_path='/'.join(sub_path),
breadcrump=mk_breadcrump(['api', self._cp_path_] + list(sub_path)),
status_code=cherrypy.response.status,
reponse_headers='\n'.join(
'{}: {}'.format(k, v) for k, v in cherrypy.response.headers.items()),
data=data,
create_form=create_form
exception=exception,
create_form=create_form,
delete_form=delete_form_template.format(path=self._cp_path_, sub_path='/'.join(
sub_path)) if sub_path else '',
params='<h2>Rrequest Params</h2><pre>{}</pre>'.format(
json.dumps(kwargs, indent=2)) if kwargs else '',
)
wrapper.exposed = True
@ -276,7 +314,7 @@ class Task(object):
sig = inspect.signature(func)
arg_list = [a for a in sig.parameters]
else:
sig = inspect.getargspec(func)
sig = getargspec(func)
arg_list = [a for a in sig.args]
for idx, arg in enumerate(arg_list):
@ -316,7 +354,7 @@ class Task(object):
if 'status' in task.ret_value:
status = task.ret_value['status']
else:
status = 500
status = getattr(ex, 'status', 500)
cherrypy.response.status = status
return task.ret_value
raise ex
@ -324,7 +362,6 @@ class Task(object):
cherrypy.response.status = 202
return {'name': self.name, 'metadata': md}
return value
wrapper.__wrapped__ = func
return wrapper
@ -336,10 +373,7 @@ class BaseControllerMeta(type):
if isinstance(thing, (types.FunctionType, types.MethodType))\
and getattr(thing, 'exposed', False):
# @cherrypy.tools.json_out() is incompatible with our browsable_api_view decorator.
cp_config = getattr(thing, '_cp_config', {})
if not cp_config.get('tools.json_out.on', False):
setattr(new_cls, a_name, browsable_api_view(thing))
setattr(new_cls, a_name, browsable_api_view(thing))
return new_cls
@ -355,23 +389,19 @@ class BaseController(object):
@classmethod
def _parse_function_args(cls, func):
# pylint: disable=deprecated-method
if sys.version_info > (3, 0): # pylint: disable=no-else-return
sig = inspect.signature(func)
cargs = [k for k, v in sig.parameters.items()
if k != 'self' and v.default is inspect.Parameter.empty and
(v.kind == inspect.Parameter.POSITIONAL_ONLY or
v.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD)]
else:
func = getattr(func, '__wrapped__', func)
args = inspect.getargspec(func)
nd = len(args.args) if not args.defaults else -len(args.defaults)
cargs = args.args[1:nd]
args = getargspec(func)
nd = len(args.args) if not args.defaults else -len(args.defaults)
cargs = args.args[1:nd]
# filter out controller path params
for idx, step in enumerate(cls._cp_path_.split('/')):
param = None
if step[0] == ':':
param = step[1:]
if step[0] == '{' and step[-1] == '}' and ':' in step[1:-1]:
param, _, _regex = step[1:-1].partition(':')
if param:
if param not in cargs:
raise Exception("function '{}' does not have the"
" positional argument '{}' in the {} "
@ -522,10 +552,7 @@ class RESTController(BaseController):
@staticmethod
def _function_args(func):
if sys.version_info > (3, 0): # pylint: disable=no-else-return
return list(inspect.signature(func).parameters.keys())
else:
return inspect.getargspec(func).args[1:] # pylint: disable=deprecated-method
return getargspec(func).args[1:]
@staticmethod
def _takes_json(func):

5
src/pybind/mgr/dashboard/controllers/cephfs.py
View File

@ -5,6 +5,7 @@ from collections import defaultdict
import cherrypy
from ..exceptions import DashboardException
from . import ApiController, AuthRequired, BaseController
from .. import mgr
from ..services.ceph_service import CephService
@ -80,7 +81,9 @@ class CephFS(BaseController):
try:
return int(fs_id)
except ValueError:
raise cherrypy.HTTPError(400, "Invalid cephfs id {}".format(fs_id))
raise DashboardException(code='invalid_cephfs_id',
msg="Invalid cephfs id {}".format(fs_id),
component='cephfs')
def _get_mds_names(self, filesystem_id=None):
names = []

22
src/pybind/mgr/dashboard/controllers/osd.py
View File

@ -1,13 +1,10 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import json
from mgr_module import CommandResult
from . import ApiController, AuthRequired, RESTController
from .. import logger, mgr
from .. import mgr
from ..services.ceph_service import CephService
from ..services.exception import handle_send_command_error
@ApiController('osd')
@ -51,20 +48,9 @@ class Osd(RESTController):
osds[str(osd['id'])] = osd
return osds
@handle_send_command_error('osd')
def get(self, svc_id):
result = CommandResult('')
mgr.send_command(result, 'osd', svc_id,
json.dumps({
'prefix': 'perf histogram dump',
}),
'')
r, outb, outs = result.wait()
if r != 0:
logger.warning('Failed to load histogram for OSD %s', svc_id)
logger.debug(outs)
histogram = outs
else:
histogram = json.loads(outb)
histogram = CephService.send_command('osd', srv_spec=svc_id, prefix='perf histogram dump')
return {
'osd_map': self.get_osd_map()[svc_id],
'osd_metadata': mgr.get_metadata('osd', svc_id),

8
src/pybind/mgr/dashboard/controllers/pool.py
View File

@ -6,6 +6,7 @@ import cherrypy
from . import ApiController, RESTController, AuthRequired
from .. import mgr
from ..services.ceph_service import CephService
from ..services.exception import handle_send_command_error
@ApiController('pool')
@ -55,12 +56,17 @@ class Pool(RESTController):
def get(self, pool_name, attrs=None, stats=False):
pools = self.list(attrs, stats)
return [pool for pool in pools if pool['pool_name'] == pool_name][0]
pool = [pool for pool in pools if pool['pool_name'] == pool_name]
if not pool:
return cherrypy.NotFound('No such pool')
return pool[0]
@handle_send_command_error('pool')
def delete(self, pool_name):
return CephService.send_command('mon', 'osd pool delete', pool=pool_name, pool2=pool_name,
sure='--yes-i-really-really-mean-it')
@handle_send_command_error('pool')
def create(self, pool, pg_num, pool_type, erasure_code_profile=None, flags=None,
application_metadata=None, rule_name=None, **kwargs):
ecp = erasure_code_profile if erasure_code_profile else None

40
src/pybind/mgr/dashboard/controllers/rbd.py
View File

@ -4,30 +4,29 @@
from __future__ import absolute_import
import math
from functools import partial
import cherrypy
import rados
import six
import rbd
from . import ApiController, AuthRequired, RESTController, Task
from .. import mgr
from ..services.ceph_service import CephService
from ..tools import ViewCache
from ..services.exception import handle_rados_error, handle_rbd_error, \
serialize_dashboard_exception
# pylint: disable=inconsistent-return-statements
def _rbd_exception_handler(ex):
if isinstance(ex, rbd.OSError):
return {'status': 409, 'detail': str(ex), 'errno': ex.errno,
'component': 'rbd'}
elif isinstance(ex, rados.OSError):
return {'status': 409, 'detail': str(ex), 'errno': ex.errno,
'component': 'rados'}
raise ex
# pylint: disable=not-callable
def RbdTask(name, metadata, wait_for):
return Task("rbd/{}".format(name), metadata, wait_for,
_rbd_exception_handler)
def composed_decorator(func):
func = handle_rados_error('pool')(func)
func = handle_rbd_error()(func)
return Task("rbd/{}".format(name), metadata, wait_for,
partial(serialize_dashboard_exception, include_http_status=True))(func)
return composed_decorator
def _rbd_call(pool_name, func, *args, **kwargs):
@ -78,9 +77,12 @@ def _format_features(features):
>>> _format_features(None) is None
True
>>> _format_features('not a list') is None
True
>>> _format_features('deep-flatten, exclusive-lock')
32
"""
if isinstance(features, six.string_types):
features = features.split(',')
if not isinstance(features, list):
return None
@ -254,9 +256,13 @@ class Rbd(RESTController):
result.append({'status': status, 'value': value, 'pool_name': pool})
return result
@handle_rbd_error()
@handle_rados_error('pool')
def list(self, pool_name=None):
return self._rbd_list(pool_name)
@handle_rbd_error()
@handle_rados_error('pool')
def get(self, pool_name, image_name):
ioctx = mgr.rados.open_ioctx(pool_name)
try:
@ -269,6 +275,8 @@ class Rbd(RESTController):
def create(self, name, pool_name, size, obj_size=None, features=None,
stripe_unit=None, stripe_count=None, data_pool=None):
size = int(size)
def _create(ioctx):
rbd_inst = rbd.RBD()

7
src/pybind/mgr/dashboard/controllers/rbd_mirroring.py
View File

@ -13,6 +13,7 @@ from . import ApiController, AuthRequired, BaseController
from .. import logger, mgr
from ..services.ceph_service import CephService
from ..tools import ViewCache
from ..services.exception import handle_rbd_error
@ViewCache()
@ -94,6 +95,7 @@ def get_daemons_and_pools(): # pylint: disable=R0915
mirror_mode = rbdctx.mirror_mode_get(ioctx)
except: # noqa pylint: disable=W0702
logger.exception("Failed to query mirror mode %s", pool_name)
mirror_mode = None
stats = {}
if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED:
@ -163,6 +165,7 @@ class RbdMirror(BaseController):
@cherrypy.expose
@cherrypy.tools.json_out()
@handle_rbd_error()
def __call__(self):
status, content_data = self._get_content_data()
return {'status': status, 'content_data': content_data}
@ -233,6 +236,7 @@ class RbdMirror(BaseController):
pass
except: # noqa pylint: disable=W0702
logger.exception("Failed to list mirror image status %s", pool_name)
raise
return data
@ -250,9 +254,6 @@ class RbdMirror(BaseController):
pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')]
_, data = get_daemons_and_pools()
if isinstance(data, Exception):
logger.exception("Failed to get rbd-mirror daemons list")
raise type(data)(str(data))
daemons = data.get('daemons', [])
pool_stats = data.get('pools', {})

2
src/pybind/mgr/dashboard/controllers/rgw.py
View File

@ -46,7 +46,7 @@ class RgwDaemon(RESTController):
}
service = CephService.get_service('rgw', svc_id)
if not service:
return daemon
raise cherrypy.NotFound('Service rgw {} is not available'.format(svc_id))
metadata = service['metadata']
status = service['status']

16
src/pybind/mgr/dashboard/controllers/summary.py
View File

@ -5,9 +5,10 @@ import json
import cherrypy
from .. import mgr
from . import AuthRequired, ApiController, BaseController
from .. import logger, mgr
from ..controllers.rbd_mirroring import get_daemons_and_pools
from ..tools import ViewCacheNoDataException
from ..services.ceph_service import CephService
from ..tools import TaskManager
@ -34,14 +35,13 @@ class Summary(BaseController):
]
def _rbd_mirroring(self):
_, data = get_daemons_and_pools()
try:
_, data = get_daemons_and_pools()
except ViewCacheNoDataException:
return {}
if isinstance(data, Exception):
logger.exception("Failed to get rbd-mirror daemons and pools")
raise type(data)(str(data))
else:
daemons = data.get('daemons', [])
pools = data.get('pools', {})
daemons = data.get('daemons', [])
pools = data.get('pools', {})
warnings = 0
errors = 0

42
src/pybind/mgr/dashboard/exceptions.py
View File

@ -2,4 +2,44 @@
from __future__ import absolute_import
# Add generic exceptions here.
class ViewCacheNoDataException(Exception):
def __init__(self):
self.status = 200
super(ViewCacheNoDataException, self).__init__('ViewCache: unable to retrieve data')
class DashboardException(Exception):
"""
Used for exceptions that are already handled and should end up as a user error.
Or, as a replacement for cherrypy.HTTPError(...)
Typically, you don't inherent from DashboardException
"""
# pylint: disable=too-many-arguments
def __init__(self, e=None, code=None, component=None, http_status_code=None, msg=None):
super(DashboardException, self).__init__(msg)
self._code = code
self.component = component
if e:
self.e = e
if http_status_code:
self.status = http_status_code
else:
self.status = 400
def __str__(self):
try:
return str(self.e)
except AttributeError:
return super(DashboardException, self).__str__()
@property
def errno(self):
return self.e.errno
@property
def code(self):
if self._code:
return str(self._code)
return str(abs(self.errno))

4
src/pybind/mgr/dashboard/module.py
View File

@ -23,6 +23,7 @@ except ImportError:
try:
import cherrypy
from cherrypy._cptools import HandlerWrapperTool
except ImportError:
# To be picked up and reported by .can_run()
cherrypy = None
@ -60,6 +61,7 @@ from .controllers import generate_routes, json_error_page
from .controllers.auth import Auth
from .tools import SessionExpireAtBrowserCloseTool, NotificationQueue, \
RequestLoggingTool, TaskManager
from .services.exception import dashboard_exception_handler
from .settings import options_command_list, options_schema_list, \
handle_option_command
@ -125,6 +127,8 @@ class SSLCherryPyConfig(object):
cherrypy.tools.authenticate = cherrypy.Tool('before_handler', Auth.check_auth)
cherrypy.tools.session_expire_at_browser_close = SessionExpireAtBrowserCloseTool()
cherrypy.tools.request_logging = RequestLoggingTool()
cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
priority=31)
# SSL initialization
cert = self.get_store("crt")

12
src/pybind/mgr/dashboard/services/ceph_service.py
View File

@ -6,6 +6,8 @@ import collections
from collections import defaultdict
import json
import rados
from mgr_module import CommandResult
try:
@ -20,6 +22,14 @@ except ImportError:
from .. import logger, mgr
class SendCommandError(rados.Error):
def __init__(self, err, prefix, argdict, errno):
self.errno = errno
self.prefix = prefix
self.argdict = argdict
super(SendCommandError, self).__init__(err)
class CephService(object):
@classmethod
def get_service_map(cls, service_name):
@ -153,7 +163,7 @@ class CephService(object):
msg = "send_command '{}' failed. (r={}, outs=\"{}\", kwargs={})".format(prefix, r, outs,
kwargs)
logger.error(msg)
raise ValueError(msg)
raise SendCommandError(outs, prefix, argdict, r)
else:
try:
return json.loads(outb)

122
src/pybind/mgr/dashboard/services/exception.py Normal file
View File

@ -0,0 +1,122 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import json
import sys
from contextlib import contextmanager
import cherrypy
import rbd
import rados
from .. import logger
from ..services.ceph_service import SendCommandError
from ..exceptions import ViewCacheNoDataException, DashboardException
from ..tools import wraps
if sys.version_info < (3, 0):
# Monkey-patch a __call__ method into @contextmanager to make
# it compatible to Python 3
from contextlib import GeneratorContextManager # pylint: disable=no-name-in-module
def init(self, *args):
if len(args) == 1:
self.gen = args[0]
elif len(args) == 3:
self.func, self.args, self.kwargs = args
else:
raise TypeError()
def enter(self):
if hasattr(self, 'func'):
self.gen = self.func(*self.args, **self.kwargs)
try:
return self.gen.next()
except StopIteration:
raise RuntimeError("generator didn't yield")
def call(self, f):
@wraps(f)
def wrapper(*args, **kwargs):
with self:
return f(*args, **kwargs)
return wrapper
GeneratorContextManager.__init__ = init
GeneratorContextManager.__enter__ = enter
GeneratorContextManager.__call__ = call
# pylint: disable=function-redefined
def contextmanager(func):
@wraps(func)
def helper(*args, **kwds):
return GeneratorContextManager(func, args, kwds)
return helper
def serialize_dashboard_exception(e, include_http_status=False, task=None):
"""
:type e: Exception
:param include_http_status: Used for Tasks, where the HTTP status code is not available.
"""
from ..tools import ViewCache
if isinstance(e, ViewCacheNoDataException):
return {'status': ViewCache.VALUE_NONE, 'value': None}
out = dict(detail=str(e))
try:
out['code'] = e.code
except AttributeError:
pass
component = getattr(e, 'component', None)
out['component'] = component if component else None
if include_http_status:
out['status'] = getattr(e, 'status', 500)
if task:
out['task'] = dict(name=task.name, metadata=task.metadata)
return out
def dashboard_exception_handler(handler, *args, **kwargs):
try:
with handle_rados_error(component=None): # make the None controller the fallback.
return handler(*args, **kwargs)
# Don't catch cherrypy.* Exceptions.
except (ViewCacheNoDataException, DashboardException) as e:
logger.exception('dashboard_exception_handler')
cherrypy.response.headers['Content-Type'] = 'application/json'
cherrypy.response.status = getattr(e, 'status', 400)
return json.dumps(serialize_dashboard_exception(e)).encode('utf-8')
@contextmanager
def handle_rbd_error():
try:
yield
except rbd.OSError as e:
raise DashboardException(e, component='rbd')
except rbd.Error as e:
raise DashboardException(e, component='rbd', code=e.__class__.__name__)
@contextmanager
def handle_rados_error(component):
try:
yield
except rados.OSError as e:
raise DashboardException(e, component=component)
except rados.Error as e:
raise DashboardException(e, component=component, code=e.__class__.__name__)
@contextmanager
def handle_send_command_error(component):
try:
yield
except SendCommandError as e:
raise DashboardException(e, component=component)

4
src/pybind/mgr/dashboard/tests/helper.py
View File

@ -7,11 +7,13 @@ import threading
import time
import cherrypy
from cherrypy._cptools import HandlerWrapperTool
from cherrypy.test import helper
from .. import logger
from ..controllers.auth import Auth
from ..controllers import json_error_page, generate_controller_routes
from ..services.exception import dashboard_exception_handler
from ..tools import SessionExpireAtBrowserCloseTool
@ -31,6 +33,8 @@ class ControllerTestCase(helper.CPWebCase):
def __init__(self, *args, **kwargs):
cherrypy.tools.authenticate = cherrypy.Tool('before_handler', Auth.check_auth)
cherrypy.tools.session_expire_at_browser_close = SessionExpireAtBrowserCloseTool()
cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
priority=31)
cherrypy.config.update({'error_page.default': json_error_page})
super(ControllerTestCase, self).__init__(*args, **kwargs)

172
src/pybind/mgr/dashboard/tests/test_exceptions.py Normal file
View File

@ -0,0 +1,172 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import time
import cherrypy
import rados
from ..services.ceph_service import SendCommandError
from ..controllers import RESTController, ApiController, Task
from .helper import ControllerTestCase
from ..services.exception import handle_rados_error, handle_send_command_error, \
serialize_dashboard_exception
from ..tools import ViewCache, TaskManager, NotificationQueue
# pylint: disable=W0613
@ApiController('foo')
class FooResource(RESTController):
@cherrypy.expose
@cherrypy.tools.json_out()
@handle_rados_error('foo')
def no_exception(self, param1, param2):
return [param1, param2]
@cherrypy.expose
@cherrypy.tools.json_out()
@handle_rados_error('foo')
def error_foo_controller(self):
raise rados.OSError('hi', errno=-42)
@cherrypy.expose
@cherrypy.tools.json_out()
@handle_send_command_error('foo')
def error_send_command(self):
raise SendCommandError('hi', 'prefix', {}, -42)
@cherrypy.expose
@cherrypy.tools.json_out()
def error_generic(self):
raise rados.Error('hi')
@cherrypy.expose
@cherrypy.tools.json_out()
def vc_no_data(self):
@ViewCache(timeout=0)
def _no_data():
time.sleep(0.2)
_no_data()
assert False
@handle_rados_error('foo')
@cherrypy.expose
@cherrypy.tools.json_out()
def vc_exception(self):
@ViewCache(timeout=10)
def _raise():
raise rados.OSError('hi', errno=-42)
_raise()
assert False
@cherrypy.expose
@cherrypy.tools.json_out()
def internal_server_error(self):
return 1/0
@handle_send_command_error('foo')
def list(self):
raise SendCommandError('list', 'prefix', {}, -42)
@cherrypy.expose
@cherrypy.tools.json_out()
@Task('task_exceptions/task_exception', {1: 2}, 1.0,
exception_handler=serialize_dashboard_exception)
@handle_rados_error('foo')
def task_exception(self):
raise rados.OSError('hi', errno=-42)
@cherrypy.expose
@cherrypy.tools.json_out()
def wait_task_exception(self):
ex, _ = TaskManager.list('task_exceptions/task_exception')
return bool(len(ex))
# pylint: disable=C0102
class Root(object):
foo = FooResource()
class RESTControllerTest(ControllerTestCase):
@classmethod
def setup_server(cls):
NotificationQueue.start_queue()
TaskManager.init()
cls.setup_controllers([FooResource])
def test_no_exception(self):
self._get('/foo/no_exception/a/b')
self.assertStatus(200)
self.assertJsonBody(
['a', 'b']
)
def test_error_foo_controller(self):
self._get('/foo/error_foo_controller')
self.assertStatus(400)
self.assertJsonBody(
{'detail': '[errno -42] hi', 'code': "42", 'component': 'foo'}
)
def test_error_send_command(self):
self._get('/foo/error_send_command')
self.assertStatus(400)
self.assertJsonBody(
{'detail': 'hi', 'code': "42", 'component': 'foo'}
)
def test_error_send_command_list(self):
self._get('/foo/')
self.assertStatus(400)
self.assertJsonBody(
{'detail': 'list', 'code': "42", 'component': 'foo'}
)
def test_error_send_command_bowsable_api(self):
self.getPage('/foo/error_send_command', headers=[('Accept', 'text/html')])
for err in ["'detail': 'hi'", "'component': 'foo'"]:
self.assertIn(err.replace("'", "\'").encode('utf-8'), self.body)
def test_error_foo_generic(self):
self._get('/foo/error_generic')
self.assertJsonBody({'detail': 'hi', 'code': 'Error', 'component': None})
self.assertStatus(400)
def test_viewcache_no_data(self):
self._get('/foo/vc_no_data')
self.assertStatus(200)
self.assertJsonBody({'status': ViewCache.VALUE_NONE, 'value': None})
def test_viewcache_exception(self):
self._get('/foo/vc_exception')
self.assertStatus(400)
self.assertJsonBody(
{'detail': '[errno -42] hi', 'code': "42", 'component': 'foo'}
)
def test_task_exception(self):
self._get('/foo/task_exception')
self.assertStatus(400)
self.assertJsonBody(
{'detail': '[errno -42] hi', 'code': "42", 'component': 'foo',
'task': {'name': 'task_exceptions/task_exception', 'metadata': {'1': 2}}}
)
self._get('/foo/wait_task_exception')
while self.jsonBody():
time.sleep(0.5)
self._get('/foo/wait_task_exception')
def test_internal_server_error(self):
self._get('/foo/internal_server_error')
self.assertStatus(500)
self.assertIn('unexpected condition', self.jsonBody()['detail'])
def test_404(self):
self._get('/foonot_found')
self.assertStatus(404)
self.assertIn('detail', self.jsonBody())

36
src/pybind/mgr/dashboard/tests/test_task.py
View File

@ -6,7 +6,9 @@ import unittest
import threading
import time
from collections import defaultdict
from functools import partial
from ..services.exception import serialize_dashboard_exception
from ..tools import NotificationQueue, TaskManager, TaskExecutor
@ -41,13 +43,11 @@ class MyTask(object):
self.handle_ex = handle_ex
self._event = threading.Event()
def _handle_exception(self, ex):
return {'status': 409, 'detail': str(ex)}
def run(self, ns, timeout=None):
args = ['dummy arg']
kwargs = {'dummy': 'arg'}
h_ex = self._handle_exception if self.handle_ex else None
h_ex = partial(serialize_dashboard_exception,
include_http_status=True) if self.handle_ex else None
if not self.is_async:
task = TaskManager.run(
ns, self.metadata(), self.task_op, args, kwargs,
@ -183,7 +183,7 @@ class TaskTest(unittest.TestCase):
state, result = task1.run('test5/task1', 0.5)
self.assertEqual(state, TaskManager.VALUE_EXECUTING)
self.assertIsNone(result)
ex_t, _ = TaskManager.list()
ex_t, _ = TaskManager.list('test5/*')
self.assertEqual(len(ex_t), 1)
self.assertEqual(ex_t[0].name, 'test5/task1')
self.assertEqual(ex_t[0].progress, 30)
@ -199,12 +199,12 @@ class TaskTest(unittest.TestCase):
self.assertEqual(task.progress, 60)
task2.resume()
self.wait_for_task('test5/task2')
ex_t, _ = TaskManager.list()
ex_t, _ = TaskManager.list('test5/*')
self.assertEqual(len(ex_t), 1)
self.assertEqual(ex_t[0].name, 'test5/task1')
task1.resume()
self.wait_for_task('test5/task1')
ex_t, _ = TaskManager.list()
ex_t, _ = TaskManager.list('test5/*')
self.assertEqual(len(ex_t), 0)
def test_task_idempotent(self):
@ -213,13 +213,13 @@ class TaskTest(unittest.TestCase):
state, result = task1.run('test6/task1', 0.5)
self.assertEqual(state, TaskManager.VALUE_EXECUTING)
self.assertIsNone(result)
ex_t, _ = TaskManager.list()
ex_t, _ = TaskManager.list('test6/*')
self.assertEqual(len(ex_t), 1)
self.assertEqual(ex_t[0].name, 'test6/task1')
state, result = task1_clone.run('test6/task1', 0.5)
self.assertEqual(state, TaskManager.VALUE_EXECUTING)
self.assertIsNone(result)
ex_t, _ = TaskManager.list()
ex_t, _ = TaskManager.list('test6/*')
self.assertEqual(len(ex_t), 1)
self.assertEqual(ex_t[0].name, 'test6/task1')
task1.resume()
@ -416,6 +416,18 @@ class TaskTest(unittest.TestCase):
self.assertEqual(fn_t[0]['progress'], 50)
self.assertFalse(fn_t[0]['success'])
self.assertIsNotNone(fn_t[0]['exception'])
self.assertEqual(fn_t[0]['exception'],
{"status": 409,
"detail": "Task Unexpected Exception"})
self.assertEqual(fn_t[0]['exception'], {
'component': None,
'detail': 'Task Unexpected Exception',
'status': 500,
'task': {
'metadata': {
'fail': True,
'handle_ex': True,
'is_async': False,
'op_seconds': 1,
'progress': 50,
'wait': False},
'name': 'test15/task1'
}
})

31
src/pybind/mgr/dashboard/tests/test_tools.py
View File

@ -3,11 +3,13 @@ from __future__ import absolute_import
import unittest
import cherrypy
from cherrypy.lib.sessions import RamSession
from mock import patch
from ..services.exception import handle_rados_error
from .helper import ControllerTestCase
from ..controllers import RESTController, ApiController
from ..controllers import RESTController, ApiController, BaseController
from ..tools import is_valid_ipv6_address, dict_contains_path
@ -42,11 +44,25 @@ class FooResourceDetail(RESTController):
return {'detail': (key, [method])}
@ApiController('rgw/proxy/{path:.*}')
class GenerateControllerRoutesController(BaseController):
@cherrypy.expose
def __call__(self, path, **params):
pass
@ApiController('fooargs')
class FooArgs(RESTController):
def set(self, code, name=None, opt1=None, opt2=None):
return {'code': code, 'name': name, 'opt1': opt1, 'opt2': opt2}
@handle_rados_error('foo')
def create(self, my_arg_name):
return my_arg_name
def list(self):
raise cherrypy.NotFound()
# pylint: disable=blacklisted-name
class Root(object):
@ -58,7 +74,8 @@ class RESTControllerTest(ControllerTestCase):
@classmethod
def setup_server(cls):
cls.setup_controllers([FooResource, FooResourceDetail, FooArgs])
cls.setup_controllers(
[FooResource, FooResourceDetail, FooArgs, GenerateControllerRoutesController])
def test_empty(self):
self._delete("/foo")
@ -134,6 +151,16 @@ class RESTControllerTest(ControllerTestCase):
method='put')
self.assertStatus(404)
def test_create_form(self):
self.getPage('/fooargs', headers=[('Accept', 'text/html')])
self.assertIn('my_arg_name', self.body.decode('utf-8'))
def test_generate_controller_routes(self):
# We just need to add this controller in setup_server():
# noinspection PyStatementEffect
# pylint: disable=pointless-statement
GenerateControllerRoutesController
class TestFunctions(unittest.TestCase):

35
src/pybind/mgr/dashboard/tools.py
View File

@ -1,6 +1,10 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import sys
import inspect
import functools
import collections
from datetime import datetime, timedelta
import fnmatch
@ -11,6 +15,7 @@ from six.moves import urllib
import cherrypy
from . import logger
from .exceptions import ViewCacheNoDataException
class RequestLoggingTool(cherrypy.Tool):
@ -197,7 +202,7 @@ class ViewCache(object):
# We have some data, but it doesn't meet freshness requirements
return ViewCache.VALUE_STALE, self.value
# We have no data, not even stale data
return ViewCache.VALUE_NONE, None
raise ViewCacheNoDataException()
def __init__(self, timeout=5):
self.timeout = timeout
@ -580,6 +585,9 @@ class Task(object):
return "Task(ns={}, md={})" \
.format(self.name, self.metadata)
def __repr__(self):
return str(self)
def _run(self):
with self.lock:
assert not self.running
@ -594,7 +602,7 @@ class Task(object):
if exception and self.ex_handler:
# pylint: disable=broad-except
try:
ret_value = self.ex_handler(exception)
ret_value = self.ex_handler(exception, task=self)
except Exception as ex:
exception = ex
with self.lock:
@ -708,3 +716,26 @@ def dict_contains_path(dct, keys):
return dict_contains_path(dct, keys)
return False
return True
if sys.version_info > (3, 0):
wraps = functools.wraps
_getargspec = inspect.getfullargspec
else:
def wraps(func):
def decorator(wrapper):
new_wrapper = functools.wraps(func)(wrapper)
new_wrapper.__wrapped__ = func # set __wrapped__ even for Python 2
return new_wrapper
return decorator
_getargspec = inspect.getargspec
def getargspec(func):
try:
while True:
func = func.__wrapped__
except AttributeError:
pass
return _getargspec(func)