diff --git a/qa/tasks/mgr/test_dashboard.py b/qa/tasks/mgr/test_dashboard.py index 3b8a2cc80c1..60c1e334ee4 100644 --- a/qa/tasks/mgr/test_dashboard.py +++ b/qa/tasks/mgr/test_dashboard.py @@ -24,7 +24,7 @@ class TestDashboard(MgrTestCase): self.mgr_cluster.mgr_fail(original_active) failed_over_uri = self._get_uri("dashboard") - log.info("After failover running at {0}".format(original_uri)) + log.info("After failover running at {0}".format(failed_over_uri)) self.assertNotEqual(original_uri, failed_over_uri) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9651c75041f..aec0681ced3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -274,6 +274,14 @@ if(WITH_PYTHON3) find_package(Python3Libs 3 REQUIRED) endif() +# the major version of the python bindings as a dependency of other +# targets +if(WITH_PYTHON2) + set(PY_BINDING_INFIX "") +else() + set(PY_BINDING_INFIX 3) +endif() + if(HAVE_XIO) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${XIO_INCLUDE_DIR}") list(APPEND EXTRALIBS ${XIO_LIBRARY} pthread) @@ -1089,13 +1097,12 @@ add_custom_target(vstart-base DEPENDS monmaptool crushtool rados - cython_rados - ) + cython${PY_BINDING_INFIX}_rados) add_custom_target(vstart DEPENDS vstart-base ceph-mds - cython_rbd) + cython${PY_BINDING_INFIX}_rbd) if(WITH_RADOSGW) add_dependencies(vstart radosgw radosgw-admin) endif(WITH_RADOSGW) @@ -1108,7 +1115,7 @@ endif(WITH_LTTNG) add_custom_target(cephfs_testing DEPENDS vstart rados - cython_modules + cython${PY_BINDING_INFIX}_modules cephfs cls_cephfs ceph-fuse diff --git a/src/init-ceph.in b/src/init-ceph.in index e8c0ada2b3a..d13032a1675 100755 --- a/src/init-ceph.in +++ b/src/init-ceph.in @@ -34,7 +34,7 @@ else ETCDIR=. ASSUME_DEV=1 CEPH_LIB=$CEPH_ROOT/build/lib - echo "$PYTHONPATH" | grep -q $CEPH_LIB || export PYTHONPATH=$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH + echo "$PYTHONPATH" | grep -q $CEPH_LIB || export PYTHONPATH=$CEPH_LIB/cython_modules/lib.@MGR_PYTHON_VERSION_MAJOR@:$PYTHONPATH echo "$LD_LIBRARY_PATH" | grep -q $CEPH_LIB || export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH echo "$DYLD_LIBRARY_PATH" | grep -q $CEPH_LIB || export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH else diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc index f3fede8de6e..fcb0b810767 100644 --- a/src/mgr/PyModule.cc +++ b/src/mgr/PyModule.cc @@ -151,12 +151,75 @@ std::string PyModule::get_site_packages() return site_packages.str(); } +#if PY_MAJOR_VERSION >= 3 +PyObject* PyModule::init_ceph_logger() +{ + auto py_logger = PyModule_Create(&ceph_logger_module); + PySys_SetObject("stderr", py_logger); + PySys_SetObject("stdout", py_logger); + return py_logger; +} +#else +void PyModule::init_ceph_logger() +{ + auto py_logger = Py_InitModule("ceph_logger", log_methods); + PySys_SetObject(const_cast("stderr"), py_logger); + PySys_SetObject(const_cast("stdout"), py_logger); +} +#endif + +#if PY_MAJOR_VERSION >= 3 +PyObject* PyModule::init_ceph_module() +#else +void PyModule::init_ceph_module() +#endif +{ + static PyMethodDef module_methods[] = { + {nullptr, nullptr, 0, nullptr} + }; +#if PY_MAJOR_VERSION >= 3 + static PyModuleDef ceph_module_def = { + PyModuleDef_HEAD_INIT, + "ceph_module", + nullptr, + -1, + module_methods, + nullptr, + nullptr, + nullptr, + nullptr + }; + PyObject *ceph_module = PyModule_Create(&ceph_module_def); +#else + PyObject *ceph_module = Py_InitModule("ceph_module", module_methods); +#endif + assert(ceph_module != nullptr); + std::map classes{ + {{"BaseMgrModule", &BaseMgrModuleType}, + {"BaseMgrStandbyModule", &BaseMgrStandbyModuleType}, + {"BasePyOSDMap", &BasePyOSDMapType}, + {"BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType}, + {"BasePyCRUSH", &BasePyCRUSHType}} + }; + for (auto [name, type] : classes) { + type->tp_new = PyType_GenericNew; + if (PyType_Ready(type) < 0) { + assert(0); + } + Py_INCREF(type); + + PyModule_AddObject(ceph_module, name, (PyObject *)type); + } +#if PY_MAJOR_VERSION >= 3 + return ceph_module; +#endif +} int PyModule::load(PyThreadState *pMainThreadState) { assert(pMainThreadState != nullptr); - // Configure sub-interpreter and construct C++-generated python classes + // Configure sub-interpreter { SafeThreadState sts(pMainThreadState); Gil gil(sts); @@ -176,19 +239,6 @@ int PyModule::load(PyThreadState *pMainThreadState) const char *argv[] = {"ceph-mgr"}; PySys_SetArgv(1, (char**)argv); #endif - - if (g_conf->daemonize) { -#if PY_MAJOR_VERSION >= 3 - auto py_logger = PyModule_Create(&ceph_logger_module); - PySys_SetObject("stderr", py_logger); - PySys_SetObject("stdout", py_logger); -#else - auto py_logger = Py_InitModule("ceph_logger", log_methods); - PySys_SetObject(const_cast("stderr"), py_logger); - PySys_SetObject(const_cast("stdout"), py_logger); -#endif - } - // Configure sys.path to include mgr_module_path string paths = (":" + get_site_packages() + ":" + g_conf->get_val("mgr_module_path")); @@ -203,46 +253,7 @@ int PyModule::load(PyThreadState *pMainThreadState) dout(10) << "Computed sys.path '" << sys_path << "'" << dendl; #endif } - - static PyMethodDef module_methods[] = { - {nullptr} - }; -#if PY_MAJOR_VERSION >= 3 - static PyModuleDef ceph_module_def = { - PyModuleDef_HEAD_INIT, - "ceph_module", - nullptr, - -1, - module_methods, - }; -#endif - - // Initialize module -#if PY_MAJOR_VERSION >= 3 - PyObject *ceph_module = PyModule_Create(&ceph_module_def); -#else - PyObject *ceph_module = Py_InitModule("ceph_module", module_methods); -#endif - assert(ceph_module != nullptr); - - auto load_class = [ceph_module](const char *name, PyTypeObject *type) - { - type->tp_new = PyType_GenericNew; - if (PyType_Ready(type) < 0) { - assert(0); - } - Py_INCREF(type); - - PyModule_AddObject(ceph_module, name, (PyObject *)type); - }; - - load_class("BaseMgrModule", &BaseMgrModuleType); - load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType); - load_class("BasePyOSDMap", &BasePyOSDMapType); - load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType); - load_class("BasePyCRUSH", &BasePyCRUSHType); } - // Environment is all good, import the external module { Gil gil(pMyThreadState); diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h index 14f418e446e..a25792f5a67 100644 --- a/src/mgr/PyModule.h +++ b/src/mgr/PyModule.h @@ -77,7 +77,13 @@ public: ~PyModule(); int load(PyThreadState *pMainThreadState); - +#if PY_MAJOR_VERSION >= 3 + static PyObject* init_ceph_logger(); + static PyObject* init_ceph_module(); +#else + static void init_ceph_logger(); + static void init_ceph_module(); +#endif /** * Extend `out` with the contents of `this->commands` diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc index 867814bfa16..82060f483f7 100644 --- a/src/mgr/PyModuleRegistry.cc +++ b/src/mgr/PyModuleRegistry.cc @@ -57,6 +57,11 @@ int PyModuleRegistry::init(const MgrMap &map) #else Py_SetProgramName(const_cast(PYTHON_EXECUTABLE)); #endif + // Add more modules + if (g_conf->get_val("daemonize")) { + PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger); + } + PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module); Py_InitializeEx(0); // Let CPython know that we will be calling it back from other diff --git a/src/pybind/mgr/balancer/__init__.py b/src/pybind/mgr/balancer/__init__.py index 79f5b86fd50..8f210ac9247 100644 --- a/src/pybind/mgr/balancer/__init__.py +++ b/src/pybind/mgr/balancer/__init__.py @@ -1,2 +1 @@ - -from module import * # NOQA +from .module import Module diff --git a/src/pybind/mgr/dashboard/__init__.py b/src/pybind/mgr/dashboard/__init__.py index 79f5b86fd50..5d6e417e4d8 100644 --- a/src/pybind/mgr/dashboard/__init__.py +++ b/src/pybind/mgr/dashboard/__init__.py @@ -1,2 +1,2 @@ -from module import * # NOQA +from .module import Module, StandbyModule diff --git a/src/pybind/mgr/dashboard/cephfs_clients.py b/src/pybind/mgr/dashboard/cephfs_clients.py index 40d79713f3b..ebbb556abcf 100644 --- a/src/pybind/mgr/dashboard/cephfs_clients.py +++ b/src/pybind/mgr/dashboard/cephfs_clients.py @@ -2,7 +2,7 @@ import json from mgr_module import CommandResult -from remote_view_cache import RemoteViewCache +from .remote_view_cache import RemoteViewCache class CephFSClients(RemoteViewCache): diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py index 0a4026d113d..f195c4dd1e6 100644 --- a/src/pybind/mgr/dashboard/module.py +++ b/src/pybind/mgr/dashboard/module.py @@ -3,12 +3,20 @@ Demonstrate writing a Ceph web interface inside a mgr module. """ +from __future__ import absolute_import + # We must share a global reference to this instance, because it is the # gatekeeper to all accesses to data from the C++ side (e.g. the REST API # request handlers need to see it) from collections import defaultdict +from functools import cmp_to_key import collections +try: + iteritems = dict.iteritems +except AttributeError: + iteritems = dict.items + _global_instance = {'plugin': None} def global_instance(): assert _global_instance['plugin'] is not None @@ -26,18 +34,21 @@ import socket import cherrypy import jinja2 -import urlparse +try: + import urlparse +except ImportError: + import urllib.parse as urlparse from mgr_module import MgrModule, MgrStandbyModule, CommandResult -from types import OsdMap, NotFound, Config, FsMap, MonMap, \ +from .types import OsdMap, NotFound, Config, FsMap, MonMap, \ PgSummary, Health, MonStatus -import rbd_iscsi -import rbd_mirroring -from rbd_ls import RbdLs, RbdPoolLs -from cephfs_clients import CephFSClients -from rgw import RGWDaemons +from . import rbd_iscsi +from . import rbd_mirroring +from .rbd_ls import RbdLs, RbdPoolLs +from .cephfs_clients import CephFSClients +from .rgw import RGWDaemons log = logging.getLogger("dashboard") @@ -68,7 +79,7 @@ def get_prefixed_url(url): def to_sorted_array(data): assert isinstance(data, dict) - return sorted(data.iteritems()) + return sorted(iteritems(data)) def prepare_url_prefix(url_prefix): """ @@ -367,7 +378,7 @@ class Module(MgrModule): ) # Find the standby replays - for gid_str, daemon_info in mdsmap['info'].iteritems(): + for gid_str, daemon_info in iteritems(mdsmap['info']): if daemon_info['state'] != "up:standby-replay": continue @@ -471,14 +482,12 @@ class Module(MgrModule): # Transform the `checks` dict into a list for the convenience # of rendering from javascript. checks = [] - for k, v in health['checks'].iteritems(): + for k, v in iteritems(health['checks']): v['type'] = k checks.append(v) - checks = sorted(checks, cmp=lambda a, b: a['severity'] > b['severity']) - + checks = sorted(checks, key=lambda x: x['severity']) health['checks'] = checks - return health def _toplevel_data(self): @@ -1208,32 +1217,28 @@ class Module(MgrModule): @cherrypy.expose def index(self, rgw_id=None): if rgw_id is not None: - template = env.get_template("rgw_detail.html") - toplevel_data = self._toplevel_data() - return template.render( - url_prefix=global_instance().url_prefix, - ceph_version=global_instance().version, - path_info='/rgw' + cherrypy.request.path_info, - toplevel_data=json.dumps(toplevel_data, indent=2), - content_data=json.dumps(self.rgw_data(rgw_id), indent=2) - ) + template = env.get_template("rgw_detail.html") + toplevel_data = self._toplevel_data() + return template.render( + url_prefix=global_instance().url_prefix, + ceph_version=global_instance().version, + path_info='/rgw' + cherrypy.request.path_info, + toplevel_data=json.dumps(toplevel_data, indent=2), + content_data=json.dumps(self.rgw_data(rgw_id), indent=2) + ) else: + # List all RGW servers + template = env.get_template("rgw.html") + toplevel_data = self._toplevel_data() + content_data = self._rgw_daemons() + return template.render( + url_prefix = global_instance().url_prefix, + ceph_version=global_instance().version, + path_info='/rgw' + cherrypy.request.path_info, + toplevel_data=json.dumps(toplevel_data, indent=2), + content_data=json.dumps(content_data, indent=2) + ) - """ List all RGW servers """ - - template = env.get_template("rgw.html") - toplevel_data = self._toplevel_data() - - content_data = self._rgw_daemons() - - return template.render( - url_prefix = global_instance().url_prefix, - ceph_version=global_instance().version, - path_info='/rgw' + cherrypy.request.path_info, - toplevel_data=json.dumps(toplevel_data, indent=2), - content_data=json.dumps(content_data, indent=2) - ) - def _rgw_daemons(self): status, data = global_instance().rgw_daemons.get() if data is None: @@ -1268,7 +1273,7 @@ class Module(MgrModule): @cherrypy.expose @cherrypy.tools.json_out() def rgw_data(self, rgw_id): - return self._rgw(rgw_id) + return self._rgw(rgw_id) cherrypy.tree.mount(Root(), get_prefixed_url("/"), conf) cherrypy.tree.mount(OSDEndpoint(), get_prefixed_url("/osd"), conf) diff --git a/src/pybind/mgr/dashboard/rbd_iscsi.py b/src/pybind/mgr/dashboard/rbd_iscsi.py index 6f5e75ed180..6152c8f8087 100644 --- a/src/pybind/mgr/dashboard/rbd_iscsi.py +++ b/src/pybind/mgr/dashboard/rbd_iscsi.py @@ -1,7 +1,8 @@ +from __future__ import absolute_import import rados import rbd -from remote_view_cache import RemoteViewCache +from .remote_view_cache import RemoteViewCache SERVICE_TYPE = 'tcmu-runner' diff --git a/src/pybind/mgr/dashboard/rbd_ls.py b/src/pybind/mgr/dashboard/rbd_ls.py index 96d362e3467..1f5108aa7cc 100644 --- a/src/pybind/mgr/dashboard/rbd_ls.py +++ b/src/pybind/mgr/dashboard/rbd_ls.py @@ -1,8 +1,9 @@ +from __future__ import absolute_import import rbd import rados -from types import OsdMap -from remote_view_cache import RemoteViewCache +from .types import OsdMap +from .remote_view_cache import RemoteViewCache class RbdPoolLs(RemoteViewCache): def _get(self): diff --git a/src/pybind/mgr/dashboard/rbd_mirroring.py b/src/pybind/mgr/dashboard/rbd_mirroring.py index e549ed51727..08e63e863e6 100644 --- a/src/pybind/mgr/dashboard/rbd_mirroring.py +++ b/src/pybind/mgr/dashboard/rbd_mirroring.py @@ -1,9 +1,10 @@ +from __future__ import absolute_import import json import re import rados import rbd -from remote_view_cache import RemoteViewCache +from .remote_view_cache import RemoteViewCache class DaemonsAndPools(RemoteViewCache): def _get(self): diff --git a/src/pybind/mgr/dashboard/rgw.py b/src/pybind/mgr/dashboard/rgw.py index 383233440cc..0e851f9e787 100644 --- a/src/pybind/mgr/dashboard/rgw.py +++ b/src/pybind/mgr/dashboard/rgw.py @@ -1,7 +1,9 @@ # -*- code: utf-8 -*- +from __future__ import absolute_import + import json -from remote_view_cache import RemoteViewCache +from .remote_view_cache import RemoteViewCache import logging diff --git a/src/pybind/mgr/influx/__init__.py b/src/pybind/mgr/influx/__init__.py index 0440e0705fb..8f210ac9247 100644 --- a/src/pybind/mgr/influx/__init__.py +++ b/src/pybind/mgr/influx/__init__.py @@ -1 +1 @@ -from module import * # NOQA +from .module import Module diff --git a/src/pybind/mgr/localpool/__init__.py b/src/pybind/mgr/localpool/__init__.py index 79f5b86fd50..8f210ac9247 100644 --- a/src/pybind/mgr/localpool/__init__.py +++ b/src/pybind/mgr/localpool/__init__.py @@ -1,2 +1 @@ - -from module import * # NOQA +from .module import Module diff --git a/src/pybind/mgr/prometheus/__init__.py b/src/pybind/mgr/prometheus/__init__.py index 6441bb42033..763a84261e6 100644 --- a/src/pybind/mgr/prometheus/__init__.py +++ b/src/pybind/mgr/prometheus/__init__.py @@ -1,2 +1,2 @@ -from module import * # NOQA +from .module import Module, StandbyModule diff --git a/src/pybind/mgr/restful/__init__.py b/src/pybind/mgr/restful/__init__.py index 0440e0705fb..8f210ac9247 100644 --- a/src/pybind/mgr/restful/__init__.py +++ b/src/pybind/mgr/restful/__init__.py @@ -1 +1 @@ -from module import * # NOQA +from .module import Module diff --git a/src/pybind/mgr/restful/api/__init__.py b/src/pybind/mgr/restful/api/__init__.py index 07d73dd8274..ebf2bd1dfcd 100644 --- a/src/pybind/mgr/restful/api/__init__.py +++ b/src/pybind/mgr/restful/api/__init__.py @@ -1,14 +1,14 @@ from pecan import expose from pecan.rest import RestController -from config import Config -from crush import Crush -from doc import Doc -from mon import Mon -from osd import Osd -from pool import Pool -from request import Request -from server import Server +from .config import Config +from .crush import Crush +from .doc import Doc +from .mon import Mon +from .osd import Osd +from .pool import Pool +from .request import Request +from .server import Server class Root(RestController): diff --git a/src/pybind/mgr/restful/api/config.py b/src/pybind/mgr/restful/api/config.py index 51fabfad4d6..4176c003c4a 100644 --- a/src/pybind/mgr/restful/api/config.py +++ b/src/pybind/mgr/restful/api/config.py @@ -1,7 +1,7 @@ from pecan import expose, request from pecan.rest import RestController -from restful import common, module +from restful import common, context from restful.decorators import auth @@ -12,7 +12,7 @@ class ConfigOsd(RestController): """ Show OSD configuration options """ - flags = module.instance.get("osd_map")['flags'] + flags = context.instance.get("osd_map")['flags'] # pause is a valid osd config command that sets pauserd,pausewr flags = flags.replace('pauserd,pausewr', 'pause') @@ -33,7 +33,7 @@ class ConfigOsd(RestController): valid_flags = set(args.keys()) & set(common.OSD_FLAGS) invalid_flags = list(set(args.keys()) - valid_flags) if invalid_flags: - module.instance.log.warn("%s not valid to set/unset" % invalid_flags) + context.instance.log.warn("%s not valid to set/unset" % invalid_flags) for flag in list(valid_flags): if args[flag]: @@ -46,7 +46,7 @@ class ConfigOsd(RestController): 'key': flag, }) - return module.instance.submit_request([commands], **kwargs) + return context.instance.submit_request([commands], **kwargs) @@ -61,7 +61,7 @@ class ConfigClusterKey(RestController): """ Show specific configuration option """ - return module.instance.get("config").get(self.key, None) + return context.instance.get("config").get(self.key, None) @@ -72,7 +72,7 @@ class ConfigCluster(RestController): """ Show all cluster configuration options """ - return module.instance.get("config") + return context.instance.get("config") @expose() diff --git a/src/pybind/mgr/restful/api/crush.py b/src/pybind/mgr/restful/api/crush.py index 11a04264da2..ee8589df0a4 100644 --- a/src/pybind/mgr/restful/api/crush.py +++ b/src/pybind/mgr/restful/api/crush.py @@ -1,7 +1,7 @@ from pecan import expose from pecan.rest import RestController -from restful import common, module +from restful import common, context from collections import defaultdict from restful.decorators import auth @@ -14,8 +14,8 @@ class CrushRule(RestController): """ Show crush rules """ - rules = module.instance.get('osd_map_crush')['rules'] - nodes = module.instance.get('osd_map_tree')['nodes'] + rules = context.instance.get('osd_map_crush')['rules'] + nodes = context.instance.get('osd_map_tree')['nodes'] for rule in rules: rule['osd_count'] = len(common.crush_rule_osds(nodes, rule)) diff --git a/src/pybind/mgr/restful/api/doc.py b/src/pybind/mgr/restful/api/doc.py index 96e4d6a3379..f1038c21b16 100644 --- a/src/pybind/mgr/restful/api/doc.py +++ b/src/pybind/mgr/restful/api/doc.py @@ -1,7 +1,7 @@ from pecan import expose from pecan.rest import RestController -from restful import module +from restful import context import restful @@ -12,4 +12,4 @@ class Doc(RestController): """ Show documentation information """ - return module.instance.get_doc_api(restful.api.Root) + return context.instance.get_doc_api(restful.api.Root) diff --git a/src/pybind/mgr/restful/api/mon.py b/src/pybind/mgr/restful/api/mon.py index 5b19aaefb29..807d6df3bf7 100644 --- a/src/pybind/mgr/restful/api/mon.py +++ b/src/pybind/mgr/restful/api/mon.py @@ -1,7 +1,7 @@ from pecan import expose, response from pecan.rest import RestController -from restful import module +from restful import context from restful.decorators import auth @@ -18,7 +18,7 @@ class MonName(RestController): """ mon = filter( lambda x: x['name'] == self.name, - module.instance.get_mons() + context.instance.get_mons() ) if len(mon) != 1: @@ -36,7 +36,7 @@ class Mon(RestController): """ Show the information for all the monitors """ - return module.instance.get_mons() + return context.instance.get_mons() @expose() diff --git a/src/pybind/mgr/restful/api/osd.py b/src/pybind/mgr/restful/api/osd.py index b42f33941f1..99f9ed4c8e7 100644 --- a/src/pybind/mgr/restful/api/osd.py +++ b/src/pybind/mgr/restful/api/osd.py @@ -1,7 +1,7 @@ from pecan import expose, request, response from pecan.rest import RestController -from restful import common, module +from restful import common, context from restful.decorators import auth @@ -16,7 +16,7 @@ class OsdIdCommand(RestController): """ Show implemented commands for the OSD id """ - osd = module.instance.get_osd_by_id(self.osd_id) + osd = context.instance.get_osd_by_id(self.osd_id) if not osd: response.status = 500 @@ -36,7 +36,7 @@ class OsdIdCommand(RestController): """ command = request.json.get('command', None) - osd = module.instance.get_osd_by_id(self.osd_id) + osd = context.instance.get_osd_by_id(self.osd_id) if not osd: response.status = 500 @@ -46,7 +46,7 @@ class OsdIdCommand(RestController): response.status = 500 return {'message': 'Command "%s" not available' % command} - return module.instance.submit_request([[{ + return context.instance.submit_request([[{ 'prefix': 'osd ' + command, 'who': str(self.osd_id) }]], **kwargs) @@ -65,7 +65,7 @@ class OsdId(RestController): """ Show the information for the OSD id """ - osd = module.instance.get_osds(ids=[str(self.osd_id)]) + osd = context.instance.get_osds(ids=[str(self.osd_id)]) if len(osd) != 1: response.status = 500 return {'message': 'Failed to identify the OSD id "%d"' % self.osd_id} @@ -112,7 +112,7 @@ class OsdId(RestController): 'weight': args['reweight'] }) - return module.instance.submit_request([commands], **kwargs) + return context.instance.submit_request([commands], **kwargs) @@ -127,7 +127,7 @@ class Osd(RestController): # TODO Filter by ids pool_id = kwargs.get('pool', None) - return module.instance.get_osds(pool_id) + return context.instance.get_osds(pool_id) @expose() diff --git a/src/pybind/mgr/restful/api/pool.py b/src/pybind/mgr/restful/api/pool.py index a59e259cb62..abf3e98133a 100644 --- a/src/pybind/mgr/restful/api/pool.py +++ b/src/pybind/mgr/restful/api/pool.py @@ -1,7 +1,7 @@ from pecan import expose, request, response from pecan.rest import RestController -from restful import common, module +from restful import common, context from restful.decorators import auth @@ -16,7 +16,7 @@ class PoolId(RestController): """ Show the information for the pool id """ - pool = module.instance.get_pool_by_id(self.pool_id) + pool = context.instance.get_pool_by_id(self.pool_id) if not pool: response.status = 500 @@ -34,14 +34,14 @@ class PoolId(RestController): """ Modify the information for the pool id """ - try: + try: args = request.json except ValueError: response.status = 400 return {'message': 'Bad request: malformed JSON or wrong Content-Type'} # Get the pool info for its name - pool = module.instance.get_pool_by_id(self.pool_id) + pool = context.instance.get_pool_by_id(self.pool_id) if not pool: response.status = 500 return {'message': 'Failed to identify the pool id "%d"' % self.pool_id} @@ -53,7 +53,7 @@ class PoolId(RestController): return {'message': 'Invalid arguments found: "%s"' % str(invalid)} # Schedule the update request - return module.instance.submit_request(common.pool_update_commands(pool['pool_name'], args), **kwargs) + return context.instance.submit_request(common.pool_update_commands(pool['pool_name'], args), **kwargs) @expose(template='json') @@ -62,13 +62,13 @@ class PoolId(RestController): """ Remove the pool data for the pool id """ - pool = module.instance.get_pool_by_id(self.pool_id) + pool = context.instance.get_pool_by_id(self.pool_id) if not pool: response.status = 500 return {'message': 'Failed to identify the pool id "%d"' % self.pool_id} - return module.instance.submit_request([[{ + return context.instance.submit_request([[{ 'prefix': 'osd pool delete', 'pool': pool['pool_name'], 'pool2': pool['pool_name'], @@ -84,7 +84,7 @@ class Pool(RestController): """ Show the information for all the pools """ - pools = module.instance.get('osd_map')['pools'] + pools = context.instance.get('osd_map')['pools'] # pgp_num is called pg_placement_num, deal with that for pool in pools: @@ -128,7 +128,7 @@ class Pool(RestController): return {'message': 'Invalid arguments found: "%s"' % str(invalid)} # Schedule the creation and update requests - return module.instance.submit_request( + return context.instance.submit_request( [[create_command]] + common.pool_update_commands(pool_name, args), **kwargs diff --git a/src/pybind/mgr/restful/api/request.py b/src/pybind/mgr/restful/api/request.py index b22b80bff8a..82c7fe16a75 100644 --- a/src/pybind/mgr/restful/api/request.py +++ b/src/pybind/mgr/restful/api/request.py @@ -1,7 +1,7 @@ from pecan import expose, request, response from pecan.rest import RestController -from restful import module +from restful import context from restful.decorators import auth, lock, paginate @@ -18,7 +18,7 @@ class RequestId(RestController): """ request = filter( lambda x: x.id == self.request_id, - module.instance.requests + context.instance.requests ) if len(request) != 1: @@ -36,9 +36,9 @@ class RequestId(RestController): """ Remove the request id from the database """ - for index in range(len(module.instance.requests)): - if module.instance.requests[index].id == self.request_id: - return module.instance.requests.pop(index) + for index in range(len(context.instance.requests)): + if context.instance.requests[index].id == self.request_id: + return context.instance.requests.pop(index) # Failed to find the job to cancel response.status = 500 @@ -54,7 +54,7 @@ class Request(RestController): """ List all the available requests """ - return module.instance.requests + return context.instance.requests @expose(template='json') @@ -64,17 +64,17 @@ class Request(RestController): """ Remove all the finished requests """ - num_requests = len(module.instance.requests) + num_requests = len(context.instance.requests) - module.instance.requests = filter( + context.instance.requests = filter( lambda x: not x.is_finished(), - module.instance.requests + context.instance.requests ) # Return the job statistics return { - 'cleaned': num_requests - len(module.instance.requests), - 'remaining': len(module.instance.requests), + 'cleaned': num_requests - len(context.instance.requests), + 'remaining': len(context.instance.requests), } @@ -84,7 +84,7 @@ class Request(RestController): """ Pass through method to create any request """ - return module.instance.submit_request([[request.json]], **kwargs) + return context.instance.submit_request([[request.json]], **kwargs) @expose() diff --git a/src/pybind/mgr/restful/api/server.py b/src/pybind/mgr/restful/api/server.py index 9e6a20784ec..8ce63493754 100644 --- a/src/pybind/mgr/restful/api/server.py +++ b/src/pybind/mgr/restful/api/server.py @@ -1,7 +1,7 @@ from pecan import expose from pecan.rest import RestController -from restful import module +from restful import context from restful.decorators import auth @@ -16,7 +16,7 @@ class ServerFqdn(RestController): """ Show the information for the server fqdn """ - return module.instance.get_server(self.fqdn) + return context.instance.get_server(self.fqdn) @@ -27,7 +27,7 @@ class Server(RestController): """ Show the information for all the servers """ - return module.instance.list_servers() + return context.instance.list_servers() @expose() diff --git a/src/pybind/mgr/restful/common.py b/src/pybind/mgr/restful/common.py index 693063a646d..f1a46aec5bd 100644 --- a/src/pybind/mgr/restful/common.py +++ b/src/pybind/mgr/restful/common.py @@ -27,10 +27,7 @@ POOL_QUOTA_PROPERTIES = [ ('quota_max_objects', 'max_objects'), ] -POOL_ARGS = POOL_PROPERTIES + map( - lambda x: x[0], - POOL_QUOTA_PROPERTIES -) +POOL_ARGS = POOL_PROPERTIES + [x for x,_ in POOL_QUOTA_PROPERTIES] # Transform command to a human readable form diff --git a/src/pybind/mgr/restful/context.py b/src/pybind/mgr/restful/context.py new file mode 100644 index 00000000000..a05ea8548df --- /dev/null +++ b/src/pybind/mgr/restful/context.py @@ -0,0 +1,2 @@ +# Global instance to share +instance = None diff --git a/src/pybind/mgr/restful/decorators.py b/src/pybind/mgr/restful/decorators.py index e5d7d62ae87..03a9448298c 100644 --- a/src/pybind/mgr/restful/decorators.py +++ b/src/pybind/mgr/restful/decorators.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import + from pecan import request, response from base64 import b64decode from functools import wraps import traceback -import module +from . import context # Handle authorization @@ -19,13 +21,13 @@ def auth(f): username, password = b64decode(request.authorization[1]).split(':') # Check that the username exists - if username not in module.instance.keys: + if username not in context.instance.keys: response.status = 401 response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"' return {'message': 'auth: No such user'} # Check the password - if module.instance.keys[username] != password: + if context.instance.keys[username] != password: response.status = 401 response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"' return {'message': 'auth: Incorrect password'} @@ -38,7 +40,7 @@ def auth(f): def lock(f): @wraps(f) def decorated(*args, **kwargs): - with module.instance.requests_lock: + with context.instance.requests_lock: return f(*args, **kwargs) return decorated diff --git a/src/pybind/mgr/restful/hooks.py b/src/pybind/mgr/restful/hooks.py index 3fd244aa85e..d677dcc2a15 100644 --- a/src/pybind/mgr/restful/hooks.py +++ b/src/pybind/mgr/restful/hooks.py @@ -1,9 +1,11 @@ +from __future__ import absolute_import + from pecan.hooks import PecanHook import traceback -import module +from . import context class ErrorHook(PecanHook): def on_error(self, stat, exc): - module.instance.log.error(str(traceback.format_exc())) + context.instance.log.error(str(traceback.format_exc())) diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py index 5125253e466..ecae22100d1 100644 --- a/src/pybind/mgr/restful/module.py +++ b/src/pybind/mgr/restful/module.py @@ -1,6 +1,7 @@ """ A RESTful API for Ceph """ +from __future__ import absolute_import import os import json @@ -12,7 +13,8 @@ import threading import traceback import socket -import common +from . import common +from . import context from uuid import uuid4 from pecan import jsonify, make_app @@ -20,11 +22,14 @@ from OpenSSL import crypto from pecan.rest import RestController from werkzeug.serving import make_server, make_ssl_devcert -from hooks import ErrorHook +from .hooks import ErrorHook from mgr_module import MgrModule, CommandResult -# Global instance to share -instance = None + +try: + iteritems = dict.iteritems +except: + iteritems = dict.items class CannotServe(Exception): @@ -51,10 +56,8 @@ class CommandsRequest(object): self.id = str(id(self)) # Filter out empty sub-requests - commands_arrays = filter( - lambda x: len(x) != 0, - commands_arrays, - ) + commands_arrays = [x for x in commands_arrays + if len(x) != 0] self.running = [] self.waiting = commands_arrays[1:] @@ -89,7 +92,7 @@ class CommandsRequest(object): results.append(result) # Run the command - instance.send_command(result, 'mon', '', json.dumps(commands[index]), tag) + context.instance.send_command(result, 'mon', '', json.dumps(commands[index]), tag) return results @@ -230,8 +233,7 @@ class Module(MgrModule): def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) - global instance - instance = self + context.instance = self self.requests = [] self.requests_lock = threading.RLock() @@ -251,7 +253,7 @@ class Module(MgrModule): self._serve() self.server.socket.close() except CannotServe as cs: - self.log.warn("server not running: {0}".format(cs.message)) + self.log.warn("server not running: %s", cs) except: self.log.error(str(traceback.format_exc())) @@ -262,7 +264,7 @@ class Module(MgrModule): def refresh_keys(self): self.keys = {} rawkeys = self.get_config_prefix('keys/') or {} - for k, v in rawkeys.iteritems(): + for k, v in iteritems(rawkeys): self.keys[k[5:]] = v # strip of keys/ prefix def _serve(self): @@ -286,7 +288,7 @@ class Module(MgrModule): cert = self.get_localized_config("crt") if cert is not None: cert_tmp = tempfile.NamedTemporaryFile() - cert_tmp.write(cert) + cert_tmp.write(cert.encode('utf-8')) cert_tmp.flush() cert_fname = cert_tmp.name else: @@ -295,7 +297,7 @@ class Module(MgrModule): pkey = self.get_localized_config("key") if pkey is not None: pkey_tmp = tempfile.NamedTemporaryFile() - pkey_tmp.write(pkey) + pkey_tmp.write(pkey.encode('utf-8')) pkey_tmp.flush() pkey_fname = pkey_tmp.name else: @@ -362,10 +364,7 @@ class Module(MgrModule): if tag == 'seq': return - request = filter( - lambda x: x.is_running(tag), - self.requests) - + request = [x for x in self.requests if x.is_running(tag)] if len(request) != 1: self.log.warn("Unknown request '%s'" % str(tag)) return @@ -438,9 +437,8 @@ class Module(MgrModule): elif command['prefix'] == "restful create-self-signed-cert": cert, pkey = self.create_self_signed_cert() - - self.set_config(self.get_mgr_id() + '/crt', cert) - self.set_config(self.get_mgr_id() + '/key', pkey) + self.set_config(self.get_mgr_id() + '/crt', cert.decode('utf-8')) + self.set_config(self.get_mgr_id() + '/key', pkey.decode('utf-8')) self.restart() return ( @@ -533,10 +531,7 @@ class Module(MgrModule): # Filter by osd ids if ids is not None: - osds = filter( - lambda x: str(x['osd']) in ids, - osds - ) + osds = [x for x in osds if str(x['osd']) in ids] # Get list of pools per osd node pools_map = self.get_osd_pools() @@ -562,19 +557,14 @@ class Module(MgrModule): # Filter by pool if pool_id: pool_id = int(pool_id) - osds = filter( - lambda x: pool_id in x['pools'], - osds - ) + osds = [x for x in osds if pool_id in x['pools']] return osds def get_osd_by_id(self, osd_id): - osd = filter( - lambda x: x['osd'] == osd_id, - self.get('osd_map')['osds'] - ) + osd = [x for x in self.get('osd_map')['osds'] + if x['osd'] == osd_id] if len(osd) != 1: return None @@ -583,10 +573,8 @@ class Module(MgrModule): def get_pool_by_id(self, pool_id): - pool = filter( - lambda x: x['pool'] == pool_id, - self.get('osd_map')['pools'], - ) + pool = [x for x in self.get('osd_map')['pools'] + if x['pool'] == pool_id] if len(pool) != 1: return None diff --git a/src/pybind/mgr/selftest/__init__.py b/src/pybind/mgr/selftest/__init__.py index 622a611b609..554aacbf946 100644 --- a/src/pybind/mgr/selftest/__init__.py +++ b/src/pybind/mgr/selftest/__init__.py @@ -1,3 +1,3 @@ -from module import * +from .module import Module diff --git a/src/pybind/mgr/status/__init__.py b/src/pybind/mgr/status/__init__.py index 79f5b86fd50..8f210ac9247 100644 --- a/src/pybind/mgr/status/__init__.py +++ b/src/pybind/mgr/status/__init__.py @@ -1,2 +1 @@ - -from module import * # NOQA +from .module import Module diff --git a/src/pybind/mgr/zabbix/__init__.py b/src/pybind/mgr/zabbix/__init__.py index 0440e0705fb..8f210ac9247 100644 --- a/src/pybind/mgr/zabbix/__init__.py +++ b/src/pybind/mgr/zabbix/__init__.py @@ -1 +1 @@ -from module import * # NOQA +from .module import Module diff --git a/src/vstart.sh b/src/vstart.sh index ba2e87f8887..6324469453a 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -16,12 +16,27 @@ if [ -n "$VSTART_DEST" ]; then CEPH_OUT_DIR=$VSTART_DEST/out fi +get_cmake_variable() { + local variable=$1 + grep "$variable" CMakeCache.txt | cut -d "=" -f 2 +} + # for running out of the CMake build directory if [ -e CMakeCache.txt ]; then # Out of tree build, learn source location from CMakeCache.txt - CEPH_ROOT=`grep ceph_SOURCE_DIR CMakeCache.txt | cut -d "=" -f 2` + CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR) CEPH_BUILD_DIR=`pwd` [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr + CEPH_MGR_PY_VERSION_MAJOR=$(get_cmake_variable MGR_PYTHON_VERSION | cut -d '.' -f1) + if [ -n "$CEPH_MGR_PY_VERSION_MAJOR" ]; then + CEPH_PY_VERSION_MAJOR=${CEPH_MGR_PY_VERSION_MAJOR} + else + if [ $(get_cmake_variable WITH_PYTHON2) = ON ]; then + CEPH_PY_VERSION_MAJOR=2 + else + CEPH_PY_VERSION_MAJOR=3 + fi + fi fi # use CEPH_BUILD_ROOT to vstart from a 'make install' @@ -30,6 +45,7 @@ if [ -n "$CEPH_BUILD_ROOT" ]; then [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes + # make install should install python extensions into PYTHONPATH elif [ -n "$CEPH_ROOT" ]; then [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin @@ -46,7 +62,7 @@ fi [ -z "$PYBIND" ] && PYBIND=./pybind -export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH +export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}:$PYTHONPATH export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH # Suppress logging for regular use that indicated that we are using a @@ -662,14 +678,21 @@ start_mgr() { host = $HOSTNAME EOF - ceph_adm config-key set mgr/dashboard/$name/server_port $MGR_PORT - DASH_URLS+="http://$IP:$MGR_PORT/" - MGR_PORT=$(($MGR_PORT + 1000)) + ceph_adm config-key set mgr/dashboard/$name/server_port $MGR_PORT + if [ $mgr -eq 1 ]; then + DASH_URLS="http://$IP:$MGR_PORT" + else + DASH_URLS+=", http://$IP:$MGR_PORT" + fi + MGR_PORT=$(($MGR_PORT + 1000)) - ceph_adm config-key set mgr/restful/$name/server_port $MGR_PORT - - RESTFUL_URLS+="https://$IP:$MGR_PORT" - MGR_PORT=$(($MGR_PORT + 1000)) + ceph_adm config-key set mgr/restful/$name/server_port $MGR_PORT + if [ $mgr -eq 1 ]; then + RESTFUL_URLS="https://$IP:$MGR_PORT" + else + RESTFUL_URLS+=", https://$IP:$MGR_PORT" + fi + MGR_PORT=$(($MGR_PORT + 1000)) echo "Starting mgr.${name}" run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS