RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-04-01 00:26:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20362 from tchaikov/wip-mgr-py3

Browse Source 
mgr: fix py3 support

Reviewed-by: Jan Fajerski <jfajerski@suse.com>
This commit is contained in:
Kefu Chai 2018-02-16 02:32:04 +08:00 committed by GitHub
commit 097e2de5f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 276 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
qa/tasks/mgr/test_dashboard.py
View File

@ -24,7 +24,7 @@ class TestDashboard(MgrTestCase):
self.mgr_cluster.mgr_fail(original_active)
failed_over_uri = self._get_uri("dashboard")
log.info("After failover running at {0}".format(original_uri))
log.info("After failover running at {0}".format(failed_over_uri))
self.assertNotEqual(original_uri, failed_over_uri)

15
src/CMakeLists.txt
View File

@ -274,6 +274,14 @@ if(WITH_PYTHON3)
find_package(Python3Libs 3 REQUIRED)
endif()
# the major version of the python bindings as a dependency of other
# targets
if(WITH_PYTHON2)
set(PY_BINDING_INFIX "")
else()
set(PY_BINDING_INFIX 3)
endif()
if(HAVE_XIO)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${XIO_INCLUDE_DIR}")
list(APPEND EXTRALIBS ${XIO_LIBRARY} pthread)
@ -1089,13 +1097,12 @@ add_custom_target(vstart-base DEPENDS
monmaptool
crushtool
rados
cython_rados
)
cython${PY_BINDING_INFIX}_rados)
add_custom_target(vstart DEPENDS
vstart-base
ceph-mds
cython_rbd)
cython${PY_BINDING_INFIX}_rbd)
if(WITH_RADOSGW)
add_dependencies(vstart radosgw radosgw-admin)
endif(WITH_RADOSGW)
@ -1108,7 +1115,7 @@ endif(WITH_LTTNG)
add_custom_target(cephfs_testing DEPENDS
vstart
rados
cython_modules
cython${PY_BINDING_INFIX}_modules
cephfs
cls_cephfs
ceph-fuse

2
src/init-ceph.in
View File

@ -34,7 +34,7 @@ else
ETCDIR=.
ASSUME_DEV=1
CEPH_LIB=$CEPH_ROOT/build/lib
echo "$PYTHONPATH" | grep -q $CEPH_LIB || export PYTHONPATH=$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH
echo "$PYTHONPATH" | grep -q $CEPH_LIB || export PYTHONPATH=$CEPH_LIB/cython_modules/lib.@MGR_PYTHON_VERSION_MAJOR@:$PYTHONPATH
echo "$LD_LIBRARY_PATH" | grep -q $CEPH_LIB || export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH
echo "$DYLD_LIBRARY_PATH" | grep -q $CEPH_LIB || export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH
else

117
src/mgr/PyModule.cc
View File

@ -151,12 +151,75 @@ std::string PyModule::get_site_packages()
return site_packages.str();
}
#if PY_MAJOR_VERSION >= 3
PyObject* PyModule::init_ceph_logger()
{
auto py_logger = PyModule_Create(&ceph_logger_module);
PySys_SetObject("stderr", py_logger);
PySys_SetObject("stdout", py_logger);
return py_logger;
}
#else
void PyModule::init_ceph_logger()
{
auto py_logger = Py_InitModule("ceph_logger", log_methods);
PySys_SetObject(const_cast<char*>("stderr"), py_logger);
PySys_SetObject(const_cast<char*>("stdout"), py_logger);
}
#endif
#if PY_MAJOR_VERSION >= 3
PyObject* PyModule::init_ceph_module()
#else
void PyModule::init_ceph_module()
#endif
{
static PyMethodDef module_methods[] = {
{nullptr, nullptr, 0, nullptr}
};
#if PY_MAJOR_VERSION >= 3
static PyModuleDef ceph_module_def = {
PyModuleDef_HEAD_INIT,
"ceph_module",
nullptr,
-1,
module_methods,
nullptr,
nullptr,
nullptr,
nullptr
};
PyObject *ceph_module = PyModule_Create(&ceph_module_def);
#else
PyObject *ceph_module = Py_InitModule("ceph_module", module_methods);
#endif
assert(ceph_module != nullptr);
std::map<const char*, PyTypeObject*> classes{
{{"BaseMgrModule", &BaseMgrModuleType},
{"BaseMgrStandbyModule", &BaseMgrStandbyModuleType},
{"BasePyOSDMap", &BasePyOSDMapType},
{"BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType},
{"BasePyCRUSH", &BasePyCRUSHType}}
};
for (auto [name, type] : classes) {
type->tp_new = PyType_GenericNew;
if (PyType_Ready(type) < 0) {
assert(0);
}
Py_INCREF(type);
PyModule_AddObject(ceph_module, name, (PyObject *)type);
}
#if PY_MAJOR_VERSION >= 3
return ceph_module;
#endif
}
int PyModule::load(PyThreadState *pMainThreadState)
{
assert(pMainThreadState != nullptr);
// Configure sub-interpreter and construct C++-generated python classes
// Configure sub-interpreter
{
SafeThreadState sts(pMainThreadState);
Gil gil(sts);
@ -176,19 +239,6 @@ int PyModule::load(PyThreadState *pMainThreadState)
const char *argv[] = {"ceph-mgr"};
PySys_SetArgv(1, (char**)argv);
#endif
if (g_conf->daemonize) {
#if PY_MAJOR_VERSION >= 3
auto py_logger = PyModule_Create(&ceph_logger_module);
PySys_SetObject("stderr", py_logger);
PySys_SetObject("stdout", py_logger);
#else
auto py_logger = Py_InitModule("ceph_logger", log_methods);
PySys_SetObject(const_cast<char*>("stderr"), py_logger);
PySys_SetObject(const_cast<char*>("stdout"), py_logger);
#endif
}
// Configure sys.path to include mgr_module_path
string paths = (":" + get_site_packages() +
":" + g_conf->get_val<std::string>("mgr_module_path"));
@ -203,46 +253,7 @@ int PyModule::load(PyThreadState *pMainThreadState)
dout(10) << "Computed sys.path '" << sys_path << "'" << dendl;
#endif
}
static PyMethodDef module_methods[] = {
{nullptr}
};
#if PY_MAJOR_VERSION >= 3
static PyModuleDef ceph_module_def = {
PyModuleDef_HEAD_INIT,
"ceph_module",
nullptr,
-1,
module_methods,
};
#endif
// Initialize module
#if PY_MAJOR_VERSION >= 3
PyObject *ceph_module = PyModule_Create(&ceph_module_def);
#else
PyObject *ceph_module = Py_InitModule("ceph_module", module_methods);
#endif
assert(ceph_module != nullptr);
auto load_class = [ceph_module](const char *name, PyTypeObject *type)
{
type->tp_new = PyType_GenericNew;
if (PyType_Ready(type) < 0) {
assert(0);
}
Py_INCREF(type);
PyModule_AddObject(ceph_module, name, (PyObject *)type);
};
load_class("BaseMgrModule", &BaseMgrModuleType);
load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType);
load_class("BasePyOSDMap", &BasePyOSDMapType);
load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType);
load_class("BasePyCRUSH", &BasePyCRUSHType);
}
// Environment is all good, import the external module
{
Gil gil(pMyThreadState);

8
src/mgr/PyModule.h
View File

@ -77,7 +77,13 @@ public:
~PyModule();
int load(PyThreadState *pMainThreadState);
#if PY_MAJOR_VERSION >= 3
static PyObject* init_ceph_logger();
static PyObject* init_ceph_module();
#else
static void init_ceph_logger();
static void init_ceph_module();
#endif
/**
* Extend `out` with the contents of `this->commands`

5
src/mgr/PyModuleRegistry.cc
View File

@ -57,6 +57,11 @@ int PyModuleRegistry::init(const MgrMap &map)
#else
Py_SetProgramName(const_cast<char*>(PYTHON_EXECUTABLE));
#endif
// Add more modules
if (g_conf->get_val<bool>("daemonize")) {
PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger);
}
PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module);
Py_InitializeEx(0);
// Let CPython know that we will be calling it back from other

3
src/pybind/mgr/balancer/__init__.py
View File

@ -1,2 +1 @@
from module import * # NOQA
from .module import Module

2
src/pybind/mgr/dashboard/__init__.py
View File

@ -1,2 +1,2 @@
from module import * # NOQA
from .module import Module, StandbyModule

2
src/pybind/mgr/dashboard/cephfs_clients.py
View File

@ -2,7 +2,7 @@
import json
from mgr_module import CommandResult
from remote_view_cache import RemoteViewCache
from .remote_view_cache import RemoteViewCache
class CephFSClients(RemoteViewCache):

81
src/pybind/mgr/dashboard/module.py
View File

@ -3,12 +3,20 @@
Demonstrate writing a Ceph web interface inside a mgr module.
"""
from __future__ import absolute_import
# We must share a global reference to this instance, because it is the
# gatekeeper to all accesses to data from the C++ side (e.g. the REST API
# request handlers need to see it)
from collections import defaultdict
from functools import cmp_to_key
import collections
try:
iteritems = dict.iteritems
except AttributeError:
iteritems = dict.items
_global_instance = {'plugin': None}
def global_instance():
assert _global_instance['plugin'] is not None
@ -26,18 +34,21 @@ import socket
import cherrypy
import jinja2
import urlparse
try:
import urlparse
except ImportError:
import urllib.parse as urlparse
from mgr_module import MgrModule, MgrStandbyModule, CommandResult
from types import OsdMap, NotFound, Config, FsMap, MonMap, \
from .types import OsdMap, NotFound, Config, FsMap, MonMap, \
PgSummary, Health, MonStatus
import rbd_iscsi
import rbd_mirroring
from rbd_ls import RbdLs, RbdPoolLs
from cephfs_clients import CephFSClients
from rgw import RGWDaemons
from . import rbd_iscsi
from . import rbd_mirroring
from .rbd_ls import RbdLs, RbdPoolLs
from .cephfs_clients import CephFSClients
from .rgw import RGWDaemons
log = logging.getLogger("dashboard")
@ -68,7 +79,7 @@ def get_prefixed_url(url):
def to_sorted_array(data):
assert isinstance(data, dict)
return sorted(data.iteritems())
return sorted(iteritems(data))
def prepare_url_prefix(url_prefix):
"""
@ -367,7 +378,7 @@ class Module(MgrModule):
)
# Find the standby replays
for gid_str, daemon_info in mdsmap['info'].iteritems():
for gid_str, daemon_info in iteritems(mdsmap['info']):
if daemon_info['state'] != "up:standby-replay":
continue
@ -471,14 +482,12 @@ class Module(MgrModule):
# Transform the `checks` dict into a list for the convenience
# of rendering from javascript.
checks = []
for k, v in health['checks'].iteritems():
for k, v in iteritems(health['checks']):
v['type'] = k
checks.append(v)
checks = sorted(checks, cmp=lambda a, b: a['severity'] > b['severity'])
checks = sorted(checks, key=lambda x: x['severity'])
health['checks'] = checks
return health
def _toplevel_data(self):
@ -1208,32 +1217,28 @@ class Module(MgrModule):
@cherrypy.expose
def index(self, rgw_id=None):
if rgw_id is not None:
template = env.get_template("rgw_detail.html")
toplevel_data = self._toplevel_data()
return template.render(
url_prefix=global_instance().url_prefix,
ceph_version=global_instance().version,
path_info='/rgw' + cherrypy.request.path_info,
toplevel_data=json.dumps(toplevel_data, indent=2),
content_data=json.dumps(self.rgw_data(rgw_id), indent=2)
)
template = env.get_template("rgw_detail.html")
toplevel_data = self._toplevel_data()
return template.render(
url_prefix=global_instance().url_prefix,
ceph_version=global_instance().version,
path_info='/rgw' + cherrypy.request.path_info,
toplevel_data=json.dumps(toplevel_data, indent=2),
content_data=json.dumps(self.rgw_data(rgw_id), indent=2)
)
else:
# List all RGW servers
template = env.get_template("rgw.html")
toplevel_data = self._toplevel_data()
content_data = self._rgw_daemons()
return template.render(
url_prefix = global_instance().url_prefix,
ceph_version=global_instance().version,
path_info='/rgw' + cherrypy.request.path_info,
toplevel_data=json.dumps(toplevel_data, indent=2),
content_data=json.dumps(content_data, indent=2)
)
""" List all RGW servers """
template = env.get_template("rgw.html")
toplevel_data = self._toplevel_data()
content_data = self._rgw_daemons()
return template.render(
url_prefix = global_instance().url_prefix,
ceph_version=global_instance().version,
path_info='/rgw' + cherrypy.request.path_info,
toplevel_data=json.dumps(toplevel_data, indent=2),
content_data=json.dumps(content_data, indent=2)
)
def _rgw_daemons(self):
status, data = global_instance().rgw_daemons.get()
if data is None:
@ -1268,7 +1273,7 @@ class Module(MgrModule):
@cherrypy.expose
@cherrypy.tools.json_out()
def rgw_data(self, rgw_id):
return self._rgw(rgw_id)
return self._rgw(rgw_id)
cherrypy.tree.mount(Root(), get_prefixed_url("/"), conf)
cherrypy.tree.mount(OSDEndpoint(), get_prefixed_url("/osd"), conf)

3
src/pybind/mgr/dashboard/rbd_iscsi.py
View File

@ -1,7 +1,8 @@
from __future__ import absolute_import
import rados
import rbd
from remote_view_cache import RemoteViewCache
from .remote_view_cache import RemoteViewCache
SERVICE_TYPE = 'tcmu-runner'

5
src/pybind/mgr/dashboard/rbd_ls.py
View File

@ -1,8 +1,9 @@
from __future__ import absolute_import
import rbd
import rados
from types import OsdMap
from remote_view_cache import RemoteViewCache
from .types import OsdMap
from .remote_view_cache import RemoteViewCache
class RbdPoolLs(RemoteViewCache):
def _get(self):

3
src/pybind/mgr/dashboard/rbd_mirroring.py
View File

@ -1,9 +1,10 @@
from __future__ import absolute_import
import json
import re
import rados
import rbd
from remote_view_cache import RemoteViewCache
from .remote_view_cache import RemoteViewCache
class DaemonsAndPools(RemoteViewCache):
def _get(self):

4
src/pybind/mgr/dashboard/rgw.py
View File

@ -1,7 +1,9 @@
# -*- code: utf-8 -*-
from __future__ import absolute_import
import json
from remote_view_cache import RemoteViewCache
from .remote_view_cache import RemoteViewCache
import logging

2
src/pybind/mgr/influx/__init__.py
View File

@ -1 +1 @@
from module import * # NOQA
from .module import Module

3
src/pybind/mgr/localpool/__init__.py
View File

@ -1,2 +1 @@
from module import * # NOQA
from .module import Module

2
src/pybind/mgr/prometheus/__init__.py
View File

@ -1,2 +1,2 @@
from module import * # NOQA
from .module import Module, StandbyModule

2
src/pybind/mgr/restful/__init__.py
View File

@ -1 +1 @@
from module import * # NOQA
from .module import Module

16
src/pybind/mgr/restful/api/__init__.py
View File

@ -1,14 +1,14 @@
from pecan import expose
from pecan.rest import RestController
from config import Config
from crush import Crush
from doc import Doc
from mon import Mon
from osd import Osd
from pool import Pool
from request import Request
from server import Server
from .config import Config
from .crush import Crush
from .doc import Doc
from .mon import Mon
from .osd import Osd
from .pool import Pool
from .request import Request
from .server import Server
class Root(RestController):

12
src/pybind/mgr/restful/api/config.py
View File

@ -1,7 +1,7 @@
from pecan import expose, request
from pecan.rest import RestController
from restful import common, module
from restful import common, context
from restful.decorators import auth
@ -12,7 +12,7 @@ class ConfigOsd(RestController):
"""
Show OSD configuration options
"""
flags = module.instance.get("osd_map")['flags']
flags = context.instance.get("osd_map")['flags']
# pause is a valid osd config command that sets pauserd,pausewr
flags = flags.replace('pauserd,pausewr', 'pause')
@ -33,7 +33,7 @@ class ConfigOsd(RestController):
valid_flags = set(args.keys()) & set(common.OSD_FLAGS)
invalid_flags = list(set(args.keys()) - valid_flags)
if invalid_flags:
module.instance.log.warn("%s not valid to set/unset" % invalid_flags)
context.instance.log.warn("%s not valid to set/unset" % invalid_flags)
for flag in list(valid_flags):
if args[flag]:
@ -46,7 +46,7 @@ class ConfigOsd(RestController):
'key': flag,
})
return module.instance.submit_request([commands], **kwargs)
return context.instance.submit_request([commands], **kwargs)
@ -61,7 +61,7 @@ class ConfigClusterKey(RestController):
"""
Show specific configuration option
"""
return module.instance.get("config").get(self.key, None)
return context.instance.get("config").get(self.key, None)
@ -72,7 +72,7 @@ class ConfigCluster(RestController):
"""
Show all cluster configuration options
"""
return module.instance.get("config")
return context.instance.get("config")
@expose()

6
src/pybind/mgr/restful/api/crush.py
View File

@ -1,7 +1,7 @@
from pecan import expose
from pecan.rest import RestController
from restful import common, module
from restful import common, context
from collections import defaultdict
from restful.decorators import auth
@ -14,8 +14,8 @@ class CrushRule(RestController):
"""
Show crush rules
"""
rules = module.instance.get('osd_map_crush')['rules']
nodes = module.instance.get('osd_map_tree')['nodes']
rules = context.instance.get('osd_map_crush')['rules']
nodes = context.instance.get('osd_map_tree')['nodes']
for rule in rules:
rule['osd_count'] = len(common.crush_rule_osds(nodes, rule))

4
src/pybind/mgr/restful/api/doc.py
View File

@ -1,7 +1,7 @@
from pecan import expose
from pecan.rest import RestController
from restful import module
from restful import context
import restful
@ -12,4 +12,4 @@ class Doc(RestController):
"""
Show documentation information
"""
return module.instance.get_doc_api(restful.api.Root)
return context.instance.get_doc_api(restful.api.Root)

6
src/pybind/mgr/restful/api/mon.py
View File

@ -1,7 +1,7 @@
from pecan import expose, response
from pecan.rest import RestController
from restful import module
from restful import context
from restful.decorators import auth
@ -18,7 +18,7 @@ class MonName(RestController):
"""
mon = filter(
lambda x: x['name'] == self.name,
module.instance.get_mons()
context.instance.get_mons()
)
if len(mon) != 1:
@ -36,7 +36,7 @@ class Mon(RestController):
"""
Show the information for all the monitors
"""
return module.instance.get_mons()
return context.instance.get_mons()
@expose()

14
src/pybind/mgr/restful/api/osd.py
View File

@ -1,7 +1,7 @@
from pecan import expose, request, response
from pecan.rest import RestController
from restful import common, module
from restful import common, context
from restful.decorators import auth
@ -16,7 +16,7 @@ class OsdIdCommand(RestController):
"""
Show implemented commands for the OSD id
"""
osd = module.instance.get_osd_by_id(self.osd_id)
osd = context.instance.get_osd_by_id(self.osd_id)
if not osd:
response.status = 500
@ -36,7 +36,7 @@ class OsdIdCommand(RestController):
"""
command = request.json.get('command', None)
osd = module.instance.get_osd_by_id(self.osd_id)
osd = context.instance.get_osd_by_id(self.osd_id)
if not osd:
response.status = 500
@ -46,7 +46,7 @@ class OsdIdCommand(RestController):
response.status = 500
return {'message': 'Command "%s" not available' % command}
return module.instance.submit_request([[{
return context.instance.submit_request([[{
'prefix': 'osd ' + command,
'who': str(self.osd_id)
}]], **kwargs)
@ -65,7 +65,7 @@ class OsdId(RestController):
"""
Show the information for the OSD id
"""
osd = module.instance.get_osds(ids=[str(self.osd_id)])
osd = context.instance.get_osds(ids=[str(self.osd_id)])
if len(osd) != 1:
response.status = 500
return {'message': 'Failed to identify the OSD id "%d"' % self.osd_id}
@ -112,7 +112,7 @@ class OsdId(RestController):
'weight': args['reweight']
})
return module.instance.submit_request([commands], **kwargs)
return context.instance.submit_request([commands], **kwargs)
@ -127,7 +127,7 @@ class Osd(RestController):
# TODO Filter by ids
pool_id = kwargs.get('pool', None)
return module.instance.get_osds(pool_id)
return context.instance.get_osds(pool_id)
@expose()

18
src/pybind/mgr/restful/api/pool.py
View File

@ -1,7 +1,7 @@
from pecan import expose, request, response
from pecan.rest import RestController
from restful import common, module
from restful import common, context
from restful.decorators import auth
@ -16,7 +16,7 @@ class PoolId(RestController):
"""
Show the information for the pool id
"""
pool = module.instance.get_pool_by_id(self.pool_id)
pool = context.instance.get_pool_by_id(self.pool_id)
if not pool:
response.status = 500
@ -34,14 +34,14 @@ class PoolId(RestController):
"""
Modify the information for the pool id
"""
try:
try:
args = request.json
except ValueError:
response.status = 400
return {'message': 'Bad request: malformed JSON or wrong Content-Type'}
# Get the pool info for its name
pool = module.instance.get_pool_by_id(self.pool_id)
pool = context.instance.get_pool_by_id(self.pool_id)
if not pool:
response.status = 500
return {'message': 'Failed to identify the pool id "%d"' % self.pool_id}
@ -53,7 +53,7 @@ class PoolId(RestController):
return {'message': 'Invalid arguments found: "%s"' % str(invalid)}
# Schedule the update request
return module.instance.submit_request(common.pool_update_commands(pool['pool_name'], args), **kwargs)
return context.instance.submit_request(common.pool_update_commands(pool['pool_name'], args), **kwargs)
@expose(template='json')
@ -62,13 +62,13 @@ class PoolId(RestController):
"""
Remove the pool data for the pool id
"""
pool = module.instance.get_pool_by_id(self.pool_id)
pool = context.instance.get_pool_by_id(self.pool_id)
if not pool:
response.status = 500
return {'message': 'Failed to identify the pool id "%d"' % self.pool_id}
return module.instance.submit_request([[{
return context.instance.submit_request([[{
'prefix': 'osd pool delete',
'pool': pool['pool_name'],
'pool2': pool['pool_name'],
@ -84,7 +84,7 @@ class Pool(RestController):
"""
Show the information for all the pools
"""
pools = module.instance.get('osd_map')['pools']
pools = context.instance.get('osd_map')['pools']
# pgp_num is called pg_placement_num, deal with that
for pool in pools:
@ -128,7 +128,7 @@ class Pool(RestController):
return {'message': 'Invalid arguments found: "%s"' % str(invalid)}
# Schedule the creation and update requests
return module.instance.submit_request(
return context.instance.submit_request(
[[create_command]] +
common.pool_update_commands(pool_name, args),
**kwargs

24
src/pybind/mgr/restful/api/request.py
View File

@ -1,7 +1,7 @@
from pecan import expose, request, response
from pecan.rest import RestController
from restful import module
from restful import context
from restful.decorators import auth, lock, paginate
@ -18,7 +18,7 @@ class RequestId(RestController):
"""
request = filter(
lambda x: x.id == self.request_id,
module.instance.requests
context.instance.requests
)
if len(request) != 1:
@ -36,9 +36,9 @@ class RequestId(RestController):
"""
Remove the request id from the database
"""
for index in range(len(module.instance.requests)):
if module.instance.requests[index].id == self.request_id:
return module.instance.requests.pop(index)
for index in range(len(context.instance.requests)):
if context.instance.requests[index].id == self.request_id:
return context.instance.requests.pop(index)
# Failed to find the job to cancel
response.status = 500
@ -54,7 +54,7 @@ class Request(RestController):
"""
List all the available requests
"""
return module.instance.requests
return context.instance.requests
@expose(template='json')
@ -64,17 +64,17 @@ class Request(RestController):
"""
Remove all the finished requests
"""
num_requests = len(module.instance.requests)
num_requests = len(context.instance.requests)
module.instance.requests = filter(
context.instance.requests = filter(
lambda x: not x.is_finished(),
module.instance.requests
context.instance.requests
)
# Return the job statistics
return {
'cleaned': num_requests - len(module.instance.requests),
'remaining': len(module.instance.requests),
'cleaned': num_requests - len(context.instance.requests),
'remaining': len(context.instance.requests),
}
@ -84,7 +84,7 @@ class Request(RestController):
"""
Pass through method to create any request
"""
return module.instance.submit_request([[request.json]], **kwargs)
return context.instance.submit_request([[request.json]], **kwargs)
@expose()

6
src/pybind/mgr/restful/api/server.py
View File

@ -1,7 +1,7 @@
from pecan import expose
from pecan.rest import RestController
from restful import module
from restful import context
from restful.decorators import auth
@ -16,7 +16,7 @@ class ServerFqdn(RestController):
"""
Show the information for the server fqdn
"""
return module.instance.get_server(self.fqdn)
return context.instance.get_server(self.fqdn)
@ -27,7 +27,7 @@ class Server(RestController):
"""
Show the information for all the servers
"""
return module.instance.list_servers()
return context.instance.list_servers()
@expose()

5
src/pybind/mgr/restful/common.py
View File

@ -27,10 +27,7 @@ POOL_QUOTA_PROPERTIES = [
('quota_max_objects', 'max_objects'),
]
POOL_ARGS = POOL_PROPERTIES + map(
lambda x: x[0],
POOL_QUOTA_PROPERTIES
)
POOL_ARGS = POOL_PROPERTIES + [x for x,_ in POOL_QUOTA_PROPERTIES]
# Transform command to a human readable form

2
src/pybind/mgr/restful/context.py Normal file
View File

@ -0,0 +1,2 @@
# Global instance to share
instance = None

10
src/pybind/mgr/restful/decorators.py
View File

@ -1,10 +1,12 @@
from __future__ import absolute_import
from pecan import request, response
from base64 import b64decode
from functools import wraps
import traceback
import module
from . import context
# Handle authorization
@ -19,13 +21,13 @@ def auth(f):
username, password = b64decode(request.authorization[1]).split(':')
# Check that the username exists
if username not in module.instance.keys:
if username not in context.instance.keys:
response.status = 401
response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"'
return {'message': 'auth: No such user'}
# Check the password
if module.instance.keys[username] != password:
if context.instance.keys[username] != password:
response.status = 401
response.headers['WWW-Authenticate'] = 'Basic realm="Login Required"'
return {'message': 'auth: Incorrect password'}
@ -38,7 +40,7 @@ def auth(f):
def lock(f):
@wraps(f)
def decorated(*args, **kwargs):
with module.instance.requests_lock:
with context.instance.requests_lock:
return f(*args, **kwargs)
return decorated

6
src/pybind/mgr/restful/hooks.py
View File

@ -1,9 +1,11 @@
from __future__ import absolute_import
from pecan.hooks import PecanHook
import traceback
import module
from . import context
class ErrorHook(PecanHook):
def on_error(self, stat, exc):
module.instance.log.error(str(traceback.format_exc()))
context.instance.log.error(str(traceback.format_exc()))

64
src/pybind/mgr/restful/module.py
View File

@ -1,6 +1,7 @@
"""
A RESTful API for Ceph
"""
from __future__ import absolute_import
import os
import json
@ -12,7 +13,8 @@ import threading
import traceback
import socket
import common
from . import common
from . import context
from uuid import uuid4
from pecan import jsonify, make_app
@ -20,11 +22,14 @@ from OpenSSL import crypto
from pecan.rest import RestController
from werkzeug.serving import make_server, make_ssl_devcert
from hooks import ErrorHook
from .hooks import ErrorHook
from mgr_module import MgrModule, CommandResult
# Global instance to share
instance = None
try:
iteritems = dict.iteritems
except:
iteritems = dict.items
class CannotServe(Exception):
@ -51,10 +56,8 @@ class CommandsRequest(object):
self.id = str(id(self))
# Filter out empty sub-requests
commands_arrays = filter(
lambda x: len(x) != 0,
commands_arrays,
)
commands_arrays = [x for x in commands_arrays
if len(x) != 0]
self.running = []
self.waiting = commands_arrays[1:]
@ -89,7 +92,7 @@ class CommandsRequest(object):
results.append(result)
# Run the command
instance.send_command(result, 'mon', '', json.dumps(commands[index]), tag)
context.instance.send_command(result, 'mon', '', json.dumps(commands[index]), tag)
return results
@ -230,8 +233,7 @@ class Module(MgrModule):
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
global instance
instance = self
context.instance = self
self.requests = []
self.requests_lock = threading.RLock()
@ -251,7 +253,7 @@ class Module(MgrModule):
self._serve()
self.server.socket.close()
except CannotServe as cs:
self.log.warn("server not running: {0}".format(cs.message))
self.log.warn("server not running: %s", cs)
except:
self.log.error(str(traceback.format_exc()))
@ -262,7 +264,7 @@ class Module(MgrModule):
def refresh_keys(self):
self.keys = {}
rawkeys = self.get_config_prefix('keys/') or {}
for k, v in rawkeys.iteritems():
for k, v in iteritems(rawkeys):
self.keys[k[5:]] = v # strip of keys/ prefix
def _serve(self):
@ -286,7 +288,7 @@ class Module(MgrModule):
cert = self.get_localized_config("crt")
if cert is not None:
cert_tmp = tempfile.NamedTemporaryFile()
cert_tmp.write(cert)
cert_tmp.write(cert.encode('utf-8'))
cert_tmp.flush()
cert_fname = cert_tmp.name
else:
@ -295,7 +297,7 @@ class Module(MgrModule):
pkey = self.get_localized_config("key")
if pkey is not None:
pkey_tmp = tempfile.NamedTemporaryFile()
pkey_tmp.write(pkey)
pkey_tmp.write(pkey.encode('utf-8'))
pkey_tmp.flush()
pkey_fname = pkey_tmp.name
else:
@ -362,10 +364,7 @@ class Module(MgrModule):
if tag == 'seq':
return
request = filter(
lambda x: x.is_running(tag),
self.requests)
request = [x for x in self.requests if x.is_running(tag)]
if len(request) != 1:
self.log.warn("Unknown request '%s'" % str(tag))
return
@ -438,9 +437,8 @@ class Module(MgrModule):
elif command['prefix'] == "restful create-self-signed-cert":
cert, pkey = self.create_self_signed_cert()
self.set_config(self.get_mgr_id() + '/crt', cert)
self.set_config(self.get_mgr_id() + '/key', pkey)
self.set_config(self.get_mgr_id() + '/crt', cert.decode('utf-8'))
self.set_config(self.get_mgr_id() + '/key', pkey.decode('utf-8'))
self.restart()
return (
@ -533,10 +531,7 @@ class Module(MgrModule):
# Filter by osd ids
if ids is not None:
osds = filter(
lambda x: str(x['osd']) in ids,
osds
)
osds = [x for x in osds if str(x['osd']) in ids]
# Get list of pools per osd node
pools_map = self.get_osd_pools()
@ -562,19 +557,14 @@ class Module(MgrModule):
# Filter by pool
if pool_id:
pool_id = int(pool_id)
osds = filter(
lambda x: pool_id in x['pools'],
osds
)
osds = [x for x in osds if pool_id in x['pools']]
return osds
def get_osd_by_id(self, osd_id):
osd = filter(
lambda x: x['osd'] == osd_id,
self.get('osd_map')['osds']
)
osd = [x for x in self.get('osd_map')['osds']
if x['osd'] == osd_id]
if len(osd) != 1:
return None
@ -583,10 +573,8 @@ class Module(MgrModule):
def get_pool_by_id(self, pool_id):
pool = filter(
lambda x: x['pool'] == pool_id,
self.get('osd_map')['pools'],
)
pool = [x for x in self.get('osd_map')['pools']
if x['pool'] == pool_id]
if len(pool) != 1:
return None

2
src/pybind/mgr/selftest/__init__.py
View File

@ -1,3 +1,3 @@
from module import *
from .module import Module

3
src/pybind/mgr/status/__init__.py
View File

@ -1,2 +1 @@
from module import * # NOQA
from .module import Module

2
src/pybind/mgr/zabbix/__init__.py
View File

@ -1 +1 @@
from module import * # NOQA
from .module import Module

41
src/vstart.sh
View File

@ -16,12 +16,27 @@ if [ -n "$VSTART_DEST" ]; then
CEPH_OUT_DIR=$VSTART_DEST/out
fi
get_cmake_variable() {
local variable=$1
grep "$variable" CMakeCache.txt | cut -d "=" -f 2
}
# for running out of the CMake build directory
if [ -e CMakeCache.txt ]; then
# Out of tree build, learn source location from CMakeCache.txt
CEPH_ROOT=`grep ceph_SOURCE_DIR CMakeCache.txt | cut -d "=" -f 2`
CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR)
CEPH_BUILD_DIR=`pwd`
[ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
CEPH_MGR_PY_VERSION_MAJOR=$(get_cmake_variable MGR_PYTHON_VERSION | cut -d '.' -f1)
if [ -n "$CEPH_MGR_PY_VERSION_MAJOR" ]; then
CEPH_PY_VERSION_MAJOR=${CEPH_MGR_PY_VERSION_MAJOR}
else
if [ $(get_cmake_variable WITH_PYTHON2) = ON ]; then
CEPH_PY_VERSION_MAJOR=2
else
CEPH_PY_VERSION_MAJOR=3
fi
fi
fi
# use CEPH_BUILD_ROOT to vstart from a 'make install'
@ -30,6 +45,7 @@ if [ -n "$CEPH_BUILD_ROOT" ]; then
[ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
[ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
[ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
# make install should install python extensions into PYTHONPATH
elif [ -n "$CEPH_ROOT" ]; then
[ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
[ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
@ -46,7 +62,7 @@ fi
[ -z "$PYBIND" ] && PYBIND=./pybind
export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.2:$PYTHONPATH
export PYTHONPATH=$PYBIND:$CEPH_LIB/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}:$PYTHONPATH
export LD_LIBRARY_PATH=$CEPH_LIB:$LD_LIBRARY_PATH
export DYLD_LIBRARY_PATH=$CEPH_LIB:$DYLD_LIBRARY_PATH
# Suppress logging for regular use that indicated that we are using a
@ -662,14 +678,21 @@ start_mgr() {
host = $HOSTNAME
EOF
ceph_adm config-key set mgr/dashboard/$name/server_port $MGR_PORT
DASH_URLS+="http://$IP:$MGR_PORT/"
MGR_PORT=$(($MGR_PORT + 1000))
ceph_adm config-key set mgr/dashboard/$name/server_port $MGR_PORT
if [ $mgr -eq 1 ]; then
DASH_URLS="http://$IP:$MGR_PORT"
else
DASH_URLS+=", http://$IP:$MGR_PORT"
fi
MGR_PORT=$(($MGR_PORT + 1000))
ceph_adm config-key set mgr/restful/$name/server_port $MGR_PORT
RESTFUL_URLS+="https://$IP:$MGR_PORT"
MGR_PORT=$(($MGR_PORT + 1000))
ceph_adm config-key set mgr/restful/$name/server_port $MGR_PORT
if [ $mgr -eq 1 ]; then
RESTFUL_URLS="https://$IP:$MGR_PORT"
else
RESTFUL_URLS+=", https://$IP:$MGR_PORT"
fi
MGR_PORT=$(($MGR_PORT + 1000))
echo "Starting mgr.${name}"
run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS