From 3d25f4611280143e256349582d1e9dfa0faf40c3 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 11 Jul 2013 17:17:39 -0700 Subject: [PATCH 1/2] ceph-rest-api: make main program be "shell" around WSGI guts Move WSGI guts to pybind/ceph_rest_api.py for importability into WSGI server frameworks. Signed-off-by: Dan Mick --- src/ceph-rest-api | 418 +----------------------------------- src/pybind/ceph_rest_api.py | 418 ++++++++++++++++++++++++++++++++++++ 2 files changed, 419 insertions(+), 417 deletions(-) create mode 100755 src/pybind/ceph_rest_api.py diff --git a/src/ceph-rest-api b/src/ceph-rest-api index 7a4db38915d..712286cd09b 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -25,423 +25,7 @@ if MYDIR.endswith('src') and \ os.execvp('python', ['python'] + sys.argv) sys.path.insert(0, os.path.join(MYDIR, 'pybind')) -import argparse -import collections -import ConfigParser -import errno -import json -import logging -import logging.handlers -import rados -import textwrap -import xml.etree.ElementTree -import xml.sax.saxutils - -import flask -from ceph_argparse import * - -# -# Globals -# - -APPNAME = '__main__' -DEFAULT_BASEURL = '/api/v0.1' -DEFAULT_ADDR = '0.0.0.0:5000' -DEFAULT_LOG_LEVEL = 'warning' -DEFAULT_CLIENTNAME = 'client.restapi' -DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' - -app = flask.Flask(APPNAME) - -LOGLEVELS = { - 'critical':logging.CRITICAL, - 'error':logging.ERROR, - 'warning':logging.WARNING, - 'info':logging.INFO, - 'debug':logging.DEBUG, -} - - -# my globals, in a named tuple for usage clarity - -glob = collections.namedtuple('gvars', - 'args cluster urls sigdict baseurl clientname') -glob.args = None -glob.cluster = None -glob.urls = {} -glob.sigdict = {} -glob.baseurl = '' -glob.clientname = '' - -def parse_args(): - parser = argparse.ArgumentParser(description="Ceph REST API webapp") - parser.add_argument('-c', '--conf', help='Ceph configuration file') - parser.add_argument('-n', '--name', help='Ceph client config/key name') - - return parser.parse_args() - -def load_conf(conffile=None): - import contextlib - - class _TrimIndentFile(object): - def __init__(self, fp): - self.fp = fp - - def readline(self): - line = self.fp.readline() - return line.lstrip(' \t') - - - def _optionxform(s): - s = s.replace('_', ' ') - s = '_'.join(s.split()) - return s - - - def parse(fp): - cfg = ConfigParser.RawConfigParser() - cfg.optionxform = _optionxform - ifp = _TrimIndentFile(fp) - cfg.readfp(ifp) - return cfg - - - def load(path): - f = file(path) - with contextlib.closing(f): - return parse(f) - - # XXX this should probably use cluster name - if conffile: - return load(conffile) - elif 'CEPH_CONF' in os.environ: - conffile = os.environ['CEPH_CONF'] - elif os.path.exists('/etc/ceph/ceph.conf'): - conffile = '/etc/ceph/ceph.conf' - elif os.path.exists(os.path.expanduser('~/.ceph/ceph.conf')): - conffile = os.path.expanduser('~/.ceph/ceph.conf') - elif os.path.exists('ceph.conf'): - conffile = 'ceph.conf' - else: - return None - - return load(conffile) - -def get_conf(cfg, key): - try: - return cfg.get(glob.clientname, 'restapi_' + key) - except ConfigParser.NoOptionError: - return None - - -# XXX this is done globally, and cluster connection kept open; there -# are facilities to pass around global info to requests and to -# tear down connections between requests if it becomes important - -def api_setup(): - """ - Initialize the running instance. Open the cluster, get the command - signatures, module,, perms, and help; stuff them away in the glob.urls - dict. - """ - - glob.args = parse_args() - - conffile = glob.args.conf or '' - if glob.args.name: - glob.clientname = glob.args.name - glob.logfile = '/var/log/ceph' + glob.clientname + '.log' - - glob.clientname = glob.args.name or DEFAULT_CLIENTNAME - glob.cluster = rados.Rados(name=glob.clientname, conffile=conffile) - glob.cluster.connect() - - cfg = load_conf(conffile) - glob.baseurl = get_conf(cfg, 'base_url') or DEFAULT_BASEURL - if glob.baseurl.endswith('/'): - glob.baseurl - addr = get_conf(cfg, 'public_addr') or DEFAULT_ADDR - addrport = addr.rsplit(':', 1) - addr = addrport[0] - if len(addrport) > 1: - port = addrport[1] - else: - port = DEFAULT_ADDR.rsplit(':', 1) - port = int(port) - - loglevel = get_conf(cfg, 'log_level') or 'warning' - logfile = get_conf(cfg, 'log_file') or glob.logfile - app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) - app.logger.setLevel(LOGLEVELS[loglevel.lower()]) - for h in app.logger.handlers: - h.setFormatter(logging.Formatter( - '%(asctime)s %(name)s %(levelname)s: %(message)s')) - - ret, outbuf, outs = json_command(glob.cluster, - prefix='get_command_descriptions') - if ret: - app.logger.error('Can\'t contact cluster for command descriptions: %s', - outs) - sys.exit(1) - - try: - glob.sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - app.logger.error('Can\'t parse command descriptions: %s', e) - sys.exit(1) - - # glob.sigdict maps "cmdNNN" to a dict containing: - # 'sig', an array of argdescs - # 'help', the helptext - # 'module', the Ceph module this command relates to - # 'perm', a 'rwx*' string representing required permissions, and also - # a hint as to whether this is a GET or POST/PUT operation - # 'avail', a comma-separated list of strings of consumers that should - # display this command (filtered by parse_json_funcsigs() above) - glob.urls = {} - for cmdnum, cmddict in glob.sigdict.iteritems(): - cmdsig = cmddict['sig'] - url, params = generate_url_and_params(cmdsig) - if url in glob.urls: - continue - else: - perm = cmddict['perm'] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - } - method_dict = {'r':['GET'], - 'w':['PUT', 'DELETE']} - for k in method_dict.iterkeys(): - if k in perm: - methods = method_dict[k] - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - app.logger.debug("urls added: %d", len(glob.urls)) - - app.add_url_rule('/', '/', - handler, methods=['GET', 'PUT']) - return addr, port - - -def generate_url_and_params(sig): - """ - Digest command signature from cluster; generate an absolute - (including glob.baseurl) endpoint from all the prefix words, - and a dictionary of non-prefix parameters - """ - - url = '' - params = [] - for desc in sig: - if desc.t == CephPrefix: - url += '/' + desc.instance.prefix - elif desc.t == CephChoices and \ - len(desc.instance.strings) == 1 and \ - desc.req and \ - not str(desc.instance).startswith('--'): - url += '/' + str(desc.instance) - else: - params.append(desc) - return glob.baseurl + url, params - - -def concise_sig_for_uri(sig): - """ - Return a generic description of how one would send a REST request for sig - """ - prefix = [] - args = [] - for d in sig: - if d.t == CephPrefix: - prefix.append(d.instance.prefix) - else: - args.append(d.name + '=' + str(d)) - sig = '/'.join(prefix) - if args: - sig += '?' + '&'.join(args) - return sig - -def show_human_help(prefix): - """ - Dump table showing commands matching prefix - """ - # XXX this really needs to be a template - #s = '' - #s += '' - #s += '' - # XXX the above mucking with css doesn't cause sensible columns. - s = '
Possible commands:
' - - possible = [] - permmap = {'r':'GET', 'rw':'PUT'} - line = '' - for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): - concise = concise_sig(cmdsig['sig']) - if concise.startswith(prefix): - line = ['\n') - s += ''.join(line) - - s += '
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) - for sigline in wrapped_sig: - line.append(flask.escape(sigline) + '\n') - line.append('') - line.append(permmap[cmdsig['perm']]) - line.append('') - line.append(flask.escape(cmdsig['help'])) - line.append('
' - if line: - return s - else: - return '' - -@app.before_request -def log_request(): - """ - For every request, log it. XXX Probably overkill for production - """ - app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) - app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - - -@app.route('/') -def root_redir(): - return flask.redirect(glob.baseurl) - -def make_response(fmt, output, statusmsg, errorcode): - """ - If formatted output, cobble up a response object that contains the - output and status wrapped in enclosing objects; if nonformatted, just - use output. Return HTTP status errorcode in any event. - """ - response = output - if fmt: - if 'json' in fmt: - try: - native_output = json.loads(output or '[]') - response = json.dumps({"output":native_output, - "status":statusmsg}) - except: - return flask.make_response("Error decoding JSON from " + - output, 500) - elif 'xml' in fmt: - # one is tempted to do this with xml.etree, but figuring out how - # to 'un-XML' the XML-dumped output so it can be reassembled into - # a piece of the tree here is beyond me right now. - #ET = xml.etree.ElementTree - #resp_elem = ET.Element('response') - #o = ET.SubElement(resp_elem, 'output') - #o.text = output - #s = ET.SubElement(resp_elem, 'status') - #s.text = statusmsg - #response = ET.tostring(resp_elem) - response = ''' - - - {0} - - - {1} - -'''.format(response, xml.sax.saxutils.escape(statusmsg)) - - return flask.make_response(response, errorcode) - -def handler(catchall_path=None, fmt=None): - """ - Main endpoint handler; generic for every endpoint - """ - - if (catchall_path): - ep = catchall_path.replace('.', '') - else: - ep = flask.request.endpoint.replace('.', '') - - if ep[0] != '/': - ep = '/' + ep - - # Extensions override Accept: headers override defaults - if not fmt: - if 'application/json' in flask.request.accept_mimetypes.values(): - fmt = 'json' - elif 'application/xml' in flask.request.accept_mimetypes.values(): - fmt = 'xml' - - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): - return make_response(fmt, '', 'Page not found', 404) - - relative_endpoint = ep[len(glob.baseurl)+1:] - prefix = ' '.join(relative_endpoint.split('/')).strip() - - # show "match as much as you gave me" help for unknown endpoints - if not ep in glob.urls: - helptext = show_human_help(prefix) - if helptext: - resp = flask.make_response(helptext, 400) - resp.headers['Content-Type'] = 'text/html' - return resp - else: - return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - - urldict = glob.urls[ep] - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - except Exception as e: - return make_response(fmt, '', str(e) + '\n', 400) - else: - # no parameters for this endpoint; complain if args are supplied - if flask.request.args: - return make_response(fmt, '', ep + 'takes no params', 400) - argdict = {} - - - argdict['format'] = fmt or 'plain' - argdict['module'] = urldict['module'] - argdict['perm'] = urldict['perm'] - - app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) - ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, - inbuf=flask.request.data, argdict=argdict) - if ret: - return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) - - response = make_response(fmt, outbuf, outs or 'OK', 200) - if fmt: - contenttype = 'application/' + fmt.replace('-pretty','') - else: - contenttype = 'text/plain' - response.headers['Content-Type'] = contenttype - return response - -# -# main -# +from ceph_rest_api import api_setup, app addr, port = api_setup() diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py new file mode 100755 index 00000000000..755ba977b04 --- /dev/null +++ b/src/pybind/ceph_rest_api.py @@ -0,0 +1,418 @@ +#!/usr/bin/python +# vim: ts=4 sw=4 smarttab expandtab + +import os +import sys +import argparse +import collections +import ConfigParser +import errno +import json +import logging +import logging.handlers +import rados +import textwrap +import xml.etree.ElementTree +import xml.sax.saxutils + +import flask +from ceph_argparse import * + +# +# Globals +# + +APPNAME = '__main__' +DEFAULT_BASEURL = '/api/v0.1' +DEFAULT_ADDR = '0.0.0.0:5000' +DEFAULT_LOG_LEVEL = 'warning' +DEFAULT_CLIENTNAME = 'client.restapi' +DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' + +app = flask.Flask(APPNAME) + +LOGLEVELS = { + 'critical':logging.CRITICAL, + 'error':logging.ERROR, + 'warning':logging.WARNING, + 'info':logging.INFO, + 'debug':logging.DEBUG, +} + + +# my globals, in a named tuple for usage clarity + +glob = collections.namedtuple('gvars', + 'args cluster urls sigdict baseurl clientname') +glob.args = None +glob.cluster = None +glob.urls = {} +glob.sigdict = {} +glob.baseurl = '' +glob.clientname = '' + +def parse_args(): + parser = argparse.ArgumentParser(description="Ceph REST API webapp") + parser.add_argument('-c', '--conf', help='Ceph configuration file') + parser.add_argument('-n', '--name', help='Ceph client config/key name') + + return parser.parse_args() + +def load_conf(conffile=None): + import contextlib + + class _TrimIndentFile(object): + def __init__(self, fp): + self.fp = fp + + def readline(self): + line = self.fp.readline() + return line.lstrip(' \t') + + + def _optionxform(s): + s = s.replace('_', ' ') + s = '_'.join(s.split()) + return s + + + def parse(fp): + cfg = ConfigParser.RawConfigParser() + cfg.optionxform = _optionxform + ifp = _TrimIndentFile(fp) + cfg.readfp(ifp) + return cfg + + + def load(path): + f = file(path) + with contextlib.closing(f): + return parse(f) + + # XXX this should probably use cluster name + if conffile: + return load(conffile) + elif 'CEPH_CONF' in os.environ: + conffile = os.environ['CEPH_CONF'] + elif os.path.exists('/etc/ceph/ceph.conf'): + conffile = '/etc/ceph/ceph.conf' + elif os.path.exists(os.path.expanduser('~/.ceph/ceph.conf')): + conffile = os.path.expanduser('~/.ceph/ceph.conf') + elif os.path.exists('ceph.conf'): + conffile = 'ceph.conf' + else: + return None + + return load(conffile) + +def get_conf(cfg, key): + try: + return cfg.get(glob.clientname, 'restapi_' + key) + except ConfigParser.NoOptionError: + return None + + +# XXX this is done globally, and cluster connection kept open; there +# are facilities to pass around global info to requests and to +# tear down connections between requests if it becomes important + +def api_setup(): + """ + Initialize the running instance. Open the cluster, get the command + signatures, module,, perms, and help; stuff them away in the glob.urls + dict. + """ + + glob.args = parse_args() + + conffile = glob.args.conf or '' + if glob.args.name: + glob.clientname = glob.args.name + glob.logfile = '/var/log/ceph' + glob.clientname + '.log' + + glob.clientname = glob.args.name or DEFAULT_CLIENTNAME + glob.cluster = rados.Rados(name=glob.clientname, conffile=conffile) + glob.cluster.connect() + + cfg = load_conf(conffile) + glob.baseurl = get_conf(cfg, 'base_url') or DEFAULT_BASEURL + if glob.baseurl.endswith('/'): + glob.baseurl + addr = get_conf(cfg, 'public_addr') or DEFAULT_ADDR + addrport = addr.rsplit(':', 1) + addr = addrport[0] + if len(addrport) > 1: + port = addrport[1] + else: + port = DEFAULT_ADDR.rsplit(':', 1) + port = int(port) + + loglevel = get_conf(cfg, 'log_level') or 'warning' + logfile = get_conf(cfg, 'log_file') or glob.logfile + app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) + app.logger.setLevel(LOGLEVELS[loglevel.lower()]) + for h in app.logger.handlers: + h.setFormatter(logging.Formatter( + '%(asctime)s %(name)s %(levelname)s: %(message)s')) + + ret, outbuf, outs = json_command(glob.cluster, + prefix='get_command_descriptions') + if ret: + app.logger.error('Can\'t contact cluster for command descriptions: %s', + outs) + sys.exit(1) + + try: + glob.sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + app.logger.error('Can\'t parse command descriptions: %s', e) + sys.exit(1) + + # glob.sigdict maps "cmdNNN" to a dict containing: + # 'sig', an array of argdescs + # 'help', the helptext + # 'module', the Ceph module this command relates to + # 'perm', a 'rwx*' string representing required permissions, and also + # a hint as to whether this is a GET or POST/PUT operation + # 'avail', a comma-separated list of strings of consumers that should + # display this command (filtered by parse_json_funcsigs() above) + glob.urls = {} + for cmdnum, cmddict in glob.sigdict.iteritems(): + cmdsig = cmddict['sig'] + url, params = generate_url_and_params(cmdsig) + if url in glob.urls: + continue + else: + perm = cmddict['perm'] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + } + method_dict = {'r':['GET'], + 'w':['PUT', 'DELETE']} + for k in method_dict.iterkeys(): + if k in perm: + methods = method_dict[k] + app.add_url_rule(url, url, handler, methods=methods) + glob.urls[url] = urldict + + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + glob.urls[url] = urldict + app.logger.debug("urls added: %d", len(glob.urls)) + + app.add_url_rule('/', '/', + handler, methods=['GET', 'PUT']) + return addr, port + + +def generate_url_and_params(sig): + """ + Digest command signature from cluster; generate an absolute + (including glob.baseurl) endpoint from all the prefix words, + and a dictionary of non-prefix parameters + """ + + url = '' + params = [] + for desc in sig: + if desc.t == CephPrefix: + url += '/' + desc.instance.prefix + elif desc.t == CephChoices and \ + len(desc.instance.strings) == 1 and \ + desc.req and \ + not str(desc.instance).startswith('--'): + url += '/' + str(desc.instance) + else: + params.append(desc) + return glob.baseurl + url, params + + +def concise_sig_for_uri(sig): + """ + Return a generic description of how one would send a REST request for sig + """ + prefix = [] + args = [] + for d in sig: + if d.t == CephPrefix: + prefix.append(d.instance.prefix) + else: + args.append(d.name + '=' + str(d)) + sig = '/'.join(prefix) + if args: + sig += '?' + '&'.join(args) + return sig + +def show_human_help(prefix): + """ + Dump table showing commands matching prefix + """ + # XXX this really needs to be a template + #s = '' + #s += '' + #s += '' + # XXX the above mucking with css doesn't cause sensible columns. + s = '
Possible commands:
' + + possible = [] + permmap = {'r':'GET', 'rw':'PUT'} + line = '' + for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): + concise = concise_sig(cmdsig['sig']) + if concise.startswith(prefix): + line = ['\n') + s += ''.join(line) + + s += '
Possible commands:MethodDescription
'] + wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) + for sigline in wrapped_sig: + line.append(flask.escape(sigline) + '\n') + line.append('') + line.append(permmap[cmdsig['perm']]) + line.append('') + line.append(flask.escape(cmdsig['help'])) + line.append('
' + if line: + return s + else: + return '' + +@app.before_request +def log_request(): + """ + For every request, log it. XXX Probably overkill for production + """ + app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) + app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) + + +@app.route('/') +def root_redir(): + return flask.redirect(glob.baseurl) + +def make_response(fmt, output, statusmsg, errorcode): + """ + If formatted output, cobble up a response object that contains the + output and status wrapped in enclosing objects; if nonformatted, just + use output. Return HTTP status errorcode in any event. + """ + response = output + if fmt: + if 'json' in fmt: + try: + native_output = json.loads(output or '[]') + response = json.dumps({"output":native_output, + "status":statusmsg}) + except: + return flask.make_response("Error decoding JSON from " + + output, 500) + elif 'xml' in fmt: + # one is tempted to do this with xml.etree, but figuring out how + # to 'un-XML' the XML-dumped output so it can be reassembled into + # a piece of the tree here is beyond me right now. + #ET = xml.etree.ElementTree + #resp_elem = ET.Element('response') + #o = ET.SubElement(resp_elem, 'output') + #o.text = output + #s = ET.SubElement(resp_elem, 'status') + #s.text = statusmsg + #response = ET.tostring(resp_elem) + response = ''' + + + {0} + + + {1} + +'''.format(response, xml.sax.saxutils.escape(statusmsg)) + + return flask.make_response(response, errorcode) + +def handler(catchall_path=None, fmt=None): + """ + Main endpoint handler; generic for every endpoint + """ + + if (catchall_path): + ep = catchall_path.replace('.', '') + else: + ep = flask.request.endpoint.replace('.', '') + + if ep[0] != '/': + ep = '/' + ep + + # Extensions override Accept: headers override defaults + if not fmt: + if 'application/json' in flask.request.accept_mimetypes.values(): + fmt = 'json' + elif 'application/xml' in flask.request.accept_mimetypes.values(): + fmt = 'xml' + + # demand that endpoint begin with glob.baseurl + if not ep.startswith(glob.baseurl): + return make_response(fmt, '', 'Page not found', 404) + + relative_endpoint = ep[len(glob.baseurl)+1:] + prefix = ' '.join(relative_endpoint.split('/')).strip() + + # show "match as much as you gave me" help for unknown endpoints + if not ep in glob.urls: + helptext = show_human_help(prefix) + if helptext: + resp = flask.make_response(helptext, 400) + resp.headers['Content-Type'] = 'text/html' + return resp + else: + return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) + + urldict = glob.urls[ep] + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + except Exception as e: + return make_response(fmt, '', str(e) + '\n', 400) + else: + # no parameters for this endpoint; complain if args are supplied + if flask.request.args: + return make_response(fmt, '', ep + 'takes no params', 400) + argdict = {} + + + argdict['format'] = fmt or 'plain' + argdict['module'] = urldict['module'] + argdict['perm'] = urldict['perm'] + + app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) + ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + inbuf=flask.request.data, argdict=argdict) + if ret: + return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) + + response = make_response(fmt, outbuf, outs or 'OK', 200) + if fmt: + contenttype = 'application/' + fmt.replace('-pretty','') + else: + contenttype = 'text/plain' + response.headers['Content-Type'] = contenttype + return response From cc109888503a4b3f3d6fd27b84e839982087fab5 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 12 Jul 2013 13:58:36 -0700 Subject: [PATCH 2/2] ceph-rest-api: separate into module and front-end for WSGI deploy To deploy ceph-rest-api within a WSGI server (apache/mod_wsgi, nginx/uwsgi, etc.), there needs to be an importable (.py) module that performs all init/config when imported. ceph-rest-api was close, but it needs to be named properly, and there's no argument passing, so it needs to get args from a fixed file or the env. Separate most of ceph-rest-api into pybind/ceph_rest_api.py, and make its arguments come from the environment, and init errors be ImportError exceptions. Recase ceph-rest-api as a thin layer that does the usual setup and arg parsing, and then sets args into the environment and imports ceph_rest_api.py, catching exceptions and reporting errors. This allows standalone execution as usual. ceph-rest-api grabs a few module globals (addr/port and the flask.app) to use after it imports. Accept cluster name, and do the ceph.conf search using cluster name in the appropriate places in the searched-for files. Also ceph_rest_api.py gets a little cleanup (fewer global variables, cleaner conf file search algorithm, better error reporting on conf load) Also: doc updates, packaging updates to include ceph_rest_api.py Signed-off-by: Dan Mick --- ceph.spec.in | 1 + doc/man/8/ceph-rest-api.rst | 47 ++++++++++++++++---- man/ceph-rest-api.8 | 51 ++++++++++++++++++---- src/Makefile.am | 3 +- src/ceph-rest-api | 46 ++++++++++++++++---- src/pybind/ceph_rest_api.py | 86 ++++++++++++++++++------------------- 6 files changed, 163 insertions(+), 71 deletions(-) diff --git a/ceph.spec.in b/ceph.spec.in index b8578d83c8a..f5718b40c93 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -562,6 +562,7 @@ fi %{python_sitelib}/rbd.py* %{python_sitelib}/cephfs.py* %{python_sitelib}/ceph_argparse.py* +%{python_sitelib}/ceph_rest_api.py* ################################################################################# %files -n rest-bench diff --git a/doc/man/8/ceph-rest-api.rst b/doc/man/8/ceph-rest-api.rst index d8727abb49a..8a87f97ce19 100644 --- a/doc/man/8/ceph-rest-api.rst +++ b/doc/man/8/ceph-rest-api.rst @@ -23,14 +23,24 @@ Options .. option:: -c/--conf *conffile* - names the ceph.conf file to use for configuration. If -c - is not specified, the configuration file is searched for in - this order: + names the ceph.conf file to use for configuration. If -c is not + specified, the default depends on the state of the --cluster option + (default 'ceph'; see below). The configuration file is searched + for in this order: * $CEPH_CONF - * /etc/ceph/ceph.conf - * ~/.ceph/ceph.conf - * ceph.conf (in the current directory) + * /etc/ceph/${cluster}.conf + * ~/.ceph/${cluster}.conf + * ${cluster}.conf (in the current directory) + + so you can also pass this option in the environment as CEPH_CONF. + +.. option:: --cluster *clustername* + + set *clustername* for use in the $cluster metavariable, for + locating the ceph.conf file. The default is 'ceph'. + You can also pass this option in the environment as + CEPH_CLUSTER_NAME. .. option:: -n/--name *name* @@ -38,14 +48,16 @@ Options client-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, - for example). The default is 'client.restapi'. - + for example). The default is 'client.restapi'. You can also + pass this option in the environment as CEPH_NAME. + Configuration parameters ======================== Supported configuration parameters include: +* **restapi client name** the 'clientname' used for auth and ceph.conf * **restapi keyring** the keyring file holding the key for 'clientname' * **restapi public addr** ip:port to listen on (default 0.0.0.0:5000) * **restapi base url** the base URL to answer requests on (default /api/v0.1) @@ -82,6 +94,25 @@ the value of **restapi base url**, and that path will give a full list of all known commands. The command set is very similar to the commands supported by the **ceph** tool. +Deployment as WSGI application +============================== + +When deploying as WSGI application (say, with Apache/mod_wsgi, +or nginx/uwsgi, or gunicorn, etc.), use the ``ceph_rest_api.py`` module +(``ceph-rest-api`` is a thin layer around this module). The standalone web +server is of course not used, so address/port configuration is done in +the WSGI server. Also, configuration switches are not passed; rather, +environment variables are used: + +* CEPH_CONF holds -c/--conf +* CEPH_CLUSTER_NAME holds --cluster +* CEPH_NAME holds -n/--name + +Any errors reading configuration or connecting to the cluster cause +ImportError to be raised with a descriptive message on import; see +your WSGI server documentation for how to see those messages in case +of problem. + Availability ============ diff --git a/man/ceph-rest-api.8 b/man/ceph-rest-api.8 index 77a9ef29c38..33425fecc00 100644 --- a/man/ceph-rest-api.8 +++ b/man/ceph-rest-api.8 @@ -1,4 +1,4 @@ -.TH "CEPH-REST-API" "8" "July 10, 2013" "dev" "Ceph" +.TH "CEPH-REST-API" "8" "July 12, 2013" "dev" "Ceph" .SH NAME ceph-rest-api \- ceph RESTlike administration server . @@ -45,19 +45,30 @@ command\-line tool through an HTTP\-accessible interface. .INDENT 0.0 .TP .B \-c/\-\-conf *conffile* -names the ceph.conf file to use for configuration. If \-c -is not specified, the configuration file is searched for in -this order: +names the ceph.conf file to use for configuration. If \-c is not +specified, the default depends on the state of the \-\-cluster option +(default \(aqceph\(aq; see below). The configuration file is searched +for in this order: .INDENT 7.0 .IP \(bu 2 $CEPH_CONF .IP \(bu 2 -/etc/ceph/ceph.conf +/etc/ceph/${cluster}.conf .IP \(bu 2 -~/.ceph/ceph.conf +~/.ceph/${cluster}.conf .IP \(bu 2 -ceph.conf (in the current directory) +${cluster}.conf (in the current directory) .UNINDENT +.sp +so you can also pass this option in the environment as CEPH_CONF. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-cluster *clustername* +set \fIclustername\fP for use in the $cluster metavariable, for +locating the ceph.conf file. The default is \(aqceph\(aq. +You can also pass this option in the environment as +CEPH_CLUSTER_NAME. .UNINDENT .INDENT 0.0 .TP @@ -66,13 +77,16 @@ specifies the client \(aqname\(aq, which is used to find the client\-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, -for example). The default is \(aqclient.restapi\(aq. +for example). The default is \(aqclient.restapi\(aq. You can also +pass this option in the environment as CEPH_NAME. .UNINDENT .SH CONFIGURATION PARAMETERS .sp Supported configuration parameters include: .INDENT 0.0 .IP \(bu 2 +\fBrestapi client name\fP the \(aqclientname\(aq used for auth and ceph.conf +.IP \(bu 2 \fBrestapi keyring\fP the keyring file holding the key for \(aqclientname\(aq .IP \(bu 2 \fBrestapi public addr\fP ip:port to listen on (default 0.0.0.0:5000) @@ -109,6 +123,27 @@ path is incomplete/partially matching. Requesting / will redirect to the value of \fBrestapi base url\fP, and that path will give a full list of all known commands. The command set is very similar to the commands supported by the \fBceph\fP tool. +.SH DEPLOYMENT AS WSGI APPLICATION +.sp +When deploying as WSGI application (say, with Apache/mod_wsgi, +or nginx/uwsgi, or gunicorn, etc.), use the \fBceph_rest_api.py\fP module +(\fBceph\-rest\-api\fP is a thin layer around this module). The standalone web +server is of course not used, so address/port configuration is done in +the WSGI server. Also, configuration switches are not passed; rather, +environment variables are used: +.INDENT 0.0 +.IP \(bu 2 +CEPH_CONF holds \-c/\-\-conf +.IP \(bu 2 +CEPH_CLUSTER_NAME holds \-\-cluster +.IP \(bu 2 +CEPH_NAME holds \-n/\-\-name +.UNINDENT +.sp +Any errors reading configuration or connecting to the cluster cause +ImportError to be raised with a descriptive message on import; see +your WSGI server documentation for how to see those messages in case +of problem. .SH AVAILABILITY .sp \fBceph\-rest\-api\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at diff --git a/src/Makefile.am b/src/Makefile.am index 3762107bfe9..f7577b639c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1694,7 +1694,8 @@ dist-hook: python_PYTHON = pybind/rados.py \ pybind/rbd.py \ pybind/cephfs.py \ - pybind/ceph_argparse.py + pybind/ceph_argparse.py \ + pybind/ceph_rest_api.py # headers... and everything else we want to include in a 'make dist' # that autotools doesn't magically identify. diff --git a/src/ceph-rest-api b/src/ceph-rest-api index 712286cd09b..a44919acd6f 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -1,6 +1,8 @@ #!/usr/bin/python # vim: ts=4 sw=4 smarttab expandtab +import argparse +import inspect import os import sys @@ -25,14 +27,40 @@ if MYDIR.endswith('src') and \ os.execvp('python', ['python'] + sys.argv) sys.path.insert(0, os.path.join(MYDIR, 'pybind')) -from ceph_rest_api import api_setup, app -addr, port = api_setup() +def parse_args(): + parser = argparse.ArgumentParser(description="Ceph REST API webapp") + parser.add_argument('-c', '--conf', help='Ceph configuration file') + parser.add_argument('--cluster', help='Ceph cluster name') + parser.add_argument('-n', '--name', help='Ceph client name') -if __name__ == '__main__': - import inspect - files = [os.path.split(fr[1])[-1] for fr in inspect.stack()] - if 'pdb.py' in files: - app.run(host=addr, port=port, debug=True, use_reloader=False, use_debugger=False) - else: - app.run(host=addr, port=port, debug=True) + return parser.parse_args() + + +# main + +parsed_args = parse_args() +if parsed_args.conf: + os.environ['CEPH_CONF'] = parsed_args.conf +if parsed_args.cluster: + os.environ['CEPH_CLUSTER_NAME'] = parsed_args.cluster +if parsed_args.name: + os.environ['CEPH_NAME'] = parsed_args.name + +# import now that env vars are available to imported module + +try: + import ceph_rest_api +except Exception as e: + print >> sys.stderr, "Error importing ceph_rest_api: ", str(e) + sys.exit(1) + +# importing ceph_rest_api has set module globals 'app', 'addr', and 'port' + +files = [os.path.split(fr[1])[-1] for fr in inspect.stack()] +if 'pdb.py' in files: + ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, + debug=True, use_reloader=False, use_debugger=False) +else: + ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, + debug=True) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 755ba977b04..4841022e1d6 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -39,28 +39,18 @@ LOGLEVELS = { 'debug':logging.DEBUG, } - # my globals, in a named tuple for usage clarity -glob = collections.namedtuple('gvars', - 'args cluster urls sigdict baseurl clientname') -glob.args = None +glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') glob.cluster = None glob.urls = {} glob.sigdict = {} glob.baseurl = '' -glob.clientname = '' -def parse_args(): - parser = argparse.ArgumentParser(description="Ceph REST API webapp") - parser.add_argument('-c', '--conf', help='Ceph configuration file') - parser.add_argument('-n', '--name', help='Ceph client config/key name') - - return parser.parse_args() - -def load_conf(conffile=None): +def load_conf(clustername='ceph', conffile=None): import contextlib + class _TrimIndentFile(object): def __init__(self, fp): self.fp = fp @@ -89,29 +79,26 @@ def load_conf(conffile=None): with contextlib.closing(f): return parse(f) - # XXX this should probably use cluster name if conffile: + # from CEPH_CONF return load(conffile) - elif 'CEPH_CONF' in os.environ: - conffile = os.environ['CEPH_CONF'] - elif os.path.exists('/etc/ceph/ceph.conf'): - conffile = '/etc/ceph/ceph.conf' - elif os.path.exists(os.path.expanduser('~/.ceph/ceph.conf')): - conffile = os.path.expanduser('~/.ceph/ceph.conf') - elif os.path.exists('ceph.conf'): - conffile = 'ceph.conf' else: - return None + for path in [ + '/etc/ceph/{0}.conf'.format(clustername), + os.path.expanduser('~/.ceph/{0}.conf'.format(clustername)), + '{0}.conf'.format(clustername), + ]: + if os.path.exists(path): + return load(path) - return load(conffile) + raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) -def get_conf(cfg, key): +def get_conf(cfg, clientname, key): try: - return cfg.get(glob.clientname, 'restapi_' + key) + return cfg.get(clientname, 'restapi_' + key) except ConfigParser.NoOptionError: return None - # XXX this is done globally, and cluster connection kept open; there # are facilities to pass around global info to requests and to # tear down connections between requests if it becomes important @@ -119,26 +106,32 @@ def get_conf(cfg, key): def api_setup(): """ Initialize the running instance. Open the cluster, get the command - signatures, module,, perms, and help; stuff them away in the glob.urls + signatures, module, perms, and help; stuff them away in the glob.urls dict. """ - glob.args = parse_args() + conffile = os.environ.get('CEPH_CONF', '') + clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') + clientname = os.environ.get('CEPH_NAME', DEFAULT_CLIENTNAME) + try: + err = '' + cfg = load_conf(clustername, conffile) + except Exception as e: + err = "Can't load Ceph conf file: " + str(e) + app.logger.critical(err) + app.logger.critical("CEPH_CONF: %s", conffile) + app.logger.critical("CEPH_CLUSTER_NAME: %s", clustername) + raise EnvironmentError(err) - conffile = glob.args.conf or '' - if glob.args.name: - glob.clientname = glob.args.name - glob.logfile = '/var/log/ceph' + glob.clientname + '.log' + client_logfile = '/var/log/ceph' + clientname + '.log' - glob.clientname = glob.args.name or DEFAULT_CLIENTNAME - glob.cluster = rados.Rados(name=glob.clientname, conffile=conffile) + glob.cluster = rados.Rados(name=clientname, conffile=conffile) glob.cluster.connect() - cfg = load_conf(conffile) - glob.baseurl = get_conf(cfg, 'base_url') or DEFAULT_BASEURL + glob.baseurl = get_conf(cfg, clientname, 'base_url') or DEFAULT_BASEURL if glob.baseurl.endswith('/'): glob.baseurl - addr = get_conf(cfg, 'public_addr') or DEFAULT_ADDR + addr = get_conf(cfg, clientname, 'public_addr') or DEFAULT_ADDR addrport = addr.rsplit(':', 1) addr = addrport[0] if len(addrport) > 1: @@ -147,8 +140,8 @@ def api_setup(): port = DEFAULT_ADDR.rsplit(':', 1) port = int(port) - loglevel = get_conf(cfg, 'log_level') or 'warning' - logfile = get_conf(cfg, 'log_file') or glob.logfile + loglevel = get_conf(cfg, clientname, 'log_level') or DEFAULT_LOG_LEVEL + logfile = get_conf(cfg, clientname, 'log_file') or client_logfile app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) app.logger.setLevel(LOGLEVELS[loglevel.lower()]) for h in app.logger.handlers: @@ -158,15 +151,16 @@ def api_setup(): ret, outbuf, outs = json_command(glob.cluster, prefix='get_command_descriptions') if ret: - app.logger.error('Can\'t contact cluster for command descriptions: %s', - outs) - sys.exit(1) + err = "Can't contact cluster for command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) try: glob.sigdict = parse_json_funcsigs(outbuf, 'rest') except Exception as e: - app.logger.error('Can\'t parse command descriptions: %s', e) - sys.exit(1) + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) # glob.sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs @@ -416,3 +410,5 @@ def handler(catchall_path=None, fmt=None): contenttype = 'text/plain' response.headers['Content-Type'] = contenttype return response + +addr, port = api_setup()