1 # vim: ts=4 sw=4 smarttab expandtab
6 import logging.handlers
10 import xml.etree.ElementTree
11 import xml.sax.saxutils
14 from ceph_argparse import \
15 ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \
16 concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \
17 validate, json_command
20 # Globals and defaults
25 DEFAULT_ID = 'restapi'
27 DEFAULT_BASEURL = '/api/v0.1'
28 DEFAULT_LOG_LEVEL = 'warning'
29 DEFAULT_LOGDIR = '/var/log/ceph'
30 # default client name will be 'client.<DEFAULT_ID>'
32 # network failure could keep the underlying json_command() waiting forever,
33 # set a timeout, so it bails out on timeout.
35 # and retry in that case.
38 # 'app' must be global for decorators, etc.
40 app = flask.Flask(APPNAME)
43 'critical': logging.CRITICAL,
44 'error': logging.ERROR,
45 'warning': logging.WARNING,
47 'debug': logging.DEBUG,
53 Find an up OSD. Return the last one that's up.
56 ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump",
57 argdict=dict(format='json'))
59 raise EnvironmentError(ret, 'Can\'t get osd dump output')
61 osddump = json.loads(outbuf)
63 raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump')
64 osds = [osd['osd'] for osd in osddump['osds'] if osd['up']]
70 METHOD_DICT = {'r': ['GET'], 'w': ['PUT', 'DELETE']}
73 def api_setup(app, conf, cluster, clientname, clientid, args):
75 This is done globally, and cluster connection kept open for
76 the lifetime of the daemon. librados should assure that even
77 if the cluster goes away and comes back, our connection remains.
79 Initialize the running instance. Open the cluster, get the command
80 signatures, module, perms, and help; stuff them away in the app.ceph_urls
81 dict. Also save app.ceph_sigdict for help() handling.
83 def get_command_descriptions(cluster, target=('mon', '')):
84 ret, outbuf, outs = json_command(cluster, target,
85 prefix='get_command_descriptions',
88 err = "Can't get command descriptions: {0}".format(outs)
90 raise EnvironmentError(ret, err)
93 sigdict = parse_json_funcsigs(outbuf, 'rest')
94 except Exception as e:
95 err = "Can't parse command descriptions: {}".format(e)
97 raise EnvironmentError(err)
100 app.ceph_cluster = cluster or 'ceph'
102 app.ceph_sigdict = {}
103 app.ceph_baseurl = ''
106 cluster = cluster or 'ceph'
107 clientid = clientid or DEFAULT_ID
108 clientname = clientname or 'client.' + clientid
110 app.ceph_cluster = rados.Rados(name=clientname, conffile=conf)
111 app.ceph_cluster.conf_parse_argv(args)
112 app.ceph_cluster.connect()
114 app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \
116 if app.ceph_baseurl.endswith('/'):
117 app.ceph_baseurl = app.ceph_baseurl[:-1]
118 addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR
124 # remove the type prefix from the conf value if any
125 for t in ('legacy:', 'msgr2:'):
126 if addr.startswith(t):
129 # remove any nonce from the conf value
130 addr = addr.split('/')[0]
131 addr, port = addr.rsplit(':', 1)
132 addr = addr or DEFAULT_ADDR
133 port = port or DEFAULT_PORT
136 loglevel = app.ceph_cluster.conf_get('restapi_log_level') \
138 # ceph has a default log file for daemons only; clients (like this)
139 # default to "". Override that for this particular client.
140 logfile = app.ceph_cluster.conf_get('log_file')
142 logfile = os.path.join(
144 '{cluster}-{clientname}.{pid}.log'.format(
146 clientname=clientname,
150 app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile))
151 app.logger.setLevel(LOGLEVELS[loglevel.lower()])
152 for h in app.logger.handlers:
153 h.setFormatter(logging.Formatter(
154 '%(asctime)s %(name)s %(levelname)s: %(message)s'))
156 app.ceph_sigdict = get_command_descriptions(app.ceph_cluster)
158 osdid = find_up_osd(app)
159 if osdid is not None:
160 osd_sigdict = get_command_descriptions(app.ceph_cluster,
161 target=('osd', int(osdid)))
163 # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict
164 maxkey = sorted(app.ceph_sigdict.keys())[-1]
165 maxkey = int(maxkey.replace('cmd', ''))
167 for k, v in osd_sigdict.iteritems():
169 newv['flavor'] = 'tell'
170 globk = 'cmd' + str(osdkey)
171 app.ceph_sigdict[globk] = newv
174 # app.ceph_sigdict maps "cmdNNN" to a dict containing:
175 # 'sig', an array of argdescs
176 # 'help', the helptext
177 # 'module', the Ceph module this command relates to
178 # 'perm', a 'rwx*' string representing required permissions, and also
179 # a hint as to whether this is a GET or POST/PUT operation
180 # 'avail', a comma-separated list of strings of consumers that should
181 # display this command (filtered by parse_json_funcsigs() above)
183 for cmdnum, cmddict in app.ceph_sigdict.iteritems():
184 cmdsig = cmddict['sig']
185 flavor = cmddict.get('flavor', 'mon')
186 url, params = generate_url_and_params(app, cmdsig, flavor)
187 perm = cmddict['perm']
188 for k in METHOD_DICT.iterkeys():
190 methods = METHOD_DICT[k]
191 urldict = {'paramsig': params,
192 'help': cmddict['help'],
193 'module': cmddict['module'],
196 'methods': methods, }
198 # app.ceph_urls contains a list of urldicts (usually only one long)
199 if url not in app.ceph_urls:
200 app.ceph_urls[url] = [urldict]
202 # If more than one, need to make union of methods of all.
203 # Method must be checked in handler
204 methodset = set(methods)
205 for old_urldict in app.ceph_urls[url]:
206 methodset |= set(old_urldict['methods'])
207 methods = list(methodset)
208 app.ceph_urls[url].append(urldict)
210 # add, or re-add, rule with all methods and urldicts
211 app.add_url_rule(url, url, handler, methods=methods)
213 app.add_url_rule(url, url, handler, methods=methods)
215 app.logger.debug("urls added: %d", len(app.ceph_urls))
217 app.add_url_rule('/<path:catchall_path>', '/<path:catchall_path>',
218 handler, methods=['GET', 'PUT'])
222 def generate_url_and_params(app, sig, flavor):
224 Digest command signature from cluster; generate an absolute
225 (including app.ceph_baseurl) endpoint from all the prefix words,
226 and a list of non-prefix param descs
231 # the OSD command descriptors don't include the 'tell <target>', so
232 # tack it onto the front of sig
234 tellsig = parse_funcsig(['tell',
235 {'name': 'target', 'type': 'CephOsdName'}])
239 # prefixes go in the URL path
240 if desc.t == CephPrefix:
241 url += '/' + desc.instance.prefix
243 # tell/<target> is a weird case; the URL includes what
244 # would everywhere else be a parameter
245 if flavor == 'tell' and ((desc.t, desc.name) ==
246 (CephOsdName, 'target')):
251 return app.ceph_baseurl + url, params
255 # end setup (import-time) functions, begin request-time functions
257 def concise_sig_for_uri(sig, flavor):
259 Return a generic description of how one would send a REST request for sig
265 ret = 'tell/<osdid-or-pgid>/'
267 if d.t == CephPrefix:
268 prefix.append(d.instance.prefix)
270 args.append(d.name + '=' + str(d))
271 ret += '/'.join(prefix)
273 ret += '?' + '&'.join(args)
277 def show_human_help(prefix):
279 Dump table showing commands matching prefix
281 # XXX There ought to be a better discovery mechanism than an HTML table
282 s = '<html><body><table border=1><th>Possible commands:</th><th>Method</th><th>Description</th>'
284 permmap = {'r': 'GET', 'rw': 'PUT', 'rx': 'GET', 'rwx': 'PUT'}
286 for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort):
287 concise = concise_sig(cmdsig['sig'])
288 flavor = cmdsig.get('flavor', 'mon')
290 concise = 'tell/<target>/' + concise
291 if concise.startswith(prefix):
293 wrapped_sig = textwrap.wrap(
294 concise_sig_for_uri(cmdsig['sig'], flavor), 40
296 for sigline in wrapped_sig:
297 line.append(flask.escape(sigline) + '\n')
298 line.append('</td><td>')
299 line.append(permmap[cmdsig['perm']])
300 line.append('</td><td>')
301 line.append(flask.escape(cmdsig['help']))
302 line.append('</td></tr>\n')
305 s += '</table></body></html>'
315 For every request, log it. XXX Probably overkill for production
317 app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string)
318 app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values())
323 return flask.redirect(app.ceph_baseurl)
326 def make_response(fmt, output, statusmsg, errorcode):
328 If formatted output, cobble up a response object that contains the
329 output and status wrapped in enclosing objects; if nonformatted, just
330 use output+status. Return HTTP status errorcode in any event.
336 native_output = json.loads(output or '[]')
337 response = json.dumps({"output": native_output,
338 "status": statusmsg})
340 return flask.make_response("Error decoding JSON from " +
344 # one is tempted to do this with xml.etree, but figuring out how
345 # to 'un-XML' the XML-dumped output so it can be reassembled into
346 # a piece of the tree here is beyond me right now.
347 # ET = xml.etree.ElementTree
348 # resp_elem = ET.Element('response')
349 # o = ET.SubElement(resp_elem, 'output')
351 # s = ET.SubElement(resp_elem, 'status')
353 # response = ET.tostring(resp_elem)
362 </response>'''.format(response, xml.sax.saxutils.escape(statusmsg))
364 if not 200 <= errorcode < 300:
365 response = response + '\n' + statusmsg + '\n'
367 return flask.make_response(response, errorcode)
370 def handler(catchall_path=None, fmt=None, target=None):
372 Main endpoint handler; generic for every endpoint, including catchall.
373 Handles the catchall, anything with <.fmt>, anything with embedded
374 <target>. Partial match or ?help cause the HTML-table
375 "show_human_help" output.
378 ep = catchall_path or flask.request.endpoint
379 ep = ep.replace('.<fmt>', '')
384 # demand that endpoint begin with app.ceph_baseurl
385 if not ep.startswith(app.ceph_baseurl):
386 return make_response(fmt, '', 'Page not found', 404)
388 rel_ep = ep[len(app.ceph_baseurl) + 1:]
390 # Extensions override Accept: headers override defaults
392 if 'application/json' in flask.request.accept_mimetypes.values():
394 elif 'application/xml' in flask.request.accept_mimetypes.values():
399 cmdtarget = 'mon', ''
402 # got tell/<target>; validate osdid or pgid
407 except ArgumentError:
410 pgidobj.valid(target)
411 except ArgumentError:
412 return flask.make_response("invalid osdid or pgid", 400)
416 cmdtarget = 'pg', pgid
419 cmdtarget = name.nametype, name.nameid
421 # prefix does not include tell/<target>/
422 prefix = ' '.join(rel_ep.split('/')[2:]).strip()
424 # non-target command: prefix is entire path
425 prefix = ' '.join(rel_ep.split('/')).strip()
427 # show "match as much as you gave me" help for unknown endpoints
428 if ep not in app.ceph_urls:
429 helptext = show_human_help(prefix)
431 resp = flask.make_response(helptext, 400)
432 resp.headers['Content-Type'] = 'text/html'
435 return make_response(fmt, '', 'Invalid endpoint ' + ep, 400)
439 for urldict in app.ceph_urls[ep]:
440 if flask.request.method not in urldict['methods']:
442 paramsig = urldict['paramsig']
444 # allow '?help' for any specifically-known endpoint
445 if 'help' in flask.request.args:
446 response = flask.make_response('{0}: {1}'.
448 concise_sig(paramsig),
450 response.headers['Content-Type'] = 'text/plain'
453 # if there are parameters for this endpoint, process them
456 for k, l in flask.request.args.iterlists():
462 # is this a valid set of params?
464 argdict = validate(args, paramsig)
467 except Exception as e:
471 if flask.request.args:
478 return make_response(fmt, '', exc + '\n', 400)
480 argdict['format'] = fmt or 'plain'
481 argdict['module'] = found['module']
482 argdict['perm'] = found['perm']
484 argdict['pgid'] = pgid
487 cmdtarget = ('mon', '')
489 app.logger.debug('sending command prefix %s argdict %s', prefix, argdict)
491 for _ in range(DEFAULT_TRIES):
492 ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix,
494 inbuf=flask.request.data,
496 timeout=DEFAULT_TIMEOUT)
497 if ret != -errno.EINTR:
500 return make_response(fmt, '',
501 'Timedout: {0} ({1})'.format(outs, ret), 504)
503 return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400)
505 response = make_response(fmt, outbuf, outs or 'OK', 200)
507 contenttype = 'application/' + fmt.replace('-pretty', '')
509 contenttype = 'text/plain'
510 response.headers['Content-Type'] = contenttype
515 # Main entry point from wrapper/WSGI server: call with cmdline args,
516 # get back the WSGI app entry point
518 def generate_app(conf, cluster, clientname, clientid, args):
519 addr, port = api_setup(app, conf, cluster, clientname, clientid, args)