--- /dev/null
+"""
+Types and routines used by the ceph CLI as well as the RESTful
+interface. These have to do with querying the daemons for
+command-description information, validating user command input against
+those descriptions, and submitting the command to the appropriate
+daemon.
+
+Copyright (C) 2013 Inktank Storage, Inc.
+
+LGPL2. See file COPYING.
+"""
+from __future__ import print_function
+import copy
+import errno
+import json
+import os
+import pprint
+import re
+import socket
+import stat
+import sys
+import threading
+import uuid
+
+
+FLAG_MGR = 8 # command is intended for mgr
+
+
+try:
+ basestring
+except NameError:
+ basestring = str
+
+
+class ArgumentError(Exception):
+ """
+ Something wrong with arguments
+ """
+ pass
+
+
+class ArgumentNumber(ArgumentError):
+ """
+ Wrong number of a repeated argument
+ """
+ pass
+
+
+class ArgumentFormat(ArgumentError):
+ """
+ Argument value has wrong format
+ """
+ pass
+
+
+class ArgumentValid(ArgumentError):
+ """
+ Argument value is otherwise invalid (doesn't match choices, for instance)
+ """
+ pass
+
+
+class ArgumentTooFew(ArgumentError):
+ """
+ Fewer arguments than descriptors in signature; may mean to continue
+ the search, so gets a special exception type
+ """
+
+
+class ArgumentPrefix(ArgumentError):
+ """
+ Special for mismatched prefix; less severe, don't report by default
+ """
+ pass
+
+
+class JsonFormat(Exception):
+ """
+ some syntactic or semantic issue with the JSON
+ """
+ pass
+
+
+class CephArgtype(object):
+ """
+ Base class for all Ceph argument types
+
+ Instantiating an object sets any validation parameters
+ (allowable strings, numeric ranges, etc.). The 'valid'
+ method validates a string against that initialized instance,
+ throwing ArgumentError if there's a problem.
+ """
+ def __init__(self, **kwargs):
+ """
+ set any per-instance validation parameters here
+ from kwargs (fixed string sets, integer ranges, etc)
+ """
+ pass
+
+ def valid(self, s, partial=False):
+ """
+ Run validation against given string s (generally one word);
+ partial means to accept partial string matches (begins-with).
+ If cool, set self.val to the value that should be returned
+ (a copy of the input string, or a numeric or boolean interpretation
+ thereof, for example)
+ if not, throw ArgumentError(msg-as-to-why)
+ """
+ self.val = s
+
+ def __repr__(self):
+ """
+ return string representation of description of type. Note,
+ this is not a representation of the actual value. Subclasses
+ probably also override __str__() to give a more user-friendly
+ 'name/type' description for use in command format help messages.
+ """
+ a = ''
+ if hasattr(self, 'typeargs'):
+ a = self.typeargs
+ return '{0}(\'{1}\')'.format(self.__class__.__name__, a)
+
+ def __str__(self):
+ """
+ where __repr__ (ideally) returns a string that could be used to
+ reproduce the object, __str__ returns one you'd like to see in
+ print messages. Use __str__ to format the argtype descriptor
+ as it would be useful in a command usage message.
+ """
+ return '<{0}>'.format(self.__class__.__name__)
+
+ def complete(self, s):
+ return []
+
+
+class CephInt(CephArgtype):
+ """
+ range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+
+ range: list of 1 or 2 ints, [min] or [min,max]
+ """
+ def __init__(self, range=''):
+ if range == '':
+ self.range = list()
+ else:
+ self.range = list(range.split('|'))
+ self.range = [int(x) for x in self.range]
+
+ def valid(self, s, partial=False):
+ try:
+ val = int(s)
+ except ValueError:
+ raise ArgumentValid("{0} doesn't represent an int".format(s))
+ if len(self.range) == 2:
+ if val < self.range[0] or val > self.range[1]:
+ raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+ elif len(self.range) == 1:
+ if val < self.range[0]:
+ raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+ self.val = val
+
+ def __str__(self):
+ r = ''
+ if len(self.range) == 1:
+ r = '[{0}-]'.format(self.range[0])
+ if len(self.range) == 2:
+ r = '[{0}-{1}]'.format(self.range[0], self.range[1])
+
+ return '<int{0}>'.format(r)
+
+
+class CephFloat(CephArgtype):
+ """
+ range-limited float type
+ range: list of 1 or 2 floats, [min] or [min, max]
+ """
+ def __init__(self, range=''):
+ if range == '':
+ self.range = list()
+ else:
+ self.range = list(range.split('|'))
+ self.range = [float(x) for x in self.range]
+
+ def valid(self, s, partial=False):
+ try:
+ val = float(s)
+ except ValueError:
+ raise ArgumentValid("{0} doesn't represent a float".format(s))
+ if len(self.range) == 2:
+ if val < self.range[0] or val > self.range[1]:
+ raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+ elif len(self.range) == 1:
+ if val < self.range[0]:
+ raise ArgumentValid("{0} not in range {1}".format(val, self.range))
+ self.val = val
+
+ def __str__(self):
+ r = ''
+ if len(self.range) == 1:
+ r = '[{0}-]'.format(self.range[0])
+ if len(self.range) == 2:
+ r = '[{0}-{1}]'.format(self.range[0], self.range[1])
+ return '<float{0}>'.format(r)
+
+
+class CephString(CephArgtype):
+ """
+ String; pretty generic. goodchars is a RE char class of valid chars
+ """
+ def __init__(self, goodchars=''):
+ from string import printable
+ try:
+ re.compile(goodchars)
+ except:
+ raise ValueError('CephString(): "{0}" is not a valid RE'.
+ format(goodchars))
+ self.goodchars = goodchars
+ self.goodset = frozenset(
+ [c for c in printable if re.match(goodchars, c)]
+ )
+
+ def valid(self, s, partial=False):
+ sset = set(s)
+ if self.goodset and not sset <= self.goodset:
+ raise ArgumentFormat("invalid chars {0} in {1}".
+ format(''.join(sset - self.goodset), s))
+ self.val = s
+
+ def __str__(self):
+ b = ''
+ if self.goodchars:
+ b += '(goodchars {0})'.format(self.goodchars)
+ return '<string{0}>'.format(b)
+
+ def complete(self, s):
+ if s == '':
+ return []
+ else:
+ return [s]
+
+
+class CephSocketpath(CephArgtype):
+ """
+ Admin socket path; check that it's readable and S_ISSOCK
+ """
+ def valid(self, s, partial=False):
+ mode = os.stat(s).st_mode
+ if not stat.S_ISSOCK(mode):
+ raise ArgumentValid('socket path {0} is not a socket'.format(s))
+ self.val = s
+
+ def __str__(self):
+ return '<admin-socket-path>'
+
+
+class CephIPAddr(CephArgtype):
+ """
+ IP address (v4 or v6) with optional port
+ """
+ def valid(self, s, partial=False):
+ # parse off port, use socket to validate addr
+ type = 6
+ if s.startswith('['):
+ type = 6
+ elif s.find('.') != -1:
+ type = 4
+ if type == 4:
+ port = s.find(':')
+ if port != -1:
+ a = s[:port]
+ p = s[port + 1:]
+ if int(p) > 65535:
+ raise ArgumentValid('{0}: invalid IPv4 port'.format(p))
+ else:
+ a = s
+ p = None
+ try:
+ socket.inet_pton(socket.AF_INET, a)
+ except:
+ raise ArgumentValid('{0}: invalid IPv4 address'.format(a))
+ else:
+ # v6
+ if s.startswith('['):
+ end = s.find(']')
+ if end == -1:
+ raise ArgumentFormat('{0} missing terminating ]'.format(s))
+ if s[end + 1] == ':':
+ try:
+ p = int(s[end + 2])
+ except:
+ raise ArgumentValid('{0}: bad port number'.format(s))
+ a = s[1:end]
+ else:
+ a = s
+ p = None
+ try:
+ socket.inet_pton(socket.AF_INET6, a)
+ except:
+ raise ArgumentValid('{0} not valid IPv6 address'.format(s))
+ if p is not None and int(p) > 65535:
+ raise ArgumentValid("{0} not a valid port number".format(p))
+ self.val = s
+ self.addr = a
+ self.port = p
+
+ def __str__(self):
+ return '<IPaddr[:port]>'
+
+
+class CephEntityAddr(CephIPAddr):
+ """
+ EntityAddress, that is, IP address[/nonce]
+ """
+ def valid(self, s, partial=False):
+ nonce = None
+ if '/' in s:
+ ip, nonce = s.split('/')
+ else:
+ ip = s
+ super(self.__class__, self).valid(ip)
+ if nonce:
+ nonce_int = None
+ try:
+ nonce_int = int(nonce)
+ except ValueError:
+ pass
+ if nonce_int is None or nonce_int < 0:
+ raise ArgumentValid(
+ '{0}: invalid entity, nonce {1} not integer > 0'.
+ format(s, nonce)
+ )
+ self.val = s
+
+ def __str__(self):
+ return '<EntityAddr>'
+
+
+class CephPoolname(CephArgtype):
+ """
+ Pool name; very little utility
+ """
+ def __str__(self):
+ return '<poolname>'
+
+
+class CephObjectname(CephArgtype):
+ """
+ Object name. Maybe should be combined with Pool name as they're always
+ present in pairs, and then could be checked for presence
+ """
+ def __str__(self):
+ return '<objectname>'
+
+
+class CephPgid(CephArgtype):
+ """
+ pgid, in form N.xxx (N = pool number, xxx = hex pgnum)
+ """
+ def valid(self, s, partial=False):
+ if s.find('.') == -1:
+ raise ArgumentFormat('pgid has no .')
+ poolid, pgnum = s.split('.', 1)
+ try:
+ poolid = int(poolid)
+ except ValueError:
+ raise ArgumentFormat('pool {0} not integer'.format(poolid))
+ if poolid < 0:
+ raise ArgumentFormat('pool {0} < 0'.format(poolid))
+ try:
+ pgnum = int(pgnum, 16)
+ except ValueError:
+ raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum))
+ self.val = s
+
+ def __str__(self):
+ return '<pgid>'
+
+
+class CephName(CephArgtype):
+ """
+ Name (type.id) where:
+ type is osd|mon|client|mds
+ id is a base10 int, if type == osd, or a string otherwise
+
+ Also accept '*'
+ """
+ def __init__(self):
+ self.nametype = None
+ self.nameid = None
+
+ def valid(self, s, partial=False):
+ if s == '*':
+ self.val = s
+ return
+ elif s == "mgr":
+ self.nametype = "mgr"
+ self.val = s
+ return
+ elif s == "mon":
+ self.nametype = "mon"
+ self.val = s
+ return
+ if s.find('.') == -1:
+ raise ArgumentFormat('CephName: no . in {0}'.format(s))
+ else:
+ t, i = s.split('.', 1)
+ if t not in ('osd', 'mon', 'client', 'mds', 'mgr'):
+ raise ArgumentValid('unknown type ' + t)
+ if t == 'osd':
+ if i != '*':
+ try:
+ i = int(i)
+ except:
+ raise ArgumentFormat('osd id ' + i + ' not integer')
+ self.nametype = t
+ self.val = s
+ self.nameid = i
+
+ def __str__(self):
+ return '<name (type.id)>'
+
+
+class CephOsdName(CephArgtype):
+ """
+ Like CephName, but specific to osds: allow <id> alone
+
+ osd.<id>, or <id>, or *, where id is a base10 int
+ """
+ def __init__(self):
+ self.nametype = None
+ self.nameid = None
+
+ def valid(self, s, partial=False):
+ if s == '*':
+ self.val = s
+ return
+ if s.find('.') != -1:
+ t, i = s.split('.', 1)
+ if t != 'osd':
+ raise ArgumentValid('unknown type ' + t)
+ else:
+ t = 'osd'
+ i = s
+ try:
+ i = int(i)
+ except:
+ raise ArgumentFormat('osd id ' + i + ' not integer')
+ if i < 0:
+ raise ArgumentFormat('osd id {0} is less than 0'.format(i))
+ self.nametype = t
+ self.nameid = i
+ self.val = i
+
+ def __str__(self):
+ return '<osdname (id|osd.id)>'
+
+
+class CephChoices(CephArgtype):
+ """
+ Set of string literals; init with valid choices
+ """
+ def __init__(self, strings='', **kwargs):
+ self.strings = strings.split('|')
+
+ def valid(self, s, partial=False):
+ if not partial:
+ if s not in self.strings:
+ # show as __str__ does: {s1|s2..}
+ raise ArgumentValid("{0} not in {1}".format(s, self))
+ self.val = s
+ return
+
+ # partial
+ for t in self.strings:
+ if t.startswith(s):
+ self.val = s
+ return
+ raise ArgumentValid("{0} not in {1}". format(s, self))
+
+ def __str__(self):
+ if len(self.strings) == 1:
+ return '{0}'.format(self.strings[0])
+ else:
+ return '{0}'.format('|'.join(self.strings))
+
+ def complete(self, s):
+ all_elems = [token for token in self.strings if token.startswith(s)]
+ return all_elems
+
+
+class CephFilepath(CephArgtype):
+ """
+ Openable file
+ """
+ def valid(self, s, partial=False):
+ try:
+ f = open(s, 'a+')
+ except Exception as e:
+ raise ArgumentValid('can\'t open {0}: {1}'.format(s, e))
+ f.close()
+ self.val = s
+
+ def __str__(self):
+ return '<outfilename>'
+
+
+class CephFragment(CephArgtype):
+ """
+ 'Fragment' ??? XXX
+ """
+ def valid(self, s, partial=False):
+ if s.find('/') == -1:
+ raise ArgumentFormat('{0}: no /'.format(s))
+ val, bits = s.split('/')
+ # XXX is this right?
+ if not val.startswith('0x'):
+ raise ArgumentFormat("{0} not a hex integer".format(val))
+ try:
+ int(val)
+ except:
+ raise ArgumentFormat('can\'t convert {0} to integer'.format(val))
+ try:
+ int(bits)
+ except:
+ raise ArgumentFormat('can\'t convert {0} to integer'.format(bits))
+ self.val = s
+
+ def __str__(self):
+ return "<CephFS fragment ID (0xvvv/bbb)>"
+
+
+class CephUUID(CephArgtype):
+ """
+ CephUUID: pretty self-explanatory
+ """
+ def valid(self, s, partial=False):
+ try:
+ uuid.UUID(s)
+ except Exception as e:
+ raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e))
+ self.val = s
+
+ def __str__(self):
+ return '<uuid>'
+
+
+class CephPrefix(CephArgtype):
+ """
+ CephPrefix: magic type for "all the first n fixed strings"
+ """
+ def __init__(self, prefix=''):
+ self.prefix = prefix
+
+ def valid(self, s, partial=False):
+ try:
+ s = str(s)
+ if isinstance(s, bytes):
+ # `prefix` can always be converted into unicode when being compared,
+ # but `s` could be anything passed by user.
+ s = s.decode('ascii')
+ except UnicodeEncodeError:
+ raise ArgumentPrefix(u"no match for {0}".format(s))
+ except UnicodeDecodeError:
+ raise ArgumentPrefix("no match for {0}".format(s))
+
+ if partial:
+ if self.prefix.startswith(s):
+ self.val = s
+ return
+ else:
+ if s == self.prefix:
+ self.val = s
+ return
+
+ raise ArgumentPrefix("no match for {0}".format(s))
+
+ def __str__(self):
+ return self.prefix
+
+ def complete(self, s):
+ if self.prefix.startswith(s):
+ return [self.prefix.rstrip(' ')]
+ else:
+ return []
+
+
+class argdesc(object):
+ """
+ argdesc(typename, name='name', n=numallowed|N,
+ req=False, helptext=helptext, **kwargs (type-specific))
+
+ validation rules:
+ typename: type(**kwargs) will be constructed
+ later, type.valid(w) will be called with a word in that position
+
+ name is used for parse errors and for constructing JSON output
+ n is a numeric literal or 'n|N', meaning "at least one, but maybe more"
+ req=False means the argument need not be present in the list
+ helptext is the associated help for the command
+ anything else are arguments to pass to the type constructor.
+
+ self.instance is an instance of type t constructed with typeargs.
+
+ valid() will later be called with input to validate against it,
+ and will store the validated value in self.instance.val for extraction.
+ """
+ def __init__(self, t, name=None, n=1, req=True, **kwargs):
+ if isinstance(t, basestring):
+ self.t = CephPrefix
+ self.typeargs = {'prefix': t}
+ self.req = True
+ else:
+ self.t = t
+ self.typeargs = kwargs
+ self.req = bool(req == True or req == 'True')
+
+ self.name = name
+ self.N = (n in ['n', 'N'])
+ if self.N:
+ self.n = 1
+ else:
+ self.n = int(n)
+ self.instance = self.t(**self.typeargs)
+
+ def __repr__(self):
+ r = 'argdesc(' + str(self.t) + ', '
+ internals = ['N', 'typeargs', 'instance', 't']
+ for (k, v) in self.__dict__.items():
+ if k.startswith('__') or k in internals:
+ pass
+ else:
+ # undo modification from __init__
+ if k == 'n' and self.N:
+ v = 'N'
+ r += '{0}={1}, '.format(k, v)
+ for (k, v) in self.typeargs.items():
+ r += '{0}={1}, '.format(k, v)
+ return r[:-2] + ')'
+
+ def __str__(self):
+ if ((self.t == CephChoices and len(self.instance.strings) == 1)
+ or (self.t == CephPrefix)):
+ s = str(self.instance)
+ else:
+ s = '{0}({1})'.format(self.name, str(self.instance))
+ if self.N:
+ s += ' [' + str(self.instance) + '...]'
+ if not self.req:
+ s = '{' + s + '}'
+ return s
+
+ def helpstr(self):
+ """
+ like str(), but omit parameter names (except for CephString,
+ which really needs them)
+ """
+ if self.t == CephString:
+ chunk = '<{0}>'.format(self.name)
+ else:
+ chunk = str(self.instance)
+ s = chunk
+ if self.N:
+ s += ' [' + chunk + '...]'
+ if not self.req:
+ s = '{' + s + '}'
+ return s
+
+ def complete(self, s):
+ return self.instance.complete(s)
+
+
+def concise_sig(sig):
+ """
+ Return string representation of sig useful for syntax reference in help
+ """
+ return ' '.join([d.helpstr() for d in sig])
+
+
+def descsort_key(sh):
+ """
+ sort descriptors by prefixes, defined as the concatenation of all simple
+ strings in the descriptor; this works out to just the leading strings.
+ """
+ return concise_sig(sh['sig'])
+
+
+def descsort(sh1, sh2):
+ """
+ Deprecated; use (key=descsort_key) instead of (cmp=descsort)
+ """
+ return cmp(descsort_key(sh1), descsort_key(sh2))
+
+
+def parse_funcsig(sig):
+ """
+ parse a single descriptor (array of strings or dicts) into a
+ dict of function descriptor/validators (objects of CephXXX type)
+ """
+ newsig = []
+ argnum = 0
+ for desc in sig:
+ argnum += 1
+ if isinstance(desc, basestring):
+ t = CephPrefix
+ desc = {'type': t, 'name': 'prefix', 'prefix': desc}
+ else:
+ # not a simple string, must be dict
+ if 'type' not in desc:
+ s = 'JSON descriptor {0} has no type'.format(sig)
+ raise JsonFormat(s)
+ # look up type string in our globals() dict; if it's an
+ # object of type `type`, it must be a
+ # locally-defined class. otherwise, we haven't a clue.
+ if desc['type'] in globals():
+ t = globals()[desc['type']]
+ if not isinstance(t, type):
+ s = 'unknown type {0}'.format(desc['type'])
+ raise JsonFormat(s)
+ else:
+ s = 'unknown type {0}'.format(desc['type'])
+ raise JsonFormat(s)
+
+ kwargs = dict()
+ for key, val in desc.items():
+ if key not in ['type', 'name', 'n', 'req']:
+ kwargs[key] = val
+ newsig.append(argdesc(t,
+ name=desc.get('name', None),
+ n=desc.get('n', 1),
+ req=desc.get('req', True),
+ **kwargs))
+ return newsig
+
+
+def parse_json_funcsigs(s, consumer):
+ """
+ A function signature is mostly an array of argdesc; it's represented
+ in JSON as
+ {
+ "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false <other param>], "help":helptext, "module":modulename, "perm":perms, "avail":availability}
+ .
+ .
+ .
+ ]
+
+ A set of sigs is in an dict mapped by a unique number:
+ {
+ "cmd1": {
+ "sig": ["type.. ], "help":helptext...
+ }
+ "cmd2"{
+ "sig": [.. ], "help":helptext...
+ }
+ }
+
+ Parse the string s and return a dict of dicts, keyed by opcode;
+ each dict contains 'sig' with the array of descriptors, and 'help'
+ with the helptext, 'module' with the module name, 'perm' with a
+ string representing required permissions in that module to execute
+ this command (and also whether it is a read or write command from
+ the cluster state perspective), and 'avail' as a hint for
+ whether the command should be advertised by CLI, REST, or both.
+ If avail does not contain 'consumer', don't include the command
+ in the returned dict.
+ """
+ try:
+ overall = json.loads(s)
+ except Exception as e:
+ print("Couldn't parse JSON {0}: {1}".format(s, e), file=sys.stderr)
+ raise e
+ sigdict = {}
+ for cmdtag, cmd in overall.items():
+ if 'sig' not in cmd:
+ s = "JSON descriptor {0} has no 'sig'".format(cmdtag)
+ raise JsonFormat(s)
+ # check 'avail' and possibly ignore this command
+ if 'avail' in cmd:
+ if consumer not in cmd['avail']:
+ continue
+ # rewrite the 'sig' item with the argdesc-ized version, and...
+ cmd['sig'] = parse_funcsig(cmd['sig'])
+ # just take everything else as given
+ sigdict[cmdtag] = cmd
+ return sigdict
+
+
+def validate_one(word, desc, partial=False):
+ """
+ validate_one(word, desc, partial=False)
+
+ validate word against the constructed instance of the type
+ in desc. May raise exception. If it returns false (and doesn't
+ raise an exception), desc.instance.val will
+ contain the validated value (in the appropriate type).
+ """
+ desc.instance.valid(word, partial)
+ desc.numseen += 1
+ if desc.N:
+ desc.n = desc.numseen + 1
+
+
+def matchnum(args, signature, partial=False):
+ """
+ matchnum(s, signature, partial=False)
+
+ Returns number of arguments matched in s against signature.
+ Can be used to determine most-likely command for full or partial
+ matches (partial applies to string matches).
+ """
+ words = args[:]
+ mysig = copy.deepcopy(signature)
+ matchcnt = 0
+ for desc in mysig:
+ setattr(desc, 'numseen', 0)
+ while desc.numseen < desc.n:
+ # if there are no more arguments, return
+ if not words:
+ return matchcnt
+ word = words.pop(0)
+
+ try:
+ # only allow partial matching if we're on the last supplied
+ # word; avoid matching foo bar and foot bar just because
+ # partial is set
+ validate_one(word, desc, partial and (len(words) == 0))
+ valid = True
+ except ArgumentError:
+ # matchnum doesn't care about type of error
+ valid = False
+
+ if not valid:
+ if not desc.req:
+ # this wasn't required, so word may match the next desc
+ words.insert(0, word)
+ break
+ else:
+ # it was required, and didn't match, return
+ return matchcnt
+ if desc.req:
+ matchcnt += 1
+ return matchcnt
+
+
+def get_next_arg(desc, args):
+ '''
+ Get either the value matching key 'desc.name' or the next arg in
+ the non-dict list. Return None if args are exhausted. Used in
+ validate() below.
+ '''
+ arg = None
+ if isinstance(args, dict):
+ arg = args.pop(desc.name, None)
+ # allow 'param=param' to be expressed as 'param'
+ if arg == '':
+ arg = desc.name
+ # Hack, or clever? If value is a list, keep the first element,
+ # push rest back onto myargs for later processing.
+ # Could process list directly, but nesting here is already bad
+ if arg and isinstance(arg, list):
+ args[desc.name] = arg[1:]
+ arg = arg[0]
+ elif args:
+ arg = args.pop(0)
+ if arg and isinstance(arg, list):
+ args = arg[1:] + args
+ arg = arg[0]
+ return arg
+
+
+def store_arg(desc, d):
+ '''
+ Store argument described by, and held in, thanks to valid(),
+ desc into the dictionary d, keyed by desc.name. Three cases:
+
+ 1) desc.N is set: value in d is a list
+ 2) prefix: multiple args are joined with ' ' into one d{} item
+ 3) single prefix or other arg: store as simple value
+
+ Used in validate() below.
+ '''
+ if desc.N:
+ # value should be a list
+ if desc.name in d:
+ d[desc.name] += [desc.instance.val]
+ else:
+ d[desc.name] = [desc.instance.val]
+ elif (desc.t == CephPrefix) and (desc.name in d):
+ # prefixes' values should be a space-joined concatenation
+ d[desc.name] += ' ' + desc.instance.val
+ else:
+ # if first CephPrefix or any other type, just set it
+ d[desc.name] = desc.instance.val
+
+
+def validate(args, signature, flags=0, partial=False):
+ """
+ validate(args, signature, flags=0, partial=False)
+
+ args is a list of either words or k,v pairs representing a possible
+ command input following format of signature. Runs a validation; no
+ exception means it's OK. Return a dict containing all arguments keyed
+ by their descriptor name, with duplicate args per name accumulated
+ into a list (or space-separated value for CephPrefix).
+
+ Mismatches of prefix are non-fatal, as this probably just means the
+ search hasn't hit the correct command. Mismatches of non-prefix
+ arguments are treated as fatal, and an exception raised.
+
+ This matching is modified if partial is set: allow partial matching
+ (with partial dict returned); in this case, there are no exceptions
+ raised.
+ """
+
+ myargs = copy.deepcopy(args)
+ mysig = copy.deepcopy(signature)
+ reqsiglen = len([desc for desc in mysig if desc.req])
+ matchcnt = 0
+ d = dict()
+ save_exception = None
+
+ for desc in mysig:
+ setattr(desc, 'numseen', 0)
+ while desc.numseen < desc.n:
+ myarg = get_next_arg(desc, myargs)
+
+ # no arg, but not required? Continue consuming mysig
+ # in case there are later required args
+ if myarg in (None, []) and not desc.req:
+ break
+
+ # out of arguments for a required param?
+ # Either return (if partial validation) or raise
+ if myarg in (None, []) and desc.req:
+ if desc.N and desc.numseen < 1:
+ # wanted N, didn't even get 1
+ if partial:
+ return d
+ raise ArgumentNumber(
+ 'saw {0} of {1}, expected at least 1'.
+ format(desc.numseen, desc)
+ )
+ elif not desc.N and desc.numseen < desc.n:
+ # wanted n, got too few
+ if partial:
+ return d
+ # special-case the "0 expected 1" case
+ if desc.numseen == 0 and desc.n == 1:
+ raise ArgumentNumber(
+ 'missing required parameter {0}'.format(desc)
+ )
+ raise ArgumentNumber(
+ 'saw {0} of {1}, expected {2}'.
+ format(desc.numseen, desc, desc.n)
+ )
+ break
+
+ # Have an arg; validate it
+ try:
+ validate_one(myarg, desc)
+ valid = True
+ except ArgumentError as e:
+ valid = False
+ exc = e
+ if not valid:
+ # argument mismatch
+ if not desc.req:
+ # if not required, just push back; it might match
+ # the next arg
+ save_exception = [ myarg, exc ]
+ myargs.insert(0, myarg)
+ break
+ else:
+ # hm, it was required, so time to return/raise
+ if partial:
+ return d
+ raise exc
+
+ # Whew, valid arg acquired. Store in dict
+ matchcnt += 1
+ store_arg(desc, d)
+ # Clear prior exception
+ save_exception = None
+
+ # Done with entire list of argdescs
+ if matchcnt < reqsiglen:
+ raise ArgumentTooFew("not enough arguments given")
+
+ if myargs and not partial:
+ if save_exception:
+ print(save_exception[0], 'not valid: ', save_exception[1], file=sys.stderr)
+ raise ArgumentError("unused arguments: " + str(myargs))
+
+ if flags & FLAG_MGR:
+ d['target'] = ('mgr','')
+
+ # Finally, success
+ return d
+
+
+def cmdsiglen(sig):
+ sigdict = sig.values()
+ assert len(sigdict) == 1
+ some_value = next(iter(sig.values()))
+ return len(some_value['sig'])
+
+
+def validate_command(sigdict, args, verbose=False):
+ """
+ turn args into a valid dictionary ready to be sent off as JSON,
+ validated against sigdict.
+ """
+ if verbose:
+ print("validate_command: " + " ".join(args), file=sys.stderr)
+ found = []
+ valid_dict = {}
+ if args:
+ # look for best match, accumulate possibles in bestcmds
+ # (so we can maybe give a more-useful error message)
+ best_match_cnt = 0
+ bestcmds = []
+ for cmdtag, cmd in sigdict.items():
+ sig = cmd['sig']
+ matched = matchnum(args, sig, partial=True)
+ if matched > best_match_cnt:
+ if verbose:
+ print("better match: {0} > {1}: {2}:{3} ".format(
+ matched, best_match_cnt, cmdtag, concise_sig(sig)
+ ), file=sys.stderr)
+ best_match_cnt = matched
+ bestcmds = [{cmdtag: cmd}]
+ elif matched == best_match_cnt:
+ if verbose:
+ print("equal match: {0} > {1}: {2}:{3} ".format(
+ matched, best_match_cnt, cmdtag, concise_sig(sig)
+ ), file=sys.stderr)
+ bestcmds.append({cmdtag: cmd})
+
+ # Sort bestcmds by number of args so we can try shortest first
+ # (relies on a cmdsig being key,val where val is a list of len 1)
+ bestcmds_sorted = sorted(bestcmds, key=cmdsiglen)
+
+ if verbose:
+ print("bestcmds_sorted: ", file=sys.stderr)
+ pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted)
+
+ # for everything in bestcmds, look for a true match
+ for cmdsig in bestcmds_sorted:
+ for cmd in cmdsig.values():
+ sig = cmd['sig']
+ try:
+ valid_dict = validate(args, sig, flags=cmd.get('flags', 0))
+ found = cmd
+ break
+ except ArgumentPrefix:
+ # ignore prefix mismatches; we just haven't found
+ # the right command yet
+ pass
+ except ArgumentTooFew:
+ # It looked like this matched the beginning, but it
+ # didn't have enough args supplied. If we're out of
+ # cmdsigs we'll fall out unfound; if we're not, maybe
+ # the next one matches completely. Whine, but pass.
+ if verbose:
+ print('Not enough args supplied for ',
+ concise_sig(sig), file=sys.stderr)
+ except ArgumentError as e:
+ # Solid mismatch on an arg (type, range, etc.)
+ # Stop now, because we have the right command but
+ # some other input is invalid
+ print("Invalid command: ", e, file=sys.stderr)
+ print(concise_sig(sig), ': ', cmd['help'], file=sys.stderr)
+ return {}
+ if found:
+ break
+
+ if not found:
+ print('no valid command found; 10 closest matches:', file=sys.stderr)
+ for cmdsig in bestcmds[:10]:
+ for (cmdtag, cmd) in cmdsig.items():
+ print(concise_sig(cmd['sig']), file=sys.stderr)
+ return None
+
+ return valid_dict
+
+
+def find_cmd_target(childargs):
+ """
+ Using a minimal validation, figure out whether the command
+ should be sent to a monitor or an osd. We do this before even
+ asking for the 'real' set of command signatures, so we can ask the
+ right daemon.
+ Returns ('osd', osdid), ('pg', pgid), ('mgr', '') or ('mon', '')
+ """
+ sig = parse_funcsig(['tell', {'name': 'target', 'type': 'CephName'}])
+ try:
+ valid_dict = validate(childargs, sig, partial=True)
+ except ArgumentError:
+ pass
+ else:
+ if len(valid_dict) == 2:
+ # revalidate to isolate type and id
+ name = CephName()
+ # if this fails, something is horribly wrong, as it just
+ # validated successfully above
+ name.valid(valid_dict['target'])
+ return name.nametype, name.nameid
+
+ sig = parse_funcsig(['tell', {'name': 'pgid', 'type': 'CephPgid'}])
+ try:
+ valid_dict = validate(childargs, sig, partial=True)
+ except ArgumentError:
+ pass
+ else:
+ if len(valid_dict) == 2:
+ # pg doesn't need revalidation; the string is fine
+ return 'pg', valid_dict['pgid']
+
+ # If we reached this far it must mean that so far we've been unable to
+ # obtain a proper target from childargs. This may mean that we are not
+ # dealing with a 'tell' command, or that the specified target is invalid.
+ # If the latter, we likely were unable to catch it because we were not
+ # really looking for it: first we tried to parse a 'CephName' (osd, mon,
+ # mds, followed by and id); given our failure to parse, we tried to parse
+ # a 'CephPgid' instead (e.g., 0.4a). Considering we got this far though
+ # we were unable to do so.
+ #
+ # We will now check if this is a tell and, if so, forcefully validate the
+ # target as a 'CephName'. This must be so because otherwise we will end
+ # up sending garbage to a monitor, which is the default target when a
+ # target is not explicitly specified.
+ # e.g.,
+ # 'ceph status' -> target is any one monitor
+ # 'ceph tell mon.* status -> target is all monitors
+ # 'ceph tell foo status -> target is invalid!
+ if len(childargs) > 1 and childargs[0] == 'tell':
+ name = CephName()
+ # CephName.valid() raises on validation error; find_cmd_target()'s
+ # caller should handle them
+ name.valid(childargs[1])
+ return name.nametype, name.nameid
+
+ sig = parse_funcsig(['pg', {'name': 'pgid', 'type': 'CephPgid'}])
+ try:
+ valid_dict = validate(childargs, sig, partial=True)
+ except ArgumentError:
+ pass
+ else:
+ if len(valid_dict) == 2:
+ return 'pg', valid_dict['pgid']
+
+ return 'mon', ''
+
+
+class RadosThread(threading.Thread):
+ def __init__(self, target, *args, **kwargs):
+ self.args = args
+ self.kwargs = kwargs
+ self.target = target
+ self.exception = None
+ threading.Thread.__init__(self)
+
+ def run(self):
+ try:
+ self.retval = self.target(*self.args, **self.kwargs)
+ except Exception as e:
+ self.exception = e
+
+
+# time in seconds between each call to t.join() for child thread
+POLL_TIME_INCR = 0.5
+
+
+def run_in_thread(target, *args, **kwargs):
+ interrupt = False
+ timeout = kwargs.pop('timeout', 0)
+ countdown = timeout
+ t = RadosThread(target, *args, **kwargs)
+
+ # allow the main thread to exit (presumably, avoid a join() on this
+ # subthread) before this thread terminates. This allows SIGINT
+ # exit of a blocked call. See below.
+ t.daemon = True
+
+ t.start()
+ try:
+ # poll for thread exit
+ while t.is_alive():
+ t.join(POLL_TIME_INCR)
+ if timeout and t.is_alive():
+ countdown = countdown - POLL_TIME_INCR
+ if countdown <= 0:
+ raise KeyboardInterrupt
+
+ t.join() # in case t exits before reaching the join() above
+ except KeyboardInterrupt:
+ # ..but allow SIGINT to terminate the waiting. Note: this
+ # relies on the Linux kernel behavior of delivering the signal
+ # to the main thread in preference to any subthread (all that's
+ # strictly guaranteed is that *some* thread that has the signal
+ # unblocked will receive it). But there doesn't seem to be
+ # any interface to create t with SIGINT blocked.
+ interrupt = True
+
+ if interrupt:
+ t.retval = -errno.EINTR, None, 'Interrupted!'
+ if t.exception:
+ raise t.exception
+ return t.retval
+
+
+def send_command_retry(*args, **kwargs):
+ while True:
+ try:
+ return send_command(*args, **kwargs)
+ except Exception as e:
+ if ('get_command_descriptions' in str(e) and
+ 'object in state configuring' in str(e)):
+ continue
+ else:
+ raise
+
+def send_command(cluster, target=('mon', ''), cmd=None, inbuf=b'', timeout=0,
+ verbose=False):
+ """
+ Send a command to a daemon using librados's
+ mon_command, osd_command, or pg_command. Any bulk input data
+ comes in inbuf.
+
+ Returns (ret, outbuf, outs); ret is the return code, outbuf is
+ the outbl "bulk useful output" buffer, and outs is any status
+ or error message (intended for stderr).
+
+ If target is osd.N, send command to that osd (except for pgid cmds)
+ """
+ cmd = cmd or []
+ try:
+ if target[0] == 'osd':
+ osdid = target[1]
+
+ if verbose:
+ print('submit {0} to osd.{1}'.format(cmd, osdid),
+ file=sys.stderr)
+ ret, outbuf, outs = run_in_thread(
+ cluster.osd_command, osdid, cmd, inbuf, timeout)
+
+ elif target[0] == 'mgr':
+ ret, outbuf, outs = run_in_thread(
+ cluster.mgr_command, cmd, inbuf, timeout)
+
+ elif target[0] == 'pg':
+ pgid = target[1]
+ # pgid will already be in the command for the pg <pgid>
+ # form, but for tell <pgid>, we need to put it in
+ if cmd:
+ cmddict = json.loads(cmd[0])
+ cmddict['pgid'] = pgid
+ else:
+ cmddict = dict(pgid=pgid)
+ cmd = [json.dumps(cmddict)]
+ if verbose:
+ print('submit {0} for pgid {1}'.format(cmd, pgid),
+ file=sys.stderr)
+ ret, outbuf, outs = run_in_thread(
+ cluster.pg_command, pgid, cmd, inbuf, timeout)
+
+ elif target[0] == 'mon':
+ if verbose:
+ print('{0} to {1}'.format(cmd, target[0]),
+ file=sys.stderr)
+ if len(target) < 2 or target[1] == '':
+ ret, outbuf, outs = run_in_thread(
+ cluster.mon_command, cmd, inbuf, timeout)
+ else:
+ ret, outbuf, outs = run_in_thread(
+ cluster.mon_command, cmd, inbuf, timeout, target[1])
+ elif target[0] == 'mds':
+ mds_spec = target[1]
+
+ if verbose:
+ print('submit {0} to mds.{1}'.format(cmd, mds_spec),
+ file=sys.stderr)
+
+ try:
+ from cephfs import LibCephFS
+ except ImportError:
+ raise RuntimeError("CephFS unavailable, have you installed libcephfs?")
+
+ filesystem = LibCephFS(cluster.conf_defaults, cluster.conffile)
+ filesystem.conf_parse_argv(cluster.parsed_args)
+
+ filesystem.init()
+ ret, outbuf, outs = \
+ filesystem.mds_command(mds_spec, cmd, inbuf)
+ filesystem.shutdown()
+ else:
+ raise ArgumentValid("Bad target type '{0}'".format(target[0]))
+
+ except Exception as e:
+ if not isinstance(e, ArgumentError):
+ raise RuntimeError('"{0}": exception {1}'.format(cmd, e))
+ else:
+ raise
+
+ return ret, outbuf, outs
+
+
+def json_command(cluster, target=('mon', ''), prefix=None, argdict=None,
+ inbuf=b'', timeout=0, verbose=False):
+ """
+ Format up a JSON command and send it with send_command() above.
+ Prefix may be supplied separately or in argdict. Any bulk input
+ data comes in inbuf.
+
+ If target is osd.N, send command to that osd (except for pgid cmds)
+ """
+ cmddict = {}
+ if prefix:
+ cmddict.update({'prefix': prefix})
+ if argdict:
+ cmddict.update(argdict)
+ if 'target' in argdict:
+ target = argdict.get('target')
+
+ # grab prefix for error messages
+ prefix = cmddict['prefix']
+
+ try:
+ if target[0] == 'osd':
+ osdtarg = CephName()
+ osdtarget = '{0}.{1}'.format(*target)
+ # prefer target from cmddict if present and valid
+ if 'target' in cmddict:
+ osdtarget = cmddict.pop('target')
+ try:
+ osdtarg.valid(osdtarget)
+ target = ('osd', osdtarg.nameid)
+ except:
+ # use the target we were originally given
+ pass
+
+ ret, outbuf, outs = send_command_retry(cluster,
+ target, [json.dumps(cmddict)],
+ inbuf, timeout, verbose)
+
+ except Exception as e:
+ if not isinstance(e, ArgumentError):
+ raise RuntimeError('"{0}": exception {1}'.format(argdict, e))
+ else:
+ raise
+
+ return ret, outbuf, outs
Code Review / stor4nfv.git / blobdiff