--- /dev/null
+import cherrypy
+import json
+import errno
+import math
+import os
+from collections import OrderedDict
+from mgr_module import MgrModule
+
+# Defaults for the Prometheus HTTP server. Can also set in config-key
+# see https://github.com/prometheus/prometheus/wiki/Default-port-allocations
+# for Prometheus exporter port registry
+
+DEFAULT_ADDR = '::'
+DEFAULT_PORT = 9283
+
+
+# cherrypy likes to sys.exit on error. don't let it take us down too!
+def os_exit_noop(*args, **kwargs):
+ pass
+
+
+os._exit = os_exit_noop
+
+
+# to access things in class Module from subclass Root. Because
+# it's a dict, the writer doesn't need to declare 'global' for access
+
+_global_instance = {'plugin': None}
+
+
+def global_instance():
+ assert _global_instance['plugin'] is not None
+ return _global_instance['plugin']
+
+
+def health_status_to_number(status):
+
+ if status == 'HEALTH_OK':
+ return 0
+ elif status == 'HEALTH_WARN':
+ return 1
+ elif status == 'HEALTH_ERR':
+ return 2
+
+PG_STATES = ['creating', 'active', 'clean', 'down', 'scrubbing', 'degraded',
+ 'inconsistent', 'peering', 'repair', 'recovering', 'forced-recovery',
+ 'backfill', 'forced-backfill', 'wait-backfill', 'backfill-toofull',
+ 'incomplete', 'stale', 'remapped', 'undersized', 'peered']
+
+DF_CLUSTER = ['total_bytes', 'total_used_bytes', 'total_objects']
+
+DF_POOL = ['max_avail', 'bytes_used', 'raw_bytes_used', 'objects', 'dirty',
+ 'quota_bytes', 'quota_objects', 'rd', 'rd_bytes', 'wr', 'wr_bytes']
+
+OSD_METADATA = ('cluster_addr', 'device_class', 'id', 'public_addr')
+
+OSD_STATUS = ['weight', 'up', 'in']
+
+POOL_METADATA = ('pool_id', 'name')
+
+DISK_OCCUPATION = ('instance', 'device', 'ceph_daemon')
+
+
+class Metric(object):
+ def __init__(self, mtype, name, desc, labels=None):
+ self.mtype = mtype
+ self.name = name
+ self.desc = desc
+ self.labelnames = labels # tuple if present
+ self.value = dict() # indexed by label values
+
+ def set(self, value, labelvalues=None):
+ # labelvalues must be a tuple
+ labelvalues = labelvalues or ('',)
+ self.value[labelvalues] = value
+
+ def str_expfmt(self):
+
+ def promethize(path):
+ ''' replace illegal metric name characters '''
+ result = path.replace('.', '_').replace('+', '_plus').replace('::', '_')
+
+ # Hyphens usually turn into underscores, unless they are
+ # trailing
+ if result.endswith("-"):
+ result = result[0:-1] + "_minus"
+ else:
+ result = result.replace("-", "_")
+
+ return "ceph_{0}".format(result)
+
+ def floatstr(value):
+ ''' represent as Go-compatible float '''
+ if value == float('inf'):
+ return '+Inf'
+ if value == float('-inf'):
+ return '-Inf'
+ if math.isnan(value):
+ return 'NaN'
+ return repr(float(value))
+
+ name = promethize(self.name)
+ expfmt = '''
+# HELP {name} {desc}
+# TYPE {name} {mtype}'''.format(
+ name=name,
+ desc=self.desc,
+ mtype=self.mtype,
+ )
+
+ for labelvalues, value in self.value.items():
+ if self.labelnames:
+ labels = zip(self.labelnames, labelvalues)
+ labels = ','.join('%s="%s"' % (k, v) for k, v in labels)
+ else:
+ labels = ''
+ if labels:
+ fmtstr = '\n{name}{{{labels}}} {value}'
+ else:
+ fmtstr = '\n{name} {value}'
+ expfmt += fmtstr.format(
+ name=name,
+ labels=labels,
+ value=floatstr(value),
+ )
+ return expfmt
+
+
+class Module(MgrModule):
+ COMMANDS = [
+ {
+ "cmd": "prometheus self-test",
+ "desc": "Run a self test on the prometheus module",
+ "perm": "rw"
+ },
+ ]
+
+ def __init__(self, *args, **kwargs):
+ super(Module, self).__init__(*args, **kwargs)
+ self.notified = False
+ self.serving = False
+ self.metrics = self._setup_static_metrics()
+ self.schema = OrderedDict()
+ _global_instance['plugin'] = self
+
+ def _stattype_to_str(self, stattype):
+
+ typeonly = stattype & self.PERFCOUNTER_TYPE_MASK
+ if typeonly == 0:
+ return 'gauge'
+ if typeonly == self.PERFCOUNTER_LONGRUNAVG:
+ # this lie matches the DaemonState decoding: only val, no counts
+ return 'counter'
+ if typeonly == self.PERFCOUNTER_COUNTER:
+ return 'counter'
+ if typeonly == self.PERFCOUNTER_HISTOGRAM:
+ return 'histogram'
+
+ return ''
+
+ def _setup_static_metrics(self):
+ metrics = {}
+ metrics['health_status'] = Metric(
+ 'untyped',
+ 'health_status',
+ 'Cluster health status'
+ )
+ metrics['mon_quorum_count'] = Metric(
+ 'gauge',
+ 'mon_quorum_count',
+ 'Monitors in quorum'
+ )
+ metrics['osd_metadata'] = Metric(
+ 'untyped',
+ 'osd_metadata',
+ 'OSD Metadata',
+ OSD_METADATA
+ )
+
+ # The reason for having this separate to OSD_METADATA is
+ # so that we can stably use the same tag names that
+ # the Prometheus node_exporter does
+ metrics['disk_occupation'] = Metric(
+ 'undef',
+ 'disk_occupation',
+ 'Associate Ceph daemon with disk used',
+ DISK_OCCUPATION
+ )
+
+ metrics['pool_metadata'] = Metric(
+ 'untyped',
+ 'pool_metadata',
+ 'POOL Metadata',
+ POOL_METADATA
+ )
+ for state in OSD_STATUS:
+ path = 'osd_{}'.format(state)
+ self.log.debug("init: creating {}".format(path))
+ metrics[path] = Metric(
+ 'untyped',
+ path,
+ 'OSD status {}'.format(state),
+ ('ceph_daemon',)
+ )
+ for state in PG_STATES:
+ path = 'pg_{}'.format(state)
+ self.log.debug("init: creating {}".format(path))
+ metrics[path] = Metric(
+ 'gauge',
+ path,
+ 'PG {}'.format(state),
+ )
+ for state in DF_CLUSTER:
+ path = 'cluster_{}'.format(state)
+ self.log.debug("init: creating {}".format(path))
+ metrics[path] = Metric(
+ 'gauge',
+ path,
+ 'DF {}'.format(state),
+ )
+ for state in DF_POOL:
+ path = 'pool_{}'.format(state)
+ self.log.debug("init: creating {}".format(path))
+ metrics[path] = Metric(
+ 'gauge',
+ path,
+ 'DF pool {}'.format(state),
+ ('pool_id',)
+ )
+
+ return metrics
+
+ def shutdown(self):
+ self.serving = False
+ pass
+
+ def get_health(self):
+ health = json.loads(self.get('health')['json'])
+ self.metrics['health_status'].set(
+ health_status_to_number(health['status'])
+ )
+
+ def get_df(self):
+ # maybe get the to-be-exported metrics from a config?
+ df = self.get('df')
+ for stat in DF_CLUSTER:
+ path = 'cluster_{}'.format(stat)
+ self.metrics[path].set(df['stats'][stat])
+
+ for pool in df['pools']:
+ for stat in DF_POOL:
+ path = 'pool_{}'.format(stat)
+ self.metrics[path].set(pool['stats'][stat], (pool['id'],))
+
+ def get_quorum_status(self):
+ mon_status = json.loads(self.get('mon_status')['json'])
+ self.metrics['mon_quorum_count'].set(len(mon_status['quorum']))
+
+ def get_pg_status(self):
+ # TODO add per pool status?
+ pg_s = self.get('pg_summary')['all']
+ reported_pg_s = [(s,v) for key, v in pg_s.items() for s in
+ key.split('+')]
+ for state, value in reported_pg_s:
+ path = 'pg_{}'.format(state)
+ self.metrics[path].set(value)
+ reported_states = [s[0] for s in reported_pg_s]
+ for state in PG_STATES:
+ path = 'pg_{}'.format(state)
+ if state not in reported_states:
+ self.metrics[path].set(0)
+
+ def get_metadata_and_osd_status(self):
+ osd_map = self.get('osd_map')
+ osd_devices = self.get('osd_map_crush')['devices']
+ for osd in osd_map['osds']:
+ id_ = osd['osd']
+ p_addr = osd['public_addr'].split(':')[0]
+ c_addr = osd['cluster_addr'].split(':')[0]
+ dev_class = next((osd for osd in osd_devices if osd['id'] == id_))
+ self.metrics['osd_metadata'].set(0, (
+ c_addr,
+ dev_class['class'],
+ id_,
+ p_addr
+ ))
+ for state in OSD_STATUS:
+ status = osd[state]
+ self.metrics['osd_{}'.format(state)].set(
+ status,
+ ('osd.{}'.format(id_),))
+
+ osd_metadata = self.get_metadata("osd", str(id_))
+ dev_keys = ("backend_filestore_dev_node", "bluestore_bdev_dev_node")
+ osd_dev_node = None
+ for dev_key in dev_keys:
+ val = osd_metadata.get(dev_key, None)
+ if val and val != "unknown":
+ osd_dev_node = val
+ break
+ osd_hostname = osd_metadata.get('hostname', None)
+ if osd_dev_node and osd_hostname:
+ self.log.debug("Got dev for osd {0}: {1}/{2}".format(
+ id_, osd_hostname, osd_dev_node))
+ self.metrics['disk_occupation'].set(0, (
+ osd_hostname,
+ osd_dev_node,
+ "osd.{0}".format(id_)
+ ))
+ else:
+ self.log.info("Missing dev node metadata for osd {0}, skipping "
+ "occupation record for this osd".format(id_))
+
+ for pool in osd_map['pools']:
+ id_ = pool['pool']
+ name = pool['pool_name']
+ self.metrics['pool_metadata'].set(0, (id_, name))
+
+ def collect(self):
+ self.get_health()
+ self.get_df()
+ self.get_quorum_status()
+ self.get_metadata_and_osd_status()
+ self.get_pg_status()
+
+ for daemon, counters in self.get_all_perf_counters().iteritems():
+ for path, counter_info in counters.items():
+ stattype = self._stattype_to_str(counter_info['type'])
+ # XXX simplify first effort: no histograms
+ # averages are already collapsed to one value for us
+ if not stattype or stattype == 'histogram':
+ self.log.debug('ignoring %s, type %s' % (path, stattype))
+ continue
+
+ if path not in self.metrics:
+ self.metrics[path] = Metric(
+ stattype,
+ path,
+ counter_info['description'],
+ ("ceph_daemon",),
+ )
+
+ self.metrics[path].set(
+ counter_info['value'],
+ (daemon,)
+ )
+
+ return self.metrics
+
+ def handle_command(self, cmd):
+ if cmd['prefix'] == 'prometheus self-test':
+ self.collect()
+ return 0, '', 'Self-test OK'
+ else:
+ return (-errno.EINVAL, '',
+ "Command not found '{0}'".format(cmd['prefix']))
+
+ def serve(self):
+
+ class Root(object):
+
+ # collapse everything to '/'
+ def _cp_dispatch(self, vpath):
+ cherrypy.request.path = ''
+ return self
+
+ def format_metrics(self, metrics):
+ formatted = ''
+ for m in metrics.values():
+ formatted += m.str_expfmt()
+ return formatted + '\n'
+
+ @cherrypy.expose
+ def index(self):
+ return '''<!DOCTYPE html>
+<html>
+ <head><title>Ceph Exporter</title></head>
+ <body>
+ <h1>Ceph Exporter</h1>
+ <p><a href='/metrics'>Metrics</a></p>
+ </body>
+</html>'''
+
+ @cherrypy.expose
+ def metrics(self):
+ metrics = global_instance().collect()
+ cherrypy.response.headers['Content-Type'] = 'text/plain'
+ if metrics:
+ return self.format_metrics(metrics)
+
+ server_addr = self.get_localized_config('server_addr', DEFAULT_ADDR)
+ server_port = self.get_localized_config('server_port', DEFAULT_PORT)
+ self.log.info(
+ "server_addr: %s server_port: %s" %
+ (server_addr, server_port)
+ )
+
+ cherrypy.config.update({
+ 'server.socket_host': server_addr,
+ 'server.socket_port': int(server_port),
+ 'engine.autoreload.on': False
+ })
+ cherrypy.tree.mount(Root(), "/")
+ cherrypy.engine.start()
+ cherrypy.engine.block()
Code Review / stor4nfv.git / blobdiff