##############################################################################
import argparse
+import collections
from flask import Flask
from flask import request
import json
-import os
+import logger as doctor_log
+import threading
+import time
+from keystoneauth1 import session
import novaclient.client as novaclient
+import identity_auth
+
+LOG = doctor_log.Logger('doctor_inspector').getLogger()
+
+
+class ThreadedResetState(threading.Thread):
+
+ def __init__(self, nova, state, server):
+ threading.Thread.__init__(self)
+ self.nova = nova
+ self.state = state
+ self.server = server
+
+ def run(self):
+ self.nova.servers.reset_state(self.server, self.state)
+ LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
class DoctorInspectorSample(object):
- nova_api_version = 2.11
+ NOVA_API_VERSION = '2.34'
+ NUMBER_OF_CLIENTS = 50
+ # TODO(tojuvone): This could be enhanced in future with dynamic
+ # reuse of self.novaclients when all threads in use and
+ # self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input
+ # argument
def __init__(self):
- self.nova = novaclient.Client(self.nova_api_version,
- os.environ['OS_USERNAME'],
- os.environ['OS_PASSWORD'],
- os.environ['OS_TENANT_NAME'],
- os.environ['OS_AUTH_URL'],
- connection_pool=True)
- # check nova is available
+ self.servers = collections.defaultdict(list)
+ self.novaclients = list()
+ auth=identity_auth.get_identity_auth()
+ sess=session.Session(auth=auth)
+ # Pool of novaclients for redundant usage
+ for i in range(self.NUMBER_OF_CLIENTS):
+ self.novaclients.append(
+ novaclient.Client(self.NOVA_API_VERSION, session=sess))
+ # Normally we use this client for non redundant API calls
+ self.nova=self.novaclients[0]
self.nova.servers.list(detailed=False)
+ self.init_servers_list()
+
+ def init_servers_list(self):
+ opts = {'all_tenants': True}
+ servers=self.nova.servers.list(search_opts=opts)
+ self.servers.clear()
+ for server in servers:
+ try:
+ host=server.__dict__.get('OS-EXT-SRV-ATTR:host')
+ self.servers[host].append(server)
+ LOG.debug('get hostname=%s from server=%s' % (host, server))
+ except Exception as e:
+ LOG.error('can not get hostname from server=%s' % server)
def disable_compute_host(self, hostname):
- opts = {'all_tenants': True, 'host': hostname}
- for server in self.nova.servers.list(detailed=False, search_opts=opts):
- self.nova.servers.reset_state(server, 'error')
+ threads = []
+ if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+ # TODO(tojuvone): This could be enhanced in future with dynamic
+ # reuse of self.novaclients when all threads in use
+ LOG.error('%d servers in %s. Can handle only %d'%(
+ self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+ for nova, server in zip(self.novaclients, self.servers[hostname]):
+ t = ThreadedResetState(nova, "error", server)
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
self.nova.services.force_down(hostname, 'nova-compute', True)
+ LOG.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
app = Flask(__name__)
@app.route('/events', methods=['POST'])
def event_posted():
- app.logger.debug('event posted')
- app.logger.debug('inspector = %s' % inspector)
- app.logger.debug('received data = %s' % request.data)
+ LOG.info('event posted at %s' % time.time())
+ LOG.info('inspector = %s' % inspector)
+ LOG.info('received data = %s' % request.data)
d = json.loads(request.data)
- hostname = d['hostname']
- event_type = d['type']
- if event_type == 'compute.host.down':
- inspector.disable_compute_host(hostname)
+ for event in d:
+ hostname = event['details']['hostname']
+ event_type = event['type']
+ if event_type == 'compute.host.down':
+ inspector.disable_compute_host(hostname)
return "OK"
def get_args():
- parser = argparse.ArgumentParser(description='Doctor Sample Monitor')
+ parser = argparse.ArgumentParser(description='Doctor Sample Inspector')
parser.add_argument('port', metavar='PORT', type=int, nargs='?',
- help='a port for inspectpr')
+ help='a port for inspector')
return parser.parse_args()
def main():
args = get_args()
- app.run(port=args.port, debug=True)
-
+ app.run(host='0.0.0.0', port=args.port)
if __name__ == '__main__':
main()