X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=tests%2Finspector.py;h=27c969e5dc6b741558ee9d44a0dbef25b4fac2bb;hb=8f72e695538c2281f923bdbf7a6d7b1d1763c70c;hp=2445c77af5fe23983036dfb1059ced42dfcf1e51;hpb=be640e1f465e4628b3b7cf73ebd954fa5deea941;p=doctor.git diff --git a/tests/inspector.py b/tests/inspector.py index 2445c77a..27c969e5 100644 --- a/tests/inspector.py +++ b/tests/inspector.py @@ -1,46 +1,98 @@ +############################################################################## +# Copyright (c) 2016 NEC Corporation and others. # -# Copyright 2016 NEC Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## import argparse +import collections from flask import Flask from flask import request import json +import logger as doctor_log import os +import threading +import time +from keystoneauth1.identity import v3 +from keystoneauth1 import session import novaclient.client as novaclient +LOG = doctor_log.Logger('doctor_inspector').getLogger() + + +class ThreadedResetState(threading.Thread): + + def __init__(self, nova, state, server): + threading.Thread.__init__(self) + self.nova = nova + self.state = state + self.server = server + + def run(self): + self.nova.servers.reset_state(self.server, self.state) + LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time())) + class DoctorInspectorSample(object): - nova_api_version = 2.11 + NOVA_API_VERSION = '2.34' + NUMBER_OF_CLIENTS = 50 + # TODO(tojuvone): This could be enhanced in future with dynamic + # reuse of self.novaclients when all threads in use and + # self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input + # argument def __init__(self): - self.nova = novaclient.Client(self.nova_api_version, - os.environ['OS_USERNAME'], - os.environ['OS_PASSWORD'], - os.environ['OS_TENANT_NAME'], - os.environ['OS_AUTH_URL'], - connection_pool=True) - # check nova is available + self.servers = collections.defaultdict(list) + self.novaclients = list() + auth = v3.Password(auth_url=os.environ['OS_AUTH_URL'], + username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + user_domain_name=os.environ['OS_USER_DOMAIN_NAME'], + project_name=os.environ['OS_PROJECT_NAME'], + project_domain_name=os.environ['OS_PROJECT_DOMAIN_NAME']) + sess=session.Session(auth=auth) + # Pool of novaclients for redundant usage + for i in range(self.NUMBER_OF_CLIENTS): + self.novaclients.append( + novaclient.Client(self.NOVA_API_VERSION, session=sess, + connection_pool=True)) + # Normally we use this client for non redundant API calls + self.nova=self.novaclients[0] self.nova.servers.list(detailed=False) + self.init_servers_list() + + def init_servers_list(self): + opts = {'all_tenants': True} + servers=self.nova.servers.list(search_opts=opts) + self.servers.clear() + for server in servers: + try: + host=server.__dict__.get('OS-EXT-SRV-ATTR:host') + self.servers[host].append(server) + LOG.debug('get hostname=%s from server=%s' % (host, server)) + except Exception as e: + LOG.error('can not get hostname from server=%s' % server) def disable_compute_host(self, hostname): - opts = {'all_tenants': True, 'host': hostname} - for server in self.nova.servers.list(detailed=False, search_opts=opts): - self.nova.servers.reset_state(server, 'error') + threads = [] + if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS: + # TODO(tojuvone): This could be enhanced in future with dynamic + # reuse of self.novaclients when all threads in use + LOG.error('%d servers in %s. Can handle only %d'%( + self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS)) + for nova, server in zip(self.novaclients, self.servers[hostname]): + t = ThreadedResetState(nova, "error", server) + t.start() + threads.append(t) + for t in threads: + t.join() self.nova.services.force_down(hostname, 'nova-compute', True) + LOG.info('doctor mark host(%s) down at %s' % (hostname, time.time())) app = Flask(__name__) @@ -49,27 +101,28 @@ inspector = DoctorInspectorSample() @app.route('/events', methods=['POST']) def event_posted(): - app.logger.debug('event posted') - app.logger.debug('inspector = %s' % inspector) - app.logger.debug('received data = %s' % request.data) + LOG.info('event posted at %s' % time.time()) + LOG.info('inspector = %s' % inspector) + LOG.info('received data = %s' % request.data) d = json.loads(request.data) - hostname = d['hostname'] - event_type = d['type'] - if event_type == 'compute.host.down': - inspector.disable_compute_host(hostname) + for event in d: + hostname = event['details']['hostname'] + event_type = event['type'] + if event_type == 'compute.host.down': + inspector.disable_compute_host(hostname) return "OK" def get_args(): - parser = argparse.ArgumentParser(description='Doctor Sample Monitor') + parser = argparse.ArgumentParser(description='Doctor Sample Inspector') parser.add_argument('port', metavar='PORT', type=int, nargs='?', - help='a port for inspectpr') + help='a port for inspector') return parser.parse_args() def main(): args = get_args() - app.run(port=args.port, debug=True) + app.run(port=args.port) if __name__ == '__main__':