1 ##############################################################################
2 # Copyright (c) 2017 ZTE Corporation and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 from flask import Flask
11 from flask import request
14 from threading import Thread
18 from doctor_tests.common import utils
19 from doctor_tests.identity_auth import get_identity_auth
20 from doctor_tests.identity_auth import get_session
21 from doctor_tests.os_clients import nova_client
22 from doctor_tests.os_clients import neutron_client
23 from doctor_tests.inspector.base import BaseInspector
26 class SampleInspector(BaseInspector):
27 event_type = 'compute.host.down'
29 def __init__(self, conf, log):
30 super(SampleInspector, self).__init__(conf, log)
31 self.inspector_url = self.get_inspector_url()
32 self.novaclients = list()
33 self._init_novaclients()
34 # Normally we use this client for non redundant API calls
35 self.nova = self.novaclients[0]
37 auth = get_identity_auth(project=self.conf.doctor_project)
38 session = get_session(auth=auth)
39 if self.conf.inspector.update_neutron_port_dp_status:
40 self.neutron = neutron_client(session)
42 self.servers = collections.defaultdict(list)
43 self.hostnames = list()
46 def _init_novaclients(self):
47 self.NUMBER_OF_CLIENTS = self.conf.instance_count
48 auth = get_identity_auth(project=self.conf.doctor_project)
49 session = get_session(auth=auth)
50 for i in range(self.NUMBER_OF_CLIENTS):
51 self.novaclients.append(
52 nova_client(self.conf.nova_version, session))
54 def _init_servers_list(self):
56 opts = {'all_tenants': True}
57 servers = self.nova.servers.list(search_opts=opts)
58 for server in servers:
60 host = server.__dict__.get('OS-EXT-SRV-ATTR:host')
61 self.servers[host].append(server)
62 self.log.debug('get hostname=%s from server=%s'
64 except Exception as e:
65 self.log.info('can not get hostname from server=%s, error=%s'
68 def get_inspector_url(self):
69 return 'http://%s:%s/events' % (self.conf.inspector.ip,
70 self.conf.inspector.port)
73 self.log.info('sample inspector start......')
74 self._init_servers_list()
75 self.app = InspectorApp(self.conf.inspector.port, self, self.log)
79 self.log.info('sample inspector stop......')
82 for hostname in self.hostnames:
83 self.nova.services.force_down(hostname, 'nova-compute', False)
86 'Content-Type': 'application/json',
87 'Accept': 'application/json',
89 url = '%s%s' % (self.inspector_url, 'shutdown') \
90 if self.inspector_url.endswith('/') else \
91 '%s%s' % (self.inspector_url, '/shutdown')
92 requests.post(url, data='', headers=headers)
94 def handle_events(self, events):
96 hostname = event['details']['hostname']
97 event_type = event['type']
98 if event_type == self.event_type:
99 self.hostnames.append(hostname)
100 thr1 = self._disable_compute_host(hostname)
101 thr2 = self._vms_reset_state('error', hostname)
102 if self.conf.inspector.update_neutron_port_dp_status:
103 thr3 = self._set_ports_data_plane_status('DOWN', hostname)
106 if self.conf.inspector.update_neutron_port_dp_status:
109 def _alarm_data_decoder(self, data):
110 if "[" in data or "{" in data:
111 # string to list or dict removing unicode
112 data = yaml.load(data.replace("u'", "'"))
115 def _alarm_traits_decoder(self, data):
116 return ({str(t[0]): self._alarm_data_decoder(str(t[2]))
117 for t in data['reason_data']['event']['traits']})
119 def maintenance(self, data):
121 payload = self._alarm_traits_decoder(data)
123 payload = ({t[0]: t[2] for t in
124 data['reason_data']['event']['traits']})
125 self.log.error('cannot parse alarm data: %s' % payload)
126 raise Exception('sample inspector cannot parse alarm.'
127 'Possibly trait data over 256 char')
128 self.log.info('sample inspector received data = %s' % payload)
130 state = payload['state']
131 host = payload['host']
133 if state == 'IN_MAINTENANCE':
134 self.log.info("sample inspector: disable %s automatic fault "
136 elif state == 'MAINTENANCE_COMPLETE':
137 self.log.info("sample inspector: enable %s automatic fault "
140 raise("sample inspector couldn't handle state: %s" % state)
143 def _disable_compute_host(self, hostname):
144 self.nova.services.force_down(hostname, 'nova-compute', True)
146 hostdown_time = time.time()
147 self.host_down_time = hostdown_time
148 self.log.info('doctor mark host(%s) down at %s'
149 % (hostname, hostdown_time))
152 def _vms_reset_state(self, state, hostname):
155 def _vm_reset_state(nova, server, state):
156 nova.servers.reset_state(server, state)
157 vmdown_time = time.time()
158 self.vm_down_time = vmdown_time
159 self.log.info('doctor mark vm(%s) error at %s'
160 % (server, vmdown_time))
163 for nova, server in zip(self.novaclients, self.servers[hostname]):
164 t = _vm_reset_state(nova, server, state)
170 def _set_ports_data_plane_status(self, status, hostname):
171 body = {'data_plane_status': status}
174 def _set_port_data_plane_status(port_id):
175 self.neutron.update_port(port_id, body)
176 self.log.info('doctor set data plane status %s on port %s'
180 params = {'binding:host_id': hostname}
181 for port_id in self.neutron.list_ports(**params):
182 t = _set_port_data_plane_status(port_id)
188 class InspectorApp(Thread):
190 def __init__(self, port, inspector, log):
191 Thread.__init__(self)
193 self.inspector = inspector
197 app = Flask('inspector')
199 @app.route('/events', methods=['PUT'])
201 self.log.info('event posted in sample inspector at %s'
203 self.log.info('sample inspector = %s' % self.inspector)
204 self.log.info('sample inspector received data = %s'
206 events = json.loads(request.data.decode('utf8'))
207 self.inspector.handle_events(events)
210 @app.route('/maintenance', methods=['POST'])
212 self.inspector.maintenance(request.json)
215 @app.route('/events/shutdown', methods=['POST'])
217 self.log.info('shutdown inspector app server at %s' % time.time())
218 func = request.environ.get('werkzeug.server.shutdown')
220 raise RuntimeError('Not running with the Werkzeug Server')
222 return 'inspector app shutting down...'
224 app.run(host="0.0.0.0", port=self.port)