1 ##############################################################################
2 # Copyright (c) 2017 ZTE Corporation and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
10 from os.path import isfile, join
16 from doctor_tests.alarm import Alarm
17 from doctor_tests.common.constants import Host
18 from doctor_tests.common.utils import match_rep_in_file
19 from doctor_tests import config
20 from doctor_tests.consumer import get_consumer
21 from doctor_tests.identity_auth import get_identity_auth
22 from doctor_tests.identity_auth import get_session
23 from doctor_tests.image import Image
24 from doctor_tests.instance import Instance
25 from doctor_tests.inspector import get_inspector
26 from doctor_tests.installer import get_installer
27 import doctor_tests.logger as doctor_log
28 from doctor_tests.network import Network
29 from doctor_tests.monitor import get_monitor
30 from doctor_tests.os_clients import nova_client
31 from doctor_tests.profiler_poc import main as profiler_main
32 from doctor_tests.scenario.common import calculate_notification_time
33 from doctor_tests.scenario.network_failure import NetworkFault
34 from doctor_tests.user import User
37 Logger = doctor_log.Logger('doctor')
38 LOG = Logger.getLogger()
39 LogFile = Logger.getLogFilename()
42 class DoctorTest(object):
44 def __init__(self, conf):
46 self.image = Image(self.conf, LOG)
47 self.user = User(self.conf, LOG)
48 self.network = Network(self.conf, LOG)
49 self.instance = Instance(self.conf, LOG)
50 self.alarm = Alarm(self.conf, LOG)
51 self.installer = get_installer(self.conf, LOG)
52 self.inspector = get_inspector(self.conf, LOG)
53 self.monitor = get_monitor(self.conf,
54 self.inspector.get_inspector_url(),
56 self.consumer = get_consumer(self.conf, LOG)
57 self.fault = NetworkFault(self.conf, self.installer, LOG)
58 auth = get_identity_auth(project=self.conf.doctor_project)
59 self.nova = nova_client(self.conf.nova_version,
60 get_session(auth=auth))
64 # prepare the cloud env
65 self.installer.setup()
67 # preparing VM image...
70 # creating test user...
73 def setup_fault_management(self):
75 self.user.update_quota()
79 self.instance.create()
80 self.instance.wait_for_vm_launch()
85 # starting doctor sample components...
86 # tbd tojuvone: move inspector and consumer to common setup
87 # when they support updating VMs via instance.create and
88 # instance.delete alarm
90 self.inspector.start()
92 self.down_host = self.get_host_info_for_random_vm()
93 self.monitor.start(self.down_host)
95 def test_fault_management(self):
97 LOG.info('doctor fault management test starting.......')
100 self.setup_fault_management()
102 # wait for aodh alarms are updated in caches for event evaluator,
103 # sleep time should be larger than event_alarm_cache_ttl
105 # (tojuvone) Fraser currently needs 120
108 # injecting host failure...
109 # NOTE (umar) add INTERFACE_NAME logic to host injection
111 self.fault.start(self.down_host)
114 # verify the test results
115 # NOTE (umar) copy remote monitor.log file when monitor=collectd
116 self.check_host_status(self.down_host.name, 'down')
118 notification_time = calculate_notification_time(LogFile)
119 if notification_time < 1 and notification_time > 0:
120 LOG.info('doctor fault management test successfully, '
121 'notification_time=%s' % notification_time)
123 LOG.error('doctor fault management test failed, '
124 'notification_time=%s' % notification_time)
127 if self.conf.profiler_type:
128 LOG.info('doctor fault management test begin to run '
132 except Exception as e:
133 LOG.error('doctor fault management test failed, '
137 self.cleanup_fault_management()
139 def _amount_compute_nodes(self):
140 services = self.nova.services.list(binary='nova-compute')
143 def test_maintenance(self):
144 cnodes = self._amount_compute_nodes()
146 # need 2 compute for redundancy and one spare to migrate
147 LOG.info('not enough compute nodes, skipping doctor '
151 LOG.info('doctor maintenance test starting.......')
152 # TODO (tojuvone) test setup and actual test
153 except Exception as e:
154 LOG.error('doctor maintenance test failed, Exception=%s' % e)
156 # TODO (tojuvone) finally: test case specific cleanup
159 """run doctor tests"""
161 LOG.info('doctor test starting.......')
162 # prepare common test env
164 if self.conf.test_case == 'all':
165 self.test_fault_management()
166 self.test_maintenance()
168 function = 'test_%s' % self.conf.test_case
169 if hasattr(self, function):
170 getattr(self, function)()
172 raise Exception('Can not find function <%s> in'
173 'DoctorTest, see config manual'
175 except Exception as e:
176 LOG.error('doctor test failed, Exception=%s' % e)
181 def get_host_info_for_random_vm(self):
182 num = random.randint(0, self.conf.instance_count - 1)
183 vm_name = "%s%d" % (self.conf.instance_basename, num)
185 servers = {getattr(server, 'name'): server
186 for server in self.nova.servers.list()}
187 server = servers.get(vm_name)
189 raise Exception('Can not find instance: vm_name(%s)' % vm_name)
190 host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
191 host_ip = self.installer.get_host_ip_from_hostname(host_name)
193 LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
194 % (host_name, host_ip, vm_name))
195 return Host(host_name, host_ip)
197 def check_host_status(self, hostname, state):
198 service = self.nova.services.list(host=hostname,
199 binary='nova-compute')
200 host_state = service[0].__dict__.get('state')
201 assert host_state == state
203 def unset_forced_down_hosts(self):
205 self.nova.services.force_down(self.down_host.name,
206 'nova-compute', False)
208 self.check_host_status(self.down_host.name, 'up')
210 def collect_logs(self):
211 self.fault.get_disable_network_log()
213 def run_profiler(self):
215 net_down_log_file = self.fault.get_disable_network_log()
216 reg = '(?<=doctor set link down at )\d+.\d+'
217 linkdown = float(match_rep_in_file(reg, net_down_log_file).group(0))
219 reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
220 vmdown = float(match_rep_in_file(reg, LogFile).group(2))
222 reg = '(.* doctor mark host.* down at )(\d+.\d+)'
223 hostdown = float(match_rep_in_file(reg, LogFile).group(2))
225 reg = '(?<=doctor monitor detected at )\d+.\d+'
226 detected = float(match_rep_in_file(reg, LogFile).group(0))
228 reg = '(?<=doctor consumer notified at )\d+.\d+'
229 notified = float(match_rep_in_file(reg, LogFile).group(0))
231 # TODO(yujunz) check the actual delay to verify time sync status
232 # expected ~1s delay from $trigger to $linkdown
233 relative_start = linkdown
234 os.environ['DOCTOR_PROFILER_T00'] = (
235 str(int((linkdown - relative_start) * 1000)))
236 os.environ['DOCTOR_PROFILER_T01'] = (
237 str(int((detected - relative_start) * 1000)))
238 os.environ['DOCTOR_PROFILER_T03'] = (
239 str(int((vmdown - relative_start) * 1000)))
240 os.environ['DOCTOR_PROFILER_T04'] = (
241 str(int((hostdown - relative_start) * 1000)))
242 os.environ['DOCTOR_PROFILER_T09'] = (
243 str(int((notified - relative_start) * 1000)))
245 profiler_main(log=LOG)
247 def cleanup_fault_management(self):
248 self.unset_forced_down_hosts()
249 self.inspector.stop()
253 self.instance.delete()
254 self.network.delete()
258 self.installer.cleanup()
261 # Kill possible hanging subprocess
262 os.killpg(0, signal.SIGKILL)
267 # TODO (tojuvone): JIRA DOCTOR-123: Test cases have some issue to always
268 # kill all subprocesses. To ensure they are killed this group is done so
269 # all processes can be killed without knowing what they are.
271 test_dir = os.path.split(os.path.realpath(__file__))[0]
272 doctor_root_dir = os.path.dirname(test_dir)
274 config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
275 config_files = [join(config_file_dir, f)
276 for f in os.listdir(config_file_dir)
277 if isfile(join(config_file_dir, f))]
279 conf = config.prepare_conf(args=sys.argv[1:],
280 config_files=config_files)
282 doctor = DoctorTest(conf)