doctor_tests/main.py

   1 ##############################################################################
   2 # Copyright (c) 2017 ZTE Corporation and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 import os
  10 from os.path import isfile, join
  11 import random
  12 import sys
  13 import time
  14
  15 from doctor_tests.alarm import Alarm
  16 from doctor_tests.common.constants import Host
  17 from doctor_tests.common.utils import match_rep_in_file
  18 from doctor_tests import config
  19 from doctor_tests.consumer import get_consumer
  20 from doctor_tests.identity_auth import get_identity_auth
  21 from doctor_tests.identity_auth import get_session
  22 from doctor_tests.image import Image
  23 from doctor_tests.instance import Instance
  24 from doctor_tests.inspector import get_inspector
  25 from doctor_tests.installer import get_installer
  26 import doctor_tests.logger as doctor_log
  27 from doctor_tests.network import Network
  28 from doctor_tests.monitor import get_monitor
  29 from doctor_tests.os_clients import nova_client
  30 from doctor_tests.profiler_poc import main as profiler_main
  31 from doctor_tests.scenario.common import calculate_notification_time
  32 from doctor_tests.scenario.network_failure import NetworkFault
  33 from doctor_tests.user import User
  34
  35
  36 Logger = doctor_log.Logger('doctor')
  37 LOG = Logger.getLogger()
  38 LogFile = Logger.getLogFilename()
  39
  40
  41 class DoctorTest(object):
  42
  43     def __init__(self, conf):
  44         self.conf = conf
  45         self.image = Image(self.conf, LOG)
  46         self.user = User(self.conf, LOG)
  47         self.network = Network(self.conf, LOG)
  48         self.instance = Instance(self.conf, LOG)
  49         self.alarm = Alarm(self.conf, LOG)
  50         self.installer = get_installer(self.conf, LOG)
  51         self.inspector = get_inspector(self.conf, LOG)
  52         self.monitor = get_monitor(self.conf,
  53                                    self.inspector.get_inspector_url(),
  54                                    LOG)
  55         self.consumer = get_consumer(self.conf, LOG)
  56         self.fault = NetworkFault(self.conf, self.installer, LOG)
  57         auth = get_identity_auth(project=self.conf.doctor_project)
  58         self.nova = nova_client(self.conf.nova_version,
  59                                 get_session(auth=auth))
  60         self.down_host = None
  61
  62     def setup(self):
  63         # prepare the cloud env
  64         self.installer.setup()
  65
  66         # preparing VM image...
  67         self.image.create()
  68
  69         # creating test user...
  70         self.user.create()
  71         self.user.update_quota()
  72
  73         # creating VM...
  74         self.network.create()
  75         self.instance.create()
  76         self.instance.wait_for_vm_launch()
  77
  78         # creating alarm...
  79         self.alarm.create()
  80
  81         # starting doctor sample components...
  82         self.inspector.start()
  83
  84         self.down_host = self.get_host_info_for_random_vm()
  85         self.monitor.start(self.down_host)
  86
  87         self.consumer.start()
  88
  89     def run(self):
  90         """run doctor test"""
  91         try:
  92             LOG.info('doctor test starting.......')
  93
  94             # prepare test env
  95             self.setup()
  96
  97             # wait for aodh alarms are updated in caches for event evaluator,
  98             # sleep time should be larger than event_alarm_cache_ttl
  99             # (default 60)
 100             time.sleep(60)
 101
 102             # injecting host failure...
 103             # NOTE (umar) add INTERFACE_NAME logic to host injection
 104
 105             self.fault.start(self.down_host)
 106             time.sleep(10)
 107
 108             # verify the test results
 109             # NOTE (umar) copy remote monitor.log file when monitor=collectd
 110             self.check_host_status(self.down_host.name, 'down')
 111
 112             notification_time = calculate_notification_time(LogFile)
 113             if notification_time < 1 and notification_time > 0:
 114                 LOG.info('doctor test successfully, notification_time=%s'
 115                          % notification_time)
 116             else:
 117                 LOG.error('doctor test failed, notification_time=%s'
 118                           % notification_time)
 119                 sys.exit(1)
 120
 121             if self.conf.profiler_type:
 122                 LOG.info('doctor test begin to run profile.......')
 123                 self.collect_logs()
 124                 self.run_profiler()
 125         except Exception as e:
 126             LOG.error('doctor test failed, Exception=%s' % e)
 127             sys.exit(1)
 128         finally:
 129             self.cleanup()
 130
 131     def get_host_info_for_random_vm(self):
 132         num = random.randint(0, self.conf.instance_count - 1)
 133         vm_name = "%s%d" % (self.conf.instance_basename, num)
 134
 135         servers = \
 136             {getattr(server, 'name'): server
 137              for server in self.nova.servers.list()}
 138         server = servers.get(vm_name)
 139         if not server:
 140             raise \
 141                 Exception('Can not find instance: vm_name(%s)' % vm_name)
 142         host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
 143         host_ip = self.installer.get_host_ip_from_hostname(host_name)
 144
 145         LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
 146                  % (host_name, host_ip, vm_name))
 147         return Host(host_name, host_ip)
 148
 149     def check_host_status(self, hostname, state):
 150         service = self.nova.services.list(host=hostname,
 151                                           binary='nova-compute')
 152         host_state = service[0].__dict__.get('state')
 153         assert host_state == state
 154
 155     def unset_forced_down_hosts(self):
 156         if self.down_host:
 157             self.nova.services.force_down(self.down_host.name,
 158                                           'nova-compute', False)
 159             time.sleep(2)
 160             self.check_host_status(self.down_host.name, 'up')
 161
 162     def collect_logs(self):
 163         self.fault.get_disable_network_log()
 164
 165     def run_profiler(self):
 166
 167         net_down_log_file = self.fault.get_disable_network_log()
 168         reg = '(?<=doctor set link down at )\d+.\d+'
 169         linkdown = float(match_rep_in_file(reg, net_down_log_file).group(0))
 170
 171         reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
 172         vmdown = float(match_rep_in_file(reg, LogFile).group(2))
 173
 174         reg = '(.* doctor mark host.* down at )(\d+.\d+)'
 175         hostdown = float(match_rep_in_file(reg, LogFile).group(2))
 176
 177         reg = '(?<=doctor monitor detected at )\d+.\d+'
 178         detected = float(match_rep_in_file(reg, LogFile).group(0))
 179
 180         reg = '(?<=doctor consumer notified at )\d+.\d+'
 181         notified = float(match_rep_in_file(reg, LogFile).group(0))
 182
 183         # TODO(yujunz) check the actual delay to verify time sync status
 184         # expected ~1s delay from $trigger to $linkdown
 185         relative_start = linkdown
 186         os.environ['DOCTOR_PROFILER_T00'] = \
 187             str(int((linkdown - relative_start) * 1000))
 188         os.environ['DOCTOR_PROFILER_T01'] = \
 189             str(int((detected - relative_start) * 1000))
 190         os.environ['DOCTOR_PROFILER_T03'] = \
 191             str(int((vmdown - relative_start) * 1000))
 192         os.environ['DOCTOR_PROFILER_T04'] = \
 193             str(int((hostdown - relative_start) * 1000))
 194         os.environ['DOCTOR_PROFILER_T09'] = \
 195             str(int((notified - relative_start) * 1000))
 196
 197         profiler_main(log=LOG)
 198
 199     def cleanup(self):
 200         self.unset_forced_down_hosts()
 201         self.inspector.stop()
 202         self.monitor.stop()
 203         self.consumer.stop()
 204         self.installer.cleanup()
 205         self.alarm.delete()
 206         self.instance.delete()
 207         self.network.delete()
 208         self.image.delete()
 209         self.fault.cleanup()
 210         self.user.delete()
 211
 212
 213 def main():
 214     """doctor main"""
 215     test_dir = os.path.split(os.path.realpath(__file__))[0]
 216     doctor_root_dir = os.path.dirname(test_dir)
 217
 218     config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
 219     config_files = [join(config_file_dir, f)
 220                     for f in os.listdir(config_file_dir)
 221                     if isfile(join(config_file_dir, f))]
 222
 223     conf = config.prepare_conf(args=sys.argv[1:],
 224                                config_files=config_files)
 225
 226     doctor = DoctorTest(conf)
 227     doctor.run()