doctor_tests/main.py

   1 ##############################################################################
   2 # Copyright (c) 2019 ZTE Corporation and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 import os
  10 from os.path import isfile, join
  11 import sys
  12 import time
  13 from traceback import format_exc
  14
  15 from doctor_tests import config
  16 from doctor_tests.identity_auth import get_identity_auth
  17 from doctor_tests.identity_auth import get_session
  18 from doctor_tests.image import Image
  19 from doctor_tests.installer import get_installer
  20 import doctor_tests.logger as doctor_log
  21 from doctor_tests.scenario.fault_management import FaultManagement
  22 from doctor_tests.os_clients import nova_client
  23 from doctor_tests.scenario.maintenance import Maintenance
  24 from doctor_tests.user import User
  25
  26
  27 Logger = doctor_log.Logger('doctor')
  28 LOG = Logger.getLogger()
  29 LogFile = Logger.getLogFilename()
  30
  31
  32 class DoctorTest(object):
  33
  34     def __init__(self, conf):
  35         self.conf = conf
  36         self.image = Image(self.conf, LOG)
  37         self.user = User(self.conf, LOG)
  38         self.installer = get_installer(self.conf, LOG)
  39         auth = get_identity_auth(project=self.conf.doctor_project)
  40         self.nova = nova_client(self.conf.nova_version,
  41                                 get_session(auth=auth))
  42
  43     def setup(self):
  44         # prepare the cloud env
  45         self.installer.setup()
  46         # preparing VM image...
  47         self.image.create()
  48
  49         # creating test user...
  50         self.user.create()
  51
  52     def test_fault_management(self):
  53         retry = 2
  54         # Retry once if notified_time is None
  55         while retry > 0:
  56             try:
  57                 self.fault_management = None
  58                 LOG.info('doctor fault management test starting.......')
  59                 transport_url = self.installer.get_transport_url()
  60                 self.fault_management = \
  61                     FaultManagement(self.conf, self.installer, self.user, LOG,
  62                                     transport_url)
  63
  64                 # prepare test env
  65                 self.fault_management.setup()
  66
  67                 # wait for aodh alarms are updated in caches for event
  68                 # evaluator,
  69                 # sleep time should be larger than event_alarm_cache_ttl
  70                 # (default 60)
  71                 # (tojuvone) Fraser currently needs 120
  72                 time.sleep(120)
  73
  74                 # injecting host failure...
  75                 # NOTE (umar) add INTERFACE_NAME logic to host injection
  76                 self.fault_management.start()
  77                 time.sleep(30)
  78
  79                 # verify the test results
  80                 # NOTE (umar) copy remote monitor.log file when
  81                 # monitor=collectd
  82                 self.fault_management.check_host_status('down')
  83                 self.fault_management.check_notification_time()
  84                 retry = 0
  85
  86             except Exception as e:
  87                 LOG.error('doctor fault management test failed, '
  88                           'Exception=%s' % e)
  89                 if 'notified_time=None' in str(e):
  90                     retry -= 1
  91                     LOG.info('doctor fault management retry')
  92                     continue
  93                 LOG.error(format_exc())
  94                 sys.exit(1)
  95             finally:
  96                 if self.fault_management is not None:
  97                     self.fault_management.cleanup()
  98
  99     def _amount_compute_nodes(self):
 100         services = self.nova.services.list(binary='nova-compute')
 101         return len(services)
 102
 103     def test_maintenance(self):
 104         cnodes = self._amount_compute_nodes()
 105         if cnodes < 3:
 106             # need 2 compute for redundancy and one spare to migrate
 107             LOG.info('not enough compute nodes, skipping doctor '
 108                      'maintenance test')
 109             return
 110         elif self.conf.installer.type not in ['apex', 'fuel', 'devstack']:
 111             LOG.info('not supported installer, skipping doctor '
 112                      'maintenance test')
 113             return
 114         try:
 115             maintenance = None
 116             LOG.info('doctor maintenance test starting.......')
 117             trasport_url = self.installer.get_transport_url()
 118             maintenance = Maintenance(trasport_url, self.conf, LOG)
 119             maintenance.setup_maintenance(self.user)
 120
 121             # wait for aodh alarms are updated in caches for event evaluator,
 122             # sleep time should be larger than event_alarm_cache_ttl
 123             # (default 60)
 124             LOG.info('wait aodh for 120s.......')
 125             time.sleep(120)
 126
 127             session_id = maintenance.start_maintenance()
 128             maintenance.wait_maintenance_complete(session_id)
 129
 130             LOG.info('doctor maintenance complete.......')
 131
 132         except Exception as e:
 133             LOG.error('doctor maintenance test failed, Exception=%s' % e)
 134             LOG.error(format_exc())
 135             sys.exit(1)
 136         finally:
 137             if maintenance is not None:
 138                 maintenance.cleanup_maintenance()
 139
 140     def run(self):
 141         """run doctor tests"""
 142         try:
 143             LOG.info('doctor test starting.......')
 144
 145             # prepare common test env
 146             self.setup()
 147
 148             if self.conf.test_case == 'all':
 149                 self.test_fault_management()
 150                 self.test_maintenance()
 151             else:
 152                 function = 'test_%s' % self.conf.test_case
 153                 if hasattr(self, function):
 154                     getattr(self, function)()
 155                 else:
 156                     raise Exception('Can not find function <%s> in'
 157                                     'DoctorTest, see config manual'
 158                                     % function)
 159         except Exception as e:
 160             LOG.error('doctor test failed, Exception=%s' % e)
 161             LOG.error(format_exc())
 162             sys.exit(1)
 163         finally:
 164             self.cleanup()
 165
 166     def cleanup(self):
 167         self.installer.cleanup()
 168         self.image.delete()
 169         self.user.delete()
 170
 171
 172 def main():
 173     """doctor main"""
 174     test_dir = os.path.split(os.path.realpath(__file__))[0]
 175     doctor_root_dir = os.path.dirname(test_dir)
 176
 177     config_file_dir = '{0}/{1}'.format(doctor_root_dir, 'etc/')
 178     config_files = [join(config_file_dir, f)
 179                     for f in os.listdir(config_file_dir)
 180                     if isfile(join(config_file_dir, f))]
 181
 182     conf = config.prepare_conf(args=sys.argv[1:],
 183                                config_files=config_files)
 184
 185     doctor = DoctorTest(conf)
 186     doctor.run()