X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=doctor_tests%2Fmain.py;h=7573faec96d6a0d92c3e13a20642dfe13a594899;hb=34b4083b4d2d579f6dbd334df06fe9394453680a;hp=79480ca1226885f1a7c21fe384dc046ef1b4f92a;hpb=eaeb3c0f9dc9e6645a159d0a78b9fc181fce53d4;p=doctor.git diff --git a/doctor_tests/main.py b/doctor_tests/main.py index 79480ca1..7573faec 100644 --- a/doctor_tests/main.py +++ b/doctor_tests/main.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. +# Copyright (c) 2019 ZTE Corporation and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -10,6 +10,7 @@ import os from os.path import isfile, join import sys import time +from traceback import format_exc from doctor_tests import config from doctor_tests.identity_auth import get_identity_auth @@ -17,8 +18,9 @@ from doctor_tests.identity_auth import get_session from doctor_tests.image import Image from doctor_tests.installer import get_installer import doctor_tests.logger as doctor_log -from doctor_tests.os_clients import nova_client from doctor_tests.scenario.fault_management import FaultManagement +from doctor_tests.os_clients import nova_client +from doctor_tests.scenario.maintenance import Maintenance from doctor_tests.user import User @@ -41,7 +43,6 @@ class DoctorTest(object): def setup(self): # prepare the cloud env self.installer.setup() - # preparing VM image... self.image.create() @@ -49,37 +50,50 @@ class DoctorTest(object): self.user.create() def test_fault_management(self): - try: - LOG.info('doctor fault management test starting.......') - - self.fault_management = \ - FaultManagement(self.conf, self.installer, self.user, LOG) - - # prepare test env - self.fault_management.setup() - - # wait for aodh alarms are updated in caches for event evaluator, - # sleep time should be larger than event_alarm_cache_ttl - # (default 60) - # (tojuvone) Fraser currently needs 120 - time.sleep(120) - - # injecting host failure... - # NOTE (umar) add INTERFACE_NAME logic to host injection - self.fault_management.start() - time.sleep(30) - - # verify the test results - # NOTE (umar) copy remote monitor.log file when monitor=collectd - self.fault_management.check_host_status('down') - self.fault_management.check_notification_time() - - except Exception as e: - LOG.error('doctor fault management test failed, ' - 'Exception=%s' % e) - sys.exit(1) - finally: - self.fault_management.cleanup() + retry = 2 + # Retry once if notified_time is None + while retry > 0: + try: + self.fault_management = None + LOG.info('doctor fault management test starting.......') + transport_url = self.installer.get_transport_url() + self.fault_management = \ + FaultManagement(self.conf, self.installer, self.user, LOG, + transport_url) + + # prepare test env + self.fault_management.setup() + + # wait for aodh alarms are updated in caches for event + # evaluator,sleep time should be larger than + # event_alarm_cache_ttl (default 60) + # (tojuvone) Fraser currently needs 120 + time.sleep(120) + + # injecting host failure... + # NOTE (umar) add INTERFACE_NAME logic to host injection + self.fault_management.start() + time.sleep(30) + + # verify the test results + # NOTE (umar) copy remote monitor.log file when + # monitor=collectd + self.fault_management.check_host_status('down') + self.fault_management.check_notification_time() + retry = 0 + + except Exception as e: + LOG.error('doctor fault management test failed, ' + 'Exception=%s' % e) + if 'notified_time=None' in str(e): + retry -= 1 + LOG.info('doctor fault management retry') + continue + LOG.error(format_exc()) + sys.exit(1) + finally: + if self.fault_management is not None: + self.fault_management.cleanup() def _amount_compute_nodes(self): services = self.nova.services.list(binary='nova-compute') @@ -92,20 +106,44 @@ class DoctorTest(object): LOG.info('not enough compute nodes, skipping doctor ' 'maintenance test') return + elif self.conf.installer.type not in ['apex', 'fuel', 'devstack']: + LOG.info('not supported installer, skipping doctor ' + 'maintenance test') + return try: + maintenance = None LOG.info('doctor maintenance test starting.......') - # TODO (tojuvone) test setup and actual test + trasport_url = self.installer.get_transport_url() + maintenance = Maintenance(trasport_url, self.conf, LOG) + maintenance.setup_maintenance(self.user) + + # wait for aodh alarms are updated in caches for event evaluator, + # sleep time should be larger than event_alarm_cache_ttl + # (default 60) + LOG.info('wait aodh for 120s.......') + time.sleep(120) + + session_id = maintenance.start_maintenance() + maintenance.wait_maintenance_complete(session_id) + + LOG.info('doctor maintenance complete.......') + except Exception as e: LOG.error('doctor maintenance test failed, Exception=%s' % e) + LOG.error(format_exc()) sys.exit(1) - # TODO (tojuvone) finally: test case specific cleanup + finally: + if maintenance is not None: + maintenance.cleanup_maintenance() def run(self): """run doctor tests""" try: LOG.info('doctor test starting.......') + # prepare common test env self.setup() + if self.conf.test_case == 'all': self.test_fault_management() self.test_maintenance() @@ -119,6 +157,7 @@ class DoctorTest(object): % function) except Exception as e: LOG.error('doctor test failed, Exception=%s' % e) + LOG.error(format_exc()) sys.exit(1) finally: self.cleanup()