def __init__(self, conf, log):
self.conf = conf
self.log = log
+ self._notified_time = None
+
+ @property
+ def notified_time(self):
+ return self._notified_time
+
+ @notified_time.setter
+ def notified_time(self, notified_time):
+ self._notified_time = notified_time
@abc.abstractmethod
def start(self):
@app.route('/failure', methods=['POST'])
def event_posted():
- self.log.info('doctor consumer notified at %s' % time.time())
+ notified_time = time.time()
+ self.log.info('doctor consumer notified at %s' % notified_time)
+ self.consumer.notified_time = notified_time
data = json.loads(request.data.decode('utf8'))
self.log.info('sample consumer received data = %s' % data)
return 'OK'
def __init__(self, conf, log):
self.conf = conf
self.log = log
+ self._host_down_time = None
+ self._vm_down_time = None
+
+ @property
+ def host_down_time(self):
+ return self._host_down_time
+
+ @host_down_time.setter
+ def host_down_time(self, host_down_time):
+ self._host_down_time = host_down_time
+
+ @property
+ def vm_down_time(self):
+ return self._vm_down_time
+
+ @vm_down_time.setter
+ def vm_down_time(self, vm_down_time):
+ self._vm_down_time = vm_down_time
@abc.abstractmethod
def get_inspector_url(self):
@utils.run_async
def _disable_compute_host(self, hostname):
self.nova.services.force_down(hostname, 'nova-compute', True)
+
+ hostdown_time = time.time()
+ self.host_down_time = hostdown_time
self.log.info('doctor mark host(%s) down at %s'
- % (hostname, time.time()))
+ % (hostname, hostdown_time))
@utils.run_async
def _vms_reset_state(self, state, hostname):
@utils.run_async
def _vm_reset_state(nova, server, state):
nova.servers.reset_state(server, state)
+ vmdown_time = time.time()
+ self.vm_down_time = vmdown_time
self.log.info('doctor mark vm(%s) error at %s'
- % (server, time.time()))
+ % (server, vmdown_time))
thrs = []
for nova, server in zip(self.novaclients, self.servers[hostname]):
##############################################################################
import os
from os.path import isfile, join
-import random
import sys
import time
-from doctor_tests.alarm import Alarm
-from doctor_tests.common.constants import Host
-from doctor_tests.common.utils import match_rep_in_file
from doctor_tests import config
-from doctor_tests.consumer import get_consumer
from doctor_tests.identity_auth import get_identity_auth
from doctor_tests.identity_auth import get_session
from doctor_tests.image import Image
-from doctor_tests.instance import Instance
-from doctor_tests.inspector import get_inspector
from doctor_tests.installer import get_installer
import doctor_tests.logger as doctor_log
-from doctor_tests.network import Network
-from doctor_tests.monitor import get_monitor
from doctor_tests.os_clients import nova_client
-from doctor_tests.profiler_poc import main as profiler_main
-from doctor_tests.scenario.common import calculate_notification_time
-from doctor_tests.scenario.network_failure import NetworkFault
+from doctor_tests.scenario.fault_management import FaultManagement
from doctor_tests.user import User
self.conf = conf
self.image = Image(self.conf, LOG)
self.user = User(self.conf, LOG)
- self.network = Network(self.conf, LOG)
- self.instance = Instance(self.conf, LOG)
- self.alarm = Alarm(self.conf, LOG)
self.installer = get_installer(self.conf, LOG)
- self.inspector = get_inspector(self.conf, LOG)
- self.monitor = get_monitor(self.conf,
- self.inspector.get_inspector_url(),
- LOG)
- self.consumer = get_consumer(self.conf, LOG)
- self.fault = NetworkFault(self.conf, self.installer, LOG)
auth = get_identity_auth(project=self.conf.doctor_project)
self.nova = nova_client(self.conf.nova_version,
get_session(auth=auth))
- self.down_host = None
def setup(self):
# prepare the cloud env
# creating test user...
self.user.create()
- def setup_fault_management(self):
- # user settings...
- self.user.update_quota()
-
- # creating VM...
- self.network.create()
- self.instance.create()
- self.instance.wait_for_vm_launch()
-
- # creating alarm...
- self.alarm.create()
-
- # starting doctor sample components...
- # tbd tojuvone: move inspector and consumer to common setup
- # when they support updating VMs via instance.create and
- # instance.delete alarm
-
- self.inspector.start()
- self.consumer.start()
- self.down_host = self.get_host_info_for_random_vm()
- self.monitor.start(self.down_host)
-
def test_fault_management(self):
try:
LOG.info('doctor fault management test starting.......')
+ self.fault_management = \
+ FaultManagement(self.conf, self.installer, self.user, LOG)
+
# prepare test env
- self.setup_fault_management()
+ self.fault_management.setup()
# wait for aodh alarms are updated in caches for event evaluator,
# sleep time should be larger than event_alarm_cache_ttl
# injecting host failure...
# NOTE (umar) add INTERFACE_NAME logic to host injection
-
- self.fault.start(self.down_host)
+ self.fault_management.start()
time.sleep(10)
# verify the test results
# NOTE (umar) copy remote monitor.log file when monitor=collectd
- self.check_host_status(self.down_host.name, 'down')
-
- notification_time = calculate_notification_time(LogFile)
- if notification_time < 1 and notification_time > 0:
- LOG.info('doctor fault management test successfully, '
- 'notification_time=%s' % notification_time)
- else:
- LOG.error('doctor fault management test failed, '
- 'notification_time=%s' % notification_time)
- sys.exit(1)
+ self.fault_management.check_host_status('down')
+ self.fault_management.check_notification_time()
- if self.conf.profiler_type:
- LOG.info('doctor fault management test begin to run '
- 'profile.......')
- self.collect_logs()
- self.run_profiler()
except Exception as e:
LOG.error('doctor fault management test failed, '
'Exception=%s' % e)
sys.exit(1)
finally:
- self.cleanup_fault_management()
+ self.fault_management.cleanup()
def _amount_compute_nodes(self):
services = self.nova.services.list(binary='nova-compute')
finally:
self.cleanup()
- def get_host_info_for_random_vm(self):
- num = random.randint(0, self.conf.instance_count - 1)
- vm_name = "%s%d" % (self.conf.instance_basename, num)
-
- servers = {getattr(server, 'name'): server
- for server in self.nova.servers.list()}
- server = servers.get(vm_name)
- if not server:
- raise Exception('Can not find instance: vm_name(%s)' % vm_name)
- host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
- host_ip = self.installer.get_host_ip_from_hostname(host_name)
-
- LOG.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
- % (host_name, host_ip, vm_name))
- return Host(host_name, host_ip)
-
- def check_host_status(self, hostname, state):
- service = self.nova.services.list(host=hostname,
- binary='nova-compute')
- host_state = service[0].__dict__.get('state')
- assert host_state == state
-
- def unset_forced_down_hosts(self):
- if self.down_host:
- self.nova.services.force_down(self.down_host.name,
- 'nova-compute', False)
- time.sleep(2)
- self.check_host_status(self.down_host.name, 'up')
-
- def collect_logs(self):
- self.fault.get_disable_network_log()
-
- def run_profiler(self):
-
- net_down_log_file = self.fault.get_disable_network_log()
- reg = '(?<=doctor set link down at )\d+.\d+'
- linkdown = float(match_rep_in_file(reg, net_down_log_file).group(0))
-
- reg = '(.* doctor mark vm.* error at )(\d+.\d+)'
- vmdown = float(match_rep_in_file(reg, LogFile).group(2))
-
- reg = '(.* doctor mark host.* down at )(\d+.\d+)'
- hostdown = float(match_rep_in_file(reg, LogFile).group(2))
-
- reg = '(?<=doctor monitor detected at )\d+.\d+'
- detected = float(match_rep_in_file(reg, LogFile).group(0))
-
- reg = '(?<=doctor consumer notified at )\d+.\d+'
- notified = float(match_rep_in_file(reg, LogFile).group(0))
-
- # TODO(yujunz) check the actual delay to verify time sync status
- # expected ~1s delay from $trigger to $linkdown
- relative_start = linkdown
- os.environ['DOCTOR_PROFILER_T00'] = (
- str(int((linkdown - relative_start) * 1000)))
- os.environ['DOCTOR_PROFILER_T01'] = (
- str(int((detected - relative_start) * 1000)))
- os.environ['DOCTOR_PROFILER_T03'] = (
- str(int((vmdown - relative_start) * 1000)))
- os.environ['DOCTOR_PROFILER_T04'] = (
- str(int((hostdown - relative_start) * 1000)))
- os.environ['DOCTOR_PROFILER_T09'] = (
- str(int((notified - relative_start) * 1000)))
-
- profiler_main(log=LOG)
-
- def cleanup_fault_management(self):
- self.unset_forced_down_hosts()
- self.inspector.stop()
- self.monitor.stop()
- self.consumer.stop()
- self.alarm.delete()
- self.instance.delete()
- self.network.delete()
- self.fault.cleanup()
-
def cleanup(self):
self.installer.cleanup()
self.image.delete()
self.conf = conf
self.log = log
self.inspector_url = inspector_url
+ self._detected_time = None
+
+ @property
+ def detected_time(self):
+ return self._detected_time
+
+ @detected_time.setter
+ def detected_time(self, detected_time):
+ self._detected_time = detected_time
@abc.abstractmethod
def start(self, host):
sock.sendto(self.ICMP_ECHO_MESSAGE, (self.ip_addr, 0))
sock.recv(4096)
except socket.timeout:
- self.log.info("doctor monitor detected at %s" % time.time())
+ detected_time = time.time()
+ self.log.info("doctor monitor detected at %s" % detected_time)
+ self.monitor.detected_time = detected_time
self.monitor.report_error(self.hostname)
self.log.info("ping timeout, quit monitoring...")
self._stopped = True
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-from doctor_tests.common.utils import match_rep_in_file
-
-
-def calculate_notification_time(log_file):
-
- reg = '(?<=doctor monitor detected at )\d+.\d+'
- result = match_rep_in_file(reg, log_file)
- if not result:
- raise Exception('Can not match detected time')
- detected = result.group(0)
-
- reg = '(?<=doctor consumer notified at )\d+.\d+'
- result = match_rep_in_file(reg, log_file)
- if not result:
- raise Exception('Can not match notified time')
- notified = result.group(0)
-
- return float(notified) - float(detected)
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 ZTE Corporation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import random
+import time
+
+from doctor_tests.alarm import Alarm
+from doctor_tests.common.constants import Host
+from doctor_tests.common.utils import get_doctor_test_root_dir
+from doctor_tests.common.utils import match_rep_in_file
+from doctor_tests.common.utils import SSHClient
+from doctor_tests.consumer import get_consumer
+from doctor_tests.identity_auth import get_identity_auth
+from doctor_tests.identity_auth import get_session
+from doctor_tests.instance import Instance
+from doctor_tests.inspector import get_inspector
+from doctor_tests.monitor import get_monitor
+from doctor_tests.network import Network
+from doctor_tests.profiler_poc import main as profiler_main
+from doctor_tests.os_clients import nova_client
+
+
+LINK_DOWN_SCRIPT = """
+#!/bin/bash -x
+dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}')
+sleep 1
+sudo ip link set $dev down
+echo "doctor set link down at" $(date "+%s.%N")
+sleep 10
+sudo ip link set $dev up
+sleep 1
+"""
+
+
+class FaultManagement(object):
+
+ def __init__(self, conf, installer, user, log):
+ self.conf = conf
+ self.log = log
+ self.user = user
+ self.installer = installer
+ auth = get_identity_auth(project=self.conf.doctor_project)
+ self.nova = nova_client(self.conf.nova_version,
+ get_session(auth=auth))
+ self.test_dir = get_doctor_test_root_dir()
+ self.down_host = None
+ self.GetLog = False
+ self.disable_network_log = None
+ self.network = Network(self.conf, log)
+ self.instance = Instance(self.conf, log)
+ self.alarm = Alarm(self.conf, log)
+ self.inspector = get_inspector(self.conf, log)
+ self.monitor = get_monitor(self.conf,
+ self.inspector.get_inspector_url(),
+ log)
+ self.consumer = get_consumer(self.conf, log)
+
+ def setup(self):
+ self.log.info('fault management setup......')
+
+ # user settings...
+ self.user.update_quota()
+
+ # creating VM...
+ self.network.create()
+ self.instance.create()
+ self.instance.wait_for_vm_launch()
+
+ # creating alarm...
+ self.alarm.create()
+
+ # starting doctor sample components...
+ # tbd tojuvone: move inspector and consumer to common setup
+ # when they support updating VMs via instance.create and
+ # instance.delete alarm
+
+ self.inspector.start()
+ self.consumer.start()
+ self.down_host = self.get_host_info_for_random_vm()
+ self.monitor.start(self.down_host)
+
+ def start(self):
+ self.log.info('fault management start......')
+ self._set_link_down(self.down_host.ip)
+ self.log.info('fault management end......')
+
+ def cleanup(self):
+ self.log.info('fault management cleanup......')
+
+ self.get_disable_network_log()
+ self.unset_forced_down_hosts()
+ self.inspector.stop()
+ self.monitor.stop()
+ self.consumer.stop()
+ self.alarm.delete()
+ self.instance.delete()
+ self.network.delete()
+
+ def get_host_info_for_random_vm(self):
+ num = random.randint(0, self.conf.instance_count - 1)
+ vm_name = "%s%d" % (self.conf.instance_basename, num)
+
+ servers = {getattr(server, 'name'): server
+ for server in self.nova.servers.list()}
+ server = servers.get(vm_name)
+ if not server:
+ raise Exception('Can not find instance: vm_name(%s)' % vm_name)
+ host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
+ host_ip = self.installer.get_host_ip_from_hostname(host_name)
+
+ self.log.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
+ % (host_name, host_ip, vm_name))
+ return Host(host_name, host_ip)
+
+ def unset_forced_down_hosts(self):
+ if self.down_host:
+ self.nova.services.force_down(self.down_host.name,
+ 'nova-compute', False)
+ time.sleep(2)
+ self.check_host_status('up')
+
+ def check_host_status(self, state):
+ service = self.nova.services.list(host=self.down_host.name,
+ binary='nova-compute')
+ host_state = service[0].__dict__.get('state')
+ assert host_state == state
+
+ def get_disable_network_log(self):
+ if self.GetLog:
+ self.log.info('Already get the disable_netork.log '
+ 'from down_host......')
+ return self.disable_network_log
+ if self.down_host is not None:
+ client = SSHClient(
+ self.down_host.ip,
+ self.installer.node_user_name,
+ key_filename=self.installer.get_ssh_key_from_installer(),
+ look_for_keys=True,
+ log=self.log)
+
+ self.disable_network_log = \
+ '{0}/{1}'.format(self.test_dir,
+ 'disable_network.log')
+ client.scp('disable_network.log',
+ self.disable_network_log,
+ method='get')
+ self.log.info('Get the disable_netork.log from'
+ 'down_host(host_name:%s, host_ip:%s)'
+ % (self.down_host.name, self.down_host.ip))
+ self.GetLog = True
+ return self.disable_network_log
+
+ def _set_link_down(self, compute_ip):
+ file_name = '{0}/{1}'.format(self.test_dir, 'disable_network.sh')
+ with open(file_name, 'w') as file:
+ file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip))
+ client = SSHClient(
+ compute_ip,
+ self.installer.node_user_name,
+ key_filename=self.installer.get_ssh_key_from_installer(),
+ look_for_keys=True,
+ log=self.log)
+ client.scp(file_name, 'disable_network.sh')
+ command = 'bash disable_network.sh > disable_network.log 2>&1 &'
+ client.ssh(command)
+
+ def check_notification_time(self):
+ if self.consumer.notified_time is None \
+ or self.monitor.detected_time is None:
+ raise Exception('doctor fault management test failed, '
+ 'detected_time=%s, notified_time=%s'
+ % (self.monitor.detected_time,
+ self.consumer.notified_time))
+ notification_time = \
+ self.consumer.notified_time - \
+ self.monitor.detected_time
+ if notification_time < 1 and notification_time > 0:
+ self.log.info('doctor fault management test successfully,'
+ 'notification_time=%s' % notification_time)
+ else:
+ raise Exception('doctor fault management test failed, '
+ 'notification_time=%s' % notification_time)
+
+ if self.conf.profiler_type:
+ self.log.info('run doctor fault management profile.......')
+ self.run_profiler()
+
+ def run_profiler(self):
+
+ net_down_log_file = self.get_disable_network_log()
+ reg = '(?<=doctor set link down at )\d+.\d+'
+ linkdown = float(match_rep_in_file(reg, net_down_log_file).group(0))
+
+ vmdown = self.inspector.vm_down_time
+ hostdown = self.inspector.host_down_time
+ detected = self.monitor.detected_time
+ notified = self.consumer.notified_time
+
+ # TODO(yujunz) check the actual delay to verify time sync status
+ # expected ~1s delay from $trigger to $linkdown
+ relative_start = linkdown
+ os.environ['DOCTOR_PROFILER_T00'] = (
+ str(int((linkdown - relative_start) * 1000)))
+ os.environ['DOCTOR_PROFILER_T01'] = (
+ str(int((detected - relative_start) * 1000)))
+ os.environ['DOCTOR_PROFILER_T03'] = (
+ str(int((vmdown - relative_start) * 1000)))
+ os.environ['DOCTOR_PROFILER_T04'] = (
+ str(int((hostdown - relative_start) * 1000)))
+ os.environ['DOCTOR_PROFILER_T09'] = (
+ str(int((notified - relative_start) * 1000)))
+
+ profiler_main(log=self.log)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2017 ZTE Corporation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-from doctor_tests.identity_auth import get_session
-from doctor_tests.os_clients import nova_client
-from doctor_tests.common.utils import get_doctor_test_root_dir
-from doctor_tests.common.utils import SSHClient
-
-LINK_DOWN_SCRIPT = """
-#!/bin/bash -x
-dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}')
-sleep 1
-sudo ip link set $dev down
-echo "doctor set link down at" $(date "+%s.%N")
-sleep 10
-sudo ip link set $dev up
-sleep 1
-"""
-
-
-class NetworkFault(object):
-
- def __init__(self, conf, installer, log):
- self.conf = conf
- self.log = log
- self.installer = installer
- self.nova = nova_client(self.conf.nova_version, get_session())
- self.test_dir = get_doctor_test_root_dir()
- self.host = None
- self.GetLog = False
- self.disable_network_log = None
-
- def start(self, host):
- self.log.info('fault inject start......')
- self._set_link_down(host.ip)
- self.host = host
- self.log.info('fault inject end......')
-
- def cleanup(self):
- self.log.info('fault inject cleanup......')
- self.get_disable_network_log()
-
- def get_disable_network_log(self):
- if self.GetLog:
- self.log.info('Already get the disable_netork.log '
- 'from down_host......')
- return self.disable_network_log
- if self.host is not None:
- client = SSHClient(
- self.host.ip,
- self.installer.node_user_name,
- key_filename=self.installer.get_ssh_key_from_installer(),
- look_for_keys=True,
- log=self.log)
-
- self.disable_network_log = '{0}/{1}'.format(self.test_dir,
- 'disable_network.log')
- client.scp('disable_network.log',
- self.disable_network_log,
- method='get')
- self.log.info('Get the disable_netork.log from'
- 'down_host(host_name:%s, host_ip:%s)'
- % (self.host.name, self.host.ip))
- self.GetLog = True
- return self.disable_network_log
-
- def _set_link_down(self, compute_ip):
- file_name = '{0}/{1}'.format(self.test_dir, 'disable_network.sh')
- with open(file_name, 'w') as file:
- file.write(LINK_DOWN_SCRIPT.format(compute_ip=compute_ip))
- client = SSHClient(
- compute_ip,
- self.installer.node_user_name,
- key_filename=self.installer.get_ssh_key_from_installer(),
- look_for_keys=True,
- log=self.log)
- client.scp(file_name, 'disable_network.sh')
- command = 'bash disable_network.sh > disable_network.log 2>&1 &'
- client.ssh(command)
Code Review / doctor.git / commitdiff