--- /dev/null
+---
+# Sample test case for ha
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+ type: ServiceHA
+ options:
+ attackers:
+ - fault_type: "bare-metal-down"
+ host: node1
+
+ monitors:
+ - monitor_type: "openstack-cmd"
+ command_name: "nova image-list"
+ monitor_time: 10
+ sla:
+ max_outage_time: 5
+ - monitor_type: "openstack-cmd"
+ command_name: "heat stack-list"
+ monitor_time: 10
+ sla:
+ max_outage_time: 5
+ - monitor_type: "openstack-cmd"
+ command_name: "neutron router-list"
+ monitor_time: 10
+ sla:
+ max_outage_time: 5
+
+ nodes:
+ node1: node1.LF
+
+ runner:
+ type: Duration
+ duration: 1
+ sla:
+ outage_time: 5
+ action: monitor
+
+
+context:
+ type: Node
+ name: LF
+ file: /root/yardstick/etc/yardstick/nodes/fuel_virtual/pod.yaml
--- /dev/null
+---
+# Sample test case for ha
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+ type: ServiceHA
+ options:
+ attackers:
+ - fault_type: "kill-process"
+ process_name: "nova-api"
+ host: node1
+
+ monitors:
+ - monitor_type: "openstack-cmd"
+ command_name: "nova image-list"
+ monitor_time: 10
+ sla:
+ max_outage_time: 5
+ - monitor_type: "process"
+ process_name: "nova-api"
+ host: node1
+ monitor_time: 10
+ sla:
+ max_recover_time: 5
+
+ nodes:
+ node1: node1.LF
+
+ runner:
+ type: Duration
+ duration: 1
+ sla:
+ outage_time: 5
+ action: monitor
+
+
+context:
+ type: Node
+ name: LF
+ file: /root/yardstick/etc/yardstick/nodes/fuel_virtual/pod.yaml
--- /dev/null
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.availability.attacker.attacker_baremetal
+
+import mock
+import unittest
+
+from yardstick.benchmark.scenarios.availability.attacker import baseattacker
+from yardstick.benchmark.scenarios.availability.attacker import attacker_baremetal
+
+@mock.patch('yardstick.benchmark.scenarios.availability.attacker.attacker_baremetal.subprocess')
+class ExecuteShellTestCase(unittest.TestCase):
+
+ def test__fun_execute_shell_command_successful(self, mock_subprocess):
+ cmd = "env"
+ mock_subprocess.check_output.return_value = (0, 'unittest')
+ exitcode, output = attacker_baremetal._execute_shell_command(cmd)
+ self.assertEqual(exitcode, 0)
+
+ def test__fun_execute_shell_command_fail_cmd_exception(self, mock_subprocess):
+ cmd = "env"
+ mock_subprocess.check_output.side_effect = RuntimeError
+ exitcode, output = attacker_baremetal._execute_shell_command(cmd)
+ self.assertEqual(exitcode, -1)
+
+
+@mock.patch('yardstick.benchmark.scenarios.availability.attacker.attacker_baremetal.ssh')
+class AttackerBaremetalTestCase(unittest.TestCase):
+
+ def setUp(self):
+ host = {
+ "ipmi_ip": "10.20.0.5",
+ "ipmi_user": "root",
+ "ipmi_pwd": "123456",
+ "ip": "10.20.0.5",
+ "user": "root",
+ "key_filename": "/root/.ssh/id_rsa"
+ }
+ self.context = {"node1": host}
+ self.attacker_cfg = {
+ 'fault_type': 'bear-metal-down',
+ 'host': 'node1',
+ }
+
+ def test__attacker_baremetal_all_successful(self, mock_ssh):
+
+ ins = attacker_baremetal.BaremetalAttacker(self.attacker_cfg, self.context)
+
+ mock_ssh.SSH().execute.return_value = (0, "running", '')
+ ins.setup()
+ ins.inject_fault()
+ ins.recover()
+
+ def test__attacker_baremetal_check_failuer(self, mock_ssh):
+
+ ins = attacker_baremetal.BaremetalAttacker(self.attacker_cfg, self.context)
+ mock_ssh.SSH().execute.return_value = (0, "error check", '')
+ ins.setup()
+
+ def test__attacker_baremetal_recover_successful(self, mock_ssh):
+
+ self.attacker_cfg["jump_host"] = 'node1'
+ self.context["node1"]["pwd"] = "123456"
+ ins = attacker_baremetal.BaremetalAttacker(self.attacker_cfg, self.context)
+
+ mock_ssh.SSH().execute.return_value = (0, "running", '')
+ ins.setup()
+ ins.recover()
from yardstick.benchmark.scenarios.availability.monitor import monitor_command
@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_command.subprocess')
-class MonitorOpenstackCmdTestCase(unittest.TestCase):
+class ExecuteShellTestCase(unittest.TestCase):
def test__fun_execute_shell_command_successful(self, mock_subprocess):
cmd = "env"
exitcode, output = monitor_command._execute_shell_command(cmd)
self.assertEqual(exitcode, -1)
- def test__monitor_command_monitor_func_successful(self, mock_subprocess):
- config = {
+@mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_command.subprocess')
+class MonitorOpenstackCmdTestCase(unittest.TestCase):
+
+ def setUp(self):
+ host = {
+ "ip": "10.20.0.5",
+ "user": "root",
+ "key_filename": "/root/.ssh/id_rsa"
+ }
+ self.context = {"node1": host}
+ self.config = {
'monitor_type': 'openstack-api',
'command_name': 'nova image-list',
'monitor_time': 1,
'sla': {'max_outage_time': 5}
}
- instance = monitor_command.MonitorOpenstackCmd(config, None)
+ def test__monitor_command_monitor_func_successful(self, mock_subprocess):
+
+ instance = monitor_command.MonitorOpenstackCmd(self.config, None)
+ instance.setup()
mock_subprocess.check_output.return_value = (0, 'unittest')
ret = instance.monitor_func()
self.assertEqual(ret, True)
def test__monitor_command_monitor_func_failure(self, mock_subprocess):
mock_subprocess.check_output.return_value = (1, 'unittest')
- config = {
- 'monitor_type': 'openstack-api',
- 'command_name': 'nova image-list',
- 'monitor_time': 1,
- 'sla': {'max_outage_time': 5}
- }
- instance = monitor_command.MonitorOpenstackCmd(config, None)
-
+ instance = monitor_command.MonitorOpenstackCmd(self.config, None)
+ instance.setup()
mock_subprocess.check_output.side_effect = RuntimeError
ret = instance.monitor_func()
self.assertEqual(ret, False)
instance._result = {"outage_time": 10}
instance.verify_SLA()
+
+ @mock.patch('yardstick.benchmark.scenarios.availability.monitor.monitor_command.ssh')
+ def test__monitor_command_ssh_monitor_successful(self, mock_ssh, mock_subprocess):
+
+ self.config["host"] = "node1"
+ instance = monitor_command.MonitorOpenstackCmd(self.config, self.context)
+ instance.setup()
+ mock_ssh.SSH().execute.return_value = (0, "0", '')
+ ret = instance.monitor_func()
ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context)
- mock_ssh.SSH().execute.return_value = (0, "running", '')
+ mock_ssh.SSH().execute.return_value = (0, "1", '')
ins.setup()
ins.monitor_func()
ins._result = {"outage_time": 0}
ins = monitor_process.MonitorProcess(self.monitor_cfg, self.context)
- mock_ssh.SSH().execute.return_value = (0, "stop", '')
+ mock_ssh.SSH().execute.return_value = (0, "0", '')
ins.setup()
ins.monitor_func()
ins._result = {"outage_time": 10}
--- /dev/null
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+import traceback
+import subprocess
+import yardstick.ssh as ssh
+from baseattacker import BaseAttacker
+
+LOG = logging.getLogger(__name__)
+
+
+def _execute_shell_command(command, stdin=None):
+ '''execute shell script with error handling'''
+ exitcode = 0
+ output = []
+ try:
+ output = subprocess.check_output(command, stdin=stdin, shell=True)
+ except Exception:
+ exitcode = -1
+ output = traceback.format_exc()
+ LOG.error("exec command '%s' error:\n " % command)
+ LOG.error(traceback.format_exc())
+
+ return exitcode, output
+
+
+class BaremetalAttacker(BaseAttacker):
+
+ __attacker_type__ = 'bare-metal-down'
+
+ def setup(self):
+ LOG.debug("config:%s context:%s" % (self._config, self._context))
+ host = self._context.get(self._config['host'], None)
+ ip = host.get("ip", None)
+ user = host.get("user", "root")
+ key_filename = host.get("key_filename", "~/.ssh/id_rsa")
+
+ self.connection = ssh.SSH(user, ip, key_filename=key_filename)
+ self.connection.wait(timeout=600)
+ LOG.debug("ssh host success!")
+ self.host_ip = ip
+
+ self.ipmi_ip = host.get("ipmi_ip", None)
+ self.ipmi_user = host.get("ipmi_user", "root")
+ self.ipmi_pwd = host.get("ipmi_pwd", None)
+
+ self.fault_cfg = BaseAttacker.attacker_cfgs.get('bare-metal-down')
+ self.check_script = self.get_script_fullpath(
+ self.fault_cfg['check_script'])
+ self.recovery_script = self.get_script_fullpath(
+ self.fault_cfg['recovery_script'])
+
+ if self.check():
+ self.setup_done = True
+
+ def check(self):
+ exit_status, stdout, stderr = self.connection.execute(
+ "/bin/sh -s {0} -W 10".format(self.host_ip),
+ stdin=open(self.check_script, "r"))
+
+ LOG.debug("check ret: %s out:%s err:%s" %
+ (exit_status, stdout, stderr))
+ if not stdout or "running" not in stdout:
+ LOG.info("the host (ipmi_ip:%s) is not running!" % self.ipmi_ip)
+ return False
+
+ return True
+
+ def inject_fault(self):
+ exit_status, stdout, stderr = self.connection.execute(
+ "shutdown -h now")
+ LOG.debug("inject fault ret: %s out:%s err:%s" %
+ (exit_status, stdout, stderr))
+ if not exit_status:
+ LOG.info("inject fault success")
+
+ def recover(self):
+ jump_host_name = self._config.get("jump_host", None)
+ self.jump_connection = None
+ if jump_host_name is not None:
+ host = self._context.get(jump_host_name, None)
+ ip = host.get("ip", None)
+ user = host.get("user", "root")
+ pwd = host.get("pwd", None)
+
+ LOG.debug("jump_host ip:%s user:%s" % (ip, user))
+ self.jump_connection = ssh.SSH(user, ip, password=pwd)
+ self.jump_connection.wait(timeout=600)
+ LOG.debug("ssh jump host success!")
+
+ if self.jump_connection is not None:
+ exit_status, stdout, stderr = self.jump_connection.execute(
+ "/bin/bash -s {0} {1} {2} {3}".format(
+ self.ipmi_ip, self.ipmi_user, self.ipmi_pwd, "on"),
+ stdin=open(self.recovery_script, "r"))
+ else:
+ exit_status, stdout = _execute_shell_command(
+ "/bin/bash -s {0} {1} {2} {3}".format(
+ self.ipmi_ip, self.ipmi_user, self.ipmi_pwd, "on"),
+ stdin=open(self.recovery_script, "r"))
+
+
+def _test(): # pragma: no cover
+ host = {
+ "ipmi_ip": "10.20.0.5",
+ "ipmi_user": "root",
+ "ipmi_pwd": "123456",
+ "ip": "10.20.0.5",
+ "user": "root",
+ "key_filename": "/root/.ssh/id_rsa"
+ }
+ context = {"node1": host}
+ attacker_cfg = {
+ 'fault_type': 'bear-metal-down',
+ 'host': 'node1',
+ }
+ ins = BaremetalAttacker(attacker_cfg, context)
+ ins.setup()
+ ins.inject_fault()
+
+
+if __name__ == '__main__': # pragma: no cover
+ _test()
+++ /dev/null
----
-# sample config file for ha test
-#
-schema: "yardstick:task:0.1"
-
-kill-process:
- inject_script: scripts/stop_service.bash
- recovery_script: scripts/start_service.bash
- check_script: scripts/check_service.bash
LOG = logging.getLogger(__name__)
attacker_conf_path = pkg_resources.resource_filename(
- "yardstick.benchmark.scenarios.availability.attacker",
+ "yardstick.benchmark.scenarios.availability",
"attacker_conf.yaml")
--- /dev/null
+---
+# sample config file for ha test
+#
+schema: "yardstick:task:0.1"
+
+kill-process:
+ check_script: ha_tools/check_process_python.bash
+ inject_script: ha_tools/fault_process_kill.bash
+ recovery_script: ha_tools/start_service.bash
+
+bare-metal-down:
+ check_script: ha_tools/check_host_ping.bash
+ recovery_script: ha_tools/ipmi_power.bash
--- /dev/null
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# check wether the host is running
+
+set -e
+
+host_ip=$1
+shift
+options="$@"
+
+ping -c 1 $options $host_ip | grep ttl | wc -l
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -ne 0 ]; then
+ exit 1
+else
+ echo "running"
+fi
--- /dev/null
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# check the status of a openstack command
+
+set -e
+
+cmd=$1
+
+source /root/openrc
+
+exec $cmd
set -e
-service_name=$1
+process_name=$1
-service $service_name status
+ps aux | grep -e .*python.*$process_name.* | grep -v grep | wc -l
--- /dev/null
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Stop process by process name
+
+set -e
+
+process_name=$1
+
+killall -9 $process_name
--- /dev/null
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Stop a service and check the service is stoped
+
+set -e
+
+ipmi_ip=$1
+ipmi_user=$2
+ipmi_pwd=$3
+
+action=$4
+ipmitool -I lanplus -H $ipmi_ip -U $ipmi_user -P $ipmi_pwd power $action
LOG = logging.getLogger(__name__)
monitor_conf_path = pkg_resources.resource_filename(
- "yardstick.benchmark.scenarios.availability.monitor",
+ "yardstick.benchmark.scenarios.availability",
"monitor_conf.yaml")
+class MonitorMgr(object):
+ """docstring for MonitorMgr"""
+ def __init__(self):
+ self._monitor_list = []
+
+ def init_monitors(self, monitor_cfgs, context):
+ LOG.debug("monitorMgr config: %s" % monitor_cfgs)
+
+ for monitor_cfg in monitor_cfgs:
+ monitor_type = monitor_cfg["monitor_type"]
+ monitor_cls = BaseMonitor.get_monitor_cls(monitor_type)
+ monitor_ins = monitor_cls(monitor_cfg, context)
+
+ self._monitor_list.append(monitor_ins)
+
+ def start_monitors(self):
+ for _monotor_instace in self._monitor_list:
+ _monotor_instace.start_monitor()
+
+ def wait_monitors(self):
+ for monitor in self._monitor_list:
+ monitor.wait_monitor()
+
+ def verify_SLA(self):
+ sla_pass = True
+ for monitor in self._monitor_list:
+ sla_pass = sla_pass & monitor.verify_SLA()
+ return sla_pass
+
+
class BaseMonitor(multiprocessing.Process):
"""docstring for BaseMonitor"""
def verify_SLA(self):
pass
-
-
-class MonitorMgr(object):
- """docstring for MonitorMgr"""
- def __init__(self):
- self._monitor_list = []
-
- def init_monitors(self, monitor_cfgs, context):
- LOG.debug("monitorMgr config: %s" % monitor_cfgs)
-
- for monitor_cfg in monitor_cfgs:
- monitor_type = monitor_cfg["monitor_type"]
- monitor_cls = BaseMonitor.get_monitor_cls(monitor_type)
- monitor_ins = monitor_cls(monitor_cfg, context)
-
- self._monitor_list.append(monitor_ins)
-
- def start_monitors(self):
- for _monotor_instace in self._monitor_list:
- _monotor_instace.start_monitor()
-
- def wait_monitors(self):
- for monitor in self._monitor_list:
- monitor.wait_monitor()
-
- def verify_SLA(self):
- sla_pass = True
- for monitor in self._monitor_list:
- sla_pass = sla_pass & monitor.verify_SLA()
- return sla_pass
import logging
import subprocess
import traceback
-
+import yardstick.ssh as ssh
import basemonitor as basemonitor
LOG = logging.getLogger(__name__)
__monitor_type__ = "openstack-cmd"
+ def setup(self):
+ self.connection = None
+ node_name = self._config.get("host", None)
+ if node_name:
+ host = self._context[node_name]
+ ip = host.get("ip", None)
+ user = host.get("user", "root")
+ key_filename = host.get("key_filename", "~/.ssh/id_rsa")
+
+ self.connection = ssh.SSH(user, ip, key_filename=key_filename)
+ self.connection.wait(timeout=600)
+ LOG.debug("ssh host success!")
+
+ self.check_script = self.get_script_fullpath(
+ "ha_tools/check_openstack_cmd.bash")
+
+ self.cmd = self._config["command_name"]
+
def monitor_func(self):
- cmd = self._config["command_name"]
- exit_status, stdout = _execute_shell_command(cmd)
+ exit_status = 0
+ if self.connection:
+ exit_status, stdout, stderr = self.connection.execute(
+ "/bin/bash -s '{0}'".format(self.cmd),
+ stdin=open(self.check_script, "r"))
+
+ LOG.debug("the ret stats: %s stdout: %s stderr: %s" %
+ (exit_status, stdout, stderr))
+ else:
+ exit_status, stdout = _execute_shell_command(self.cmd)
if exit_status:
return False
return True
def _test(): # pragma: no cover
host = {
- "ip": "10.20.0.5",
+ "ip": "192.168.235.22",
"user": "root",
"key_filename": "/root/.ssh/id_rsa"
}
'monitor_type': 'openstack-cmd',
'command_name': 'nova image-list',
'monitor_time': 1,
- 'SLA': {'max_outage_time': 5}
+ 'host': 'node1',
+ 'sla': {'max_outage_time': 5}
}
monitor_configs.append(config)
p.init_monitors(monitor_configs, context)
p.start_monitors()
p.wait_monitors()
- p.verify()
+ p.verify_SLA()
if __name__ == '__main__': # pragma: no cover
+++ /dev/null
----
-# sample config file for ha test
-#
-schema: "yardstick:task:0.1"
-
-kill-process:
- inject_script: scripts/stop_service.bash
- recovery_script: scripts/start_service.bash
- check_script: scripts/check_service.bash
self.connection.wait(timeout=600)
LOG.debug("ssh host success!")
self.check_script = self.get_script_fullpath(
- "script_tools/check_service.bash")
+ "ha_tools/check_process_python.bash")
self.process_name = self._config["process_name"]
def monitor_func(self):
exit_status, stdout, stderr = self.connection.execute(
"/bin/sh -s {0}".format(self.process_name),
stdin=open(self.check_script, "r"))
+ if not stdout or int(stdout) <= 0:
+ LOG.info("the process (%s) is not running!" % self.process_name)
+ return False
- if stdout and "running" in stdout:
- LOG.info("check the envrioment success!")
- return True
- else:
- LOG.error(
- "the host envrioment is error, stdout:%s, stderr:%s" %
- (stdout, stderr))
- return False
+ return True
def verify_SLA(self):
+ LOG.debug("the _result:%s" % self._result)
outage_time = self._result.get('outage_time', None)
max_outage_time = self._config["sla"]["max_recover_time"]
if outage_time > max_outage_time:
'process_name': 'nova-api',
'host': "node1",
'monitor_time': 1,
- 'SLA': {'max_recover_time': 5}
+ 'sla': {'max_recover_time': 5}
}
monitor_configs.append(config)
p.init_monitors(monitor_configs, context)
p.start_monitors()
p.wait_monitors()
- p.verify()
+ p.verify_SLA()
if __name__ == '__main__': # pragma: no cover
Code Review / yardstick.git / commitdiff