Merge "Refactor "utils.parse_ini_file" testing"
[yardstick.git] / yardstick / tests / unit / benchmark / scenarios / networking / test_vsperf.py
1 # Copyright 2016 Intel Corporation.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #   http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import mock
16 import unittest
17 import subprocess
18 import yardstick.ssh as ssh
19
20 from yardstick.benchmark.scenarios.networking import vsperf
21 from yardstick import exceptions as y_exc
22
23
24 class VsperfTestCase(unittest.TestCase):
25
26     def setUp(self):
27         self.context_cfg = {
28             "host": {
29                 "ip": "10.229.47.137",
30                 "user": "ubuntu",
31                 "password": "ubuntu",
32             },
33         }
34         self.scenario_cfg = {
35             'options': {
36                 'testname': 'p2p_rfc2544_continuous',
37                 'traffic_type': 'continuous',
38                 'frame_size': '64',
39                 'bidirectional': 'True',
40                 'iload': 100,
41                 'trafficgen_port1': 'eth1',
42                 'trafficgen_port2': 'eth3',
43                 'external_bridge': 'br-ex',
44                 'conf_file': 'vsperf-yardstick.conf',
45                 'setup_script': 'setup_yardstick.sh',
46                 'test_params': 'TRAFFICGEN_DURATION=30;',
47             },
48             'sla': {
49                 'metrics': 'throughput_rx_fps',
50                 'throughput_rx_fps': 500000,
51                 'action': 'monitor',
52             }
53         }
54
55         self._mock_SSH = mock.patch.object(ssh, 'SSH')
56         self.mock_SSH = self._mock_SSH.start()
57         self.mock_SSH.from_node().execute.return_value = (0, '', '')
58
59         self._mock_subprocess_call = mock.patch.object(subprocess, 'call')
60         self.mock_subprocess_call = self._mock_subprocess_call.start()
61         self.mock_subprocess_call.return_value = None
62
63         self.addCleanup(self._stop_mock)
64
65         self.scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
66
67     def _stop_mock(self):
68         self._mock_SSH.stop()
69         self._mock_subprocess_call.stop()
70
71     def test_setup(self):
72         self.scenario.setup()
73         self.assertIsNotNone(self.scenario.client)
74         self.assertTrue(self.scenario.setup_done)
75
76     def test_setup_tg_port_not_set(self):
77         del self.scenario_cfg['options']['trafficgen_port1']
78         del self.scenario_cfg['options']['trafficgen_port2']
79         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
80         scenario.setup()
81
82         self.mock_subprocess_call.assert_called_once_with(
83             'setup_yardstick.sh setup', shell=True)
84         self.assertIsNone(scenario.tg_port1)
85         self.assertIsNone(scenario.tg_port2)
86         self.assertIsNotNone(scenario.client)
87         self.assertTrue(scenario.setup_done)
88
89     def test_setup_no_setup_script(self):
90         del self.scenario_cfg['options']['setup_script']
91         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
92         scenario.setup()
93
94         self.mock_subprocess_call.assert_has_calls(
95             (mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth1"',
96                        shell=True),
97              mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth3"',
98                        shell=True)))
99         self.assertEqual(2, self.mock_subprocess_call.call_count)
100         self.assertIsNone(scenario.setup_script)
101         self.assertIsNotNone(scenario.client)
102         self.assertTrue(scenario.setup_done)
103
104     def test_run_ok(self):
105         self.scenario.setup()
106
107         self.mock_SSH.from_node().execute.return_value = (
108             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
109
110         result = {}
111         self.scenario.run(result)
112
113         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
114
115     def test_run_ok_setup_not_done(self):
116         self.mock_SSH.from_node().execute.return_value = (
117             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
118
119         result = {}
120         self.scenario.run(result)
121
122         self.assertTrue(self.scenario.setup_done)
123         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
124
125     def test_run_failed_vsperf_execution(self):
126         self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
127                                                          (1, '', ''))
128
129         with self.assertRaises(RuntimeError):
130             self.scenario.run({})
131         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 2)
132
133     def test_run_failed_csv_report(self):
134         self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
135                                                          (0, '', ''),
136                                                          (1, '', ''))
137
138         with self.assertRaises(RuntimeError):
139             self.scenario.run({})
140         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 3)
141
142     def test_run_sla_fail(self):
143         self.mock_SSH.from_node().execute.return_value = (
144             0, 'throughput_rx_fps\r\n123456.000\r\n', '')
145
146         with self.assertRaises(y_exc.SLAValidationError) as raised:
147             self.scenario.run({})
148
149         self.assertTrue('VSPERF_throughput_rx_fps(123456.000000) < '
150                         'SLA_throughput_rx_fps(500000.000000)'
151                         in str(raised.exception))
152
153     def test_run_sla_fail_metric_not_collected(self):
154         self.mock_SSH.from_node().execute.return_value = (
155             0, 'nonexisting_metric\r\n14797660.000\r\n', '')
156
157         with self.assertRaises(y_exc.SLAValidationError) as raised:
158             self.scenario.run({})
159
160         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
161                         in str(raised.exception))
162
163     def test_run_sla_fail_metric_not_defined_in_sla(self):
164         del self.scenario_cfg['sla']['throughput_rx_fps']
165         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
166         scenario.setup()
167
168         self.mock_SSH.from_node().execute.return_value = (
169             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
170
171         with self.assertRaises(y_exc.SLAValidationError) as raised:
172             scenario.run({})
173         self.assertTrue('throughput_rx_fps is not defined in SLA'
174                         in str(raised.exception))
175
176     def test_teardown(self):
177         self.scenario.setup()
178         self.assertIsNotNone(self.scenario.client)
179         self.assertTrue(self.scenario.setup_done)
180
181         self.scenario.teardown()
182         self.assertFalse(self.scenario.setup_done)
183
184     def test_teardown_tg_port_not_set(self):
185         del self.scenario_cfg['options']['trafficgen_port1']
186         del self.scenario_cfg['options']['trafficgen_port2']
187         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
188         scenario.teardown()
189
190         self.mock_subprocess_call.assert_called_once_with(
191             'setup_yardstick.sh teardown', shell=True)
192         self.assertFalse(scenario.setup_done)
193
194     def test_teardown_no_setup_script(self):
195         del self.scenario_cfg['options']['setup_script']
196         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
197         scenario.teardown()
198
199         self.mock_subprocess_call.assert_has_calls(
200             (mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth1"',
201                        shell=True),
202              mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth3"',
203                        shell=True)))
204         self.assertEqual(2, self.mock_subprocess_call.call_count)
205         self.assertFalse(scenario.setup_done)