Merge "Add unit test file for ArithmeticRunner"
[yardstick.git] / yardstick / tests / unit / benchmark / scenarios / networking / test_vsperf.py
1 # Copyright 2016 Intel Corporation.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #   http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import mock
16 import unittest
17 import subprocess
18 import yardstick.ssh as ssh
19
20 from yardstick.benchmark.scenarios.networking import vsperf
21 from yardstick import exceptions as y_exc
22
23
24 class VsperfTestCase(unittest.TestCase):
25
26     def setUp(self):
27         self.context_cfg = {
28             "host": {
29                 "ip": "10.229.47.137",
30                 "user": "ubuntu",
31                 "password": "ubuntu",
32             },
33         }
34         self.scenario_cfg = {
35             'options': {
36                 'testname': 'p2p_rfc2544_continuous',
37                 'traffic_type': 'continuous',
38                 'frame_size': '64',
39                 'bidirectional': 'True',
40                 'iload': 100,
41                 'trafficgen_port1': 'eth1',
42                 'trafficgen_port2': 'eth3',
43                 'external_bridge': 'br-ex',
44                 'conf_file': 'vsperf-yardstick.conf',
45                 'setup_script': 'setup_yardstick.sh',
46                 'test_params': 'TRAFFICGEN_DURATION=30;',
47             },
48             'sla': {
49                 'metrics': 'throughput_rx_fps',
50                 'throughput_rx_fps': 500000,
51                 'action': 'monitor',
52             }
53         }
54
55         self._mock_SSH = mock.patch.object(ssh, 'SSH')
56         self.mock_SSH = self._mock_SSH.start()
57         self.mock_SSH.from_node().execute.return_value = (
58             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
59
60         self._mock_subprocess_call = mock.patch.object(subprocess, 'call')
61         self.mock_subprocess_call = self._mock_subprocess_call.start()
62         self.mock_subprocess_call.return_value = None
63
64         self.addCleanup(self._stop_mock)
65
66         self.scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
67
68     def _stop_mock(self):
69         self._mock_SSH.stop()
70         self._mock_subprocess_call.stop()
71
72     def test_setup(self):
73         self.scenario.setup()
74         self.assertIsNotNone(self.scenario.client)
75         self.assertTrue(self.scenario.setup_done)
76
77     def test_setup_tg_port_not_set(self):
78         del self.scenario_cfg['options']['trafficgen_port1']
79         del self.scenario_cfg['options']['trafficgen_port2']
80         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
81         scenario.setup()
82
83         self.mock_subprocess_call.assert_called_once_with(
84             'setup_yardstick.sh setup', shell=True)
85         self.assertIsNone(scenario.tg_port1)
86         self.assertIsNone(scenario.tg_port2)
87         self.assertIsNotNone(scenario.client)
88         self.assertTrue(scenario.setup_done)
89
90     def test_setup_no_setup_script(self):
91         del self.scenario_cfg['options']['setup_script']
92         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
93         scenario.setup()
94
95         self.mock_subprocess_call.assert_has_calls(
96             (mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth1"',
97                        shell=True),
98              mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth3"',
99                        shell=True)))
100         self.assertEqual(2, self.mock_subprocess_call.call_count)
101         self.assertIsNone(scenario.setup_script)
102         self.assertIsNotNone(scenario.client)
103         self.assertTrue(scenario.setup_done)
104
105     def test_run_ok(self):
106         self.scenario.setup()
107
108         result = {}
109         self.scenario.run(result)
110
111         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
112
113     def test_run_ok_setup_not_done(self):
114         result = {}
115         self.scenario.run(result)
116
117         self.assertTrue(self.scenario.setup_done)
118         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
119
120     def test_run_ssh_command_call_counts(self):
121         self.scenario.run({})
122
123         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 2)
124         self.mock_SSH.from_node().run.assert_called_once()
125
126     def test_run_sla_fail(self):
127         self.mock_SSH.from_node().execute.return_value = (
128             0, 'throughput_rx_fps\r\n123456.000\r\n', '')
129
130         with self.assertRaises(y_exc.SLAValidationError) as raised:
131             self.scenario.run({})
132
133         self.assertTrue('VSPERF_throughput_rx_fps(123456.000000) < '
134                         'SLA_throughput_rx_fps(500000.000000)'
135                         in str(raised.exception))
136
137     def test_run_sla_fail_metric_not_collected(self):
138         self.mock_SSH.from_node().execute.return_value = (
139             0, 'nonexisting_metric\r\n14797660.000\r\n', '')
140
141         with self.assertRaises(y_exc.SLAValidationError) as raised:
142             self.scenario.run({})
143
144         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
145                         in str(raised.exception))
146
147     def test_run_faulty_result_csv(self):
148         self.mock_SSH.from_node().execute.return_value = (
149             0, 'faulty output not csv', '')
150
151         with self.assertRaises(y_exc.SLAValidationError) as raised:
152             self.scenario.run({})
153
154         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
155                         in str(raised.exception))
156
157     def test_run_sla_fail_metric_not_defined_in_sla(self):
158         del self.scenario_cfg['sla']['throughput_rx_fps']
159         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
160         scenario.setup()
161
162         with self.assertRaises(y_exc.SLAValidationError) as raised:
163             scenario.run({})
164         self.assertTrue('throughput_rx_fps is not defined in SLA'
165                         in str(raised.exception))
166
167     def test_teardown(self):
168         self.scenario.setup()
169         self.assertIsNotNone(self.scenario.client)
170         self.assertTrue(self.scenario.setup_done)
171
172         self.scenario.teardown()
173         self.assertFalse(self.scenario.setup_done)
174
175     def test_teardown_tg_port_not_set(self):
176         del self.scenario_cfg['options']['trafficgen_port1']
177         del self.scenario_cfg['options']['trafficgen_port2']
178         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
179         scenario.teardown()
180
181         self.mock_subprocess_call.assert_called_once_with(
182             'setup_yardstick.sh teardown', shell=True)
183         self.assertFalse(scenario.setup_done)
184
185     def test_teardown_no_setup_script(self):
186         del self.scenario_cfg['options']['setup_script']
187         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
188         scenario.teardown()
189
190         self.mock_subprocess_call.assert_has_calls(
191             (mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth1"',
192                        shell=True),
193              mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth3"',
194                        shell=True)))
195         self.assertEqual(2, self.mock_subprocess_call.call_count)
196         self.assertFalse(scenario.setup_done)