yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py

   1 # Copyright 2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import mock
  16 import unittest
  17 import subprocess
  18 import yardstick.ssh as ssh
  19
  20 from yardstick.benchmark.scenarios.networking import vsperf
  21 from yardstick import exceptions as y_exc
  22
  23
  24 class VsperfTestCase(unittest.TestCase):
  25
  26     def setUp(self):
  27         self.context_cfg = {
  28             "host": {
  29                 "ip": "10.229.47.137",
  30                 "user": "ubuntu",
  31                 "password": "ubuntu",
  32             },
  33         }
  34         self.scenario_cfg = {
  35             'options': {
  36                 'testname': 'p2p_rfc2544_continuous',
  37                 'traffic_type': 'continuous',
  38                 'frame_size': '64',
  39                 'bidirectional': 'True',
  40                 'iload': 100,
  41                 'trafficgen_port1': 'eth1',
  42                 'trafficgen_port2': 'eth3',
  43                 'external_bridge': 'br-ex',
  44                 'conf_file': 'vsperf-yardstick.conf',
  45                 'setup_script': 'setup_yardstick.sh',
  46                 'test_params': 'TRAFFICGEN_DURATION=30;',
  47             },
  48             'sla': {
  49                 'metrics': 'throughput_rx_fps',
  50                 'throughput_rx_fps': 500000,
  51                 'action': 'monitor',
  52             }
  53         }
  54
  55         self._mock_SSH = mock.patch.object(ssh, 'SSH')
  56         self.mock_SSH = self._mock_SSH.start()
  57         self.mock_SSH.from_node().execute.return_value = (0, '', '')
  58
  59         self._mock_subprocess_call = mock.patch.object(subprocess, 'call')
  60         self.mock_subprocess_call = self._mock_subprocess_call.start()
  61         self.mock_subprocess_call.return_value = None
  62
  63         self.addCleanup(self._stop_mock)
  64
  65         self.scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  66
  67     def _stop_mock(self):
  68         self._mock_SSH.stop()
  69         self._mock_subprocess_call.stop()
  70
  71     def test_setup(self):
  72         self.scenario.setup()
  73         self.assertIsNotNone(self.scenario.client)
  74         self.assertTrue(self.scenario.setup_done)
  75
  76     def test_setup_tg_port_not_set(self):
  77         del self.scenario_cfg['options']['trafficgen_port1']
  78         del self.scenario_cfg['options']['trafficgen_port2']
  79         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  80         scenario.setup()
  81
  82         self.mock_subprocess_call.assert_called_once_with(
  83             'setup_yardstick.sh setup', shell=True)
  84         self.assertIsNone(scenario.tg_port1)
  85         self.assertIsNone(scenario.tg_port2)
  86         self.assertIsNotNone(scenario.client)
  87         self.assertTrue(scenario.setup_done)
  88
  89     def test_setup_no_setup_script(self):
  90         del self.scenario_cfg['options']['setup_script']
  91         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  92         scenario.setup()
  93
  94         self.mock_subprocess_call.assert_has_calls(
  95             (mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth1"',
  96                        shell=True),
  97              mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth3"',
  98                        shell=True)))
  99         self.assertEqual(2, self.mock_subprocess_call.call_count)
 100         self.assertIsNone(scenario.setup_script)
 101         self.assertIsNotNone(scenario.client)
 102         self.assertTrue(scenario.setup_done)
 103
 104     def test_run_ok(self):
 105         self.scenario.setup()
 106
 107         self.mock_SSH.from_node().execute.return_value = (
 108             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 109
 110         result = {}
 111         self.scenario.run(result)
 112
 113         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 114
 115     def test_run_ok_setup_not_done(self):
 116         self.mock_SSH.from_node().execute.return_value = (
 117             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 118
 119         result = {}
 120         self.scenario.run(result)
 121
 122         self.assertTrue(self.scenario.setup_done)
 123         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 124
 125     def test_run_failed_vsperf_execution(self):
 126         self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
 127                                                          (1, '', ''))
 128
 129         with self.assertRaises(RuntimeError):
 130             self.scenario.run({})
 131         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 2)
 132
 133     def test_run_failed_csv_report(self):
 134         self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
 135                                                          (0, '', ''),
 136                                                          (1, '', ''))
 137
 138         with self.assertRaises(RuntimeError):
 139             self.scenario.run({})
 140         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 3)
 141
 142     def test_run_sla_fail(self):
 143         self.mock_SSH.from_node().execute.return_value = (
 144             0, 'throughput_rx_fps\r\n123456.000\r\n', '')
 145
 146         with self.assertRaises(y_exc.SLAValidationError) as raised:
 147             self.scenario.run({})
 148
 149         self.assertTrue('VSPERF_throughput_rx_fps(123456.000000) < '
 150                         'SLA_throughput_rx_fps(500000.000000)'
 151                         in str(raised.exception))
 152
 153     def test_run_sla_fail_metric_not_collected(self):
 154         self.mock_SSH.from_node().execute.return_value = (
 155             0, 'nonexisting_metric\r\n14797660.000\r\n', '')
 156
 157         with self.assertRaises(y_exc.SLAValidationError) as raised:
 158             self.scenario.run({})
 159
 160         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
 161                         in str(raised.exception))
 162
 163     def test_run_sla_fail_metric_not_defined_in_sla(self):
 164         del self.scenario_cfg['sla']['throughput_rx_fps']
 165         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 166         scenario.setup()
 167
 168         self.mock_SSH.from_node().execute.return_value = (
 169             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 170
 171         with self.assertRaises(y_exc.SLAValidationError) as raised:
 172             scenario.run({})
 173         self.assertTrue('throughput_rx_fps is not defined in SLA'
 174                         in str(raised.exception))
 175
 176     def test_teardown(self):
 177         self.scenario.setup()
 178         self.assertIsNotNone(self.scenario.client)
 179         self.assertTrue(self.scenario.setup_done)
 180
 181         self.scenario.teardown()
 182         self.assertFalse(self.scenario.setup_done)
 183
 184     def test_teardown_tg_port_not_set(self):
 185         del self.scenario_cfg['options']['trafficgen_port1']
 186         del self.scenario_cfg['options']['trafficgen_port2']
 187         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 188         scenario.teardown()
 189
 190         self.mock_subprocess_call.assert_called_once_with(
 191             'setup_yardstick.sh teardown', shell=True)
 192         self.assertFalse(scenario.setup_done)
 193
 194     def test_teardown_no_setup_script(self):
 195         del self.scenario_cfg['options']['setup_script']
 196         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 197         scenario.teardown()
 198
 199         self.mock_subprocess_call.assert_has_calls(
 200             (mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth1"',
 201                        shell=True),
 202              mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth3"',
 203                        shell=True)))
 204         self.assertEqual(2, self.mock_subprocess_call.call_count)
 205         self.assertFalse(scenario.setup_done)