yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py

   1 # Copyright 2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import mock
  16 import unittest
  17 import subprocess
  18 import yardstick.ssh as ssh
  19
  20 from yardstick.benchmark.scenarios.networking import vsperf
  21 from yardstick import exceptions as y_exc
  22
  23
  24 class VsperfTestCase(unittest.TestCase):
  25
  26     def setUp(self):
  27         self.context_cfg = {
  28             "host": {
  29                 "ip": "10.229.47.137",
  30                 "user": "ubuntu",
  31                 "password": "ubuntu",
  32             },
  33         }
  34         self.scenario_cfg = {
  35             'options': {
  36                 'testname': 'p2p_rfc2544_continuous',
  37                 'traffic_type': 'continuous',
  38                 'frame_size': '64',
  39                 'bidirectional': 'True',
  40                 'iload': 100,
  41                 'trafficgen_port1': 'eth1',
  42                 'trafficgen_port2': 'eth3',
  43                 'external_bridge': 'br-ex',
  44                 'conf_file': 'vsperf-yardstick.conf',
  45                 'setup_script': 'setup_yardstick.sh',
  46                 'test_params': 'TRAFFICGEN_DURATION=30;',
  47             },
  48             'sla': {
  49                 'metrics': 'throughput_rx_fps',
  50                 'throughput_rx_fps': 500000,
  51                 'action': 'monitor',
  52             }
  53         }
  54
  55         self._mock_SSH = mock.patch.object(ssh, 'SSH')
  56         self.mock_SSH = self._mock_SSH.start()
  57         self.mock_SSH.from_node().execute.return_value = (
  58             0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
  59
  60         self._mock_subprocess_call = mock.patch.object(subprocess, 'call')
  61         self.mock_subprocess_call = self._mock_subprocess_call.start()
  62         self.mock_subprocess_call.return_value = None
  63
  64         self.addCleanup(self._stop_mock)
  65
  66         self.scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  67
  68     def _stop_mock(self):
  69         self._mock_SSH.stop()
  70         self._mock_subprocess_call.stop()
  71
  72     def test_setup(self):
  73         self.scenario.setup()
  74         self.assertIsNotNone(self.scenario.client)
  75         self.assertTrue(self.scenario.setup_done)
  76
  77     def test_setup_tg_port_not_set(self):
  78         del self.scenario_cfg['options']['trafficgen_port1']
  79         del self.scenario_cfg['options']['trafficgen_port2']
  80         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  81         scenario.setup()
  82
  83         self.mock_subprocess_call.assert_called_once_with(
  84             'setup_yardstick.sh setup', shell=True)
  85         self.assertIsNone(scenario.tg_port1)
  86         self.assertIsNone(scenario.tg_port2)
  87         self.assertIsNotNone(scenario.client)
  88         self.assertTrue(scenario.setup_done)
  89
  90     def test_setup_no_setup_script(self):
  91         del self.scenario_cfg['options']['setup_script']
  92         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
  93         scenario.setup()
  94
  95         self.mock_subprocess_call.assert_has_calls(
  96             (mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth1"',
  97                        shell=True),
  98              mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth3"',
  99                        shell=True)))
 100         self.assertEqual(2, self.mock_subprocess_call.call_count)
 101         self.assertIsNone(scenario.setup_script)
 102         self.assertIsNotNone(scenario.client)
 103         self.assertTrue(scenario.setup_done)
 104
 105     def test_run_ok(self):
 106         self.scenario.setup()
 107
 108         result = {}
 109         self.scenario.run(result)
 110
 111         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 112
 113     def test_run_ok_setup_not_done(self):
 114         result = {}
 115         self.scenario.run(result)
 116
 117         self.assertTrue(self.scenario.setup_done)
 118         self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 119
 120     def test_run_ssh_command_call_counts(self):
 121         self.scenario.run({})
 122
 123         self.assertEqual(self.mock_SSH.from_node().execute.call_count, 2)
 124         self.mock_SSH.from_node().run.assert_called_once()
 125
 126     def test_run_sla_fail(self):
 127         self.mock_SSH.from_node().execute.return_value = (
 128             0, 'throughput_rx_fps\r\n123456.000\r\n', '')
 129
 130         with self.assertRaises(y_exc.SLAValidationError) as raised:
 131             self.scenario.run({})
 132
 133         self.assertTrue('VSPERF_throughput_rx_fps(123456.000000) < '
 134                         'SLA_throughput_rx_fps(500000.000000)'
 135                         in str(raised.exception))
 136
 137     def test_run_sla_fail_metric_not_collected(self):
 138         self.mock_SSH.from_node().execute.return_value = (
 139             0, 'nonexisting_metric\r\n14797660.000\r\n', '')
 140
 141         with self.assertRaises(y_exc.SLAValidationError) as raised:
 142             self.scenario.run({})
 143
 144         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
 145                         in str(raised.exception))
 146
 147     def test_run_faulty_result_csv(self):
 148         self.mock_SSH.from_node().execute.return_value = (
 149             0, 'faulty output not csv', '')
 150
 151         with self.assertRaises(y_exc.SLAValidationError) as raised:
 152             self.scenario.run({})
 153
 154         self.assertTrue('throughput_rx_fps was not collected by VSPERF'
 155                         in str(raised.exception))
 156
 157     def test_run_sla_fail_metric_not_defined_in_sla(self):
 158         del self.scenario_cfg['sla']['throughput_rx_fps']
 159         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 160         scenario.setup()
 161
 162         with self.assertRaises(y_exc.SLAValidationError) as raised:
 163             scenario.run({})
 164         self.assertTrue('throughput_rx_fps is not defined in SLA'
 165                         in str(raised.exception))
 166
 167     def test_teardown(self):
 168         self.scenario.setup()
 169         self.assertIsNotNone(self.scenario.client)
 170         self.assertTrue(self.scenario.setup_done)
 171
 172         self.scenario.teardown()
 173         self.assertFalse(self.scenario.setup_done)
 174
 175     def test_teardown_tg_port_not_set(self):
 176         del self.scenario_cfg['options']['trafficgen_port1']
 177         del self.scenario_cfg['options']['trafficgen_port2']
 178         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 179         scenario.teardown()
 180
 181         self.mock_subprocess_call.assert_called_once_with(
 182             'setup_yardstick.sh teardown', shell=True)
 183         self.assertFalse(scenario.setup_done)
 184
 185     def test_teardown_no_setup_script(self):
 186         del self.scenario_cfg['options']['setup_script']
 187         scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
 188         scenario.teardown()
 189
 190         self.mock_subprocess_call.assert_has_calls(
 191             (mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth1"',
 192                        shell=True),
 193              mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth3"',
 194                        shell=True)))
 195         self.assertEqual(2, self.mock_subprocess_call.call_count)
 196         self.assertFalse(scenario.setup_done)