yardstick/tests/unit/benchmark/runner/test_search.py

   1 # Copyright (c) 2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import time
  16
  17 import mock
  18 import unittest
  19
  20 from yardstick.tests.unit import STL_MOCKS
  21
  22 STLClient = mock.MagicMock()
  23 stl_patch = mock.patch.dict("sys.modules", STL_MOCKS)
  24 stl_patch.start()
  25
  26 if stl_patch:
  27     from yardstick.benchmark.runners.search import SearchRunner
  28     from yardstick.benchmark.runners.search import SearchRunnerHelper
  29
  30
  31 class TestSearchRunnerHelper(unittest.TestCase):
  32
  33     def test___call__(self):
  34         cls = mock.MagicMock()
  35         aborted = mock.MagicMock()
  36         scenario_cfg = {
  37             'runner': {},
  38         }
  39
  40         benchmark = cls()
  41         method = getattr(benchmark, 'my_method')
  42         helper = SearchRunnerHelper(
  43             cls, 'my_method', scenario_cfg, {}, aborted)
  44
  45         with helper.get_benchmark_instance():
  46             helper()
  47
  48         self.assertEqual(method.call_count, 1)
  49
  50     def test___call___error(self):
  51         cls = mock.MagicMock()
  52         aborted = mock.MagicMock()
  53         scenario_cfg = {
  54             'runner': {},
  55         }
  56
  57         helper = SearchRunnerHelper(
  58             cls, 'my_method', scenario_cfg, {}, aborted)
  59
  60         with self.assertRaises(RuntimeError):
  61             helper()
  62
  63     @mock.patch.object(time, 'sleep')
  64     @mock.patch.object(time, 'time')
  65     def test_is_not_done(self, mock_time, *args):
  66         cls = mock.MagicMock()
  67         aborted = mock.MagicMock()
  68         scenario_cfg = {
  69             'runner': {},
  70         }
  71
  72         mock_time.side_effect = range(1000)
  73
  74         helper = SearchRunnerHelper(
  75             cls, 'my_method', scenario_cfg, {}, aborted)
  76
  77         index = -1
  78         for index in helper.is_not_done():
  79             if index >= 10:
  80                 break
  81
  82         self.assertGreaterEqual(index, 10)
  83
  84     @mock.patch.object(time, 'sleep')
  85     def test_is_not_done_immediate_stop(self, *args):
  86         cls = mock.MagicMock()
  87         aborted = mock.MagicMock()
  88         scenario_cfg = {
  89             'runner': {
  90                 'run_step': '',
  91             },
  92         }
  93
  94         helper = SearchRunnerHelper(
  95             cls, 'my_method', scenario_cfg, {}, aborted)
  96
  97         index = -1
  98         for index in helper.is_not_done():
  99             if index >= 10:
 100                 break
 101
 102         self.assertEqual(index, -1)
 103
 104
 105 class TestSearchRunner(unittest.TestCase):
 106
 107     def test__worker_run_once(self):
 108         def update(*args):
 109             args[-1].update(data)
 110
 111         data = {
 112             'key1': {
 113                 'inner1': 'value1',
 114                 'done': 0,
 115             },
 116             'key2': {
 117                 'done': None,
 118             },
 119         }
 120
 121         runner = SearchRunner({})
 122         runner.worker_helper = mock.MagicMock(side_effect=update)
 123
 124         self.assertFalse(runner._worker_run_once('sequence 1'))
 125
 126     def test__worker_run_once_done(self):
 127         def update(*args):
 128             args[-1].update(data)
 129
 130         data = {
 131             'key1': {
 132                 'inner1': 'value1',
 133                 'done': 0,
 134             },
 135             'key2': {
 136                 'done': None,
 137             },
 138             'key3': {
 139                 'done': True,
 140             },
 141             'key4': [],
 142             'key5': 'value5',
 143         }
 144
 145         runner = SearchRunner({})
 146         runner.worker_helper = mock.MagicMock(side_effect=update)
 147
 148         self.assertTrue(runner._worker_run_once('sequence 1'))
 149
 150     def test__worker_run_once_assertion_error_assert(self):
 151         runner = SearchRunner({})
 152         runner.sla_action = 'assert'
 153         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 154
 155         with self.assertRaises(AssertionError):
 156             runner._worker_run_once('sequence 1')
 157
 158     def test__worker_run_once_assertion_error_monitor(self):
 159         runner = SearchRunner({})
 160         runner.sla_action = 'monitor'
 161         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 162
 163         self.assertFalse(runner._worker_run_once('sequence 1'))
 164
 165     def test__worker_run_once_non_assertion_error_none(self):
 166         runner = SearchRunner({})
 167         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 168
 169         self.assertTrue(runner._worker_run_once('sequence 1'))
 170
 171     def test__worker_run_once_non_assertion_error(self):
 172         runner = SearchRunner({})
 173         runner.sla_action = 'monitor'
 174         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 175
 176         self.assertFalse(runner._worker_run_once('sequence 1'))
 177
 178     def test__worker_run(self):
 179         cls = mock.MagicMock()
 180         scenario_cfg = {
 181             'runner': {'interval': 0, 'timeout': 1},
 182         }
 183
 184         runner = SearchRunner({})
 185         runner._worker_run_once = mock.MagicMock(side_effect=[0, 0, 1])
 186
 187         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 188
 189     def test__worker_run_immediate_stop(self):
 190         cls = mock.MagicMock()
 191         scenario_cfg = {
 192             'runner': {
 193                 'run_step': '',
 194             },
 195         }
 196
 197         runner = SearchRunner({})
 198         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 199
 200     @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
 201     def test__run_benchmark(self, mock_multi_process):
 202         cls = mock.MagicMock()
 203         scenario_cfg = {
 204             'runner': {},
 205         }
 206
 207         runner = SearchRunner({})
 208         runner._run_benchmark(cls, 'my_method', scenario_cfg, {})
 209         self.assertEqual(mock_multi_process.Process.call_count, 1)