yardstick/tests/unit/benchmark/runner/test_search.py

   1 # Copyright (c) 2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import time
  16
  17 import mock
  18 import unittest
  19
  20 from yardstick.benchmark.runners.search import SearchRunner
  21 from yardstick.benchmark.runners.search import SearchRunnerHelper
  22 from yardstick.common import exceptions as y_exc
  23
  24
  25 class TestSearchRunnerHelper(unittest.TestCase):
  26
  27     def test___call__(self):
  28         cls = mock.MagicMock()
  29         aborted = mock.MagicMock()
  30         scenario_cfg = {
  31             'runner': {},
  32         }
  33
  34         benchmark = cls()
  35         method = getattr(benchmark, 'my_method')
  36         helper = SearchRunnerHelper(
  37             cls, 'my_method', scenario_cfg, {}, aborted)
  38
  39         with helper.get_benchmark_instance():
  40             helper()
  41
  42         method.assert_called_once()
  43
  44     def test___call___error(self):
  45         cls = mock.MagicMock()
  46         aborted = mock.MagicMock()
  47         scenario_cfg = {
  48             'runner': {},
  49         }
  50
  51         helper = SearchRunnerHelper(
  52             cls, 'my_method', scenario_cfg, {}, aborted)
  53
  54         with self.assertRaises(RuntimeError):
  55             helper()
  56
  57     @mock.patch.object(time, 'sleep')
  58     @mock.patch.object(time, 'time')
  59     def test_is_not_done(self, mock_time, *args):
  60         cls = mock.MagicMock()
  61         aborted = mock.MagicMock()
  62         scenario_cfg = {
  63             'runner': {},
  64         }
  65
  66         mock_time.side_effect = range(1000)
  67
  68         helper = SearchRunnerHelper(
  69             cls, 'my_method', scenario_cfg, {}, aborted)
  70
  71         index = -1
  72         for index in helper.is_not_done():
  73             if index >= 10:
  74                 break
  75
  76         self.assertGreaterEqual(index, 10)
  77
  78     @mock.patch.object(time, 'sleep')
  79     def test_is_not_done_immediate_stop(self, *args):
  80         cls = mock.MagicMock()
  81         aborted = mock.MagicMock()
  82         scenario_cfg = {
  83             'runner': {
  84                 'run_step': '',
  85             },
  86         }
  87
  88         helper = SearchRunnerHelper(
  89             cls, 'my_method', scenario_cfg, {}, aborted)
  90
  91         index = -1
  92         for index in helper.is_not_done():
  93             if index >= 10:
  94                 break
  95
  96         self.assertEqual(index, -1)
  97
  98
  99 class TestSearchRunner(unittest.TestCase):
 100
 101     def test__worker_run_once(self):
 102         def update(*args):
 103             args[-1].update(data)
 104
 105         data = {
 106             'key1': {
 107                 'inner1': 'value1',
 108                 'done': 0,
 109             },
 110             'key2': {
 111                 'done': None,
 112             },
 113         }
 114
 115         runner = SearchRunner({})
 116         runner.worker_helper = mock.MagicMock(side_effect=update)
 117
 118         self.assertFalse(runner._worker_run_once('sequence 1'))
 119
 120     def test__worker_run_once_done(self):
 121         def update(*args):
 122             args[-1].update(data)
 123
 124         data = {
 125             'key1': {
 126                 'inner1': 'value1',
 127                 'done': 0,
 128             },
 129             'key2': {
 130                 'done': None,
 131             },
 132             'key3': {
 133                 'done': True,
 134             },
 135             'key4': [],
 136             'key5': 'value5',
 137         }
 138
 139         runner = SearchRunner({})
 140         runner.worker_helper = mock.MagicMock(side_effect=update)
 141
 142         self.assertTrue(runner._worker_run_once('sequence 1'))
 143
 144     def test__worker_run_once_assertion_error_assert(self):
 145         runner = SearchRunner({})
 146         runner.sla_action = 'assert'
 147         runner.worker_helper = mock.MagicMock(side_effect=y_exc.SLAValidationError)
 148
 149         with self.assertRaises(y_exc.SLAValidationError):
 150             runner._worker_run_once('sequence 1')
 151
 152     def test__worker_run_once_assertion_error_monitor(self):
 153         runner = SearchRunner({})
 154         runner.sla_action = 'monitor'
 155         runner.worker_helper = mock.MagicMock(side_effect=y_exc.SLAValidationError)
 156
 157         self.assertFalse(runner._worker_run_once('sequence 1'))
 158
 159     def test__worker_run_once_non_assertion_error_none(self):
 160         runner = SearchRunner({})
 161         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 162
 163         self.assertTrue(runner._worker_run_once('sequence 1'))
 164
 165     def test__worker_run_once_non_assertion_error(self):
 166         runner = SearchRunner({})
 167         runner.sla_action = 'monitor'
 168         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 169
 170         self.assertFalse(runner._worker_run_once('sequence 1'))
 171
 172     def test__worker_run(self):
 173         cls = mock.MagicMock()
 174         scenario_cfg = {
 175             'runner': {'interval': 0, 'timeout': 1},
 176         }
 177
 178         runner = SearchRunner({})
 179         runner._worker_run_once = mock.MagicMock(side_effect=[0, 0, 1])
 180
 181         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 182
 183     def test__worker_run_immediate_stop(self):
 184         cls = mock.MagicMock()
 185         scenario_cfg = {
 186             'runner': {
 187                 'run_step': '',
 188             },
 189         }
 190
 191         runner = SearchRunner({})
 192         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 193
 194     @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
 195     def test__run_benchmark(self, mock_multi_process):
 196         cls = mock.MagicMock()
 197         scenario_cfg = {
 198             'runner': {},
 199         }
 200
 201         runner = SearchRunner({})
 202         runner._run_benchmark(cls, 'my_method', scenario_cfg, {})
 203         mock_multi_process.Process.assert_called_once()