yardstick/tests/unit/benchmark/runner/test_search.py

   1 # Copyright (c) 2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import time
  16
  17 import mock
  18 import unittest
  19
  20 from yardstick.benchmark.runners.search import SearchRunner
  21 from yardstick.benchmark.runners.search import SearchRunnerHelper
  22 from yardstick.common import exceptions as y_exc
  23
  24
  25 class TestSearchRunnerHelper(unittest.TestCase):
  26
  27     def test___call__(self):
  28         scenario_cfg = {
  29             'runner': {},
  30         }
  31
  32         benchmark = mock.Mock()
  33         method = getattr(benchmark(), 'my_method')
  34         helper = SearchRunnerHelper(
  35             benchmark, 'my_method', scenario_cfg, {}, mock.Mock())
  36
  37         with helper.get_benchmark_instance():
  38             helper()
  39
  40         method.assert_called_once()
  41
  42     def test___call___error(self):
  43         scenario_cfg = {
  44             'runner': {},
  45         }
  46
  47         helper = SearchRunnerHelper(
  48             mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock())
  49
  50         with self.assertRaises(RuntimeError):
  51             helper()
  52
  53     @mock.patch.object(time, 'sleep')
  54     @mock.patch.object(time, 'time')
  55     def test_is_not_done(self, mock_time, *args):
  56         scenario_cfg = {
  57             'runner': {},
  58         }
  59
  60         mock_time.side_effect = range(1000)
  61
  62         helper = SearchRunnerHelper(
  63             mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock())
  64
  65         index = -1
  66         for index in helper.is_not_done():
  67             if index >= 10:
  68                 break
  69
  70         self.assertGreaterEqual(index, 10)
  71
  72     @mock.patch.object(time, 'sleep')
  73     def test_is_not_done_immediate_stop(self, *args):
  74         scenario_cfg = {
  75             'runner': {
  76                 'run_step': '',
  77             },
  78         }
  79
  80         helper = SearchRunnerHelper(
  81             mock.Mock(), 'my_method', scenario_cfg, {}, mock.Mock())
  82
  83         index = -1
  84         for index in helper.is_not_done():
  85             if index >= 10:
  86                 break
  87
  88         self.assertEqual(index, -1)
  89
  90
  91 class TestSearchRunner(unittest.TestCase):
  92
  93     def test__worker_run_once(self):
  94         def update(*args):
  95             args[-1].update(data)
  96
  97         data = {
  98             'key1': {
  99                 'inner1': 'value1',
 100                 'done': 0,
 101             },
 102             'key2': {
 103                 'done': None,
 104             },
 105         }
 106
 107         runner = SearchRunner({})
 108         runner.worker_helper = mock.Mock(side_effect=update)
 109
 110         self.assertFalse(runner._worker_run_once('sequence 1'))
 111
 112     def test__worker_run_once_done(self):
 113         def update(*args):
 114             args[-1].update(data)
 115
 116         data = {
 117             'key1': {
 118                 'inner1': 'value1',
 119                 'done': 0,
 120             },
 121             'key2': {
 122                 'done': None,
 123             },
 124             'key3': {
 125                 'done': True,
 126             },
 127             'key4': [],
 128             'key5': 'value5',
 129         }
 130
 131         runner = SearchRunner({})
 132         runner.worker_helper = mock.Mock(side_effect=update)
 133
 134         self.assertTrue(runner._worker_run_once('sequence 1'))
 135
 136     def test__worker_run_once_assertion_error_assert(self):
 137         runner = SearchRunner({})
 138         runner.sla_action = 'assert'
 139         runner.worker_helper = mock.Mock(side_effect=y_exc.SLAValidationError)
 140
 141         with self.assertRaises(y_exc.SLAValidationError):
 142             runner._worker_run_once('sequence 1')
 143
 144     def test__worker_run_once_assertion_error_monitor(self):
 145         runner = SearchRunner({})
 146         runner.sla_action = 'monitor'
 147         runner.worker_helper = mock.Mock(side_effect=y_exc.SLAValidationError)
 148
 149         self.assertFalse(runner._worker_run_once('sequence 1'))
 150
 151     def test__worker_run_once_non_assertion_error_none(self):
 152         runner = SearchRunner({})
 153         runner.worker_helper = mock.Mock(side_effect=RuntimeError)
 154
 155         self.assertTrue(runner._worker_run_once('sequence 1'))
 156
 157     def test__worker_run_once_non_assertion_error(self):
 158         runner = SearchRunner({})
 159         runner.sla_action = 'monitor'
 160         runner.worker_helper = mock.Mock(side_effect=RuntimeError)
 161
 162         self.assertFalse(runner._worker_run_once('sequence 1'))
 163
 164     def test__worker_run(self):
 165         scenario_cfg = {
 166             'runner': {'interval': 0, 'timeout': 1},
 167         }
 168
 169         runner = SearchRunner({})
 170         runner._worker_run_once = mock.Mock(side_effect=[0, 0, 1])
 171
 172         runner._worker_run(mock.Mock(), 'my_method', scenario_cfg, {})
 173
 174     def test__worker_run_immediate_stop(self):
 175         scenario_cfg = {
 176             'runner': {
 177                 'run_step': '',
 178             },
 179         }
 180
 181         runner = SearchRunner({})
 182         runner._worker_run(mock.Mock(), 'my_method', scenario_cfg, {})
 183
 184     @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
 185     def test__run_benchmark(self, mock_multi_process):
 186         scenario_cfg = {
 187             'runner': {},
 188         }
 189
 190         runner = SearchRunner({})
 191         runner._run_benchmark(mock.Mock(), 'my_method', scenario_cfg, {})
 192         mock_multi_process.Process.assert_called_once()