tests/unit/benchmark/runner/test_search.py

   1 # Copyright (c) 2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 #
  15
  16 from __future__ import absolute_import
  17
  18 import unittest
  19 import mock
  20
  21 from tests.unit import STL_MOCKS
  22
  23 STLClient = mock.MagicMock()
  24 stl_patch = mock.patch.dict("sys.modules", STL_MOCKS)
  25 stl_patch.start()
  26
  27 if stl_patch:
  28     from yardstick.benchmark.runners.search import SearchRunner
  29     from yardstick.benchmark.runners.search import SearchRunnerHelper
  30
  31
  32 class TestSearchRunnerHelper(unittest.TestCase):
  33
  34     def test___call__(self):
  35         cls = mock.MagicMock()
  36         aborted = mock.MagicMock()
  37         scenario_cfg = {
  38             'runner': {},
  39         }
  40
  41         benchmark = cls()
  42         method = getattr(benchmark, 'my_method')
  43         helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
  44
  45         with helper.get_benchmark_instance():
  46             helper()
  47
  48         self.assertEqual(method.call_count, 1)
  49
  50     def test___call___error(self):
  51         cls = mock.MagicMock()
  52         aborted = mock.MagicMock()
  53         scenario_cfg = {
  54             'runner': {},
  55         }
  56
  57         helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
  58
  59         with self.assertRaises(RuntimeError):
  60             helper()
  61
  62     @mock.patch('yardstick.benchmark.runners.search.time')
  63     def test_is_not_done(self, mock_time):
  64         cls = mock.MagicMock()
  65         aborted = mock.MagicMock()
  66         scenario_cfg = {
  67             'runner': {},
  68         }
  69
  70         mock_time.time.side_effect = range(1000)
  71
  72         helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
  73
  74         index = -1
  75         for index in helper.is_not_done():
  76             if index >= 10:
  77                 break
  78
  79         self.assertGreaterEqual(index, 10)
  80
  81     @mock.patch('yardstick.benchmark.runners.search.time')
  82     def test_is_not_done_immediate_stop(self, mock_time):
  83         cls = mock.MagicMock()
  84         aborted = mock.MagicMock()
  85         scenario_cfg = {
  86             'runner': {
  87                 'run_step': '',
  88             },
  89         }
  90
  91         helper = SearchRunnerHelper(cls, 'my_method', scenario_cfg, {}, aborted)
  92
  93         index = -1
  94         for index in helper.is_not_done():
  95             if index >= 10:
  96                 break
  97
  98         self.assertEqual(index, -1)
  99
 100 class TestSearchRunner(unittest.TestCase):
 101
 102     def test__worker_run_once(self):
 103         def update(*args):
 104             args[-1].update(data)
 105
 106         data = {
 107             'key1': {
 108                 'inner1': 'value1',
 109                 'done': 0,
 110             },
 111             'key2': {
 112                 'done': None,
 113             },
 114         }
 115
 116         runner = SearchRunner({})
 117         runner.worker_helper = mock.MagicMock(side_effect=update)
 118
 119         self.assertFalse(runner._worker_run_once('sequence 1'))
 120
 121     def test__worker_run_once_done(self):
 122         def update(*args):
 123             args[-1].update(data)
 124
 125         data = {
 126             'key1': {
 127                 'inner1': 'value1',
 128                 'done': 0,
 129             },
 130             'key2': {
 131                 'done': None,
 132             },
 133             'key3': {
 134                 'done': True,
 135             },
 136             'key4': [],
 137             'key5': 'value5',
 138         }
 139
 140         runner = SearchRunner({})
 141         runner.worker_helper = mock.MagicMock(side_effect=update)
 142
 143         self.assertTrue(runner._worker_run_once('sequence 1'))
 144
 145     def test__worker_run_once_assertion_error_assert(self):
 146         runner = SearchRunner({})
 147         runner.sla_action = 'assert'
 148         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 149
 150         with self.assertRaises(AssertionError):
 151             runner._worker_run_once('sequence 1')
 152
 153     def test__worker_run_once_assertion_error_monitor(self):
 154         runner = SearchRunner({})
 155         runner.sla_action = 'monitor'
 156         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 157
 158         self.assertFalse(runner._worker_run_once('sequence 1'))
 159
 160     def test__worker_run_once_non_assertion_error_none(self):
 161         runner = SearchRunner({})
 162         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 163
 164         self.assertTrue(runner._worker_run_once('sequence 1'))
 165
 166     def test__worker_run_once_non_assertion_error(self):
 167         runner = SearchRunner({})
 168         runner.sla_action = 'monitor'
 169         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 170
 171         self.assertFalse(runner._worker_run_once('sequence 1'))
 172
 173     def test__worker_run(self):
 174         cls = mock.MagicMock()
 175         scenario_cfg = {
 176             'runner': {'interval': 0, 'timeout': 1},
 177         }
 178
 179         runner = SearchRunner({})
 180         runner._worker_run_once = mock.MagicMock(side_effect=[0, 0, 1])
 181
 182         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 183
 184     def test__worker_run_immediate_stop(self):
 185         cls = mock.MagicMock()
 186         scenario_cfg = {
 187             'runner': {
 188                 'run_step': '',
 189             },
 190         }
 191
 192         runner = SearchRunner({})
 193         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 194
 195     @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
 196     def test__run_benchmark(self, mock_multi_process):
 197         cls = mock.MagicMock()
 198         scenario_cfg = {
 199             'runner': {},
 200         }
 201
 202         runner = SearchRunner({})
 203         runner._run_benchmark(cls, 'my_method', scenario_cfg, {})
 204         self.assertEqual(mock_multi_process.Process.call_count, 1)