yardstick/tests/unit/benchmark/runner/test_search.py

   1 # Copyright (c) 2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import time
  16
  17 import mock
  18 import unittest
  19
  20 from yardstick.benchmark.runners.search import SearchRunner
  21 from yardstick.benchmark.runners.search import SearchRunnerHelper
  22
  23
  24 class TestSearchRunnerHelper(unittest.TestCase):
  25
  26     def test___call__(self):
  27         cls = mock.MagicMock()
  28         aborted = mock.MagicMock()
  29         scenario_cfg = {
  30             'runner': {},
  31         }
  32
  33         benchmark = cls()
  34         method = getattr(benchmark, 'my_method')
  35         helper = SearchRunnerHelper(
  36             cls, 'my_method', scenario_cfg, {}, aborted)
  37
  38         with helper.get_benchmark_instance():
  39             helper()
  40
  41         self.assertEqual(method.call_count, 1)
  42
  43     def test___call___error(self):
  44         cls = mock.MagicMock()
  45         aborted = mock.MagicMock()
  46         scenario_cfg = {
  47             'runner': {},
  48         }
  49
  50         helper = SearchRunnerHelper(
  51             cls, 'my_method', scenario_cfg, {}, aborted)
  52
  53         with self.assertRaises(RuntimeError):
  54             helper()
  55
  56     @mock.patch.object(time, 'sleep')
  57     @mock.patch.object(time, 'time')
  58     def test_is_not_done(self, mock_time, *args):
  59         cls = mock.MagicMock()
  60         aborted = mock.MagicMock()
  61         scenario_cfg = {
  62             'runner': {},
  63         }
  64
  65         mock_time.side_effect = range(1000)
  66
  67         helper = SearchRunnerHelper(
  68             cls, 'my_method', scenario_cfg, {}, aborted)
  69
  70         index = -1
  71         for index in helper.is_not_done():
  72             if index >= 10:
  73                 break
  74
  75         self.assertGreaterEqual(index, 10)
  76
  77     @mock.patch.object(time, 'sleep')
  78     def test_is_not_done_immediate_stop(self, *args):
  79         cls = mock.MagicMock()
  80         aborted = mock.MagicMock()
  81         scenario_cfg = {
  82             'runner': {
  83                 'run_step': '',
  84             },
  85         }
  86
  87         helper = SearchRunnerHelper(
  88             cls, 'my_method', scenario_cfg, {}, aborted)
  89
  90         index = -1
  91         for index in helper.is_not_done():
  92             if index >= 10:
  93                 break
  94
  95         self.assertEqual(index, -1)
  96
  97
  98 class TestSearchRunner(unittest.TestCase):
  99
 100     def test__worker_run_once(self):
 101         def update(*args):
 102             args[-1].update(data)
 103
 104         data = {
 105             'key1': {
 106                 'inner1': 'value1',
 107                 'done': 0,
 108             },
 109             'key2': {
 110                 'done': None,
 111             },
 112         }
 113
 114         runner = SearchRunner({})
 115         runner.worker_helper = mock.MagicMock(side_effect=update)
 116
 117         self.assertFalse(runner._worker_run_once('sequence 1'))
 118
 119     def test__worker_run_once_done(self):
 120         def update(*args):
 121             args[-1].update(data)
 122
 123         data = {
 124             'key1': {
 125                 'inner1': 'value1',
 126                 'done': 0,
 127             },
 128             'key2': {
 129                 'done': None,
 130             },
 131             'key3': {
 132                 'done': True,
 133             },
 134             'key4': [],
 135             'key5': 'value5',
 136         }
 137
 138         runner = SearchRunner({})
 139         runner.worker_helper = mock.MagicMock(side_effect=update)
 140
 141         self.assertTrue(runner._worker_run_once('sequence 1'))
 142
 143     def test__worker_run_once_assertion_error_assert(self):
 144         runner = SearchRunner({})
 145         runner.sla_action = 'assert'
 146         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 147
 148         with self.assertRaises(AssertionError):
 149             runner._worker_run_once('sequence 1')
 150
 151     def test__worker_run_once_assertion_error_monitor(self):
 152         runner = SearchRunner({})
 153         runner.sla_action = 'monitor'
 154         runner.worker_helper = mock.MagicMock(side_effect=AssertionError)
 155
 156         self.assertFalse(runner._worker_run_once('sequence 1'))
 157
 158     def test__worker_run_once_non_assertion_error_none(self):
 159         runner = SearchRunner({})
 160         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 161
 162         self.assertTrue(runner._worker_run_once('sequence 1'))
 163
 164     def test__worker_run_once_non_assertion_error(self):
 165         runner = SearchRunner({})
 166         runner.sla_action = 'monitor'
 167         runner.worker_helper = mock.MagicMock(side_effect=RuntimeError)
 168
 169         self.assertFalse(runner._worker_run_once('sequence 1'))
 170
 171     def test__worker_run(self):
 172         cls = mock.MagicMock()
 173         scenario_cfg = {
 174             'runner': {'interval': 0, 'timeout': 1},
 175         }
 176
 177         runner = SearchRunner({})
 178         runner._worker_run_once = mock.MagicMock(side_effect=[0, 0, 1])
 179
 180         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 181
 182     def test__worker_run_immediate_stop(self):
 183         cls = mock.MagicMock()
 184         scenario_cfg = {
 185             'runner': {
 186                 'run_step': '',
 187             },
 188         }
 189
 190         runner = SearchRunner({})
 191         runner._worker_run(cls, 'my_method', scenario_cfg, {})
 192
 193     @mock.patch('yardstick.benchmark.runners.search.multiprocessing')
 194     def test__run_benchmark(self, mock_multi_process):
 195         cls = mock.MagicMock()
 196         scenario_cfg = {
 197             'runner': {},
 198         }
 199
 200         runner = SearchRunner({})
 201         runner._run_benchmark(cls, 'my_method', scenario_cfg, {})
 202         self.assertEqual(mock_multi_process.Process.call_count, 1)