yardstick/tests/unit/benchmark/runner/test_arithmetic.py

   1 ##############################################################################
   2 # Copyright (c) 2018 Nokia and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 import mock
  11 import unittest
  12 import multiprocessing
  13 import os
  14 import time
  15
  16 from yardstick.benchmark.runners import arithmetic
  17
  18
  19 class ArithmeticRunnerTest(unittest.TestCase):
  20     class MyMethod(object):
  21         def __init__(self):
  22             self.count = 101
  23
  24         def __call__(self, data):
  25             self.count += 1
  26             data['my_key'] = self.count
  27             return self.count
  28
  29     def setUp(self):
  30         self.scenario_cfg = {
  31             'runner': {
  32                 'interval': 0,
  33                 'iter_type': 'nested_for_loops',
  34                 'iterators': [
  35                     {
  36                         'name': 'stride',
  37                         'start': 64,
  38                         'stop': 128,
  39                         'step': 64
  40                     },
  41                     {
  42                         'name': 'size',
  43                         'start': 500,
  44                         'stop': 2000,
  45                         'step': 500
  46                     }
  47                 ]
  48             },
  49             'type': 'some_type'
  50         }
  51
  52         self.benchmark = mock.Mock()
  53         self.benchmark_cls = mock.Mock(return_value=self.benchmark)
  54
  55     def _assert_defaults__worker_process_run_setup_and_teardown(self):
  56         self.benchmark_cls.assert_called_once_with(self.scenario_cfg, {})
  57         self.benchmark.setup.assert_called_once()
  58         self.benchmark.teardown.assert_called_once()
  59
  60     @mock.patch.object(os, 'getpid')
  61     @mock.patch.object(multiprocessing, 'Process')
  62     def test__run_benchmark_called_with(self, mock_multiprocessing_process,
  63                                         mock_os_getpid):
  64         mock_os_getpid.return_value = 101
  65
  66         runner = arithmetic.ArithmeticRunner({})
  67         benchmark_cls = mock.Mock()
  68         runner._run_benchmark(benchmark_cls, 'my_method', self.scenario_cfg,
  69                               {})
  70         mock_multiprocessing_process.assert_called_once_with(
  71             name='Arithmetic-some_type-101',
  72             target=arithmetic._worker_process,
  73             args=(runner.result_queue, benchmark_cls, 'my_method',
  74                   self.scenario_cfg, {}, runner.aborted, runner.output_queue))
  75
  76     @mock.patch.object(os, 'getpid')
  77     def test__worker_process_runner_id(self, mock_os_getpid):
  78         mock_os_getpid.return_value = 101
  79
  80         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
  81                                    'my_method', self.scenario_cfg, {},
  82                                    multiprocessing.Event(), mock.Mock())
  83
  84         self.assertEqual(self.scenario_cfg['runner']['runner_id'], 101)
  85
  86     @mock.patch.object(time, 'sleep')
  87     def test__worker_process_calls_nested_for_loops(self, mock_time_sleep):
  88         self.scenario_cfg['runner']['interval'] = 99
  89
  90         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
  91                                    'my_method', self.scenario_cfg, {},
  92                                    multiprocessing.Event(), mock.Mock())
  93
  94         self._assert_defaults__worker_process_run_setup_and_teardown()
  95         self.benchmark.my_method.assert_has_calls([mock.call({})] * 8)
  96         self.assertEqual(self.benchmark.my_method.call_count, 8)
  97         mock_time_sleep.assert_has_calls([mock.call(99)] * 8)
  98         self.assertEqual(mock_time_sleep.call_count, 8)
  99
 100     @mock.patch.object(time, 'sleep')
 101     def test__worker_process_calls_tuple_loops(self, mock_time_sleep):
 102         self.scenario_cfg['runner']['interval'] = 99
 103         self.scenario_cfg['runner']['iter_type'] = 'tuple_loops'
 104
 105         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 106                                    'my_method', self.scenario_cfg, {},
 107                                    multiprocessing.Event(), mock.Mock())
 108
 109         self._assert_defaults__worker_process_run_setup_and_teardown()
 110         self.benchmark.my_method.assert_has_calls([mock.call({})] * 2)
 111         self.assertEqual(self.benchmark.my_method.call_count, 2)
 112         mock_time_sleep.assert_has_calls([mock.call(99)] * 2)
 113         self.assertEqual(mock_time_sleep.call_count, 2)
 114
 115     def test__worker_process_stored_options_nested_for_loops(self):
 116         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 117                                    'my_method', self.scenario_cfg, {},
 118                                    multiprocessing.Event(), mock.Mock())
 119
 120         self.assertDictEqual(self.scenario_cfg['options'],
 121                              {'stride': 128, 'size': 2000})
 122
 123     def test__worker_process_stored_options_tuple_loops(self):
 124         self.scenario_cfg['runner']['iter_type'] = 'tuple_loops'
 125
 126         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 127                                    'my_method', self.scenario_cfg, {},
 128                                    multiprocessing.Event(), mock.Mock())
 129
 130         self.assertDictEqual(self.scenario_cfg['options'],
 131                              {'stride': 128, 'size': 1000})
 132
 133     def test__worker_process_aborted_set_early(self):
 134         aborted = multiprocessing.Event()
 135         aborted.set()
 136         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 137                                    'my_method', self.scenario_cfg, {},
 138                                    aborted, mock.Mock())
 139
 140         self._assert_defaults__worker_process_run_setup_and_teardown()
 141         self.assertEqual(self.scenario_cfg['options'], {})
 142         self.benchmark.my_method.assert_not_called()
 143
 144     def test__worker_process_output_queue_nested_for_loops(self):
 145         self.benchmark.my_method = self.MyMethod()
 146
 147         output_queue = multiprocessing.Queue()
 148         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 149                                    'my_method', self.scenario_cfg, {},
 150                                    multiprocessing.Event(), output_queue)
 151         time.sleep(0.01)
 152
 153         self._assert_defaults__worker_process_run_setup_and_teardown()
 154         self.assertEqual(self.benchmark.my_method.count, 109)
 155         result = []
 156         while not output_queue.empty():
 157             result.append(output_queue.get())
 158         self.assertListEqual(result, [102, 103, 104, 105, 106, 107, 108, 109])
 159
 160     def test__worker_process_output_queue_tuple_loops(self):
 161         self.scenario_cfg['runner']['iter_type'] = 'tuple_loops'
 162         self.benchmark.my_method = self.MyMethod()
 163
 164         output_queue = multiprocessing.Queue()
 165         arithmetic._worker_process(mock.Mock(), self.benchmark_cls,
 166                                    'my_method', self.scenario_cfg, {},
 167                                    multiprocessing.Event(), output_queue)
 168         time.sleep(0.01)
 169
 170         self._assert_defaults__worker_process_run_setup_and_teardown()
 171         self.assertEqual(self.benchmark.my_method.count, 103)
 172         result = []
 173         while not output_queue.empty():
 174             result.append(output_queue.get())
 175         self.assertListEqual(result, [102, 103])
 176
 177     def test__worker_process_queue_nested_for_loops(self):
 178         self.benchmark.my_method = self.MyMethod()
 179
 180         queue = multiprocessing.Queue()
 181         timestamp = time.time()
 182         arithmetic._worker_process(queue, self.benchmark_cls, 'my_method',
 183                                    self.scenario_cfg, {},
 184                                    multiprocessing.Event(), mock.Mock())
 185         time.sleep(0.01)
 186
 187         self._assert_defaults__worker_process_run_setup_and_teardown()
 188         self.assertEqual(self.benchmark.my_method.count, 109)
 189         count = 0
 190         while not queue.empty():
 191             count += 1
 192             result = queue.get()
 193             self.assertEqual(result['errors'], '')
 194             self.assertEqual(result['data'], {'my_key': count + 101})
 195             self.assertEqual(result['sequence'], count)
 196             self.assertGreater(result['timestamp'], timestamp)
 197             timestamp = result['timestamp']
 198
 199     def test__worker_process_queue_tuple_loops(self):
 200         self.scenario_cfg['runner']['iter_type'] = 'tuple_loops'
 201         self.benchmark.my_method = self.MyMethod()
 202
 203         queue = multiprocessing.Queue()
 204         timestamp = time.time()
 205         arithmetic._worker_process(queue, self.benchmark_cls, 'my_method',
 206                                    self.scenario_cfg, {},
 207                                    multiprocessing.Event(), mock.Mock())
 208         time.sleep(0.01)
 209
 210         self._assert_defaults__worker_process_run_setup_and_teardown()
 211         self.assertEqual(self.benchmark.my_method.count, 103)
 212         count = 0
 213         while not queue.empty():
 214             count += 1
 215             result = queue.get()
 216             self.assertEqual(result['errors'], '')
 217             self.assertEqual(result['data'], {'my_key': count + 101})
 218             self.assertEqual(result['sequence'], count)
 219             self.assertGreater(result['timestamp'], timestamp)
 220             timestamp = result['timestamp']