yardstick/vTC/apexlake/tests/benchmarking_unit_test.py

   1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 __author__ = 'vmriccox'
  16
  17
  18 import unittest
  19 import mock
  20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
  21 from experimental_framework.data_manager import DataManager
  22 from experimental_framework.deployment_unit import DeploymentUnit
  23 import experimental_framework.common as common
  24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
  25     RFC2544ThroughputBenchmark
  26
  27
  28 class DummyDataManager(DataManager):
  29
  30     def __init__(self, experiment_directory):
  31         self.experiment_directory = experiment_directory
  32         self.experiments = dict()
  33         self.new_exp_counter = 0
  34         self.add_bench_counter = 0
  35         self.close_experiment_1_counter = 0
  36         self.close_experiment_2_counter = 0
  37         self.generate_csv_counter = 0
  38
  39     def create_new_experiment(self, experiment_name, get_counter=None):
  40         if not get_counter:
  41             self.new_exp_counter += 1
  42         else:
  43             return self.new_exp_counter
  44
  45     def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
  46         if not get_counter:
  47             self.add_bench_counter += 1
  48         else:
  49             return self.add_bench_counter
  50
  51     def close_experiment(self, experiment, get_counter=None):
  52         if get_counter:
  53             return [self.close_experiment_1_counter,
  54                     self.close_experiment_2_counter]
  55         if experiment == 'VTC_base_single_vm_wait_1':
  56             self.close_experiment_1_counter += 1
  57         if experiment == 'VTC_base_single_vm_wait_2':
  58             self.close_experiment_2_counter += 1
  59
  60     def generate_result_csv_file(self, get_counter=None):
  61         if get_counter:
  62             return self.generate_csv_counter
  63         else:
  64             self.generate_csv_counter += 1
  65
  66     def add_metadata(self, experiment_name, metadata):
  67         pass
  68
  69     def add_configuration(self, experiment_name, configuration):
  70         pass
  71
  72     def add_data_points(self, experiment_name, benchmark_name, result):
  73         pass
  74
  75
  76 class Dummy_2544(RFC2544ThroughputBenchmark):
  77
  78     def __init__(self, name, params):
  79         self.name = name
  80         self.init_counter = 0
  81         self.finalize_counter = 0
  82         self.run_counter = 0
  83         self.params = params
  84
  85     def init(self, get_counter=None):
  86         if get_counter:
  87             return self.init_counter
  88         else:
  89             self.init_counter += 1
  90
  91     def finalize(self, get_counter=None):
  92         if get_counter:
  93             return self.finalize_counter
  94         else:
  95             self.finalize_counter += 1
  96
  97     def run(self, get_counter=None):
  98         if get_counter:
  99             return self.run_counter
 100         else:
 101             self.run_counter += 1
 102         return {'throughput': 10}
 103
 104
 105 class DummyDeploymentUnit(DeploymentUnit):
 106
 107     def __init__(self, openstack_credentials):
 108         pass
 109
 110     def deploy_heat_template(self, template_file, stack_name, parameters,
 111                              attempt=0):
 112         return False
 113
 114
 115 class TestBenchmarkingUnit(unittest.TestCase):
 116
 117     def setUp(self):
 118         pass
 119
 120     def tearDown(self):
 121         pass
 122
 123     @mock.patch('time.time')
 124     @mock.patch('experimental_framework.common.get_template_dir')
 125     @mock.patch('experimental_framework.data_manager.DataManager',
 126                 side_effect=DummyDataManager)
 127     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 128     @mock.patch('experimental_framework.benchmarking_unit.heat.'
 129                 'get_all_heat_templates')
 130     def test___init__(self, mock_heat, mock_dep_unit, mock_data_manager,
 131                       mock_temp_dir, mock_time):
 132         mock_heat.return_value = list()
 133         mock_time.return_value = '12345'
 134         mock_temp_dir.return_value = 'tests/data/results/'
 135         common.TEMPLATE_FILE_EXTENSION = '.ext'
 136         common.RESULT_DIR = 'tests/data/results/'
 137         heat_template_name = 'name'
 138         openstack_credentials = {
 139             'name': 'aaa',
 140             'surname': 'bbb'
 141         }
 142         heat_template_parameters = {
 143             'param_1': 'name_1',
 144             'param_2': 'name_2'
 145         }
 146         iterations = 1
 147         benchmarks = ['bench_1', 'bench_2']
 148         bu = BenchmarkingUnit(heat_template_name,
 149                               openstack_credentials,
 150                               heat_template_parameters,
 151                               iterations,
 152                               benchmarks)
 153         self.assertEqual(bu.required_benchmarks, benchmarks)
 154         bu.heat_template_parameters = heat_template_parameters
 155         mock_data_manager.assert_called_once_with('tests/data/results/12345')
 156         mock_dep_unit.assert_called_once_with(openstack_credentials)
 157         mock_heat.assert_called_once_with('tests/data/results/', '.ext')
 158
 159     @mock.patch('experimental_framework.benchmarks.'
 160                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 161     @mock.patch('time.time')
 162     @mock.patch('experimental_framework.common.get_template_dir')
 163     @mock.patch('experimental_framework.data_manager.DataManager',
 164                 side_effect=DummyDataManager)
 165     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 166     @mock.patch('experimental_framework.benchmarking_unit.'
 167                 'heat.get_all_heat_templates')
 168     def test_initialize_for_success(self, mock_heat, mock_dep_unit,
 169                                     mock_data_manager, mock_temp_dir,
 170                                     mock_time, mock_rfc2544):
 171         mock_heat.return_value = list()
 172         mock_time.return_value = '12345'
 173         mock_temp_dir.return_value = 'tests/data/test_templates/'
 174         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 175         common.RESULT_DIR = 'tests/data/results/'
 176
 177         heat_template_name = 'VTC_base_single_vm_wait_'
 178         openstack_credentials = {
 179             'name': 'aaa',
 180             'surname': 'bbb'
 181         }
 182         heat_template_parameters = {
 183             'param_1': 'name_1',
 184             'param_2': 'name_2'
 185         }
 186         iterations = 1
 187         benchmarks = [
 188             {
 189                 'name':
 190                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 191                 'params': dict()
 192             }
 193         ]
 194         bu = BenchmarkingUnit(heat_template_name,
 195                               openstack_credentials,
 196                               heat_template_parameters,
 197                               iterations,
 198                               benchmarks)
 199         self.assertEqual(bu.required_benchmarks, benchmarks)
 200         bu.heat_template_parameters = heat_template_parameters
 201         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 202                              'VTC_base_single_vm_wait_2.yaml']
 203         bu.initialize()
 204         self.assertTrue(len(bu.benchmarks) == 1)
 205         self.assertEqual(bu.benchmarks[0].__class__,
 206                          Dummy_2544)
 207         self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
 208         self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
 209
 210     @mock.patch('experimental_framework.benchmarks.'
 211                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 212     @mock.patch('time.time')
 213     @mock.patch('experimental_framework.common.get_template_dir')
 214     @mock.patch('experimental_framework.data_manager.DataManager',
 215                 side_effect=DummyDataManager)
 216     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 217     @mock.patch('experimental_framework.benchmarking_unit.'
 218                 'heat.get_all_heat_templates')
 219     def test_finalize_for_success(
 220             self, mock_heat, mock_dep_unit, mock_data_manager, mock_temp_dir,
 221             mock_time, mock_rfc2544):
 222         mock_heat.return_value = list()
 223         mock_time.return_value = '12345'
 224         mock_temp_dir.return_value = 'tests/data/test_templates/'
 225         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 226         common.RESULT_DIR = 'tests/data/results/'
 227
 228         heat_template_name = 'VTC_base_single_vm_wait_'
 229         openstack_credentials = {
 230             'name': 'aaa',
 231             'surname': 'bbb'
 232         }
 233         heat_template_parameters = {
 234             'param_1': 'name_1',
 235             'param_2': 'name_2'
 236         }
 237         iterations = 1
 238         benchmarks = [
 239             {
 240                 'name':
 241                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 242                 'params': dict()
 243             }
 244         ]
 245         bu = BenchmarkingUnit(heat_template_name,
 246                               openstack_credentials,
 247                               heat_template_parameters,
 248                               iterations,
 249                               benchmarks)
 250         bu.heat_template_parameters = heat_template_parameters
 251         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 252                              'VTC_base_single_vm_wait_2.yaml']
 253         bu.finalize()
 254         # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
 255         self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
 256
 257     @mock.patch('experimental_framework.common.push_data_influxdb')
 258     @mock.patch('experimental_framework.common.LOG')
 259     @mock.patch('experimental_framework.benchmarks.'
 260                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 261     @mock.patch('time.time')
 262     @mock.patch('experimental_framework.common.get_template_dir')
 263     @mock.patch('experimental_framework.data_manager.DataManager',
 264                 side_effect=DummyDataManager)
 265     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 266     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 267     @mock.patch('experimental_framework.benchmarking_unit.'
 268                 'heat.get_all_heat_templates')
 269     def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
 270                                         mock_dep_unit, mock_data_manager,
 271                                         mock_temp_dir, mock_time,
 272                                         mock_rfc2544, mock_log, mock_influx):
 273         mock_heat.return_value = list()
 274         mock_time.return_value = '12345'
 275         mock_temp_dir.return_value = 'tests/data/test_templates/'
 276         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 277         common.RESULT_DIR = 'tests/data/results/'
 278         common.INFLUXDB_IP = 'InfluxIP'
 279         common.INFLUXDB_PORT = '8086'
 280         common.INFLUXDB_DB_NAME = 'test_db'
 281
 282         heat_template_name = 'VTC_base_single_vm_wait_'
 283         openstack_credentials = {
 284             'name': 'aaa',
 285             'surname': 'bbb'
 286         }
 287         heat_template_parameters = {
 288             'param_1': 'name_1',
 289             'param_2': 'name_2'
 290         }
 291         iterations = 1
 292         benchmarks = [
 293             {
 294                 'name':
 295                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 296                 'params': dict()
 297             }
 298         ]
 299         bu = BenchmarkingUnit(heat_template_name,
 300                               openstack_credentials,
 301                               heat_template_parameters,
 302                               iterations,
 303                               benchmarks)
 304         bu.data_manager = DummyDataManager('tests/data/results/12345')
 305         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 306                              'VTC_base_single_vm_wait_2.yaml']
 307         bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
 308         bu.run_benchmarks()
 309         self.assertEqual(bu.benchmarks[0].init(True), 2)
 310         self.assertEqual(bu.benchmarks[0].finalize(True), 2)
 311         self.assertEqual(bu.benchmarks[0].run(True), 2)
 312         # expected_metric = \
 313         #     'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
 314         #     'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
 315         #     'param1=val1 value=10 12345000000000'
 316         # mock_influx.assert_called_with(expected_metric)
 317
 318     @mock.patch('experimental_framework.common.LOG')
 319     @mock.patch('experimental_framework.benchmarks.'
 320                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 321     @mock.patch('time.time')
 322     @mock.patch('experimental_framework.common.get_template_dir')
 323     @mock.patch('experimental_framework.data_manager.DataManager',
 324                 side_effect=DummyDataManager)
 325     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 326     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 327     @mock.patch('experimental_framework.benchmarking_unit.'
 328                 'heat.get_all_heat_templates')
 329     def test_run_benchmarks_2_for_success(
 330             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 331             mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
 332             mock_log):
 333         mock_heat.return_value = list()
 334         mock_time.return_value = '12345'
 335         mock_temp_dir.return_value = 'tests/data/test_templates/'
 336         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 337         common.RESULT_DIR = 'tests/data/results/'
 338
 339         heat_template_name = 'VTC_base_single_vm_wait_'
 340         openstack_credentials = {
 341             'name': 'aaa',
 342             'surname': 'bbb'
 343         }
 344         heat_template_parameters = {
 345             'param_1': 'name_1',
 346             'param_2': 'name_2'
 347         }
 348         iterations = 1
 349         benchmarks = [
 350             {
 351                 'name':
 352                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 353                 'params': dict()
 354             }
 355         ]
 356         bu = BenchmarkingUnit(heat_template_name,
 357                               openstack_credentials,
 358                               heat_template_parameters,
 359                               iterations,
 360                               benchmarks)
 361         bu.data_manager = DummyDataManager('tests/data/results/12345')
 362         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 363                              'VTC_base_single_vm_wait_2.yaml']
 364         bu.benchmarks = [Dummy_2544('dummy', dict())]
 365         common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
 366         bu.run_benchmarks()
 367         self.assertEqual(bu.benchmarks[0].init(True), 2)
 368         self.assertEqual(bu.benchmarks[0].finalize(True), 0)
 369         self.assertEqual(bu.benchmarks[0].run(True), 0)
 370
 371     @mock.patch('experimental_framework.common.LOG')
 372     @mock.patch('experimental_framework.benchmarks.'
 373                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 374     @mock.patch('time.time')
 375     @mock.patch('experimental_framework.common.get_template_dir')
 376     @mock.patch('experimental_framework.data_manager.DataManager',
 377                 side_effect=DummyDataManager)
 378     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 379     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 380     @mock.patch('experimental_framework.benchmarking_unit.'
 381                 'heat.get_all_heat_templates')
 382     def test_get_benchmark_name_for_success(
 383             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 384             mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
 385             mock_log):
 386         mock_heat.return_value = list()
 387         mock_time.return_value = '12345'
 388         mock_temp_dir.return_value = 'tests/data/test_templates/'
 389         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 390         common.RESULT_DIR = 'tests/data/results/'
 391
 392         heat_template_name = 'VTC_base_single_vm_wait_'
 393         openstack_credentials = {
 394             'name': 'aaa',
 395             'surname': 'bbb'
 396         }
 397         heat_template_parameters = {
 398             'param_1': 'name_1',
 399             'param_2': 'name_2'
 400         }
 401         iterations = 1
 402         benchmarks = [
 403             {
 404                 'name':
 405                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 406                 'params': dict()
 407             }
 408         ]
 409         bu = BenchmarkingUnit(heat_template_name,
 410                               openstack_credentials,
 411                               heat_template_parameters,
 412                               iterations,
 413                               benchmarks)
 414
 415         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
 416         output = bu.get_benchmark_name(
 417             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 418         self.assertEqual(expected, output)
 419
 420         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
 421         output = bu.get_benchmark_name(
 422             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 423         self.assertEqual(expected, output)
 424
 425     @mock.patch('experimental_framework.common.LOG')
 426     @mock.patch('experimental_framework.benchmarks.'
 427                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 428     @mock.patch('time.time')
 429     @mock.patch('experimental_framework.common.get_template_dir')
 430     @mock.patch('experimental_framework.data_manager.DataManager',
 431                 side_effect=DummyDataManager)
 432     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 433     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 434     @mock.patch('experimental_framework.benchmarking_unit.'
 435                 'heat.get_all_heat_templates')
 436     def test_get_required_benchmarks_for_success(
 437             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 438             mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
 439             mock_log):
 440         mock_heat.return_value = list()
 441         mock_time.return_value = '12345'
 442         mock_temp_dir.return_value = 'tests/data/test_templates/'
 443         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 444         common.RESULT_DIR = 'tests/data/results/'
 445         openstack_credentials = {
 446             'name': 'aaa',
 447             'surname': 'bbb'
 448         }
 449         heat_template_parameters = {
 450             'param_1': 'name_1',
 451             'param_2': 'name_2'
 452         }
 453         iterations = 1
 454         benchmarks = [
 455             {
 456                 'name':
 457                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 458                 'params': dict()
 459             }
 460         ]
 461         bu = BenchmarkingUnit('',
 462                               openstack_credentials,
 463                               heat_template_parameters,
 464                               iterations,
 465                               benchmarks)
 466         req_benchs = \
 467             ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
 468         output = bu.get_required_benchmarks(req_benchs)
 469         self.assertEqual(len(req_benchs), 1)
 470         self.assertEqual(output[0].__class__, Dummy_2544)