yardstick/vTC/apexlake/tests/benchmarking_unit_test.py

   1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 __author__ = 'vmriccox'
  16
  17
  18 import unittest
  19 import mock
  20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
  21 # from experimental_framework.data_manager import DataManager
  22 from experimental_framework.deployment_unit import DeploymentUnit
  23 import experimental_framework.common as common
  24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
  25     RFC2544ThroughputBenchmark
  26
  27
  28 # class DummyDataManager(DataManager):
  29 #
  30 #     def __init__(self, experiment_directory):
  31 #         self.experiment_directory = experiment_directory
  32 #         self.experiments = dict()
  33 #         self.new_exp_counter = 0
  34 #         self.add_bench_counter = 0
  35 #         self.close_experiment_1_counter = 0
  36 #         self.close_experiment_2_counter = 0
  37 #         self.generate_csv_counter = 0
  38 #
  39 #     def create_new_experiment(self, experiment_name, get_counter=None):
  40 #         if not get_counter:
  41 #             self.new_exp_counter += 1
  42 #         else:
  43 #             return self.new_exp_counter
  44 #
  45 #   def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
  46 #         if not get_counter:
  47 #             self.add_bench_counter += 1
  48 #         else:
  49 #             return self.add_bench_counter
  50 #
  51 #     def close_experiment(self, experiment, get_counter=None):
  52 #         if get_counter:
  53 #             return [self.close_experiment_1_counter,
  54 #                     self.close_experiment_2_counter]
  55 #         if experiment == 'VTC_base_single_vm_wait_1':
  56 #             self.close_experiment_1_counter += 1
  57 #         if experiment == 'VTC_base_single_vm_wait_2':
  58 #             self.close_experiment_2_counter += 1
  59 #
  60 #     def generate_result_csv_file(self, get_counter=None):
  61 #         if get_counter:
  62 #             return self.generate_csv_counter
  63 #         else:
  64 #             self.generate_csv_counter += 1
  65 #
  66 #     def add_metadata(self, experiment_name, metadata):
  67 #         pass
  68 #
  69 #     def add_configuration(self, experiment_name, configuration):
  70 #         pass
  71 #
  72 #     def add_data_points(self, experiment_name, benchmark_name, result):
  73 #         pass
  74
  75
  76 class Dummy_2544(RFC2544ThroughputBenchmark):
  77
  78     def __init__(self, name, params):
  79         self.name = name
  80         self.init_counter = 0
  81         self.finalize_counter = 0
  82         self.run_counter = 0
  83         self.params = params
  84
  85     def init(self, get_counter=None):
  86         if get_counter:
  87             return self.init_counter
  88         else:
  89             self.init_counter += 1
  90
  91     def finalize(self, get_counter=None):
  92         if get_counter:
  93             return self.finalize_counter
  94         else:
  95             self.finalize_counter += 1
  96
  97     def run(self, get_counter=None):
  98         if get_counter:
  99             return self.run_counter
 100         else:
 101             self.run_counter += 1
 102         return {'throughput': 10}
 103
 104
 105 class DummyDeploymentUnit(DeploymentUnit):
 106
 107     def __init__(self, openstack_credentials):
 108         pass
 109
 110     def deploy_heat_template(self, template_file, stack_name, parameters,
 111                              attempt=0):
 112         return False
 113
 114
 115 class TestBenchmarkingUnit(unittest.TestCase):
 116
 117     def setUp(self):
 118         pass
 119
 120     def tearDown(self):
 121         pass
 122
 123     @mock.patch('time.time')
 124     @mock.patch('experimental_framework.common.get_template_dir')
 125     # @mock.patch('experimental_framework.data_manager.DataManager',
 126     #             side_effect=DummyDataManager)
 127     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 128     @mock.patch('experimental_framework.benchmarking_unit.heat.'
 129                 'get_all_heat_templates')
 130     def test___init__(self, mock_heat, mock_dep_unit,
 131                       # mock_data_manager,
 132                       mock_temp_dir, mock_time):
 133         mock_heat.return_value = list()
 134         mock_time.return_value = '12345'
 135         mock_temp_dir.return_value = 'tests/data/results/'
 136         common.TEMPLATE_FILE_EXTENSION = '.ext'
 137         common.RESULT_DIR = 'tests/data/results/'
 138         heat_template_name = 'name'
 139         openstack_credentials = {
 140             'name': 'aaa',
 141             'surname': 'bbb'
 142         }
 143         heat_template_parameters = {
 144             'param_1': 'name_1',
 145             'param_2': 'name_2'
 146         }
 147         iterations = 1
 148         benchmarks = ['bench_1', 'bench_2']
 149         bu = BenchmarkingUnit(heat_template_name,
 150                               openstack_credentials,
 151                               heat_template_parameters,
 152                               iterations,
 153                               benchmarks)
 154         self.assertEqual(bu.required_benchmarks, benchmarks)
 155         bu.heat_template_parameters = heat_template_parameters
 156         # mock_data_manager.assert_called_once_with('tests/data/results/12345')
 157         mock_dep_unit.assert_called_once_with(openstack_credentials)
 158         mock_heat.assert_called_once_with('tests/data/results/', '.ext')
 159
 160     @mock.patch('experimental_framework.benchmarks.'
 161                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 162     @mock.patch('time.time')
 163     @mock.patch('experimental_framework.common.get_template_dir')
 164     # @mock.patch('experimental_framework.data_manager.DataManager',
 165     #             side_effect=DummyDataManager)
 166     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 167     @mock.patch('experimental_framework.benchmarking_unit.'
 168                 'heat.get_all_heat_templates')
 169     def test_initialize_for_success(self, mock_heat, mock_dep_unit,
 170                                     # mock_data_manager,
 171                                     mock_temp_dir,
 172                                     mock_time, mock_rfc2544):
 173         mock_heat.return_value = list()
 174         mock_time.return_value = '12345'
 175         mock_temp_dir.return_value = 'tests/data/test_templates/'
 176         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 177         common.RESULT_DIR = 'tests/data/results/'
 178
 179         heat_template_name = 'VTC_base_single_vm_wait_'
 180         openstack_credentials = {
 181             'name': 'aaa',
 182             'surname': 'bbb'
 183         }
 184         heat_template_parameters = {
 185             'param_1': 'name_1',
 186             'param_2': 'name_2'
 187         }
 188         iterations = 1
 189         benchmarks = [
 190             {
 191                 'name':
 192                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 193                 'params': dict()
 194             }
 195         ]
 196         bu = BenchmarkingUnit(heat_template_name,
 197                               openstack_credentials,
 198                               heat_template_parameters,
 199                               iterations,
 200                               benchmarks)
 201         self.assertEqual(bu.required_benchmarks, benchmarks)
 202         bu.heat_template_parameters = heat_template_parameters
 203         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 204                              'VTC_base_single_vm_wait_2.yaml']
 205         bu.initialize()
 206         self.assertTrue(len(bu.benchmarks) == 1)
 207         self.assertEqual(bu.benchmarks[0].__class__,
 208                          Dummy_2544)
 209         # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
 210         # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
 211
 212     @mock.patch('experimental_framework.benchmarks.'
 213                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 214     @mock.patch('time.time')
 215     @mock.patch('experimental_framework.common.get_template_dir')
 216     # @mock.patch('experimental_framework.data_manager.DataManager',
 217     #             side_effect=DummyDataManager)
 218     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 219     @mock.patch('experimental_framework.benchmarking_unit.'
 220                 'heat.get_all_heat_templates')
 221     def test_finalize_for_success(
 222             self, mock_heat, mock_dep_unit,
 223             # mock_data_manager,
 224             mock_temp_dir, mock_time, mock_rfc2544):
 225         mock_heat.return_value = list()
 226         mock_time.return_value = '12345'
 227         mock_temp_dir.return_value = 'tests/data/test_templates/'
 228         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 229         common.RESULT_DIR = 'tests/data/results/'
 230
 231         heat_template_name = 'VTC_base_single_vm_wait_'
 232         openstack_credentials = {
 233             'name': 'aaa',
 234             'surname': 'bbb'
 235         }
 236         heat_template_parameters = {
 237             'param_1': 'name_1',
 238             'param_2': 'name_2'
 239         }
 240         iterations = 1
 241         benchmarks = [
 242             {
 243                 'name':
 244                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 245                 'params': dict()
 246             }
 247         ]
 248         bu = BenchmarkingUnit(heat_template_name,
 249                               openstack_credentials,
 250                               heat_template_parameters,
 251                               iterations,
 252                               benchmarks)
 253         bu.heat_template_parameters = heat_template_parameters
 254         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 255                              'VTC_base_single_vm_wait_2.yaml']
 256         bu.finalize()
 257         # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
 258         # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
 259
 260     @mock.patch('experimental_framework.common.push_data_influxdb')
 261     @mock.patch('experimental_framework.common.LOG')
 262     @mock.patch('experimental_framework.benchmarks.'
 263                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 264     @mock.patch('time.time')
 265     @mock.patch('experimental_framework.common.get_template_dir')
 266     # @mock.patch('experimental_framework.data_manager.DataManager',
 267     #             side_effect=DummyDataManager)
 268     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 269     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 270     @mock.patch('experimental_framework.benchmarking_unit.'
 271                 'heat.get_all_heat_templates')
 272     def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
 273                                         mock_dep_unit,
 274                                         # mock_data_manager,
 275                                         mock_temp_dir, mock_time,
 276                                         mock_rfc2544, mock_log, mock_influx):
 277         mock_heat.return_value = list()
 278         mock_time.return_value = '12345'
 279         mock_temp_dir.return_value = 'tests/data/test_templates/'
 280         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 281         common.RESULT_DIR = 'tests/data/results/'
 282         common.INFLUXDB_IP = 'InfluxIP'
 283         common.INFLUXDB_PORT = '8086'
 284         common.INFLUXDB_DB_NAME = 'test_db'
 285
 286         heat_template_name = 'VTC_base_single_vm_wait_'
 287         openstack_credentials = {
 288             'name': 'aaa',
 289             'surname': 'bbb'
 290         }
 291         heat_template_parameters = {
 292             'param_1': 'name_1',
 293             'param_2': 'name_2'
 294         }
 295         iterations = 1
 296         benchmarks = [
 297             {
 298                 'name':
 299                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 300                 'params': dict()
 301             }
 302         ]
 303         bu = BenchmarkingUnit(heat_template_name,
 304                               openstack_credentials,
 305                               heat_template_parameters,
 306                               iterations,
 307                               benchmarks)
 308         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 309         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 310                              'VTC_base_single_vm_wait_2.yaml']
 311         bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
 312         bu.run_benchmarks()
 313         self.assertEqual(bu.benchmarks[0].init(True), 2)
 314         self.assertEqual(bu.benchmarks[0].finalize(True), 2)
 315         self.assertEqual(bu.benchmarks[0].run(True), 2)
 316         # expected_metric = \
 317         #     'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
 318         #     'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
 319         #     'param1=val1 value=10 12345000000000'
 320         # mock_influx.assert_called_with(expected_metric)
 321
 322     @mock.patch('experimental_framework.common.LOG')
 323     @mock.patch('experimental_framework.benchmarks.'
 324                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 325     @mock.patch('time.time')
 326     @mock.patch('experimental_framework.common.get_template_dir')
 327     # @mock.patch('experimental_framework.data_manager.DataManager',
 328     #             side_effect=DummyDataManager)
 329     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 330     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 331     @mock.patch('experimental_framework.benchmarking_unit.'
 332                 'heat.get_all_heat_templates')
 333     def test_run_benchmarks_2_for_success(
 334             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 335             # mock_data_manager,
 336             mock_temp_dir, mock_time, mock_rfc2544,
 337             mock_log):
 338         mock_heat.return_value = list()
 339         mock_time.return_value = '12345'
 340         mock_temp_dir.return_value = 'tests/data/test_templates/'
 341         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 342         common.RESULT_DIR = 'tests/data/results/'
 343
 344         heat_template_name = 'VTC_base_single_vm_wait_'
 345         openstack_credentials = {
 346             'name': 'aaa',
 347             'surname': 'bbb'
 348         }
 349         heat_template_parameters = {
 350             'param_1': 'name_1',
 351             'param_2': 'name_2'
 352         }
 353         iterations = 1
 354         benchmarks = [
 355             {
 356                 'name':
 357                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 358                 'params': dict()
 359             }
 360         ]
 361         bu = BenchmarkingUnit(heat_template_name,
 362                               openstack_credentials,
 363                               heat_template_parameters,
 364                               iterations,
 365                               benchmarks)
 366         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 367         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 368                              'VTC_base_single_vm_wait_2.yaml']
 369         bu.benchmarks = [Dummy_2544('dummy', dict())]
 370         common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
 371         bu.run_benchmarks()
 372         self.assertEqual(bu.benchmarks[0].init(True), 2)
 373         self.assertEqual(bu.benchmarks[0].finalize(True), 0)
 374         self.assertEqual(bu.benchmarks[0].run(True), 0)
 375
 376     @mock.patch('experimental_framework.common.LOG')
 377     @mock.patch('experimental_framework.benchmarks.'
 378                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 379     @mock.patch('time.time')
 380     @mock.patch('experimental_framework.common.get_template_dir')
 381     # @mock.patch('experimental_framework.data_manager.DataManager',
 382     #             side_effect=DummyDataManager)
 383     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 384     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 385     @mock.patch('experimental_framework.benchmarking_unit.'
 386                 'heat.get_all_heat_templates')
 387     def test_get_benchmark_name_for_success(
 388             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 389             # mock_data_manager,
 390             mock_temp_dir, mock_time, mock_rfc2544,
 391             mock_log):
 392         mock_heat.return_value = list()
 393         mock_time.return_value = '12345'
 394         mock_temp_dir.return_value = 'tests/data/test_templates/'
 395         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 396         common.RESULT_DIR = 'tests/data/results/'
 397
 398         heat_template_name = 'VTC_base_single_vm_wait_'
 399         openstack_credentials = {
 400             'name': 'aaa',
 401             'surname': 'bbb'
 402         }
 403         heat_template_parameters = {
 404             'param_1': 'name_1',
 405             'param_2': 'name_2'
 406         }
 407         iterations = 1
 408         benchmarks = [
 409             {
 410                 'name':
 411                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 412                 'params': dict()
 413             }
 414         ]
 415         bu = BenchmarkingUnit(heat_template_name,
 416                               openstack_credentials,
 417                               heat_template_parameters,
 418                               iterations,
 419                               benchmarks)
 420
 421         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
 422         output = bu.get_benchmark_name(
 423             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 424         self.assertEqual(expected, output)
 425
 426         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
 427         output = bu.get_benchmark_name(
 428             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 429         self.assertEqual(expected, output)
 430
 431     @mock.patch('experimental_framework.common.LOG')
 432     @mock.patch('experimental_framework.benchmarks.'
 433                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 434     @mock.patch('time.time')
 435     @mock.patch('experimental_framework.common.get_template_dir')
 436     # @mock.patch('experimental_framework.data_manager.DataManager',
 437     #             side_effect=DummyDataManager)
 438     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 439     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 440     @mock.patch('experimental_framework.benchmarking_unit.'
 441                 'heat.get_all_heat_templates')
 442     def test_get_required_benchmarks_for_success(
 443             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 444             # mock_data_manager,
 445             mock_temp_dir, mock_time, mock_rfc2544,
 446             mock_log):
 447         mock_heat.return_value = list()
 448         mock_time.return_value = '12345'
 449         mock_temp_dir.return_value = 'tests/data/test_templates/'
 450         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 451         common.RESULT_DIR = 'tests/data/results/'
 452         openstack_credentials = {
 453             'name': 'aaa',
 454             'surname': 'bbb'
 455         }
 456         heat_template_parameters = {
 457             'param_1': 'name_1',
 458             'param_2': 'name_2'
 459         }
 460         iterations = 1
 461         benchmarks = [
 462             {
 463                 'name':
 464                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 465                 'params': dict()
 466             }
 467         ]
 468         bu = BenchmarkingUnit('',
 469                               openstack_credentials,
 470                               heat_template_parameters,
 471                               iterations,
 472                               benchmarks)
 473         req_benchs = \
 474             ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
 475         output = bu.get_required_benchmarks(req_benchs)
 476         self.assertEqual(len(req_benchs), 1)
 477         self.assertEqual(output[0].__class__, Dummy_2544)