yardstick/vTC/apexlake/tests/benchmarking_unit_test.py

   1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 from __future__ import absolute_import
  15 import os
  16 import unittest
  17 import mock
  18
  19 from experimental_framework import APEX_LAKE_ROOT
  20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
  21 # from experimental_framework.data_manager import DataManager
  22 from experimental_framework.deployment_unit import DeploymentUnit
  23 import experimental_framework.common as common
  24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
  25     RFC2544ThroughputBenchmark
  26
  27 __author__ = 'vmriccox'
  28
  29
  30 # class DummyDataManager(DataManager):
  31 #
  32 #     def __init__(self, experiment_directory):
  33 #         self.experiment_directory = experiment_directory
  34 #         self.experiments = dict()
  35 #         self.new_exp_counter = 0
  36 #         self.add_bench_counter = 0
  37 #         self.close_experiment_1_counter = 0
  38 #         self.close_experiment_2_counter = 0
  39 #         self.generate_csv_counter = 0
  40 #
  41 #     def create_new_experiment(self, experiment_name, get_counter=None):
  42 #         if not get_counter:
  43 #             self.new_exp_counter += 1
  44 #         else:
  45 #             return self.new_exp_counter
  46 #
  47 #   def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
  48 #         if not get_counter:
  49 #             self.add_bench_counter += 1
  50 #         else:
  51 #             return self.add_bench_counter
  52 #
  53 #     def close_experiment(self, experiment, get_counter=None):
  54 #         if get_counter:
  55 #             return [self.close_experiment_1_counter,
  56 #                     self.close_experiment_2_counter]
  57 #         if experiment == 'VTC_base_single_vm_wait_1':
  58 #             self.close_experiment_1_counter += 1
  59 #         if experiment == 'VTC_base_single_vm_wait_2':
  60 #             self.close_experiment_2_counter += 1
  61 #
  62 #     def generate_result_csv_file(self, get_counter=None):
  63 #         if get_counter:
  64 #             return self.generate_csv_counter
  65 #         else:
  66 #             self.generate_csv_counter += 1
  67 #
  68 #     def add_metadata(self, experiment_name, metadata):
  69 #         pass
  70 #
  71 #     def add_configuration(self, experiment_name, configuration):
  72 #         pass
  73 #
  74 #     def add_data_points(self, experiment_name, benchmark_name, result):
  75 #         pass
  76
  77
  78 class Dummy_2544(RFC2544ThroughputBenchmark):
  79
  80     def __init__(self, name, params):
  81         self.name = name
  82         self.init_counter = 0
  83         self.finalize_counter = 0
  84         self.run_counter = 0
  85         self.params = params
  86
  87     def init(self, get_counter=None):
  88         if get_counter:
  89             return self.init_counter
  90         else:
  91             self.init_counter += 1
  92
  93     def finalize(self, get_counter=None):
  94         if get_counter:
  95             return self.finalize_counter
  96         else:
  97             self.finalize_counter += 1
  98
  99     def run(self, get_counter=None):
 100         if get_counter:
 101             return self.run_counter
 102         else:
 103             self.run_counter += 1
 104         return {'throughput': 10}
 105
 106
 107 class DummyDeploymentUnit(DeploymentUnit):
 108
 109     def __init__(self, openstack_credentials):
 110         pass
 111
 112     def deploy_heat_template(self, template_file, stack_name, parameters,
 113                              attempt=0):
 114         return False
 115
 116
 117 class TestBenchmarkingUnit(unittest.TestCase):
 118
 119     def setUp(self):
 120         pass
 121
 122     def tearDown(self):
 123         pass
 124
 125     @mock.patch('time.time')
 126     @mock.patch('experimental_framework.common.get_template_dir')
 127     # @mock.patch('experimental_framework.data_manager.DataManager',
 128     #             side_effect=DummyDataManager)
 129     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 130     @mock.patch('experimental_framework.benchmarking_unit.heat.'
 131                 'get_all_heat_templates')
 132     def test___init__(self, mock_heat, mock_dep_unit,
 133                       # mock_data_manager,
 134                       mock_temp_dir, mock_time):
 135         mock_heat.return_value = list()
 136         mock_time.return_value = '12345'
 137         mock_temp_dir.return_value = 'tests/data/results/'
 138         common.TEMPLATE_FILE_EXTENSION = '.ext'
 139         common.RESULT_DIR = 'tests/data/results/'
 140         heat_template_name = 'name'
 141         openstack_credentials = {
 142             'name': 'aaa',
 143             'surname': 'bbb'
 144         }
 145         heat_template_parameters = {
 146             'param_1': 'name_1',
 147             'param_2': 'name_2'
 148         }
 149         iterations = 1
 150         benchmarks = ['bench_1', 'bench_2']
 151         bu = BenchmarkingUnit(heat_template_name,
 152                               openstack_credentials,
 153                               heat_template_parameters,
 154                               iterations,
 155                               benchmarks)
 156         self.assertEqual(bu.required_benchmarks, benchmarks)
 157         bu.heat_template_parameters = heat_template_parameters
 158         # mock_data_manager.assert_called_once_with('tests/data/results/12345')
 159         mock_dep_unit.assert_called_once_with(openstack_credentials)
 160         mock_heat.assert_called_once_with('tests/data/results/', '.ext')
 161
 162     @mock.patch('experimental_framework.benchmarks.'
 163                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 164     @mock.patch('time.time')
 165     @mock.patch('experimental_framework.common.get_template_dir')
 166     # @mock.patch('experimental_framework.data_manager.DataManager',
 167     #             side_effect=DummyDataManager)
 168     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 169     @mock.patch('experimental_framework.benchmarking_unit.'
 170                 'heat.get_all_heat_templates')
 171     def test_initialize_for_success(self, mock_heat, mock_dep_unit,
 172                                     # mock_data_manager,
 173                                     mock_temp_dir,
 174                                     mock_time, mock_rfc2544):
 175         mock_heat.return_value = list()
 176         mock_time.return_value = '12345'
 177         mock_temp_dir.return_value = 'tests/data/test_templates/'
 178         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 179         common.RESULT_DIR = 'tests/data/results/'
 180
 181         heat_template_name = 'VTC_base_single_vm_wait_'
 182         openstack_credentials = {
 183             'name': 'aaa',
 184             'surname': 'bbb'
 185         }
 186         heat_template_parameters = {
 187             'param_1': 'name_1',
 188             'param_2': 'name_2'
 189         }
 190         iterations = 1
 191         benchmarks = [
 192             {
 193                 'name':
 194                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 195                 'params': dict()
 196             }
 197         ]
 198         bu = BenchmarkingUnit(heat_template_name,
 199                               openstack_credentials,
 200                               heat_template_parameters,
 201                               iterations,
 202                               benchmarks)
 203         self.assertEqual(bu.required_benchmarks, benchmarks)
 204         bu.heat_template_parameters = heat_template_parameters
 205         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 206                              'VTC_base_single_vm_wait_2.yaml']
 207         bu.initialize()
 208         self.assertTrue(len(bu.benchmarks) == 1)
 209         self.assertEqual(bu.benchmarks[0].__class__,
 210                          Dummy_2544)
 211         # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
 212         # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
 213
 214     @mock.patch('experimental_framework.benchmarks.'
 215                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 216     @mock.patch('time.time')
 217     @mock.patch('experimental_framework.common.get_template_dir')
 218     # @mock.patch('experimental_framework.data_manager.DataManager',
 219     #             side_effect=DummyDataManager)
 220     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 221     @mock.patch('experimental_framework.benchmarking_unit.'
 222                 'heat.get_all_heat_templates')
 223     def test_finalize_for_success(
 224             self, mock_heat, mock_dep_unit,
 225             # mock_data_manager,
 226             mock_temp_dir, mock_time, mock_rfc2544):
 227         mock_heat.return_value = list()
 228         mock_time.return_value = '12345'
 229         mock_temp_dir.return_value = 'tests/data/test_templates/'
 230         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 231         common.RESULT_DIR = 'tests/data/results/'
 232
 233         heat_template_name = 'VTC_base_single_vm_wait_'
 234         openstack_credentials = {
 235             'name': 'aaa',
 236             'surname': 'bbb'
 237         }
 238         heat_template_parameters = {
 239             'param_1': 'name_1',
 240             'param_2': 'name_2'
 241         }
 242         iterations = 1
 243         benchmarks = [
 244             {
 245                 'name':
 246                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 247                 'params': dict()
 248             }
 249         ]
 250         bu = BenchmarkingUnit(heat_template_name,
 251                               openstack_credentials,
 252                               heat_template_parameters,
 253                               iterations,
 254                               benchmarks)
 255         bu.heat_template_parameters = heat_template_parameters
 256         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 257                              'VTC_base_single_vm_wait_2.yaml']
 258         bu.finalize()
 259         # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
 260         # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
 261
 262     @mock.patch('experimental_framework.common.push_data_influxdb')
 263     @mock.patch('experimental_framework.common.LOG')
 264     @mock.patch('experimental_framework.benchmarks.'
 265                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 266     @mock.patch('time.time')
 267     @mock.patch('experimental_framework.common.get_template_dir')
 268     # @mock.patch('experimental_framework.data_manager.DataManager',
 269     #             side_effect=DummyDataManager)
 270     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 271     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 272     @mock.patch('experimental_framework.benchmarking_unit.'
 273                 'heat.get_all_heat_templates')
 274     def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
 275                                         mock_dep_unit,
 276                                         # mock_data_manager,
 277                                         mock_temp_dir, mock_time,
 278                                         mock_rfc2544, mock_log, mock_influx):
 279         mock_heat.return_value = list()
 280         mock_time.return_value = '12345'
 281         mock_temp_dir.return_value = os.path.join(APEX_LAKE_ROOT,
 282                                                   'tests/data/test_templates/')
 283         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 284         common.RESULT_DIR = 'tests/data/results/'
 285         common.INFLUXDB_IP = 'InfluxIP'
 286         common.INFLUXDB_PORT = '8086'
 287         common.INFLUXDB_DB_NAME = 'test_db'
 288
 289         heat_template_name = 'VTC_base_single_vm_wait_'
 290         openstack_credentials = {
 291             'name': 'aaa',
 292             'surname': 'bbb'
 293         }
 294         heat_template_parameters = {
 295             'param_1': 'name_1',
 296             'param_2': 'name_2'
 297         }
 298         iterations = 1
 299         benchmarks = [
 300             {
 301                 'name':
 302                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 303                 'params': dict()
 304             }
 305         ]
 306         bu = BenchmarkingUnit(heat_template_name,
 307                               openstack_credentials,
 308                               heat_template_parameters,
 309                               iterations,
 310                               benchmarks)
 311         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 312         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 313                              'VTC_base_single_vm_wait_2.yaml']
 314         bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
 315         bu.run_benchmarks()
 316         self.assertEqual(bu.benchmarks[0].init(True), 2)
 317         self.assertEqual(bu.benchmarks[0].finalize(True), 2)
 318         self.assertEqual(bu.benchmarks[0].run(True), 2)
 319         # expected_metric = \
 320         #     'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
 321         #     'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
 322         #     'param1=val1 value=10 12345000000000'
 323         # mock_influx.assert_called_with(expected_metric)
 324
 325     @mock.patch('experimental_framework.common.LOG')
 326     @mock.patch('experimental_framework.benchmarks.'
 327                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 328     @mock.patch('time.time')
 329     @mock.patch('experimental_framework.common.get_template_dir')
 330     # @mock.patch('experimental_framework.data_manager.DataManager',
 331     #             side_effect=DummyDataManager)
 332     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 333     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 334     @mock.patch('experimental_framework.benchmarking_unit.'
 335                 'heat.get_all_heat_templates')
 336     def test_run_benchmarks_2_for_success(
 337             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 338             # mock_data_manager,
 339             mock_temp_dir, mock_time, mock_rfc2544,
 340             mock_log):
 341         mock_heat.return_value = list()
 342         mock_time.return_value = '12345'
 343         mock_temp_dir.return_value = os.path.join(APEX_LAKE_ROOT,
 344                                                   'tests/data/test_templates/')
 345         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 346         common.RESULT_DIR = 'tests/data/results/'
 347
 348         heat_template_name = 'VTC_base_single_vm_wait_'
 349         openstack_credentials = {
 350             'name': 'aaa',
 351             'surname': 'bbb'
 352         }
 353         heat_template_parameters = {
 354             'param_1': 'name_1',
 355             'param_2': 'name_2'
 356         }
 357         iterations = 1
 358         benchmarks = [
 359             {
 360                 'name':
 361                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 362                 'params': dict()
 363             }
 364         ]
 365         bu = BenchmarkingUnit(heat_template_name,
 366                               openstack_credentials,
 367                               heat_template_parameters,
 368                               iterations,
 369                               benchmarks)
 370         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 371         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 372                              'VTC_base_single_vm_wait_2.yaml']
 373         bu.benchmarks = [Dummy_2544('dummy', dict())]
 374         common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
 375         bu.run_benchmarks()
 376         self.assertEqual(bu.benchmarks[0].init(True), 2)
 377         self.assertEqual(bu.benchmarks[0].finalize(True), 0)
 378         self.assertEqual(bu.benchmarks[0].run(True), 0)
 379
 380     @mock.patch('experimental_framework.common.LOG')
 381     @mock.patch('experimental_framework.benchmarks.'
 382                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 383     @mock.patch('time.time')
 384     @mock.patch('experimental_framework.common.get_template_dir')
 385     # @mock.patch('experimental_framework.data_manager.DataManager',
 386     #             side_effect=DummyDataManager)
 387     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 388     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 389     @mock.patch('experimental_framework.benchmarking_unit.'
 390                 'heat.get_all_heat_templates')
 391     def test_get_benchmark_name_for_success(
 392             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 393             # mock_data_manager,
 394             mock_temp_dir, mock_time, mock_rfc2544,
 395             mock_log):
 396         mock_heat.return_value = list()
 397         mock_time.return_value = '12345'
 398         mock_temp_dir.return_value = 'tests/data/test_templates/'
 399         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 400         common.RESULT_DIR = 'tests/data/results/'
 401
 402         heat_template_name = 'VTC_base_single_vm_wait_'
 403         openstack_credentials = {
 404             'name': 'aaa',
 405             'surname': 'bbb'
 406         }
 407         heat_template_parameters = {
 408             'param_1': 'name_1',
 409             'param_2': 'name_2'
 410         }
 411         iterations = 1
 412         benchmarks = [
 413             {
 414                 'name':
 415                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 416                 'params': dict()
 417             }
 418         ]
 419         bu = BenchmarkingUnit(heat_template_name,
 420                               openstack_credentials,
 421                               heat_template_parameters,
 422                               iterations,
 423                               benchmarks)
 424
 425         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
 426         output = bu.get_benchmark_name(
 427             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 428         self.assertEqual(expected, output)
 429
 430         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
 431         output = bu.get_benchmark_name(
 432             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 433         self.assertEqual(expected, output)
 434
 435     @mock.patch('experimental_framework.common.LOG')
 436     @mock.patch('experimental_framework.benchmarks.'
 437                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 438     @mock.patch('time.time')
 439     @mock.patch('experimental_framework.common.get_template_dir')
 440     # @mock.patch('experimental_framework.data_manager.DataManager',
 441     #             side_effect=DummyDataManager)
 442     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 443     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 444     @mock.patch('experimental_framework.benchmarking_unit.'
 445                 'heat.get_all_heat_templates')
 446     def test_get_required_benchmarks_for_success(
 447             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 448             # mock_data_manager,
 449             mock_temp_dir, mock_time, mock_rfc2544,
 450             mock_log):
 451         mock_heat.return_value = list()
 452         mock_time.return_value = '12345'
 453         mock_temp_dir.return_value = 'tests/data/test_templates/'
 454         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 455         common.RESULT_DIR = 'tests/data/results/'
 456         openstack_credentials = {
 457             'name': 'aaa',
 458             'surname': 'bbb'
 459         }
 460         heat_template_parameters = {
 461             'param_1': 'name_1',
 462             'param_2': 'name_2'
 463         }
 464         iterations = 1
 465         benchmarks = [
 466             {
 467                 'name':
 468                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 469                 'params': dict()
 470             }
 471         ]
 472         bu = BenchmarkingUnit('',
 473                               openstack_credentials,
 474                               heat_template_parameters,
 475                               iterations,
 476                               benchmarks)
 477         req_benchs = \
 478             ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
 479         output = bu.get_required_benchmarks(req_benchs)
 480         self.assertEqual(len(req_benchs), 1)
 481         self.assertEqual(output[0].__class__, Dummy_2544)