yardstick/vTC/apexlake/tests/benchmarking_unit_test.py

   1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import unittest
  16 import mock
  17 from experimental_framework.benchmarking_unit import BenchmarkingUnit
  18 # from experimental_framework.data_manager import DataManager
  19 from experimental_framework.deployment_unit import DeploymentUnit
  20 import experimental_framework.common as common
  21 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
  22     RFC2544ThroughputBenchmark
  23
  24 __author__ = 'vmriccox'
  25
  26
  27 # class DummyDataManager(DataManager):
  28 #
  29 #     def __init__(self, experiment_directory):
  30 #         self.experiment_directory = experiment_directory
  31 #         self.experiments = dict()
  32 #         self.new_exp_counter = 0
  33 #         self.add_bench_counter = 0
  34 #         self.close_experiment_1_counter = 0
  35 #         self.close_experiment_2_counter = 0
  36 #         self.generate_csv_counter = 0
  37 #
  38 #     def create_new_experiment(self, experiment_name, get_counter=None):
  39 #         if not get_counter:
  40 #             self.new_exp_counter += 1
  41 #         else:
  42 #             return self.new_exp_counter
  43 #
  44 #   def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
  45 #         if not get_counter:
  46 #             self.add_bench_counter += 1
  47 #         else:
  48 #             return self.add_bench_counter
  49 #
  50 #     def close_experiment(self, experiment, get_counter=None):
  51 #         if get_counter:
  52 #             return [self.close_experiment_1_counter,
  53 #                     self.close_experiment_2_counter]
  54 #         if experiment == 'VTC_base_single_vm_wait_1':
  55 #             self.close_experiment_1_counter += 1
  56 #         if experiment == 'VTC_base_single_vm_wait_2':
  57 #             self.close_experiment_2_counter += 1
  58 #
  59 #     def generate_result_csv_file(self, get_counter=None):
  60 #         if get_counter:
  61 #             return self.generate_csv_counter
  62 #         else:
  63 #             self.generate_csv_counter += 1
  64 #
  65 #     def add_metadata(self, experiment_name, metadata):
  66 #         pass
  67 #
  68 #     def add_configuration(self, experiment_name, configuration):
  69 #         pass
  70 #
  71 #     def add_data_points(self, experiment_name, benchmark_name, result):
  72 #         pass
  73
  74
  75 class Dummy_2544(RFC2544ThroughputBenchmark):
  76
  77     def __init__(self, name, params):
  78         self.name = name
  79         self.init_counter = 0
  80         self.finalize_counter = 0
  81         self.run_counter = 0
  82         self.params = params
  83
  84     def init(self, get_counter=None):
  85         if get_counter:
  86             return self.init_counter
  87         else:
  88             self.init_counter += 1
  89
  90     def finalize(self, get_counter=None):
  91         if get_counter:
  92             return self.finalize_counter
  93         else:
  94             self.finalize_counter += 1
  95
  96     def run(self, get_counter=None):
  97         if get_counter:
  98             return self.run_counter
  99         else:
 100             self.run_counter += 1
 101         return {'throughput': 10}
 102
 103
 104 class DummyDeploymentUnit(DeploymentUnit):
 105
 106     def __init__(self, openstack_credentials):
 107         pass
 108
 109     def deploy_heat_template(self, template_file, stack_name, parameters,
 110                              attempt=0):
 111         return False
 112
 113
 114 class TestBenchmarkingUnit(unittest.TestCase):
 115
 116     def setUp(self):
 117         pass
 118
 119     def tearDown(self):
 120         pass
 121
 122     @mock.patch('time.time')
 123     @mock.patch('experimental_framework.common.get_template_dir')
 124     # @mock.patch('experimental_framework.data_manager.DataManager',
 125     #             side_effect=DummyDataManager)
 126     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 127     @mock.patch('experimental_framework.benchmarking_unit.heat.'
 128                 'get_all_heat_templates')
 129     def test___init__(self, mock_heat, mock_dep_unit,
 130                       # mock_data_manager,
 131                       mock_temp_dir, mock_time):
 132         mock_heat.return_value = list()
 133         mock_time.return_value = '12345'
 134         mock_temp_dir.return_value = 'tests/data/results/'
 135         common.TEMPLATE_FILE_EXTENSION = '.ext'
 136         common.RESULT_DIR = 'tests/data/results/'
 137         heat_template_name = 'name'
 138         openstack_credentials = {
 139             'name': 'aaa',
 140             'surname': 'bbb'
 141         }
 142         heat_template_parameters = {
 143             'param_1': 'name_1',
 144             'param_2': 'name_2'
 145         }
 146         iterations = 1
 147         benchmarks = ['bench_1', 'bench_2']
 148         bu = BenchmarkingUnit(heat_template_name,
 149                               openstack_credentials,
 150                               heat_template_parameters,
 151                               iterations,
 152                               benchmarks)
 153         self.assertEqual(bu.required_benchmarks, benchmarks)
 154         bu.heat_template_parameters = heat_template_parameters
 155         # mock_data_manager.assert_called_once_with('tests/data/results/12345')
 156         mock_dep_unit.assert_called_once_with(openstack_credentials)
 157         mock_heat.assert_called_once_with('tests/data/results/', '.ext')
 158
 159     @mock.patch('experimental_framework.benchmarks.'
 160                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 161     @mock.patch('time.time')
 162     @mock.patch('experimental_framework.common.get_template_dir')
 163     # @mock.patch('experimental_framework.data_manager.DataManager',
 164     #             side_effect=DummyDataManager)
 165     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 166     @mock.patch('experimental_framework.benchmarking_unit.'
 167                 'heat.get_all_heat_templates')
 168     def test_initialize_for_success(self, mock_heat, mock_dep_unit,
 169                                     # mock_data_manager,
 170                                     mock_temp_dir,
 171                                     mock_time, mock_rfc2544):
 172         mock_heat.return_value = list()
 173         mock_time.return_value = '12345'
 174         mock_temp_dir.return_value = 'tests/data/test_templates/'
 175         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 176         common.RESULT_DIR = 'tests/data/results/'
 177
 178         heat_template_name = 'VTC_base_single_vm_wait_'
 179         openstack_credentials = {
 180             'name': 'aaa',
 181             'surname': 'bbb'
 182         }
 183         heat_template_parameters = {
 184             'param_1': 'name_1',
 185             'param_2': 'name_2'
 186         }
 187         iterations = 1
 188         benchmarks = [
 189             {
 190                 'name':
 191                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 192                 'params': dict()
 193             }
 194         ]
 195         bu = BenchmarkingUnit(heat_template_name,
 196                               openstack_credentials,
 197                               heat_template_parameters,
 198                               iterations,
 199                               benchmarks)
 200         self.assertEqual(bu.required_benchmarks, benchmarks)
 201         bu.heat_template_parameters = heat_template_parameters
 202         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 203                              'VTC_base_single_vm_wait_2.yaml']
 204         bu.initialize()
 205         self.assertTrue(len(bu.benchmarks) == 1)
 206         self.assertEqual(bu.benchmarks[0].__class__,
 207                          Dummy_2544)
 208         # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
 209         # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
 210
 211     @mock.patch('experimental_framework.benchmarks.'
 212                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 213     @mock.patch('time.time')
 214     @mock.patch('experimental_framework.common.get_template_dir')
 215     # @mock.patch('experimental_framework.data_manager.DataManager',
 216     #             side_effect=DummyDataManager)
 217     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 218     @mock.patch('experimental_framework.benchmarking_unit.'
 219                 'heat.get_all_heat_templates')
 220     def test_finalize_for_success(
 221             self, mock_heat, mock_dep_unit,
 222             # mock_data_manager,
 223             mock_temp_dir, mock_time, mock_rfc2544):
 224         mock_heat.return_value = list()
 225         mock_time.return_value = '12345'
 226         mock_temp_dir.return_value = 'tests/data/test_templates/'
 227         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 228         common.RESULT_DIR = 'tests/data/results/'
 229
 230         heat_template_name = 'VTC_base_single_vm_wait_'
 231         openstack_credentials = {
 232             'name': 'aaa',
 233             'surname': 'bbb'
 234         }
 235         heat_template_parameters = {
 236             'param_1': 'name_1',
 237             'param_2': 'name_2'
 238         }
 239         iterations = 1
 240         benchmarks = [
 241             {
 242                 'name':
 243                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 244                 'params': dict()
 245             }
 246         ]
 247         bu = BenchmarkingUnit(heat_template_name,
 248                               openstack_credentials,
 249                               heat_template_parameters,
 250                               iterations,
 251                               benchmarks)
 252         bu.heat_template_parameters = heat_template_parameters
 253         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 254                              'VTC_base_single_vm_wait_2.yaml']
 255         bu.finalize()
 256         # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
 257         # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
 258
 259     @mock.patch('experimental_framework.common.push_data_influxdb')
 260     @mock.patch('experimental_framework.common.LOG')
 261     @mock.patch('experimental_framework.benchmarks.'
 262                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 263     @mock.patch('time.time')
 264     @mock.patch('experimental_framework.common.get_template_dir')
 265     # @mock.patch('experimental_framework.data_manager.DataManager',
 266     #             side_effect=DummyDataManager)
 267     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 268     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 269     @mock.patch('experimental_framework.benchmarking_unit.'
 270                 'heat.get_all_heat_templates')
 271     def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
 272                                         mock_dep_unit,
 273                                         # mock_data_manager,
 274                                         mock_temp_dir, mock_time,
 275                                         mock_rfc2544, mock_log, mock_influx):
 276         mock_heat.return_value = list()
 277         mock_time.return_value = '12345'
 278         mock_temp_dir.return_value = 'tests/data/test_templates/'
 279         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 280         common.RESULT_DIR = 'tests/data/results/'
 281         common.INFLUXDB_IP = 'InfluxIP'
 282         common.INFLUXDB_PORT = '8086'
 283         common.INFLUXDB_DB_NAME = 'test_db'
 284
 285         heat_template_name = 'VTC_base_single_vm_wait_'
 286         openstack_credentials = {
 287             'name': 'aaa',
 288             'surname': 'bbb'
 289         }
 290         heat_template_parameters = {
 291             'param_1': 'name_1',
 292             'param_2': 'name_2'
 293         }
 294         iterations = 1
 295         benchmarks = [
 296             {
 297                 'name':
 298                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 299                 'params': dict()
 300             }
 301         ]
 302         bu = BenchmarkingUnit(heat_template_name,
 303                               openstack_credentials,
 304                               heat_template_parameters,
 305                               iterations,
 306                               benchmarks)
 307         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 308         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 309                              'VTC_base_single_vm_wait_2.yaml']
 310         bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
 311         bu.run_benchmarks()
 312         self.assertEqual(bu.benchmarks[0].init(True), 2)
 313         self.assertEqual(bu.benchmarks[0].finalize(True), 2)
 314         self.assertEqual(bu.benchmarks[0].run(True), 2)
 315         # expected_metric = \
 316         #     'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
 317         #     'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
 318         #     'param1=val1 value=10 12345000000000'
 319         # mock_influx.assert_called_with(expected_metric)
 320
 321     @mock.patch('experimental_framework.common.LOG')
 322     @mock.patch('experimental_framework.benchmarks.'
 323                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 324     @mock.patch('time.time')
 325     @mock.patch('experimental_framework.common.get_template_dir')
 326     # @mock.patch('experimental_framework.data_manager.DataManager',
 327     #             side_effect=DummyDataManager)
 328     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 329     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 330     @mock.patch('experimental_framework.benchmarking_unit.'
 331                 'heat.get_all_heat_templates')
 332     def test_run_benchmarks_2_for_success(
 333             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 334             # mock_data_manager,
 335             mock_temp_dir, mock_time, mock_rfc2544,
 336             mock_log):
 337         mock_heat.return_value = list()
 338         mock_time.return_value = '12345'
 339         mock_temp_dir.return_value = 'tests/data/test_templates/'
 340         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 341         common.RESULT_DIR = 'tests/data/results/'
 342
 343         heat_template_name = 'VTC_base_single_vm_wait_'
 344         openstack_credentials = {
 345             'name': 'aaa',
 346             'surname': 'bbb'
 347         }
 348         heat_template_parameters = {
 349             'param_1': 'name_1',
 350             'param_2': 'name_2'
 351         }
 352         iterations = 1
 353         benchmarks = [
 354             {
 355                 'name':
 356                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 357                 'params': dict()
 358             }
 359         ]
 360         bu = BenchmarkingUnit(heat_template_name,
 361                               openstack_credentials,
 362                               heat_template_parameters,
 363                               iterations,
 364                               benchmarks)
 365         # bu.data_manager = DummyDataManager('tests/data/results/12345')
 366         bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
 367                              'VTC_base_single_vm_wait_2.yaml']
 368         bu.benchmarks = [Dummy_2544('dummy', dict())]
 369         common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
 370         bu.run_benchmarks()
 371         self.assertEqual(bu.benchmarks[0].init(True), 2)
 372         self.assertEqual(bu.benchmarks[0].finalize(True), 0)
 373         self.assertEqual(bu.benchmarks[0].run(True), 0)
 374
 375     @mock.patch('experimental_framework.common.LOG')
 376     @mock.patch('experimental_framework.benchmarks.'
 377                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 378     @mock.patch('time.time')
 379     @mock.patch('experimental_framework.common.get_template_dir')
 380     # @mock.patch('experimental_framework.data_manager.DataManager',
 381     #             side_effect=DummyDataManager)
 382     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 383     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 384     @mock.patch('experimental_framework.benchmarking_unit.'
 385                 'heat.get_all_heat_templates')
 386     def test_get_benchmark_name_for_success(
 387             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 388             # mock_data_manager,
 389             mock_temp_dir, mock_time, mock_rfc2544,
 390             mock_log):
 391         mock_heat.return_value = list()
 392         mock_time.return_value = '12345'
 393         mock_temp_dir.return_value = 'tests/data/test_templates/'
 394         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 395         common.RESULT_DIR = 'tests/data/results/'
 396
 397         heat_template_name = 'VTC_base_single_vm_wait_'
 398         openstack_credentials = {
 399             'name': 'aaa',
 400             'surname': 'bbb'
 401         }
 402         heat_template_parameters = {
 403             'param_1': 'name_1',
 404             'param_2': 'name_2'
 405         }
 406         iterations = 1
 407         benchmarks = [
 408             {
 409                 'name':
 410                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 411                 'params': dict()
 412             }
 413         ]
 414         bu = BenchmarkingUnit(heat_template_name,
 415                               openstack_credentials,
 416                               heat_template_parameters,
 417                               iterations,
 418                               benchmarks)
 419
 420         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
 421         output = bu.get_benchmark_name(
 422             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 423         self.assertEqual(expected, output)
 424
 425         expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
 426         output = bu.get_benchmark_name(
 427             'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
 428         self.assertEqual(expected, output)
 429
 430     @mock.patch('experimental_framework.common.LOG')
 431     @mock.patch('experimental_framework.benchmarks.'
 432                 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
 433     @mock.patch('time.time')
 434     @mock.patch('experimental_framework.common.get_template_dir')
 435     # @mock.patch('experimental_framework.data_manager.DataManager',
 436     #             side_effect=DummyDataManager)
 437     @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
 438     @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
 439     @mock.patch('experimental_framework.benchmarking_unit.'
 440                 'heat.get_all_heat_templates')
 441     def test_get_required_benchmarks_for_success(
 442             self, mock_heat, mock_common_dep_unit, mock_dep_unit,
 443             # mock_data_manager,
 444             mock_temp_dir, mock_time, mock_rfc2544,
 445             mock_log):
 446         mock_heat.return_value = list()
 447         mock_time.return_value = '12345'
 448         mock_temp_dir.return_value = 'tests/data/test_templates/'
 449         common.TEMPLATE_FILE_EXTENSION = '.yaml'
 450         common.RESULT_DIR = 'tests/data/results/'
 451         openstack_credentials = {
 452             'name': 'aaa',
 453             'surname': 'bbb'
 454         }
 455         heat_template_parameters = {
 456             'param_1': 'name_1',
 457             'param_2': 'name_2'
 458         }
 459         iterations = 1
 460         benchmarks = [
 461             {
 462                 'name':
 463                     'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
 464                 'params': dict()
 465             }
 466         ]
 467         bu = BenchmarkingUnit('',
 468                               openstack_credentials,
 469                               heat_template_parameters,
 470                               iterations,
 471                               benchmarks)
 472         req_benchs = \
 473             ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
 474         output = bu.get_required_benchmarks(req_benchs)
 475         self.assertEqual(len(req_benchs), 1)
 476         self.assertEqual(output[0].__class__, Dummy_2544)