functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import subprocess
  14 import unittest
  15
  16 import mock
  17 import munch
  18 from xtesting.core import testcase
  19
  20 from functest.opnfv_tests.openstack.rally import rally
  21 from functest.utils import config
  22
  23
  24 class OSRallyTesting(unittest.TestCase):
  25     # pylint: disable=too-many-public-methods
  26     def setUp(self):
  27         with mock.patch('os_client_config.get_config') as mock_get_config, \
  28                 mock.patch('shade.OpenStackCloud') as mock_shade, \
  29                 mock.patch('functest.core.tenantnetwork.NewProject') \
  30                 as mock_new_project:
  31             self.rally_base = rally.RallyBase()
  32             self.rally_base.image = munch.Munch(name='foo')
  33             self.rally_base.flavor = munch.Munch(name='foo')
  34             self.rally_base.flavor_alt = munch.Munch(name='bar')
  35         self.assertTrue(mock_get_config.called)
  36         self.assertTrue(mock_shade.called)
  37         self.assertTrue(mock_new_project.called)
  38
  39     def test_build_task_args_missing_floating_network(self):
  40         os.environ['OS_AUTH_URL'] = ''
  41         self.rally_base.ext_net = None
  42         task_args = self.rally_base.build_task_args('test_name')
  43         self.assertEqual(task_args['floating_network'], '')
  44
  45     def test_build_task_args_missing_net_id(self):
  46         os.environ['OS_AUTH_URL'] = ''
  47         self.rally_base.network = None
  48         task_args = self.rally_base.build_task_args('test_name')
  49         self.assertEqual(task_args['netid'], '')
  50
  51     @staticmethod
  52     def check_scenario_file(value):
  53         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  54         if yaml_file in value:
  55             return False
  56         return True
  57
  58     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  59     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  60         mock_func.side_effect = self.check_scenario_file
  61         with self.assertRaises(Exception):
  62             self.rally_base._prepare_test_list('test_file_name')
  63         mock_func.assert_called()
  64
  65     @staticmethod
  66     def check_temp_dir(value):
  67         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  68         if yaml_file in value:
  69             return True
  70         return False
  71
  72     @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  73                 'RallyBase.get_verifier_deployment_id', return_value='foo')
  74     @mock.patch('subprocess.check_output')
  75     def test_create_rally_deployment(self, mock_exec, mock_get_id):
  76         # pylint: disable=unused-argument
  77         self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
  78         calls = [
  79             mock.call(['rally', 'deployment', 'destroy', '--deployment',
  80                        str(getattr(config.CONF, 'rally_deployment_name'))]),
  81             mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
  82                        str(getattr(config.CONF, 'rally_deployment_name'))],
  83                       env=None),
  84             mock.call(['rally', 'deployment', 'check'])]
  85         mock_exec.assert_has_calls(calls)
  86
  87     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  88     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  89     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  90                 'apply_blacklist')
  91     def test_prepare_test_list_missing_temp_dir(
  92             self, mock_method, mock_os_makedirs, mock_path_exists):
  93         mock_path_exists.side_effect = self.check_temp_dir
  94
  95         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  96         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  97         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  98                          ret_val)
  99         mock_path_exists.assert_called()
 100         mock_method.assert_called()
 101         mock_os_makedirs.assert_called()
 102
 103     def test_get_task_id_default(self):
 104         cmd_raw = b'Task 1: started'
 105         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
 106                          '1')
 107
 108     def test_get_task_id_missing_id(self):
 109         cmd_raw = b''
 110         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
 111                          None)
 112
 113     def test_task_succeed_fail(self):
 114         json_raw = json.dumps({})
 115         self.assertEqual(self.rally_base.task_succeed(json_raw),
 116                          False)
 117         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
 118         self.assertEqual(self.rally_base.task_succeed(json_raw),
 119                          False)
 120
 121     def test_task_succeed_success(self):
 122         json_raw = json.dumps({'tasks': [{'status': 'finished',
 123                                           'pass_sla': True}]})
 124         self.assertEqual(self.rally_base.task_succeed(json_raw),
 125                          True)
 126
 127     @mock.patch('six.moves.builtins.open', mock.mock_open())
 128     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 129                 return_value={'scenario': [
 130                     {'scenarios': ['test_scenario'],
 131                      'tests': ['test']},
 132                     {'scenarios': ['other_scenario'],
 133                      'tests': ['other_test']}]})
 134     def test_excl_scenario_default(self, mock_func):
 135         os.environ['INSTALLER_TYPE'] = 'test_installer'
 136         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 137         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 138         mock_func.assert_called()
 139
 140     @mock.patch('six.moves.builtins.open', mock.mock_open())
 141     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 142                 return_value={'scenario': [
 143                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 144                      'tests': ['test1']},
 145                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 146                      'tests': ['test2']},
 147                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 148                      'tests': ['test3']},
 149                     {'scenarios': ['^os-'],
 150                      'tests': ['test4']},
 151                     {'scenarios': ['other_scenario'],
 152                      'tests': ['test0a']},
 153                     {'scenarios': [''],  # empty scenario
 154                      'tests': ['test0b']}]})
 155     def test_excl_scenario_regex(self, mock_func):
 156         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 157         self.assertEqual(self.rally_base.excl_scenario(),
 158                          ['test1', 'test2', 'test3', 'test4'])
 159         mock_func.assert_called()
 160
 161     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 162     def test_excl_scenario_exception(self, mock_open):
 163         self.assertEqual(self.rally_base.excl_scenario(), [])
 164         mock_open.assert_called()
 165
 166     @mock.patch('six.moves.builtins.open', mock.mock_open())
 167     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 168                 return_value={'functionality': [
 169                     {'functions': ['no_migration'], 'tests': ['test']}]})
 170     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 171                 '_migration_supported', return_value=False)
 172     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 173                 '_network_trunk_supported', return_value=False)
 174     def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
 175         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 176         self.assertEqual(self.rally_base.excl_func(), ['test'])
 177         mock_func.assert_called()
 178         mock_trunk.assert_called()
 179         mock_yaml_load.assert_called()
 180
 181     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 182     def test_excl_func_exception(self, mock_open):
 183         self.assertEqual(self.rally_base.excl_func(), [])
 184         mock_open.assert_called()
 185
 186     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 187     def test_file_is_empty_default(self, mock_os_stat):
 188         attrs = {'st_size': 10}
 189         mock_os_stat.return_value.configure_mock(**attrs)
 190         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 191                          False)
 192         mock_os_stat.assert_called()
 193
 194     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 195                 side_effect=Exception)
 196     def test_file_is_empty_exception(self, mock_os_stat):
 197         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 198         mock_os_stat.assert_called()
 199
 200     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 201                 return_value=False)
 202     def test_run_task_missing_task_file(self, mock_path_exists):
 203         with self.assertRaises(Exception):
 204             self.rally_base.prepare_run()
 205         mock_path_exists.assert_called()
 206
 207     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 208                 '_prepare_test_list', return_value='test_file_name')
 209     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 210                 'file_is_empty', return_value=True)
 211     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 212     def test_prepare_task_no_tests_for_scenario(
 213             self, mock_logger_info, mock_file_empty, mock_prep_list):
 214         self.rally_base.prepare_task('test_name')
 215         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 216                                          'test_name')
 217         mock_file_empty.assert_called()
 218         mock_prep_list.assert_called()
 219
 220     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 221                 '_prepare_test_list', return_value='test_file_name')
 222     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 223                 'file_is_empty', return_value=False)
 224     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 225                 'build_task_args', return_value={})
 226     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 227                 'get_task_id', return_value=None)
 228     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 229                 return_value=True)
 230     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 232     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 233         # pylint: disable=unused-argument
 234         with self.assertRaises(Exception):
 235             self.rally_base.run_task('test_name')
 236         text = 'Failed to retrieve task_id'
 237         mock_logger_error.assert_any_call(text)
 238
 239     @mock.patch('six.moves.builtins.open', mock.mock_open())
 240     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 241                 '_prepare_test_list', return_value='test_file_name')
 242     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 243                 'file_is_empty', return_value=False)
 244     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 245                 'build_task_args', return_value={})
 246     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 247                 'get_task_id', return_value='1')
 248     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 249                 'task_succeed', return_value=True)
 250     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 251                 return_value=True)
 252     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 253     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 254     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 256     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 257                 '_save_results')
 258     def test_run_task_default(self, mock_save_res, *args):
 259         # pylint: disable=unused-argument
 260         self.rally_base.run_task('test_name')
 261         mock_save_res.assert_called()
 262
 263     @mock.patch('six.moves.builtins.open', mock.mock_open())
 264     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 265                 'task_succeed', return_value=True)
 266     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 267                 return_value=True)
 268     @mock.patch('subprocess.check_output')
 269     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 270     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 271     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 272     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 273                 '_append_summary')
 274     def test_save_results(self, mock_summary, *args):
 275         # pylint: disable=unused-argument
 276         self.rally_base._save_results('test_name', '1234')
 277         mock_summary.assert_called()
 278
 279     def test_prepare_run_testname_invalid(self):
 280         self.rally_base.TESTS = ['test1', 'test2']
 281         with self.assertRaises(Exception):
 282             self.rally_base.prepare_run(tests=['test'])
 283
 284     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
 285     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
 286     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copytree')
 287     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.rmtree')
 288     def test_prepare_run_flavor_alt_creation_failed(self, *args):
 289         # pylint: disable=unused-argument
 290         self.rally_base.TESTS = ['test1', 'test2']
 291         with mock.patch.object(self.rally_base.cloud,
 292                                'list_hypervisors') as mock_list_hyperv, \
 293             mock.patch.object(self.rally_base, 'create_flavor_alt',
 294                               side_effect=Exception) \
 295                 as mock_create_flavor:
 296             with self.assertRaises(Exception):
 297                 self.rally_base.prepare_run(tests=['test1'])
 298             mock_list_hyperv.assert_called_once()
 299             mock_create_flavor.assert_called_once()
 300
 301     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 302                 'prepare_task', return_value=True)
 303     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 304                 'run_task')
 305     def test_run_tests_all(self, mock_run_task, mock_prepare_task):
 306         self.rally_base.tests = ['test1', 'test2']
 307         self.rally_base.run_tests()
 308         mock_prepare_task.assert_any_call('test1')
 309         mock_prepare_task.assert_any_call('test2')
 310         mock_run_task.assert_any_call('test1')
 311         mock_run_task.assert_any_call('test2')
 312
 313     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 314                 'prepare_task', return_value=True)
 315     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 316                 'run_task')
 317     def test_run_tests_default(self, mock_run_task, mock_prepare_task):
 318         self.rally_base.tests = ['test1', 'test2']
 319         self.rally_base.run_tests()
 320         mock_prepare_task.assert_any_call('test1')
 321         mock_prepare_task.assert_any_call('test2')
 322         mock_run_task.assert_any_call('test1')
 323         mock_run_task.assert_any_call('test2')
 324
 325     def test_clean_up_default(self):
 326         with mock.patch.object(self.rally_base.orig_cloud,
 327                                'delete_flavor') as mock_delete_flavor:
 328             self.rally_base.flavor_alt = mock.Mock()
 329             self.rally_base.clean()
 330             self.assertEqual(mock_delete_flavor.call_count, 1)
 331
 332     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 333                 'create_rally_deployment')
 334     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 335                 'prepare_run')
 336     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 337                 'run_tests')
 338     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 339                 '_generate_report')
 340     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 341                 'export_task')
 342     def test_run_default(self, *args):
 343         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 344         for func in args:
 345             func.assert_called()
 346
 347     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 348                 'create_rally_deployment', side_effect=Exception)
 349     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 350         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 351         mock_create_rally_dep.assert_called()
 352
 353     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 354                 'create_rally_deployment', return_value=mock.Mock())
 355     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 356                 'prepare_run', side_effect=Exception)
 357     def test_run_exception_prepare_run(self, mock_prep_env, *args):
 358         # pylint: disable=unused-argument
 359         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 360         mock_prep_env.assert_called()
 361
 362     def test_append_summary(self):
 363         json_dict = {
 364             'tasks': [{
 365                 'subtasks': [{
 366                     'title': 'sub_task',
 367                     'workloads': [{
 368                         'full_duration': 1.23,
 369                         'data': [{
 370                             'error': []
 371                         }]
 372                     }, {
 373                         'full_duration': 2.78,
 374                         'data': [{
 375                             'error': ['err']
 376                         }]
 377                     }]
 378                 }]
 379             }]
 380         }
 381         self.rally_base._append_summary(json.dumps(json_dict), "foo_test")
 382         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 383         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 384         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 385         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 386         self.assertEqual(self.rally_base.summary[0]['success'], [])
 387         self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task'])
 388
 389     def test_is_successful_false(self):
 390         with mock.patch('six.moves.builtins.super') as mock_super:
 391             self.rally_base.summary = [{"task_status": True},
 392                                        {"task_status": False}]
 393             self.assertEqual(self.rally_base.is_successful(),
 394                              testcase.TestCase.EX_TESTCASE_FAILED)
 395             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 396
 397     def test_is_successful_true(self):
 398         with mock.patch('six.moves.builtins.super') as mock_super:
 399             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 400             self.rally_base.summary = [{"task_status": True},
 401                                        {"task_status": True}]
 402             self.assertEqual(self.rally_base.is_successful(), 424)
 403             mock_super(rally.RallyBase, self).is_successful.assert_called()
 404
 405     @mock.patch('subprocess.check_output',
 406                 side_effect=subprocess.CalledProcessError('', ''))
 407     def test_export_task_ko(self, *args):
 408         file_name = "{}/{}.html".format(
 409             self.rally_base.results_dir, self.rally_base.case_name)
 410         with self.assertRaises(subprocess.CalledProcessError):
 411             self.rally_base.export_task(file_name)
 412         cmd = ["rally", "task", "export", "--type", "html", "--deployment",
 413                str(getattr(config.CONF, 'rally_deployment_name')),
 414                "--to", file_name]
 415         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 416
 417     @mock.patch('subprocess.check_output', return_value=None)
 418     def test_export_task(self, *args):
 419         file_name = "{}/{}.html".format(
 420             self.rally_base.results_dir, self.rally_base.case_name)
 421         self.assertEqual(self.rally_base.export_task(file_name), None)
 422         cmd = ["rally", "task", "export", "--type", "html", "--deployment",
 423                str(getattr(config.CONF, 'rally_deployment_name')),
 424                "--to", file_name]
 425         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 426
 427     @mock.patch('subprocess.check_output',
 428                 side_effect=subprocess.CalledProcessError('', ''))
 429     def test_verify_report_ko(self, *args):
 430         file_name = "{}/{}.html".format(
 431             self.rally_base.results_dir, self.rally_base.case_name)
 432         with self.assertRaises(subprocess.CalledProcessError):
 433             self.rally_base.verify_report(file_name, "1")
 434         cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
 435                "--to", file_name]
 436         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 437
 438     @mock.patch('subprocess.check_output', return_value=None)
 439     def test_verify_report(self, *args):
 440         file_name = "{}/{}.html".format(
 441             self.rally_base.results_dir, self.rally_base.case_name)
 442         self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
 443         cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
 444                "--to", file_name]
 445         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 446
 447
 448 if __name__ == "__main__":
 449     logging.disable(logging.CRITICAL)
 450     unittest.main(verbosity=2)