functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import unittest
  14
  15 import mock
  16 import munch
  17 from xtesting.core import testcase
  18
  19 from functest.opnfv_tests.openstack.rally import rally
  20
  21
  22 class OSRallyTesting(unittest.TestCase):
  23     # pylint: disable=too-many-public-methods
  24     def setUp(self):
  25         with mock.patch('os_client_config.get_config') as mock_get_config, \
  26                 mock.patch('shade.OpenStackCloud') as mock_shade:
  27             self.rally_base = rally.RallyBase()
  28             self.rally_base.image = munch.Munch(name='foo')
  29             self.rally_base.flavor = munch.Munch(name='foo')
  30             self.rally_base.flavor_alt = munch.Munch(name='bar')
  31         self.assertTrue(mock_get_config.called)
  32         self.assertTrue(mock_shade.called)
  33
  34     def test_build_task_args_missing_floating_network(self):
  35         os.environ['OS_AUTH_URL'] = ''
  36         self.rally_base.ext_net = None
  37         task_args = self.rally_base._build_task_args('test_file_name')
  38         self.assertEqual(task_args['floating_network'], '')
  39
  40     def test_build_task_args_missing_net_id(self):
  41         os.environ['OS_AUTH_URL'] = ''
  42         self.rally_base.network = None
  43         task_args = self.rally_base._build_task_args('test_file_name')
  44         self.assertEqual(task_args['netid'], '')
  45
  46     @staticmethod
  47     def check_scenario_file(value):
  48         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  49         if yaml_file in value:
  50             return False
  51         return True
  52
  53     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  54     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  55         mock_func.side_effect = self.check_scenario_file
  56         with self.assertRaises(Exception):
  57             self.rally_base._prepare_test_list('test_file_name')
  58         mock_func.assert_called()
  59
  60     @staticmethod
  61     def check_temp_dir(value):
  62         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  63         if yaml_file in value:
  64             return True
  65         return False
  66
  67     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  68     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  69     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  70                 '_apply_blacklist')
  71     def test_prepare_test_list_missing_temp_dir(
  72             self, mock_method, mock_os_makedirs, mock_path_exists):
  73         mock_path_exists.side_effect = self.check_temp_dir
  74
  75         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  76         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  77         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  78                          ret_val)
  79         mock_path_exists.assert_called()
  80         mock_method.assert_called()
  81         mock_os_makedirs.assert_called()
  82
  83     def test_get_task_id_default(self):
  84         cmd_raw = 'Task 1: started'
  85         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  86                          '1')
  87
  88     def test_get_task_id_missing_id(self):
  89         cmd_raw = ''
  90         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  91                          None)
  92
  93     def test_task_succeed_fail(self):
  94         json_raw = json.dumps({})
  95         self.assertEqual(self.rally_base.task_succeed(json_raw),
  96                          False)
  97         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
  98         self.assertEqual(self.rally_base.task_succeed(json_raw),
  99                          False)
 100
 101     def test_task_succeed_success(self):
 102         json_raw = json.dumps({'tasks': [{'status': 'finished',
 103                                           'pass_sla': True}]})
 104         self.assertEqual(self.rally_base.task_succeed(json_raw),
 105                          True)
 106
 107     @mock.patch('six.moves.builtins.open', mock.mock_open())
 108     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 109                 return_value={'scenario': [
 110                     {'scenarios': ['test_scenario'],
 111                      'installers': ['test_installer'],
 112                      'tests': ['test']},
 113                     {'scenarios': ['other_scenario'],
 114                      'installers': ['test_installer'],
 115                      'tests': ['other_test']}]})
 116     def test_excl_scenario_default(self, mock_func):
 117         os.environ['INSTALLER_TYPE'] = 'test_installer'
 118         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 119         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 120         mock_func.assert_called()
 121
 122     @mock.patch('six.moves.builtins.open', mock.mock_open())
 123     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 124                 return_value={'scenario': [
 125                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 126                      'installers': ['test_installer'],
 127                      'tests': ['test1']},
 128                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 129                      'installers': ['test_installer'],
 130                      'tests': ['test2']},
 131                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 132                      'installers': ['test_installer'],
 133                      'tests': ['test3']},
 134                     {'scenarios': ['^os-'],
 135                      'installers': ['test_installer'],
 136                      'tests': ['test4']},
 137                     {'scenarios': ['other_scenario'],
 138                      'installers': ['test_installer'],
 139                      'tests': ['test0a']},
 140                     {'scenarios': [''],  # empty scenario
 141                      'installers': ['test_installer'],
 142                      'tests': ['test0b']}]})
 143     def test_excl_scenario_regex(self, mock_func):
 144         os.environ['INSTALLER_TYPE'] = 'test_installer'
 145         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 146         self.assertEqual(self.rally_base.excl_scenario(),
 147                          ['test1', 'test2', 'test3', 'test4'])
 148         mock_func.assert_called()
 149
 150     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 151     def test_excl_scenario_exception(self, mock_open):
 152         self.assertEqual(self.rally_base.excl_scenario(), [])
 153         mock_open.assert_called()
 154
 155     @mock.patch('six.moves.builtins.open', mock.mock_open())
 156     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 157                 return_value={'functionality': [
 158                     {'functions': ['no_migration'], 'tests': ['test']}]})
 159     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 160                 '_migration_supported', return_value=False)
 161     def test_excl_func_default(self, mock_func, mock_yaml_load):
 162         os.environ['INSTALLER_TYPE'] = 'test_installer'
 163         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 164         self.assertEqual(self.rally_base.excl_func(), ['test'])
 165         mock_func.assert_called()
 166         mock_yaml_load.assert_called()
 167
 168     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 169     def test_excl_func_exception(self, mock_open):
 170         self.assertEqual(self.rally_base.excl_func(), [])
 171         mock_open.assert_called()
 172
 173     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 174     def test_file_is_empty_default(self, mock_os_stat):
 175         attrs = {'st_size': 10}
 176         mock_os_stat.return_value.configure_mock(**attrs)
 177         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 178                          False)
 179         mock_os_stat.assert_called()
 180
 181     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 182                 side_effect=Exception)
 183     def test_file_is_empty_exception(self, mock_os_stat):
 184         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 185         mock_os_stat.assert_called()
 186
 187     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 188                 return_value=False)
 189     def test_run_task_missing_task_file(self, mock_path_exists):
 190         with self.assertRaises(Exception):
 191             self.rally_base._run_task('test_name')
 192         mock_path_exists.assert_called()
 193
 194     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 195                 return_value=True)
 196     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 197                 '_prepare_test_list', return_value='test_file_name')
 198     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 199                 'file_is_empty', return_value=True)
 200     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 201     def test_run_task_no_tests_for_scenario(self, mock_logger_info,
 202                                             mock_file_empty, mock_prep_list,
 203                                             mock_path_exists):
 204         self.rally_base._run_task('test_name')
 205         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 206                                          'test_name')
 207         mock_file_empty.assert_called()
 208         mock_prep_list.assert_called()
 209         mock_path_exists.assert_called()
 210
 211     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 212                 '_prepare_test_list', return_value='test_file_name')
 213     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 214                 'file_is_empty', return_value=False)
 215     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 216                 '_build_task_args', return_value={})
 217     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 218                 'get_task_id', return_value=None)
 219     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 220                 return_value=True)
 221     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 222     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 223     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 224         # pylint: disable=unused-argument
 225         with self.assertRaises(Exception):
 226             self.rally_base._run_task('test_name')
 227         text = 'Failed to retrieve task_id'
 228         mock_logger_error.assert_any_call(text)
 229
 230     @mock.patch('six.moves.builtins.open', mock.mock_open())
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 232                 '_prepare_test_list', return_value='test_file_name')
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 234                 'file_is_empty', return_value=False)
 235     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 236                 '_build_task_args', return_value={})
 237     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 238                 'get_task_id', return_value='1')
 239     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 240                 'task_succeed', return_value=True)
 241     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 242                 return_value=True)
 243     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 244     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 245     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 246     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 247     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 248                 '_save_results')
 249     def test_run_task_default(self, mock_save_res, *args):
 250         # pylint: disable=unused-argument
 251         self.rally_base._run_task('test_name')
 252         mock_save_res.assert_called()
 253
 254     @mock.patch('six.moves.builtins.open', mock.mock_open())
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 256                 'task_succeed', return_value=True)
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 258                 return_value=True)
 259     @mock.patch('subprocess.check_output')
 260     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 261     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 262     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 263     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 264                 '_append_summary')
 265     def test_save_results(self, mock_summary, *args):
 266         # pylint: disable=unused-argument
 267         self.rally_base._save_results('test_name', '1234')
 268         mock_summary.assert_called()
 269
 270     def test_prepare_env_testname_invalid(self):
 271         self.rally_base.TESTS = ['test1', 'test2']
 272         self.rally_base.test_name = 'test'
 273         with self.assertRaises(Exception):
 274             self.rally_base._prepare_env()
 275
 276     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 277                 'get_external_network')
 278     def test_prepare_env_flavor_alt_creation_failed(self, *args):
 279         # pylint: disable=unused-argument
 280         self.rally_base.TESTS = ['test1', 'test2']
 281         self.rally_base.test_name = 'test1'
 282         with mock.patch.object(self.rally_base.cloud,
 283                                'list_hypervisors') as mock_list_hyperv, \
 284             mock.patch.object(self.rally_base.cloud,
 285                               'set_flavor_specs') as mock_set_flavor_specs, \
 286             mock.patch.object(self.rally_base.cloud, 'create_flavor',
 287                               side_effect=Exception) \
 288                 as mock_create_flavor:
 289             with self.assertRaises(Exception):
 290                 self.rally_base._prepare_env()
 291             mock_list_hyperv.assert_called_once()
 292             mock_create_flavor.assert_called_once()
 293             mock_set_flavor_specs.assert_not_called()
 294
 295     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 296                 '_run_task')
 297     def test_run_tests_all(self, mock_run_task):
 298         self.rally_base.TESTS = ['test1', 'test2']
 299         self.rally_base.test_name = 'all'
 300         self.rally_base._run_tests()
 301         mock_run_task.assert_any_call('test1')
 302         mock_run_task.assert_any_call('test2')
 303
 304     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 305                 '_run_task')
 306     def test_run_tests_default(self, mock_run_task):
 307         self.rally_base.TESTS = ['test1', 'test2']
 308         self.rally_base.test_name = 'test1'
 309         self.rally_base._run_tests()
 310         mock_run_task.assert_any_call('test1')
 311
 312     def test_clean_up_default(self):
 313         with mock.patch.object(self.rally_base.cloud,
 314                                'delete_flavor') as mock_delete_flavor:
 315             self.rally_base.flavor_alt = mock.Mock()
 316             self.rally_base.clean()
 317             self.assertEqual(mock_delete_flavor.call_count, 1)
 318
 319     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 320                 'create_rally_deployment')
 321     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 322                 '_prepare_env')
 323     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 324                 '_run_tests')
 325     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 326                 '_generate_report')
 327     def test_run_default(self, *args):
 328         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 329         for func in args:
 330             func.assert_called()
 331
 332     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 333                 'create_rally_deployment', side_effect=Exception)
 334     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 335         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 336         mock_create_rally_dep.assert_called()
 337
 338     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 339                 'create_rally_deployment', return_value=mock.Mock())
 340     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 341                 '_prepare_env', side_effect=Exception)
 342     def test_run_exception_prepare_env(self, mock_prep_env, *args):
 343         # pylint: disable=unused-argument
 344         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 345         mock_prep_env.assert_called()
 346
 347     def test_append_summary(self):
 348         text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \
 349                '1.23,"data": [{"error": []}]}]},{"workloads": ' \
 350                '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}'
 351         self.rally_base._append_summary(text, "foo_test")
 352         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 353         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 354         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 355         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 356
 357     def test_is_successful_false(self):
 358         with mock.patch('six.moves.builtins.super') as mock_super:
 359             self.rally_base.summary = [{"task_status": True},
 360                                        {"task_status": False}]
 361             self.assertEqual(self.rally_base.is_successful(),
 362                              testcase.TestCase.EX_TESTCASE_FAILED)
 363             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 364
 365     def test_is_successful_true(self):
 366         with mock.patch('six.moves.builtins.super') as mock_super:
 367             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 368             self.rally_base.summary = [{"task_status": True},
 369                                        {"task_status": True}]
 370             self.assertEqual(self.rally_base.is_successful(), 424)
 371             mock_super(rally.RallyBase, self).is_successful.assert_called()
 372
 373
 374 if __name__ == "__main__":
 375     logging.disable(logging.CRITICAL)
 376     unittest.main(verbosity=2)