functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import unittest
  14
  15 import mock
  16 import munch
  17 from xtesting.core import testcase
  18
  19 from functest.opnfv_tests.openstack.rally import rally
  20
  21
  22 class OSRallyTesting(unittest.TestCase):
  23     # pylint: disable=too-many-public-methods
  24     def setUp(self):
  25         with mock.patch('os_client_config.get_config') as mock_get_config, \
  26                 mock.patch('shade.OpenStackCloud') as mock_shade, \
  27                 mock.patch('functest.core.tenantnetwork.NewProject') \
  28                 as mock_new_project:
  29             self.rally_base = rally.RallyBase()
  30             self.rally_base.image = munch.Munch(name='foo')
  31             self.rally_base.flavor = munch.Munch(name='foo')
  32             self.rally_base.flavor_alt = munch.Munch(name='bar')
  33         self.assertTrue(mock_get_config.called)
  34         self.assertTrue(mock_shade.called)
  35         self.assertTrue(mock_new_project.called)
  36
  37     def test_build_task_args_missing_floating_network(self):
  38         os.environ['OS_AUTH_URL'] = ''
  39         self.rally_base.ext_net = None
  40         task_args = self.rally_base._build_task_args('test_file_name')
  41         self.assertEqual(task_args['floating_network'], '')
  42
  43     def test_build_task_args_missing_net_id(self):
  44         os.environ['OS_AUTH_URL'] = ''
  45         self.rally_base.network = None
  46         task_args = self.rally_base._build_task_args('test_file_name')
  47         self.assertEqual(task_args['netid'], '')
  48
  49     @staticmethod
  50     def check_scenario_file(value):
  51         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  52         if yaml_file in value:
  53             return False
  54         return True
  55
  56     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  57     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  58         mock_func.side_effect = self.check_scenario_file
  59         with self.assertRaises(Exception):
  60             self.rally_base._prepare_test_list('test_file_name')
  61         mock_func.assert_called()
  62
  63     @staticmethod
  64     def check_temp_dir(value):
  65         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  66         if yaml_file in value:
  67             return True
  68         return False
  69
  70     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  71     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  72     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  73                 '_apply_blacklist')
  74     def test_prepare_test_list_missing_temp_dir(
  75             self, mock_method, mock_os_makedirs, mock_path_exists):
  76         mock_path_exists.side_effect = self.check_temp_dir
  77
  78         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  79         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  80         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  81                          ret_val)
  82         mock_path_exists.assert_called()
  83         mock_method.assert_called()
  84         mock_os_makedirs.assert_called()
  85
  86     def test_get_task_id_default(self):
  87         cmd_raw = 'Task 1: started'
  88         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  89                          '1')
  90
  91     def test_get_task_id_missing_id(self):
  92         cmd_raw = ''
  93         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  94                          None)
  95
  96     def test_task_succeed_fail(self):
  97         json_raw = json.dumps({})
  98         self.assertEqual(self.rally_base.task_succeed(json_raw),
  99                          False)
 100         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
 101         self.assertEqual(self.rally_base.task_succeed(json_raw),
 102                          False)
 103
 104     def test_task_succeed_success(self):
 105         json_raw = json.dumps({'tasks': [{'status': 'finished',
 106                                           'pass_sla': True}]})
 107         self.assertEqual(self.rally_base.task_succeed(json_raw),
 108                          True)
 109
 110     @mock.patch('six.moves.builtins.open', mock.mock_open())
 111     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 112                 return_value={'scenario': [
 113                     {'scenarios': ['test_scenario'],
 114                      'tests': ['test']},
 115                     {'scenarios': ['other_scenario'],
 116                      'tests': ['other_test']}]})
 117     def test_excl_scenario_default(self, mock_func):
 118         os.environ['INSTALLER_TYPE'] = 'test_installer'
 119         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 120         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 121         mock_func.assert_called()
 122
 123     @mock.patch('six.moves.builtins.open', mock.mock_open())
 124     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 125                 return_value={'scenario': [
 126                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 127                      'tests': ['test1']},
 128                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 129                      'tests': ['test2']},
 130                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 131                      'tests': ['test3']},
 132                     {'scenarios': ['^os-'],
 133                      'tests': ['test4']},
 134                     {'scenarios': ['other_scenario'],
 135                      'tests': ['test0a']},
 136                     {'scenarios': [''],  # empty scenario
 137                      'tests': ['test0b']}]})
 138     def test_excl_scenario_regex(self, mock_func):
 139         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 140         self.assertEqual(self.rally_base.excl_scenario(),
 141                          ['test1', 'test2', 'test3', 'test4'])
 142         mock_func.assert_called()
 143
 144     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 145     def test_excl_scenario_exception(self, mock_open):
 146         self.assertEqual(self.rally_base.excl_scenario(), [])
 147         mock_open.assert_called()
 148
 149     @mock.patch('six.moves.builtins.open', mock.mock_open())
 150     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 151                 return_value={'functionality': [
 152                     {'functions': ['no_migration'], 'tests': ['test']}]})
 153     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 154                 '_migration_supported', return_value=False)
 155     def test_excl_func_default(self, mock_func, mock_yaml_load):
 156         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 157         self.assertEqual(self.rally_base.excl_func(), ['test'])
 158         mock_func.assert_called()
 159         mock_yaml_load.assert_called()
 160
 161     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 162     def test_excl_func_exception(self, mock_open):
 163         self.assertEqual(self.rally_base.excl_func(), [])
 164         mock_open.assert_called()
 165
 166     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 167     def test_file_is_empty_default(self, mock_os_stat):
 168         attrs = {'st_size': 10}
 169         mock_os_stat.return_value.configure_mock(**attrs)
 170         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 171                          False)
 172         mock_os_stat.assert_called()
 173
 174     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 175                 side_effect=Exception)
 176     def test_file_is_empty_exception(self, mock_os_stat):
 177         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 178         mock_os_stat.assert_called()
 179
 180     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 181                 return_value=False)
 182     def test_run_task_missing_task_file(self, mock_path_exists):
 183         with self.assertRaises(Exception):
 184             self.rally_base._run_task('test_name')
 185         mock_path_exists.assert_called()
 186
 187     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 188                 return_value=True)
 189     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 190                 '_prepare_test_list', return_value='test_file_name')
 191     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 192                 'file_is_empty', return_value=True)
 193     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 194     def test_run_task_no_tests_for_scenario(self, mock_logger_info,
 195                                             mock_file_empty, mock_prep_list,
 196                                             mock_path_exists):
 197         self.rally_base._run_task('test_name')
 198         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 199                                          'test_name')
 200         mock_file_empty.assert_called()
 201         mock_prep_list.assert_called()
 202         mock_path_exists.assert_called()
 203
 204     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 205                 '_prepare_test_list', return_value='test_file_name')
 206     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 207                 'file_is_empty', return_value=False)
 208     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 209                 '_build_task_args', return_value={})
 210     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 211                 'get_task_id', return_value=None)
 212     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 213                 return_value=True)
 214     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 215     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 216     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 217         # pylint: disable=unused-argument
 218         with self.assertRaises(Exception):
 219             self.rally_base._run_task('test_name')
 220         text = 'Failed to retrieve task_id'
 221         mock_logger_error.assert_any_call(text)
 222
 223     @mock.patch('six.moves.builtins.open', mock.mock_open())
 224     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 225                 '_prepare_test_list', return_value='test_file_name')
 226     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 227                 'file_is_empty', return_value=False)
 228     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 229                 '_build_task_args', return_value={})
 230     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 231                 'get_task_id', return_value='1')
 232     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 233                 'task_succeed', return_value=True)
 234     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 235                 return_value=True)
 236     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 237     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 238     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 239     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 240     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 241                 '_save_results')
 242     def test_run_task_default(self, mock_save_res, *args):
 243         # pylint: disable=unused-argument
 244         self.rally_base._run_task('test_name')
 245         mock_save_res.assert_called()
 246
 247     @mock.patch('six.moves.builtins.open', mock.mock_open())
 248     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 249                 'task_succeed', return_value=True)
 250     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 251                 return_value=True)
 252     @mock.patch('subprocess.check_output')
 253     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 254     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 256     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 257                 '_append_summary')
 258     def test_save_results(self, mock_summary, *args):
 259         # pylint: disable=unused-argument
 260         self.rally_base._save_results('test_name', '1234')
 261         mock_summary.assert_called()
 262
 263     def test_prepare_env_testname_invalid(self):
 264         self.rally_base.TESTS = ['test1', 'test2']
 265         self.rally_base.test_name = 'test'
 266         with self.assertRaises(Exception):
 267             self.rally_base._prepare_env()
 268
 269     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 270                 'get_external_network')
 271     def test_prepare_env_flavor_alt_creation_failed(self, *args):
 272         # pylint: disable=unused-argument
 273         self.rally_base.TESTS = ['test1', 'test2']
 274         self.rally_base.test_name = 'test1'
 275         with mock.patch.object(self.rally_base.cloud,
 276                                'list_hypervisors') as mock_list_hyperv, \
 277             mock.patch.object(self.rally_base, 'create_flavor_alt',
 278                               side_effect=Exception) \
 279                 as mock_create_flavor:
 280             with self.assertRaises(Exception):
 281                 self.rally_base._prepare_env()
 282             mock_list_hyperv.assert_called_once()
 283             mock_create_flavor.assert_called_once()
 284
 285     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 286                 '_run_task')
 287     def test_run_tests_all(self, mock_run_task):
 288         self.rally_base.TESTS = ['test1', 'test2']
 289         self.rally_base.test_name = 'all'
 290         self.rally_base._run_tests()
 291         mock_run_task.assert_any_call('test1')
 292         mock_run_task.assert_any_call('test2')
 293
 294     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 295                 '_run_task')
 296     def test_run_tests_default(self, mock_run_task):
 297         self.rally_base.TESTS = ['test1', 'test2']
 298         self.rally_base.test_name = 'test1'
 299         self.rally_base._run_tests()
 300         mock_run_task.assert_any_call('test1')
 301
 302     def test_clean_up_default(self):
 303         with mock.patch.object(self.rally_base.orig_cloud,
 304                                'delete_flavor') as mock_delete_flavor:
 305             self.rally_base.flavor_alt = mock.Mock()
 306             self.rally_base.clean()
 307             self.assertEqual(mock_delete_flavor.call_count, 1)
 308
 309     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 310                 'create_rally_deployment')
 311     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 312                 '_prepare_env')
 313     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 314                 '_run_tests')
 315     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 316                 '_generate_report')
 317     def test_run_default(self, *args):
 318         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 319         for func in args:
 320             func.assert_called()
 321
 322     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 323                 'create_rally_deployment', side_effect=Exception)
 324     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 325         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 326         mock_create_rally_dep.assert_called()
 327
 328     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 329                 'create_rally_deployment', return_value=mock.Mock())
 330     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 331                 '_prepare_env', side_effect=Exception)
 332     def test_run_exception_prepare_env(self, mock_prep_env, *args):
 333         # pylint: disable=unused-argument
 334         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 335         mock_prep_env.assert_called()
 336
 337     def test_append_summary(self):
 338         text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \
 339                '1.23,"data": [{"error": []}]}]},{"workloads": ' \
 340                '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}'
 341         self.rally_base._append_summary(text, "foo_test")
 342         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 343         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 344         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 345         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 346
 347     def test_is_successful_false(self):
 348         with mock.patch('six.moves.builtins.super') as mock_super:
 349             self.rally_base.summary = [{"task_status": True},
 350                                        {"task_status": False}]
 351             self.assertEqual(self.rally_base.is_successful(),
 352                              testcase.TestCase.EX_TESTCASE_FAILED)
 353             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 354
 355     def test_is_successful_true(self):
 356         with mock.patch('six.moves.builtins.super') as mock_super:
 357             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 358             self.rally_base.summary = [{"task_status": True},
 359                                        {"task_status": True}]
 360             self.assertEqual(self.rally_base.is_successful(), 424)
 361             mock_super(rally.RallyBase, self).is_successful.assert_called()
 362
 363
 364 if __name__ == "__main__":
 365     logging.disable(logging.CRITICAL)
 366     unittest.main(verbosity=2)