functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring
   9
  10 import json
  11 import logging
  12 import os
  13 import unittest
  14
  15 import mock
  16
  17 from functest.core import testcase
  18 from functest.opnfv_tests.openstack.rally import rally
  19 from functest.utils.constants import CONST
  20
  21 from snaps.openstack.os_credentials import OSCreds
  22
  23
  24 class OSRallyTesting(unittest.TestCase):
  25     def setUp(self):
  26         os_creds = OSCreds(
  27             username='user', password='pass',
  28             auth_url='http://foo.com:5000/v3', project_name='bar')
  29         with mock.patch('snaps.openstack.tests.openstack_tests.'
  30                         'get_credentials', return_value=os_creds) as m:
  31             self.rally_base = rally.RallyBase()
  32         self.assertTrue(m.called)
  33
  34     def test_build_task_args_missing_floating_network(self):
  35         CONST.__setattr__('OS_AUTH_URL', None)
  36         self.rally_base.ext_net_name = ''
  37         task_args = self.rally_base._build_task_args('test_file_name')
  38         self.assertEqual(task_args['floating_network'], '')
  39
  40     def test_build_task_args_missing_net_id(self):
  41         CONST.__setattr__('OS_AUTH_URL', None)
  42         self.rally_base.priv_net_id = ''
  43         task_args = self.rally_base._build_task_args('test_file_name')
  44         self.assertEqual(task_args['netid'], '')
  45
  46     @staticmethod
  47     def check_scenario_file(value):
  48         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  49         if yaml_file in value:
  50             return False
  51         return True
  52
  53     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  54     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  55         mock_func.side_effect = self.check_scenario_file
  56         with self.assertRaises(Exception):
  57             self.rally_base._prepare_test_list('test_file_name')
  58         mock_func.assert_called()
  59
  60     @staticmethod
  61     def check_temp_dir(value):
  62         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  63         if yaml_file in value:
  64             return True
  65         return False
  66
  67     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  68     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  69     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  70                 '_apply_blacklist')
  71     def test_prepare_test_list_missing_temp_dir(
  72             self, mock_method, mock_os_makedirs, mock_path_exists):
  73         mock_path_exists.side_effect = self.check_temp_dir
  74
  75         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  76         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  77         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  78                          ret_val)
  79         mock_path_exists.assert_called()
  80         mock_method.assert_called()
  81         mock_os_makedirs.assert_called()
  82
  83     def test_get_task_id_default(self):
  84         cmd_raw = 'Task 1: started'
  85         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  86                          '1')
  87
  88     def test_get_task_id_missing_id(self):
  89         cmd_raw = ''
  90         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  91                          None)
  92
  93     def test_task_succeed_fail(self):
  94         json_raw = json.dumps([None])
  95         self.assertEqual(self.rally_base.task_succeed(json_raw),
  96                          False)
  97         json_raw = json.dumps([{'result': [{'error': ['test_error']}]}])
  98         self.assertEqual(self.rally_base.task_succeed(json_raw),
  99                          False)
 100
 101     def test_task_succeed_success(self):
 102         json_raw = json.dumps('')
 103         self.assertEqual(self.rally_base.task_succeed(json_raw),
 104                          True)
 105
 106     def test_get_cmd_output(self):
 107         proc = mock.Mock()
 108         proc.stdout.__iter__ = mock.Mock(return_value=iter(['line1', 'line2']))
 109         self.assertEqual(self.rally_base.get_cmd_output(proc),
 110                          'line1line2')
 111
 112     @mock.patch('__builtin__.open', mock.mock_open())
 113     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 114                 return_value={'scenario': [
 115                     {'scenarios': ['test_scenario'],
 116                      'installers': ['test_installer'],
 117                      'tests': ['test']},
 118                     {'scenarios': ['other_scenario'],
 119                      'installers': ['test_installer'],
 120                      'tests': ['other_test']}]})
 121     def test_excl_scenario_default(self, mock_func):
 122         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 123         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 124         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 125         mock_func.assert_called()
 126
 127     @mock.patch('__builtin__.open', mock.mock_open())
 128     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 129                 return_value={'scenario': [
 130                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 131                      'installers': ['test_installer'],
 132                      'tests': ['test1']},
 133                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 134                      'installers': ['test_installer'],
 135                      'tests': ['test2']},
 136                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 137                      'installers': ['test_installer'],
 138                      'tests': ['test3']},
 139                     {'scenarios': ['^os-'],
 140                      'installers': ['test_installer'],
 141                      'tests': ['test4']},
 142                     {'scenarios': ['other_scenario'],
 143                      'installers': ['test_installer'],
 144                      'tests': ['test0a']},
 145                     {'scenarios': [''],  # empty scenario
 146                      'installers': ['test_installer'],
 147                      'tests': ['test0b']}]})
 148     def test_excl_scenario_regex(self, mock_func):
 149         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 150         CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
 151         self.assertEqual(self.rally_base.excl_scenario(),
 152                          ['test1', 'test2', 'test3', 'test4'])
 153         mock_func.assert_called()
 154
 155     @mock.patch('__builtin__.open', side_effect=Exception)
 156     def test_excl_scenario_exception(self, mock_open):
 157         self.assertEqual(self.rally_base.excl_scenario(), [])
 158         mock_open.assert_called()
 159
 160     @mock.patch('__builtin__.open', mock.mock_open())
 161     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 162                 return_value={'functionality': [
 163                     {'functions': ['no_migration'], 'tests': ['test']}]})
 164     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 165                 '_migration_supported', return_value=False)
 166     def test_excl_func_default(self, mock_func, mock_yaml_load):
 167         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 168         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 169         self.assertEqual(self.rally_base.excl_func(), ['test'])
 170         mock_func.assert_called()
 171         mock_yaml_load.assert_called()
 172
 173     @mock.patch('__builtin__.open', side_effect=Exception)
 174     def test_excl_func_exception(self, mock_open):
 175         self.assertEqual(self.rally_base.excl_func(), [])
 176         mock_open.assert_called()
 177
 178     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 179     def test_file_is_empty_default(self, mock_os_stat):
 180         attrs = {'st_size': 10}
 181         mock_os_stat.return_value.configure_mock(**attrs)
 182         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 183                          False)
 184         mock_os_stat.assert_called()
 185
 186     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 187                 side_effect=Exception)
 188     def test_file_is_empty_exception(self, mock_os_stat):
 189         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 190         mock_os_stat.assert_called()
 191
 192     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 193                 return_value=False)
 194     def test_run_task_missing_task_file(self, mock_path_exists):
 195         with self.assertRaises(Exception):
 196             self.rally_base._run_task('test_name')
 197         mock_path_exists.assert_called()
 198
 199     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 200                 return_value=True)
 201     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 202                 '_prepare_test_list', return_value='test_file_name')
 203     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 204                 'file_is_empty', return_value=True)
 205     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 206     def test_run_task_no_tests_for_scenario(self, mock_logger_info,
 207                                             mock_file_empty, mock_prep_list,
 208                                             mock_path_exists):
 209         self.rally_base._run_task('test_name')
 210         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 211                                          'test_name')
 212         mock_file_empty.assert_called()
 213         mock_prep_list.assert_called()
 214         mock_path_exists.assert_called()
 215
 216     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 217                 '_prepare_test_list', return_value='test_file_name')
 218     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 219                 'file_is_empty', return_value=False)
 220     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 221                 '_build_task_args', return_value={})
 222     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 223                 '_get_output')
 224     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 225                 'get_task_id', return_value=None)
 226     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 227                 'get_cmd_output', return_value='')
 228     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 229                 return_value=True)
 230     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 232     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 233         self.rally_base._run_task('test_name')
 234         text = 'Failed to retrieve task_id, validating task...'
 235         mock_logger_error.assert_any_call(text)
 236
 237     @mock.patch('__builtin__.open', mock.mock_open())
 238     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 239                 '_prepare_test_list', return_value='test_file_name')
 240     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 241                 'file_is_empty', return_value=False)
 242     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 243                 '_build_task_args', return_value={})
 244     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 245                 '_get_output')
 246     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 247                 'get_task_id', return_value='1')
 248     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 249                 'get_cmd_output', return_value='')
 250     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 251                 'task_succeed', return_value=True)
 252     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 253                 return_value=True)
 254     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 256     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 258     def test_run_task_default(self, mock_logger_error, mock_logger_info,
 259                               *args):
 260         self.rally_base._run_task('test_name')
 261         text = 'Test scenario: "test_name" OK.\n'
 262         mock_logger_info.assert_any_call(text)
 263         mock_logger_error.assert_not_called()
 264
 265     def test_prepare_env_testname_invalid(self):
 266         self.rally_base.TESTS = ['test1', 'test2']
 267         self.rally_base.test_name = 'test'
 268         with self.assertRaises(Exception):
 269             self.rally_base._prepare_env()
 270
 271     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 272                 'get_active_compute_cnt')
 273     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 274                 'get_ext_net_name', return_value='test_net_name')
 275     @mock.patch('snaps.openstack.utils.deploy_utils.create_image',
 276                 return_value=None)
 277     def test_prepare_env_image_missing(
 278             self, mock_get_img, mock_get_net, mock_get_comp_cnt):
 279         self.rally_base.TESTS = ['test1', 'test2']
 280         self.rally_base.test_name = 'test1'
 281         with self.assertRaises(Exception):
 282             self.rally_base._prepare_env()
 283         mock_get_img.assert_called()
 284         mock_get_net.assert_called()
 285         mock_get_comp_cnt.assert_called()
 286
 287     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 288                 'get_active_compute_cnt')
 289     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 290                 'get_ext_net_name', return_value='test_net_name')
 291     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 292     @mock.patch('snaps.openstack.utils.deploy_utils.create_network',
 293                 return_value=None)
 294     def test_prepare_env_network_creation_failed(
 295             self, mock_create_net, mock_get_img, mock_get_net,
 296             mock_get_comp_cnt):
 297         self.rally_base.TESTS = ['test1', 'test2']
 298         self.rally_base.test_name = 'test1'
 299         with self.assertRaises(Exception):
 300             self.rally_base._prepare_env()
 301         mock_create_net.assert_called()
 302         mock_get_img.assert_called()
 303         mock_get_net.assert_called()
 304         mock_get_comp_cnt.assert_called()
 305
 306     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 307                 'get_active_compute_cnt')
 308     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 309                 'get_ext_net_name', return_value='test_net_name')
 310     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 311     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 312     @mock.patch('snaps.openstack.utils.deploy_utils.create_router',
 313                 return_value=None)
 314     def test_prepare_env_router_creation_failed(
 315             self, mock_create_router, mock_create_net, mock_get_img,
 316             mock_get_net, mock_get_comp_cnt):
 317         self.rally_base.TESTS = ['test1', 'test2']
 318         self.rally_base.test_name = 'test1'
 319         with self.assertRaises(Exception):
 320             self.rally_base._prepare_env()
 321         mock_create_net.assert_called()
 322         mock_get_img.assert_called()
 323         mock_get_net.assert_called()
 324         mock_create_router.assert_called()
 325         mock_get_comp_cnt.assert_called()
 326
 327     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 328                 'get_active_compute_cnt')
 329     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 330                 'get_ext_net_name', return_value='test_net_name')
 331     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 332     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 333     @mock.patch('snaps.openstack.utils.deploy_utils.create_router')
 334     @mock.patch('snaps.openstack.create_flavor.OpenStackFlavor.create',
 335                 return_value=None)
 336     def test_prepare_env_flavor_creation_failed(
 337             self, mock_create_flavor, mock_create_router, mock_create_net,
 338             mock_get_img, mock_get_net, mock_get_comp_cnt):
 339         self.rally_base.TESTS = ['test1', 'test2']
 340         self.rally_base.test_name = 'test1'
 341         with self.assertRaises(Exception):
 342             self.rally_base._prepare_env()
 343         mock_create_net.assert_called()
 344         mock_get_img.assert_called()
 345         mock_get_net.assert_called()
 346         mock_create_router.assert_called()
 347         mock_get_comp_cnt.assert_called()
 348         mock_create_flavor.assert_called_once()
 349
 350     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 351                 'get_active_compute_cnt')
 352     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 353                 'get_ext_net_name', return_value='test_net_name')
 354     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 355     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 356     @mock.patch('snaps.openstack.utils.deploy_utils.create_router')
 357     @mock.patch('snaps.openstack.create_flavor.OpenStackFlavor.create',
 358                 side_effect=[mock.Mock, None])
 359     def test_prepare_env_flavor_alt_creation_failed(
 360             self, mock_create_flavor, mock_create_router, mock_create_net,
 361             mock_get_img, mock_get_net, mock_get_comp_cnt):
 362         self.rally_base.TESTS = ['test1', 'test2']
 363         self.rally_base.test_name = 'test1'
 364         with self.assertRaises(Exception):
 365             self.rally_base._prepare_env()
 366         mock_create_net.assert_called()
 367         mock_get_img.assert_called()
 368         mock_get_net.assert_called()
 369         mock_create_router.assert_called()
 370         mock_get_comp_cnt.assert_called()
 371         self.assertEqual(mock_create_flavor.call_count, 2)
 372
 373     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 374                 '_run_task')
 375     def test_run_tests_all(self, mock_run_task):
 376         self.rally_base.TESTS = ['test1', 'test2']
 377         self.rally_base.test_name = 'all'
 378         self.rally_base._run_tests()
 379         mock_run_task.assert_any_call('test1')
 380         mock_run_task.assert_any_call('test2')
 381
 382     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 383                 '_run_task')
 384     def test_run_tests_default(self, mock_run_task):
 385         self.rally_base.TESTS = ['test1', 'test2']
 386         self.rally_base.test_name = 'test1'
 387         self.rally_base._run_tests()
 388         mock_run_task.assert_any_call('test1')
 389
 390     def test_clean_up_default(self):
 391         creator1 = mock.Mock()
 392         creator2 = mock.Mock()
 393         self.rally_base.creators = [creator1, creator2]
 394         self.rally_base._clean_up()
 395         self.assertTrue(creator1.clean.called)
 396         self.assertTrue(creator2.clean.called)
 397
 398     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 399                 'create_rally_deployment')
 400     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 401                 '_prepare_env')
 402     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 403                 '_run_tests')
 404     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 405                 '_generate_report')
 406     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 407                 '_clean_up')
 408     def test_run_default(self, *args):
 409         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 410         map(lambda m: m.assert_called(), args)
 411
 412     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 413                 'create_rally_deployment', side_effect=Exception)
 414     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 415         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 416         mock_create_rally_dep.assert_called()
 417
 418     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 419                 '_prepare_env', side_effect=Exception)
 420     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 421                 'create_rally_deployment', return_value=mock.Mock())
 422     def test_run_exception_prepare_env(self, mock_create_rally_dep,
 423                                        mock_prep_env):
 424         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 425         mock_prep_env.assert_called()
 426
 427
 428 if __name__ == "__main__":
 429     logging.disable(logging.CRITICAL)
 430     unittest.main(verbosity=2)