functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import unittest
  14
  15 import mock
  16
  17 from functest.core import testcase
  18 from functest.opnfv_tests.openstack.rally import rally
  19 from functest.utils.constants import CONST
  20
  21 from snaps.openstack.os_credentials import OSCreds
  22
  23
  24 class OSRallyTesting(unittest.TestCase):
  25     # pylint: disable=too-many-public-methods
  26     def setUp(self):
  27         os_creds = OSCreds(
  28             username='user', password='pass',
  29             auth_url='http://foo.com:5000/v3', project_name='bar')
  30         with mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
  31                         'get_credentials',
  32                         return_value=os_creds) as mock_get_creds:
  33             self.rally_base = rally.RallyBase()
  34         self.assertTrue(mock_get_creds.called)
  35
  36     def test_build_task_args_missing_floating_network(self):
  37         CONST.__setattr__('OS_AUTH_URL', None)
  38         self.rally_base.ext_net_name = ''
  39         task_args = self.rally_base._build_task_args('test_file_name')
  40         self.assertEqual(task_args['floating_network'], '')
  41
  42     def test_build_task_args_missing_net_id(self):
  43         CONST.__setattr__('OS_AUTH_URL', None)
  44         self.rally_base.priv_net_id = ''
  45         task_args = self.rally_base._build_task_args('test_file_name')
  46         self.assertEqual(task_args['netid'], '')
  47
  48     @staticmethod
  49     def check_scenario_file(value):
  50         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  51         if yaml_file in value:
  52             return False
  53         return True
  54
  55     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  56     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  57         mock_func.side_effect = self.check_scenario_file
  58         with self.assertRaises(Exception):
  59             self.rally_base._prepare_test_list('test_file_name')
  60         mock_func.assert_called()
  61
  62     @staticmethod
  63     def check_temp_dir(value):
  64         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  65         if yaml_file in value:
  66             return True
  67         return False
  68
  69     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  70     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  71     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  72                 '_apply_blacklist')
  73     def test_prepare_test_list_missing_temp_dir(
  74             self, mock_method, mock_os_makedirs, mock_path_exists):
  75         mock_path_exists.side_effect = self.check_temp_dir
  76
  77         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  78         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  79         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  80                          ret_val)
  81         mock_path_exists.assert_called()
  82         mock_method.assert_called()
  83         mock_os_makedirs.assert_called()
  84
  85     def test_get_task_id_default(self):
  86         cmd_raw = 'Task 1: started'
  87         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  88                          '1')
  89
  90     def test_get_task_id_missing_id(self):
  91         cmd_raw = ''
  92         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  93                          None)
  94
  95     def test_task_succeed_fail(self):
  96         json_raw = json.dumps([None])
  97         self.assertEqual(self.rally_base.task_succeed(json_raw),
  98                          False)
  99         json_raw = json.dumps([{'result': [{'error': ['test_error']}]}])
 100         self.assertEqual(self.rally_base.task_succeed(json_raw),
 101                          False)
 102
 103     def test_task_succeed_success(self):
 104         json_raw = json.dumps('')
 105         self.assertEqual(self.rally_base.task_succeed(json_raw),
 106                          True)
 107
 108     def test_get_cmd_output(self):
 109         proc = mock.Mock()
 110         proc.stdout.__iter__ = mock.Mock(return_value=iter(['line1', 'line2']))
 111         self.assertEqual(self.rally_base.get_cmd_output(proc),
 112                          'line1line2')
 113
 114     @mock.patch('__builtin__.open', mock.mock_open())
 115     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 116                 return_value={'scenario': [
 117                     {'scenarios': ['test_scenario'],
 118                      'installers': ['test_installer'],
 119                      'tests': ['test']},
 120                     {'scenarios': ['other_scenario'],
 121                      'installers': ['test_installer'],
 122                      'tests': ['other_test']}]})
 123     def test_excl_scenario_default(self, mock_func):
 124         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 125         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 126         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 127         mock_func.assert_called()
 128
 129     @mock.patch('__builtin__.open', mock.mock_open())
 130     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 131                 return_value={'scenario': [
 132                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 133                      'installers': ['test_installer'],
 134                      'tests': ['test1']},
 135                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 136                      'installers': ['test_installer'],
 137                      'tests': ['test2']},
 138                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 139                      'installers': ['test_installer'],
 140                      'tests': ['test3']},
 141                     {'scenarios': ['^os-'],
 142                      'installers': ['test_installer'],
 143                      'tests': ['test4']},
 144                     {'scenarios': ['other_scenario'],
 145                      'installers': ['test_installer'],
 146                      'tests': ['test0a']},
 147                     {'scenarios': [''],  # empty scenario
 148                      'installers': ['test_installer'],
 149                      'tests': ['test0b']}]})
 150     def test_excl_scenario_regex(self, mock_func):
 151         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 152         CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
 153         self.assertEqual(self.rally_base.excl_scenario(),
 154                          ['test1', 'test2', 'test3', 'test4'])
 155         mock_func.assert_called()
 156
 157     @mock.patch('__builtin__.open', side_effect=Exception)
 158     def test_excl_scenario_exception(self, mock_open):
 159         self.assertEqual(self.rally_base.excl_scenario(), [])
 160         mock_open.assert_called()
 161
 162     @mock.patch('__builtin__.open', mock.mock_open())
 163     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 164                 return_value={'functionality': [
 165                     {'functions': ['no_migration'], 'tests': ['test']}]})
 166     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 167                 '_migration_supported', return_value=False)
 168     def test_excl_func_default(self, mock_func, mock_yaml_load):
 169         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 170         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 171         self.assertEqual(self.rally_base.excl_func(), ['test'])
 172         mock_func.assert_called()
 173         mock_yaml_load.assert_called()
 174
 175     @mock.patch('__builtin__.open', side_effect=Exception)
 176     def test_excl_func_exception(self, mock_open):
 177         self.assertEqual(self.rally_base.excl_func(), [])
 178         mock_open.assert_called()
 179
 180     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 181     def test_file_is_empty_default(self, mock_os_stat):
 182         attrs = {'st_size': 10}
 183         mock_os_stat.return_value.configure_mock(**attrs)
 184         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 185                          False)
 186         mock_os_stat.assert_called()
 187
 188     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 189                 side_effect=Exception)
 190     def test_file_is_empty_exception(self, mock_os_stat):
 191         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 192         mock_os_stat.assert_called()
 193
 194     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 195                 return_value=False)
 196     def test_run_task_missing_task_file(self, mock_path_exists):
 197         with self.assertRaises(Exception):
 198             self.rally_base._run_task('test_name')
 199         mock_path_exists.assert_called()
 200
 201     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 202                 return_value=True)
 203     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 204                 '_prepare_test_list', return_value='test_file_name')
 205     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 206                 'file_is_empty', return_value=True)
 207     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 208     def test_run_task_no_tests_for_scenario(self, mock_logger_info,
 209                                             mock_file_empty, mock_prep_list,
 210                                             mock_path_exists):
 211         self.rally_base._run_task('test_name')
 212         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 213                                          'test_name')
 214         mock_file_empty.assert_called()
 215         mock_prep_list.assert_called()
 216         mock_path_exists.assert_called()
 217
 218     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 219                 '_prepare_test_list', return_value='test_file_name')
 220     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 221                 'file_is_empty', return_value=False)
 222     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 223                 '_build_task_args', return_value={})
 224     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 225                 '_append_summary')
 226     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 227                 'get_task_id', return_value=None)
 228     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 229                 'get_cmd_output', return_value='')
 230     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 231                 return_value=True)
 232     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 234     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 235         # pylint: disable=unused-argument
 236         self.rally_base._run_task('test_name')
 237         text = 'Failed to retrieve task_id, validating task...'
 238         mock_logger_error.assert_any_call(text)
 239
 240     @mock.patch('__builtin__.open', mock.mock_open())
 241     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 242                 '_prepare_test_list', return_value='test_file_name')
 243     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 244                 'file_is_empty', return_value=False)
 245     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 246                 '_build_task_args', return_value={})
 247     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 248                 '_append_summary')
 249     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 250                 'get_task_id', return_value='1')
 251     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 252                 'get_cmd_output', return_value='')
 253     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 254                 'task_succeed', return_value=True)
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 256                 return_value=True)
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 258     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 259     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 260     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 261     def test_run_task_default(self, mock_logger_error, mock_logger_info,
 262                               *args):
 263         # pylint: disable=unused-argument
 264         self.rally_base._run_task('test_name')
 265         text = 'Test scenario: "test_name" OK.\n'
 266         mock_logger_info.assert_any_call(text)
 267         mock_logger_error.assert_not_called()
 268
 269     def test_prepare_env_testname_invalid(self):
 270         self.rally_base.TESTS = ['test1', 'test2']
 271         self.rally_base.test_name = 'test'
 272         with self.assertRaises(Exception):
 273             self.rally_base._prepare_env()
 274
 275     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 276                 'get_active_compute_cnt')
 277     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 278                 'get_ext_net_name', return_value='test_net_name')
 279     @mock.patch('snaps.openstack.utils.deploy_utils.create_image',
 280                 return_value=None)
 281     def test_prepare_env_image_missing(
 282             self, mock_get_img, mock_get_net, mock_get_comp_cnt):
 283         self.rally_base.TESTS = ['test1', 'test2']
 284         self.rally_base.test_name = 'test1'
 285         with self.assertRaises(Exception):
 286             self.rally_base._prepare_env()
 287         mock_get_img.assert_called()
 288         mock_get_net.assert_called()
 289         mock_get_comp_cnt.assert_called()
 290
 291     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 292                 'get_active_compute_cnt')
 293     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 294                 'get_ext_net_name', return_value='test_net_name')
 295     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 296     @mock.patch('snaps.openstack.utils.deploy_utils.create_network',
 297                 return_value=None)
 298     def test_prepare_env_network_creation_failed(
 299             self, mock_create_net, mock_get_img, mock_get_net,
 300             mock_get_comp_cnt):
 301         self.rally_base.TESTS = ['test1', 'test2']
 302         self.rally_base.test_name = 'test1'
 303         with self.assertRaises(Exception):
 304             self.rally_base._prepare_env()
 305         mock_create_net.assert_called()
 306         mock_get_img.assert_called()
 307         mock_get_net.assert_called()
 308         mock_get_comp_cnt.assert_called()
 309
 310     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 311                 'get_active_compute_cnt')
 312     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 313                 'get_ext_net_name', return_value='test_net_name')
 314     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 315     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 316     @mock.patch('snaps.openstack.utils.deploy_utils.create_router',
 317                 return_value=None)
 318     def test_prepare_env_router_creation_failed(self, *args):
 319         self.rally_base.TESTS = ['test1', 'test2']
 320         self.rally_base.test_name = 'test1'
 321         with self.assertRaises(Exception):
 322             self.rally_base._prepare_env()
 323         for func in args:
 324             func.assert_called()
 325
 326     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 327                 'get_active_compute_cnt')
 328     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 329                 'get_ext_net_name', return_value='test_net_name')
 330     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 331     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 332     @mock.patch('snaps.openstack.utils.deploy_utils.create_router')
 333     @mock.patch('snaps.openstack.create_flavor.OpenStackFlavor.create',
 334                 return_value=None)
 335     def test_prepare_env_flavor_creation_failed(self, mock_create_flavor,
 336                                                 *args):
 337         self.rally_base.TESTS = ['test1', 'test2']
 338         self.rally_base.test_name = 'test1'
 339         with self.assertRaises(Exception):
 340             self.rally_base._prepare_env()
 341         for func in args:
 342             func.assert_called()
 343         mock_create_flavor.assert_called_once()
 344
 345     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 346                 'get_active_compute_cnt')
 347     @mock.patch('functest.opnfv_tests.openstack.snaps.snaps_utils.'
 348                 'get_ext_net_name', return_value='test_net_name')
 349     @mock.patch('snaps.openstack.utils.deploy_utils.create_image')
 350     @mock.patch('snaps.openstack.utils.deploy_utils.create_network')
 351     @mock.patch('snaps.openstack.utils.deploy_utils.create_router')
 352     @mock.patch('snaps.openstack.create_flavor.OpenStackFlavor.create',
 353                 side_effect=[mock.Mock, None])
 354     def test_prepare_env_flavor_alt_creation_failed(self, mock_create_flavor,
 355                                                     *args):
 356         self.rally_base.TESTS = ['test1', 'test2']
 357         self.rally_base.test_name = 'test1'
 358         with self.assertRaises(Exception):
 359             self.rally_base._prepare_env()
 360         for func in args:
 361             func.assert_called()
 362         self.assertEqual(mock_create_flavor.call_count, 2)
 363
 364     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 365                 '_run_task')
 366     def test_run_tests_all(self, mock_run_task):
 367         self.rally_base.TESTS = ['test1', 'test2']
 368         self.rally_base.test_name = 'all'
 369         self.rally_base._run_tests()
 370         mock_run_task.assert_any_call('test1')
 371         mock_run_task.assert_any_call('test2')
 372
 373     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 374                 '_run_task')
 375     def test_run_tests_default(self, mock_run_task):
 376         self.rally_base.TESTS = ['test1', 'test2']
 377         self.rally_base.test_name = 'test1'
 378         self.rally_base._run_tests()
 379         mock_run_task.assert_any_call('test1')
 380
 381     def test_clean_up_default(self):
 382         creator1 = mock.Mock()
 383         creator2 = mock.Mock()
 384         self.rally_base.creators = [creator1, creator2]
 385         self.rally_base._clean_up()
 386         self.assertTrue(creator1.clean.called)
 387         self.assertTrue(creator2.clean.called)
 388
 389     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 390                 'create_rally_deployment')
 391     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 392                 '_prepare_env')
 393     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 394                 '_run_tests')
 395     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 396                 '_generate_report')
 397     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 398                 '_clean_up')
 399     def test_run_default(self, *args):
 400         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 401         for func in args:
 402             func.assert_called()
 403
 404     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 405                 'create_rally_deployment', side_effect=Exception)
 406     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 407         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 408         mock_create_rally_dep.assert_called()
 409
 410     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 411                 'create_rally_deployment', return_value=mock.Mock())
 412     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 413                 '_prepare_env', side_effect=Exception)
 414     def test_run_exception_prepare_env(self, mock_prep_env, *args):
 415         # pylint: disable=unused-argument
 416         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 417         mock_prep_env.assert_called()
 418
 419     def test_append_summary(self):
 420         text = '[{"result":[{"error":[]},{"error":["err"]}],' \
 421                '"full_duration": 17.312026}]'
 422         self.rally_base._append_summary(text, "foo_test")
 423         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 424         self.assertEqual(self.rally_base.summary[0]['overall_duration'],
 425                          17.312026)
 426         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 427         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 428
 429
 430 if __name__ == "__main__":
 431     logging.disable(logging.CRITICAL)
 432     unittest.main(verbosity=2)