functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import unittest
  14
  15 import mock
  16 import munch
  17 from xtesting.core import testcase
  18
  19 from functest.opnfv_tests.openstack.rally import rally
  20
  21
  22 class OSRallyTesting(unittest.TestCase):
  23     # pylint: disable=too-many-public-methods
  24     def setUp(self):
  25         with mock.patch('os_client_config.get_config') as mock_get_config, \
  26                 mock.patch('shade.OpenStackCloud') as mock_shade, \
  27                 mock.patch('functest.core.tenantnetwork.NewProject') \
  28                 as mock_new_project:
  29             self.rally_base = rally.RallyBase()
  30             self.rally_base.image = munch.Munch(name='foo')
  31             self.rally_base.flavor = munch.Munch(name='foo')
  32             self.rally_base.flavor_alt = munch.Munch(name='bar')
  33             self.rally_base.test_name = 'all'
  34         self.assertTrue(mock_get_config.called)
  35         self.assertTrue(mock_shade.called)
  36         self.assertTrue(mock_new_project.called)
  37
  38     def test_build_task_args_missing_floating_network(self):
  39         os.environ['OS_AUTH_URL'] = ''
  40         self.rally_base.ext_net = None
  41         task_args = self.rally_base._build_task_args('test_file_name')
  42         self.assertEqual(task_args['floating_network'], '')
  43
  44     def test_build_task_args_missing_net_id(self):
  45         os.environ['OS_AUTH_URL'] = ''
  46         self.rally_base.network = None
  47         task_args = self.rally_base._build_task_args('test_file_name')
  48         self.assertEqual(task_args['netid'], '')
  49
  50     @staticmethod
  51     def check_scenario_file(value):
  52         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  53         if yaml_file in value:
  54             return False
  55         return True
  56
  57     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  58     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  59         mock_func.side_effect = self.check_scenario_file
  60         with self.assertRaises(Exception):
  61             self.rally_base._prepare_test_list('test_file_name')
  62         mock_func.assert_called()
  63
  64     @staticmethod
  65     def check_temp_dir(value):
  66         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  67         if yaml_file in value:
  68             return True
  69         return False
  70
  71     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  72     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  73     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  74                 '_apply_blacklist')
  75     def test_prepare_test_list_missing_temp_dir(
  76             self, mock_method, mock_os_makedirs, mock_path_exists):
  77         mock_path_exists.side_effect = self.check_temp_dir
  78
  79         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  80         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  81         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  82                          ret_val)
  83         mock_path_exists.assert_called()
  84         mock_method.assert_called()
  85         mock_os_makedirs.assert_called()
  86
  87     def test_get_task_id_default(self):
  88         cmd_raw = 'Task 1: started'
  89         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  90                          '1')
  91
  92     def test_get_task_id_missing_id(self):
  93         cmd_raw = ''
  94         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  95                          None)
  96
  97     def test_task_succeed_fail(self):
  98         json_raw = json.dumps({})
  99         self.assertEqual(self.rally_base.task_succeed(json_raw),
 100                          False)
 101         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
 102         self.assertEqual(self.rally_base.task_succeed(json_raw),
 103                          False)
 104
 105     def test_task_succeed_success(self):
 106         json_raw = json.dumps({'tasks': [{'status': 'finished',
 107                                           'pass_sla': True}]})
 108         self.assertEqual(self.rally_base.task_succeed(json_raw),
 109                          True)
 110
 111     @mock.patch('six.moves.builtins.open', mock.mock_open())
 112     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 113                 return_value={'scenario': [
 114                     {'scenarios': ['test_scenario'],
 115                      'tests': ['test']},
 116                     {'scenarios': ['other_scenario'],
 117                      'tests': ['other_test']}]})
 118     def test_excl_scenario_default(self, mock_func):
 119         os.environ['INSTALLER_TYPE'] = 'test_installer'
 120         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 121         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 122         mock_func.assert_called()
 123
 124     @mock.patch('six.moves.builtins.open', mock.mock_open())
 125     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 126                 return_value={'scenario': [
 127                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 128                      'tests': ['test1']},
 129                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 130                      'tests': ['test2']},
 131                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 132                      'tests': ['test3']},
 133                     {'scenarios': ['^os-'],
 134                      'tests': ['test4']},
 135                     {'scenarios': ['other_scenario'],
 136                      'tests': ['test0a']},
 137                     {'scenarios': [''],  # empty scenario
 138                      'tests': ['test0b']}]})
 139     def test_excl_scenario_regex(self, mock_func):
 140         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 141         self.assertEqual(self.rally_base.excl_scenario(),
 142                          ['test1', 'test2', 'test3', 'test4'])
 143         mock_func.assert_called()
 144
 145     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 146     def test_excl_scenario_exception(self, mock_open):
 147         self.assertEqual(self.rally_base.excl_scenario(), [])
 148         mock_open.assert_called()
 149
 150     @mock.patch('six.moves.builtins.open', mock.mock_open())
 151     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 152                 return_value={'functionality': [
 153                     {'functions': ['no_migration'], 'tests': ['test']}]})
 154     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 155                 '_migration_supported', return_value=False)
 156     def test_excl_func_default(self, mock_func, mock_yaml_load):
 157         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 158         self.assertEqual(self.rally_base.excl_func(), ['test'])
 159         mock_func.assert_called()
 160         mock_yaml_load.assert_called()
 161
 162     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 163     def test_excl_func_exception(self, mock_open):
 164         self.assertEqual(self.rally_base.excl_func(), [])
 165         mock_open.assert_called()
 166
 167     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 168     def test_file_is_empty_default(self, mock_os_stat):
 169         attrs = {'st_size': 10}
 170         mock_os_stat.return_value.configure_mock(**attrs)
 171         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 172                          False)
 173         mock_os_stat.assert_called()
 174
 175     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 176                 side_effect=Exception)
 177     def test_file_is_empty_exception(self, mock_os_stat):
 178         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 179         mock_os_stat.assert_called()
 180
 181     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 182                 return_value=False)
 183     def test_run_task_missing_task_file(self, mock_path_exists):
 184         with self.assertRaises(Exception):
 185             self.rally_base.prepare_run()
 186         mock_path_exists.assert_called()
 187
 188     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 189                 '_prepare_test_list', return_value='test_file_name')
 190     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 191                 'file_is_empty', return_value=True)
 192     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 193     def test_prepare_task_no_tests_for_scenario(
 194             self, mock_logger_info, mock_file_empty, mock_prep_list):
 195         self.rally_base.prepare_task('test_name')
 196         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 197                                          'test_name')
 198         mock_file_empty.assert_called()
 199         mock_prep_list.assert_called()
 200
 201     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 202                 '_prepare_test_list', return_value='test_file_name')
 203     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 204                 'file_is_empty', return_value=False)
 205     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 206                 '_build_task_args', return_value={})
 207     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 208                 'get_task_id', return_value=None)
 209     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 210                 return_value=True)
 211     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 212     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 213     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 214         # pylint: disable=unused-argument
 215         with self.assertRaises(Exception):
 216             self.rally_base.run_task('test_name')
 217         text = 'Failed to retrieve task_id'
 218         mock_logger_error.assert_any_call(text)
 219
 220     @mock.patch('six.moves.builtins.open', mock.mock_open())
 221     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 222                 '_prepare_test_list', return_value='test_file_name')
 223     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 224                 'file_is_empty', return_value=False)
 225     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 226                 '_build_task_args', return_value={})
 227     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 228                 'get_task_id', return_value='1')
 229     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 230                 'task_succeed', return_value=True)
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 232                 return_value=True)
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 234     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 235     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 236     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 237     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 238                 '_save_results')
 239     def test_run_task_default(self, mock_save_res, *args):
 240         # pylint: disable=unused-argument
 241         self.rally_base.run_task('test_name')
 242         mock_save_res.assert_called()
 243
 244     @mock.patch('six.moves.builtins.open', mock.mock_open())
 245     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 246                 'task_succeed', return_value=True)
 247     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 248                 return_value=True)
 249     @mock.patch('subprocess.check_output')
 250     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 251     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 252     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 253     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 254                 '_append_summary')
 255     def test_save_results(self, mock_summary, *args):
 256         # pylint: disable=unused-argument
 257         self.rally_base._save_results('test_name', '1234')
 258         mock_summary.assert_called()
 259
 260     def test_prepare_run_testname_invalid(self):
 261         self.rally_base.TESTS = ['test1', 'test2']
 262         self.rally_base.test_name = 'test'
 263         with self.assertRaises(Exception):
 264             self.rally_base.prepare_run()
 265
 266     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 267                 'get_external_network')
 268     def test_prepare_run_flavor_alt_creation_failed(self, *args):
 269         # pylint: disable=unused-argument
 270         self.rally_base.TESTS = ['test1', 'test2']
 271         self.rally_base.test_name = 'test1'
 272         with mock.patch.object(self.rally_base.cloud,
 273                                'list_hypervisors') as mock_list_hyperv, \
 274             mock.patch.object(self.rally_base, 'create_flavor_alt',
 275                               side_effect=Exception) \
 276                 as mock_create_flavor:
 277             with self.assertRaises(Exception):
 278                 self.rally_base.prepare_run()
 279             mock_list_hyperv.assert_called_once()
 280             mock_create_flavor.assert_called_once()
 281
 282     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 283                 'prepare_task', return_value=True)
 284     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 285                 'run_task')
 286     def test_run_tests_all(self, mock_run_task, mock_prepare_task):
 287         self.rally_base.tests = ['test1', 'test2']
 288         self.rally_base.test_name = 'all'
 289         self.rally_base.run_tests()
 290         mock_prepare_task.assert_any_call('test1')
 291         mock_prepare_task.assert_any_call('test2')
 292         mock_run_task.assert_any_call('test1')
 293         mock_run_task.assert_any_call('test2')
 294
 295     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 296                 'prepare_task', return_value=True)
 297     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 298                 'run_task')
 299     def test_run_tests_default(self, mock_run_task, mock_prepare_task):
 300         self.rally_base.tests = ['test1', 'test2']
 301         self.rally_base.run_tests()
 302         mock_prepare_task.assert_any_call('test1')
 303         mock_prepare_task.assert_any_call('test2')
 304         mock_run_task.assert_any_call('test1')
 305         mock_run_task.assert_any_call('test2')
 306
 307     def test_clean_up_default(self):
 308         with mock.patch.object(self.rally_base.orig_cloud,
 309                                'delete_flavor') as mock_delete_flavor:
 310             self.rally_base.flavor_alt = mock.Mock()
 311             self.rally_base.clean()
 312             self.assertEqual(mock_delete_flavor.call_count, 1)
 313
 314     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 315                 'create_rally_deployment')
 316     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 317                 'prepare_run')
 318     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 319                 'run_tests')
 320     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 321                 '_generate_report')
 322     def test_run_default(self, *args):
 323         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 324         for func in args:
 325             func.assert_called()
 326
 327     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 328                 'create_rally_deployment', side_effect=Exception)
 329     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 330         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 331         mock_create_rally_dep.assert_called()
 332
 333     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 334                 'create_rally_deployment', return_value=mock.Mock())
 335     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 336                 'prepare_run', side_effect=Exception)
 337     def test_run_exception_prepare_run(self, mock_prep_env, *args):
 338         # pylint: disable=unused-argument
 339         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 340         mock_prep_env.assert_called()
 341
 342     def test_append_summary(self):
 343         text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \
 344                '1.23,"data": [{"error": []}]}]},{"workloads": ' \
 345                '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}'
 346         self.rally_base._append_summary(text, "foo_test")
 347         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 348         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 349         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 350         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 351
 352     def test_is_successful_false(self):
 353         with mock.patch('six.moves.builtins.super') as mock_super:
 354             self.rally_base.summary = [{"task_status": True},
 355                                        {"task_status": False}]
 356             self.assertEqual(self.rally_base.is_successful(),
 357                              testcase.TestCase.EX_TESTCASE_FAILED)
 358             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 359
 360     def test_is_successful_true(self):
 361         with mock.patch('six.moves.builtins.super') as mock_super:
 362             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 363             self.rally_base.summary = [{"task_status": True},
 364                                        {"task_status": True}]
 365             self.assertEqual(self.rally_base.is_successful(), 424)
 366             mock_super(rally.RallyBase, self).is_successful.assert_called()
 367
 368
 369 if __name__ == "__main__":
 370     logging.disable(logging.CRITICAL)
 371     unittest.main(verbosity=2)