functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import subprocess
  14 import unittest
  15
  16 import mock
  17 import munch
  18 from xtesting.core import testcase
  19
  20 from functest.opnfv_tests.openstack.rally import rally
  21 from functest.utils import config
  22
  23
  24 class OSRallyTesting(unittest.TestCase):
  25     # pylint: disable=too-many-public-methods
  26     def setUp(self):
  27         with mock.patch('os_client_config.get_config') as mock_get_config, \
  28                 mock.patch('shade.OpenStackCloud') as mock_shade, \
  29                 mock.patch('functest.core.tenantnetwork.NewProject') \
  30                 as mock_new_project:
  31             self.rally_base = rally.RallyBase()
  32             self.rally_base.image = munch.Munch(name='foo')
  33             self.rally_base.flavor = munch.Munch(name='foo')
  34             self.rally_base.flavor_alt = munch.Munch(name='bar')
  35         self.assertTrue(mock_get_config.called)
  36         self.assertTrue(mock_shade.called)
  37         self.assertTrue(mock_new_project.called)
  38
  39     def test_build_task_args_missing_floating_network(self):
  40         os.environ['OS_AUTH_URL'] = ''
  41         self.rally_base.ext_net = None
  42         task_args = self.rally_base.build_task_args('test_name')
  43         self.assertEqual(task_args['floating_network'], '')
  44
  45     def test_build_task_args_missing_net_id(self):
  46         os.environ['OS_AUTH_URL'] = ''
  47         self.rally_base.network = None
  48         task_args = self.rally_base.build_task_args('test_name')
  49         self.assertEqual(task_args['netid'], '')
  50
  51     @staticmethod
  52     def check_scenario_file(value):
  53         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  54         if yaml_file in value:
  55             return False
  56         return True
  57
  58     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  59     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  60         mock_func.side_effect = self.check_scenario_file
  61         with self.assertRaises(Exception):
  62             self.rally_base._prepare_test_list('test_file_name')
  63         mock_func.assert_called()
  64
  65     @staticmethod
  66     def check_temp_dir(value):
  67         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  68         if yaml_file in value:
  69             return True
  70         return False
  71
  72     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  73     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  74     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  75                 'apply_blacklist')
  76     def test_prepare_test_list_missing_temp_dir(
  77             self, mock_method, mock_os_makedirs, mock_path_exists):
  78         mock_path_exists.side_effect = self.check_temp_dir
  79
  80         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  81         ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  82         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
  83                          ret_val)
  84         mock_path_exists.assert_called()
  85         mock_method.assert_called()
  86         mock_os_makedirs.assert_called()
  87
  88     def test_get_task_id_default(self):
  89         cmd_raw = 'Task 1: started'
  90         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  91                          '1')
  92
  93     def test_get_task_id_missing_id(self):
  94         cmd_raw = ''
  95         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  96                          None)
  97
  98     def test_task_succeed_fail(self):
  99         json_raw = json.dumps({})
 100         self.assertEqual(self.rally_base.task_succeed(json_raw),
 101                          False)
 102         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
 103         self.assertEqual(self.rally_base.task_succeed(json_raw),
 104                          False)
 105
 106     def test_task_succeed_success(self):
 107         json_raw = json.dumps({'tasks': [{'status': 'finished',
 108                                           'pass_sla': True}]})
 109         self.assertEqual(self.rally_base.task_succeed(json_raw),
 110                          True)
 111
 112     @mock.patch('six.moves.builtins.open', mock.mock_open())
 113     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 114                 return_value={'scenario': [
 115                     {'scenarios': ['test_scenario'],
 116                      'tests': ['test']},
 117                     {'scenarios': ['other_scenario'],
 118                      'tests': ['other_test']}]})
 119     def test_excl_scenario_default(self, mock_func):
 120         os.environ['INSTALLER_TYPE'] = 'test_installer'
 121         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 122         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 123         mock_func.assert_called()
 124
 125     @mock.patch('six.moves.builtins.open', mock.mock_open())
 126     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 127                 return_value={'scenario': [
 128                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 129                      'tests': ['test1']},
 130                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 131                      'tests': ['test2']},
 132                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 133                      'tests': ['test3']},
 134                     {'scenarios': ['^os-'],
 135                      'tests': ['test4']},
 136                     {'scenarios': ['other_scenario'],
 137                      'tests': ['test0a']},
 138                     {'scenarios': [''],  # empty scenario
 139                      'tests': ['test0b']}]})
 140     def test_excl_scenario_regex(self, mock_func):
 141         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 142         self.assertEqual(self.rally_base.excl_scenario(),
 143                          ['test1', 'test2', 'test3', 'test4'])
 144         mock_func.assert_called()
 145
 146     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 147     def test_excl_scenario_exception(self, mock_open):
 148         self.assertEqual(self.rally_base.excl_scenario(), [])
 149         mock_open.assert_called()
 150
 151     @mock.patch('six.moves.builtins.open', mock.mock_open())
 152     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 153                 return_value={'functionality': [
 154                     {'functions': ['no_migration'], 'tests': ['test']}]})
 155     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 156                 '_migration_supported', return_value=False)
 157     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 158                 '_network_trunk_supported', return_value=False)
 159     def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
 160         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 161         self.assertEqual(self.rally_base.excl_func(), ['test'])
 162         mock_func.assert_called()
 163         mock_trunk.assert_called()
 164         mock_yaml_load.assert_called()
 165
 166     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 167     def test_excl_func_exception(self, mock_open):
 168         self.assertEqual(self.rally_base.excl_func(), [])
 169         mock_open.assert_called()
 170
 171     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 172     def test_file_is_empty_default(self, mock_os_stat):
 173         attrs = {'st_size': 10}
 174         mock_os_stat.return_value.configure_mock(**attrs)
 175         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 176                          False)
 177         mock_os_stat.assert_called()
 178
 179     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 180                 side_effect=Exception)
 181     def test_file_is_empty_exception(self, mock_os_stat):
 182         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 183         mock_os_stat.assert_called()
 184
 185     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 186                 return_value=False)
 187     def test_run_task_missing_task_file(self, mock_path_exists):
 188         with self.assertRaises(Exception):
 189             self.rally_base.prepare_run()
 190         mock_path_exists.assert_called()
 191
 192     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 193                 '_prepare_test_list', return_value='test_file_name')
 194     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 195                 'file_is_empty', return_value=True)
 196     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 197     def test_prepare_task_no_tests_for_scenario(
 198             self, mock_logger_info, mock_file_empty, mock_prep_list):
 199         self.rally_base.prepare_task('test_name')
 200         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 201                                          'test_name')
 202         mock_file_empty.assert_called()
 203         mock_prep_list.assert_called()
 204
 205     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 206                 '_prepare_test_list', return_value='test_file_name')
 207     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 208                 'file_is_empty', return_value=False)
 209     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 210                 'build_task_args', return_value={})
 211     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 212                 'get_task_id', return_value=None)
 213     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 214                 return_value=True)
 215     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 216     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 217     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 218         # pylint: disable=unused-argument
 219         with self.assertRaises(Exception):
 220             self.rally_base.run_task('test_name')
 221         text = 'Failed to retrieve task_id'
 222         mock_logger_error.assert_any_call(text)
 223
 224     @mock.patch('six.moves.builtins.open', mock.mock_open())
 225     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 226                 '_prepare_test_list', return_value='test_file_name')
 227     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 228                 'file_is_empty', return_value=False)
 229     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 230                 'build_task_args', return_value={})
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 232                 'get_task_id', return_value='1')
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 234                 'task_succeed', return_value=True)
 235     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 236                 return_value=True)
 237     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 238     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 239     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 240     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 241     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 242                 '_save_results')
 243     def test_run_task_default(self, mock_save_res, *args):
 244         # pylint: disable=unused-argument
 245         self.rally_base.run_task('test_name')
 246         mock_save_res.assert_called()
 247
 248     @mock.patch('six.moves.builtins.open', mock.mock_open())
 249     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 250                 'task_succeed', return_value=True)
 251     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 252                 return_value=True)
 253     @mock.patch('subprocess.check_output')
 254     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 256     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 258                 '_append_summary')
 259     def test_save_results(self, mock_summary, *args):
 260         # pylint: disable=unused-argument
 261         self.rally_base._save_results('test_name', '1234')
 262         mock_summary.assert_called()
 263
 264     def test_prepare_run_testname_invalid(self):
 265         self.rally_base.TESTS = ['test1', 'test2']
 266         with self.assertRaises(Exception):
 267             self.rally_base.prepare_run(tests=['test'])
 268
 269     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
 270     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
 271     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copytree')
 272     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.rmtree')
 273     def test_prepare_run_flavor_alt_creation_failed(self, *args):
 274         # pylint: disable=unused-argument
 275         self.rally_base.TESTS = ['test1', 'test2']
 276         with mock.patch.object(self.rally_base.cloud,
 277                                'list_hypervisors') as mock_list_hyperv, \
 278             mock.patch.object(self.rally_base, 'create_flavor_alt',
 279                               side_effect=Exception) \
 280                 as mock_create_flavor:
 281             with self.assertRaises(Exception):
 282                 self.rally_base.prepare_run(tests=['test1'])
 283             mock_list_hyperv.assert_called_once()
 284             mock_create_flavor.assert_called_once()
 285
 286     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 287                 'prepare_task', return_value=True)
 288     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 289                 'run_task')
 290     def test_run_tests_all(self, mock_run_task, mock_prepare_task):
 291         self.rally_base.tests = ['test1', 'test2']
 292         self.rally_base.run_tests()
 293         mock_prepare_task.assert_any_call('test1')
 294         mock_prepare_task.assert_any_call('test2')
 295         mock_run_task.assert_any_call('test1')
 296         mock_run_task.assert_any_call('test2')
 297
 298     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 299                 'prepare_task', return_value=True)
 300     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 301                 'run_task')
 302     def test_run_tests_default(self, mock_run_task, mock_prepare_task):
 303         self.rally_base.tests = ['test1', 'test2']
 304         self.rally_base.run_tests()
 305         mock_prepare_task.assert_any_call('test1')
 306         mock_prepare_task.assert_any_call('test2')
 307         mock_run_task.assert_any_call('test1')
 308         mock_run_task.assert_any_call('test2')
 309
 310     def test_clean_up_default(self):
 311         with mock.patch.object(self.rally_base.orig_cloud,
 312                                'delete_flavor') as mock_delete_flavor:
 313             self.rally_base.flavor_alt = mock.Mock()
 314             self.rally_base.clean()
 315             self.assertEqual(mock_delete_flavor.call_count, 1)
 316
 317     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 318                 'create_rally_deployment')
 319     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 320                 'prepare_run')
 321     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 322                 'run_tests')
 323     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 324                 '_generate_report')
 325     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 326                 'generate_html_report')
 327     def test_run_default(self, *args):
 328         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 329         for func in args:
 330             func.assert_called()
 331
 332     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 333                 'create_rally_deployment', side_effect=Exception)
 334     def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
 335         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 336         mock_create_rally_dep.assert_called()
 337
 338     @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
 339                 'create_rally_deployment', return_value=mock.Mock())
 340     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 341                 'prepare_run', side_effect=Exception)
 342     def test_run_exception_prepare_run(self, mock_prep_env, *args):
 343         # pylint: disable=unused-argument
 344         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 345         mock_prep_env.assert_called()
 346
 347     def test_append_summary(self):
 348         text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \
 349                '1.23,"data": [{"error": []}]}]},{"workloads": ' \
 350                '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}'
 351         self.rally_base._append_summary(text, "foo_test")
 352         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 353         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 354         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 355         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 356
 357     def test_is_successful_false(self):
 358         with mock.patch('six.moves.builtins.super') as mock_super:
 359             self.rally_base.summary = [{"task_status": True},
 360                                        {"task_status": False}]
 361             self.assertEqual(self.rally_base.is_successful(),
 362                              testcase.TestCase.EX_TESTCASE_FAILED)
 363             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 364
 365     def test_is_successful_true(self):
 366         with mock.patch('six.moves.builtins.super') as mock_super:
 367             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 368             self.rally_base.summary = [{"task_status": True},
 369                                        {"task_status": True}]
 370             self.assertEqual(self.rally_base.is_successful(), 424)
 371             mock_super(rally.RallyBase, self).is_successful.assert_called()
 372
 373     @mock.patch('subprocess.check_output',
 374                 side_effect=subprocess.CalledProcessError('', ''))
 375     def test_generate_html_report_ko(self, *args):
 376         with self.assertRaises(subprocess.CalledProcessError):
 377             self.rally_base.generate_html_report()
 378         cmd = ["rally", "task", "report", "--deployment",
 379                str(getattr(config.CONF, 'rally_deployment_name')),
 380                "--out", "{}/{}.html".format(
 381                    self.rally_base.results_dir, self.rally_base.case_name)]
 382         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 383
 384     @mock.patch('subprocess.check_output', return_value=None)
 385     def test_generate_html_report(self, *args):
 386         self.assertEqual(self.rally_base.generate_html_report(), None)
 387         cmd = ["rally", "task", "report", "--deployment",
 388                str(getattr(config.CONF, 'rally_deployment_name')),
 389                "--out", "{}/{}.html".format(
 390                    self.rally_base.results_dir, self.rally_base.case_name)]
 391         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 392
 393
 394 if __name__ == "__main__":
 395     logging.disable(logging.CRITICAL)
 396     unittest.main(verbosity=2)