functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 # pylint: disable=missing-docstring,protected-access,invalid-name
   9
  10 import json
  11 import logging
  12 import os
  13 import subprocess
  14 import unittest
  15
  16 import mock
  17 import munch
  18 from xtesting.core import testcase
  19
  20 from functest.opnfv_tests.openstack.rally import rally
  21 from functest.utils import config
  22
  23
  24 class OSRallyTesting(unittest.TestCase):
  25     # pylint: disable=too-many-public-methods
  26     def setUp(self):
  27         with mock.patch('os_client_config.get_config') as mock_get_config, \
  28                 mock.patch('shade.OpenStackCloud') as mock_shade, \
  29                 mock.patch('functest.core.tenantnetwork.NewProject') \
  30                 as mock_new_project:
  31             self.rally_base = rally.RallyBase()
  32             self.rally_base.image = munch.Munch(name='foo')
  33             self.rally_base.flavor = munch.Munch(name='foo')
  34             self.rally_base.flavor_alt = munch.Munch(name='bar')
  35         self.assertTrue(mock_get_config.called)
  36         self.assertTrue(mock_shade.called)
  37         self.assertTrue(mock_new_project.called)
  38
  39     def test_build_task_args_missing_floating_network(self):
  40         os.environ['OS_AUTH_URL'] = ''
  41         self.rally_base.ext_net = None
  42         task_args = self.rally_base.build_task_args('test_name')
  43         self.assertEqual(task_args['floating_network'], '')
  44
  45     def test_build_task_args_missing_net_id(self):
  46         os.environ['OS_AUTH_URL'] = ''
  47         self.rally_base.network = None
  48         task_args = self.rally_base.build_task_args('test_name')
  49         self.assertEqual(task_args['netid'], '')
  50
  51     @staticmethod
  52     def check_scenario_file(value):
  53         yaml_file = 'opnfv-test_file_name.yaml'
  54         if yaml_file in value:
  55             return False
  56         return True
  57
  58     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  59     def test_prepare_test_list_missing_scenario_file(self, mock_func):
  60         mock_func.side_effect = self.check_scenario_file
  61         with self.assertRaises(Exception):
  62             self.rally_base._prepare_test_list('test_file_name')
  63         mock_func.assert_called()
  64
  65     @staticmethod
  66     def check_temp_dir(value):
  67         yaml_file = 'opnfv-test_file_name.yaml'
  68         if yaml_file in value:
  69             return True
  70         return False
  71
  72     @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  73                 'RallyBase.get_verifier_deployment_id', return_value='foo')
  74     @mock.patch('subprocess.check_output')
  75     def test_create_rally_deployment(self, mock_exec, mock_get_id):
  76         # pylint: disable=unused-argument
  77         self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
  78         calls = [
  79             mock.call(['rally', 'deployment', 'destroy', '--deployment',
  80                        str(getattr(config.CONF, 'rally_deployment_name'))]),
  81             mock.call().decode("utf-8"),
  82             mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
  83                        str(getattr(config.CONF, 'rally_deployment_name'))],
  84                       env=None),
  85             mock.call().decode("utf-8"),
  86             mock.call(['rally', 'deployment', 'check']),
  87             mock.call().decode("utf-8")]
  88         mock_exec.assert_has_calls(calls)
  89
  90     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
  91     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
  92     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
  93                 'apply_blacklist')
  94     def test_prepare_test_list_missing_temp_dir(
  95             self, mock_method, mock_os_makedirs, mock_path_exists):
  96         mock_path_exists.side_effect = self.check_temp_dir
  97
  98         yaml_file = 'opnfv-test_file_name.yaml'
  99         ret_val = os.path.join(self.rally_base.temp_dir, yaml_file)
 100         self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
 101                          ret_val)
 102         mock_path_exists.assert_called()
 103         mock_method.assert_called()
 104         mock_os_makedirs.assert_called()
 105
 106     @mock.patch('subprocess.check_output', return_value=b'1\n')
 107     def test_get_task_id_default(self, *args):
 108         tag = 'nova'
 109         self.assertEqual(self.rally_base.get_task_id(tag), '1')
 110         args[0].assert_called_with(
 111             ['rally', 'task', 'list', '--tag', tag, '--uuids-only'])
 112
 113     @mock.patch('subprocess.check_output', return_value=b'\n')
 114     def test_get_task_id_missing_id(self, *args):
 115         tag = 'nova'
 116         self.assertEqual(self.rally_base.get_task_id(tag), '')
 117         args[0].assert_called_with(
 118             ['rally', 'task', 'list', '--tag', tag, '--uuids-only'])
 119
 120     def test_task_succeed_fail(self):
 121         json_raw = json.dumps({})
 122         self.assertEqual(self.rally_base.task_succeed(json_raw),
 123                          False)
 124         json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
 125         self.assertEqual(self.rally_base.task_succeed(json_raw),
 126                          False)
 127
 128     def test_task_succeed_success(self):
 129         json_raw = json.dumps({'tasks': [{'status': 'finished',
 130                                           'pass_sla': True}]})
 131         self.assertEqual(self.rally_base.task_succeed(json_raw),
 132                          True)
 133
 134     @mock.patch('six.moves.builtins.open', mock.mock_open())
 135     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 136                 return_value={'scenario': [
 137                     {'scenarios': ['test_scenario'],
 138                      'tests': ['test']},
 139                     {'scenarios': ['other_scenario'],
 140                      'tests': ['other_test']}]})
 141     def test_excl_scenario_default(self, mock_func):
 142         os.environ['INSTALLER_TYPE'] = 'test_installer'
 143         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 144         self.assertEqual(self.rally_base.excl_scenario(), ['test'])
 145         mock_func.assert_called()
 146
 147     @mock.patch('six.moves.builtins.open', mock.mock_open())
 148     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 149                 return_value={'scenario': [
 150                     {'scenarios': ['^os-[^-]+-featT-modeT$'],
 151                      'tests': ['test1']},
 152                     {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 153                      'tests': ['test2']},
 154                     {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 155                      'tests': ['test3']},
 156                     {'scenarios': ['^os-'],
 157                      'tests': ['test4']},
 158                     {'scenarios': ['other_scenario'],
 159                      'tests': ['test0a']},
 160                     {'scenarios': [''],  # empty scenario
 161                      'tests': ['test0b']}]})
 162     def test_excl_scenario_regex(self, mock_func):
 163         os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
 164         self.assertEqual(self.rally_base.excl_scenario(),
 165                          ['test1', 'test2', 'test3', 'test4'])
 166         mock_func.assert_called()
 167
 168     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 169     def test_excl_scenario_exception(self, mock_open):
 170         self.assertEqual(self.rally_base.excl_scenario(), [])
 171         mock_open.assert_called()
 172
 173     @mock.patch('six.moves.builtins.open', mock.mock_open())
 174     @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
 175                 return_value={'functionality': [
 176                     {'functions': ['no_migration'], 'tests': ['test']}]})
 177     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 178                 '_migration_supported', return_value=False)
 179     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 180                 '_network_trunk_supported', return_value=False)
 181     def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
 182         os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
 183         self.assertEqual(self.rally_base.excl_func(), ['test'])
 184         mock_func.assert_called()
 185         mock_trunk.assert_called()
 186         mock_yaml_load.assert_called()
 187
 188     @mock.patch('six.moves.builtins.open', side_effect=Exception)
 189     def test_excl_func_exception(self, mock_open):
 190         self.assertEqual(self.rally_base.excl_func(), [])
 191         mock_open.assert_called()
 192
 193     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
 194     def test_file_is_empty_default(self, mock_os_stat):
 195         attrs = {'st_size': 10}
 196         mock_os_stat.return_value.configure_mock(**attrs)
 197         self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 198                          False)
 199         mock_os_stat.assert_called()
 200
 201     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
 202                 side_effect=Exception)
 203     def test_file_is_empty_exception(self, mock_os_stat):
 204         self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
 205         mock_os_stat.assert_called()
 206
 207     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 208                 return_value=False)
 209     def test_run_task_missing_task_file(self, mock_path_exists):
 210         with self.assertRaises(Exception):
 211             self.rally_base.prepare_run()
 212         mock_path_exists.assert_called()
 213
 214     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 215                 '_prepare_test_list', return_value='test_file_name')
 216     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 217                 'file_is_empty', return_value=True)
 218     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 219     def test_prepare_task_no_tests_for_scenario(
 220             self, mock_logger_info, mock_file_empty, mock_prep_list):
 221         self.rally_base.prepare_task('test_name')
 222         mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
 223                                          'test_name')
 224         mock_file_empty.assert_called()
 225         mock_prep_list.assert_called()
 226
 227     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 228                 '_prepare_test_list', return_value='test_file_name')
 229     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 230                 'file_is_empty', return_value=False)
 231     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 232                 'build_task_args', return_value={})
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 234                 'get_task_id', return_value=None)
 235     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 236                 return_value=True)
 237     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 238     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 239     def test_run_task_taskid_missing(self, mock_logger_error, *args):
 240         # pylint: disable=unused-argument
 241         with self.assertRaises(Exception):
 242             self.rally_base.run_task('test_name')
 243         text = 'Failed to retrieve task_id'
 244         mock_logger_error.assert_any_call(text)
 245
 246     @mock.patch('six.moves.builtins.open', mock.mock_open())
 247     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 248                 '_prepare_test_list', return_value='test_file_name')
 249     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 250                 'file_is_empty', return_value=False)
 251     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 252                 'build_task_args', return_value={})
 253     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 254                 'get_task_id', return_value='1')
 255     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 256                 'task_succeed', return_value=True)
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 258                 return_value=True)
 259     @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
 260     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 261     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 262     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
 263     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 264                 '_save_results')
 265     def test_run_task_default(self, mock_save_res, *args):
 266         # pylint: disable=unused-argument
 267         self.rally_base.run_task('test_name')
 268         mock_save_res.assert_called()
 269
 270     @mock.patch('six.moves.builtins.open', mock.mock_open())
 271     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 272                 'task_succeed', return_value=True)
 273     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
 274                 return_value=True)
 275     @mock.patch('subprocess.check_output')
 276     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
 277     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
 278     @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
 279     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 280                 '_append_summary')
 281     def test_save_results(self, mock_summary, *args):
 282         # pylint: disable=unused-argument
 283         self.rally_base._save_results('test_name', '1234')
 284         mock_summary.assert_called()
 285
 286     def test_prepare_run_testname_invalid(self):
 287         self.rally_base.stests = ['test1', 'test2']
 288         with self.assertRaises(Exception):
 289             self.rally_base.prepare_run(tests=['test'])
 290
 291     @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
 292     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
 293     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copytree')
 294     @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.rmtree')
 295     def test_prepare_run_flavor_alt_creation_failed(self, *args):
 296         # pylint: disable=unused-argument
 297         self.rally_base.stests = ['test1', 'test2']
 298         with mock.patch.object(self.rally_base, 'count_hypervisors') \
 299             as mock_list_hyperv, \
 300             mock.patch.object(self.rally_base, 'create_flavor_alt',
 301                               side_effect=Exception) \
 302                 as mock_create_flavor:
 303             with self.assertRaises(Exception):
 304                 self.rally_base.prepare_run(tests=['test1'])
 305             mock_list_hyperv.assert_called_once()
 306             mock_create_flavor.assert_called_once()
 307
 308     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 309                 'prepare_task', return_value=True)
 310     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 311                 'run_task')
 312     def test_run_tests_all(self, mock_run_task, mock_prepare_task):
 313         self.rally_base.tests = ['test1', 'test2']
 314         self.rally_base.run_tests()
 315         mock_prepare_task.assert_any_call('test1')
 316         mock_prepare_task.assert_any_call('test2')
 317         mock_run_task.assert_any_call('test1')
 318         mock_run_task.assert_any_call('test2')
 319
 320     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 321                 'prepare_task', return_value=True)
 322     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 323                 'run_task')
 324     def test_run_tests_default(self, mock_run_task, mock_prepare_task):
 325         self.rally_base.tests = ['test1', 'test2']
 326         self.rally_base.run_tests()
 327         mock_prepare_task.assert_any_call('test1')
 328         mock_prepare_task.assert_any_call('test2')
 329         mock_run_task.assert_any_call('test1')
 330         mock_run_task.assert_any_call('test2')
 331
 332     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 333                 'clean_rally_logs')
 334     def test_clean_up_default(self, *args):
 335         with mock.patch.object(self.rally_base.orig_cloud,
 336                                'delete_flavor') as mock_delete_flavor:
 337             self.rally_base.flavor_alt = mock.Mock()
 338             self.rally_base.clean()
 339             self.assertEqual(mock_delete_flavor.call_count, 1)
 340             args[0].assert_called_once_with()
 341
 342     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 343                 'update_rally_logs')
 344     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 345                 'create_rally_deployment')
 346     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 347                 'prepare_run')
 348     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 349                 'run_tests')
 350     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 351                 '_generate_report')
 352     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 353                 'export_task')
 354     def test_run_default(self, *args):
 355         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
 356         for func in args:
 357             func.assert_called()
 358
 359     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 360                 'update_rally_logs')
 361     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 362                 'create_rally_deployment', side_effect=Exception)
 363     def test_run_exception_create_rally_dep(self, *args):
 364         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 365         args[0].assert_called()
 366         args[1].assert_called_once_with(self.rally_base.res_dir)
 367
 368     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 369                 'update_rally_logs')
 370     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 371                 'create_rally_deployment', return_value=mock.Mock())
 372     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
 373                 'prepare_run', side_effect=Exception)
 374     def test_run_exception_prepare_run(self, mock_prep_env, *args):
 375         # pylint: disable=unused-argument
 376         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
 377         mock_prep_env.assert_called()
 378         args[1].assert_called_once_with(self.rally_base.res_dir)
 379
 380     def test_append_summary(self):
 381         json_dict = {
 382             'tasks': [{
 383                 'subtasks': [{
 384                     'title': 'sub_task',
 385                     'workloads': [{
 386                         'full_duration': 1.23,
 387                         'data': [{
 388                             'error': []
 389                         }]
 390                     }, {
 391                         'full_duration': 2.78,
 392                         'data': [{
 393                             'error': ['err']
 394                         }]
 395                     }]
 396                 }]
 397             }]
 398         }
 399         self.rally_base._append_summary(json.dumps(json_dict), "foo_test")
 400         self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
 401         self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
 402         self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
 403         self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
 404         self.assertEqual(self.rally_base.summary[0]['success'], [])
 405         self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task'])
 406
 407     def test_is_successful_false(self):
 408         with mock.patch('six.moves.builtins.super') as mock_super:
 409             self.rally_base.summary = [{"task_status": True},
 410                                        {"task_status": False}]
 411             self.assertEqual(self.rally_base.is_successful(),
 412                              testcase.TestCase.EX_TESTCASE_FAILED)
 413             mock_super(rally.RallyBase, self).is_successful.assert_not_called()
 414
 415     def test_is_successful_true(self):
 416         with mock.patch('six.moves.builtins.super') as mock_super:
 417             mock_super(rally.RallyBase, self).is_successful.return_value = 424
 418             self.rally_base.summary = [{"task_status": True},
 419                                        {"task_status": True}]
 420             self.assertEqual(self.rally_base.is_successful(), 424)
 421             mock_super(rally.RallyBase, self).is_successful.assert_called()
 422
 423     @mock.patch('subprocess.check_output',
 424                 side_effect=subprocess.CalledProcessError('', ''))
 425     def test_export_task_ko(self, *args):
 426         file_name = (f"{self.rally_base.results_dir}/"
 427                      f"{self.rally_base.case_name}.html")
 428         with self.assertRaises(subprocess.CalledProcessError):
 429             self.rally_base.export_task(file_name)
 430         cmd = ["rally", "task", "export", "--type", "html", "--deployment",
 431                str(getattr(config.CONF, 'rally_deployment_name')),
 432                "--to", file_name]
 433         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 434
 435     @mock.patch('subprocess.check_output', return_value=b'')
 436     def test_export_task(self, *args):
 437         file_name = (f"{self.rally_base.results_dir}/"
 438                      f"{self.rally_base.case_name}.html")
 439         self.assertEqual(self.rally_base.export_task(file_name), None)
 440         cmd = ["rally", "task", "export", "--type", "html", "--deployment",
 441                str(getattr(config.CONF, 'rally_deployment_name')),
 442                "--to", file_name]
 443         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 444
 445     @mock.patch('subprocess.check_output',
 446                 side_effect=subprocess.CalledProcessError('', ''))
 447     def test_verify_report_ko(self, *args):
 448         file_name = (f"{self.rally_base.results_dir}/"
 449                      f"{self.rally_base.case_name}.html")
 450         with self.assertRaises(subprocess.CalledProcessError):
 451             self.rally_base.verify_report(file_name, "1")
 452         cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
 453                "--to", file_name]
 454         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 455
 456     @mock.patch('subprocess.check_output', return_value=b'')
 457     def test_verify_report(self, *args):
 458         file_name = (f"{self.rally_base.results_dir}/"
 459                      f"{self.rally_base.case_name}.html")
 460         self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
 461         cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
 462                "--to", file_name]
 463         args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
 464
 465
 466 if __name__ == "__main__":
 467     logging.disable(logging.CRITICAL)
 468     unittest.main(verbosity=2)