functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 import json
   9 import logging
  10 import os
  11 import unittest
  12
  13 import mock
  14
  15 from functest.core import testcase
  16 from functest.opnfv_tests.openstack.rally import rally
  17 from functest.utils.constants import CONST
  18
  19
  20 class OSRallyTesting(unittest.TestCase):
  21
  22     def setUp(self):
  23         self.nova_client = mock.Mock()
  24         self.neutron_client = mock.Mock()
  25         self.cinder_client = mock.Mock()
  26         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  27                         'os_utils.get_nova_client',
  28                         return_value=self.nova_client), \
  29             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  30                        'os_utils.get_neutron_client',
  31                        return_value=self.neutron_client), \
  32             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  33                        'os_utils.get_cinder_client',
  34                        return_value=self.cinder_client):
  35             self.rally_base = rally.RallyBase()
  36             self.rally_base.network_dict['net_id'] = 'test_net_id'
  37             self.polling_iter = 2
  38
  39     def test_build_task_args_missing_floating_network(self):
  40         CONST.__setattr__('OS_AUTH_URL', None)
  41         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  42                         'os_utils.get_external_net',
  43                         return_value=None):
  44             task_args = self.rally_base._build_task_args('test_file_name')
  45             self.assertEqual(task_args['floating_network'], '')
  46
  47     def test_build_task_args_missing_net_id(self):
  48         CONST.__setattr__('OS_AUTH_URL', None)
  49         self.rally_base.network_dict['net_id'] = ''
  50         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  51                         'os_utils.get_external_net',
  52                         return_value='test_floating_network'):
  53             task_args = self.rally_base._build_task_args('test_file_name')
  54             self.assertEqual(task_args['netid'], '')
  55
  56     def check_scenario_file(self, value):
  57         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  58         if yaml_file in value:
  59             return False
  60         return True
  61
  62     def test_prepare_test_list_missing_scenario_file(self):
  63         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  64                         'os.path.exists',
  65                         side_effect=self.check_scenario_file), \
  66                 self.assertRaises(Exception):
  67             self.rally_base._prepare_test_list('test_file_name')
  68
  69     def check_temp_dir(self, value):
  70         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  71         if yaml_file in value:
  72             return True
  73         return False
  74
  75     def test_prepare_test_list_missing_temp_dir(self):
  76         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  77                         'os.path.exists',
  78                         side_effect=self.check_temp_dir), \
  79             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  80                        'os.makedirs') as mock_os_makedir, \
  81             mock.patch.object(self.rally_base, 'apply_blacklist',
  82                               return_value=mock.Mock()) as mock_method:
  83             yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  84             ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  85             self.assertEqual(self.rally_base.
  86                              _prepare_test_list('test_file_name'),
  87                              ret_val)
  88             self.assertTrue(mock_method.called)
  89             self.assertTrue(mock_os_makedir.called)
  90
  91     def test_get_task_id_default(self):
  92         cmd_raw = 'Task 1: started'
  93         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  94                          '1')
  95
  96     def test_get_task_id_missing_id(self):
  97         cmd_raw = ''
  98         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
  99                          None)
 100
 101     def test_task_succeed_fail(self):
 102         json_raw = json.dumps([None])
 103         self.assertEqual(self.rally_base.task_succeed(json_raw),
 104                          False)
 105         json_raw = json.dumps([{'result': [{'error': ['test_error']}]}])
 106         self.assertEqual(self.rally_base.task_succeed(json_raw),
 107                          False)
 108
 109     def test_task_succeed_success(self):
 110         json_raw = json.dumps('')
 111         self.assertEqual(self.rally_base.task_succeed(json_raw),
 112                          True)
 113
 114     def polling(self):
 115         if self.polling_iter == 0:
 116             return "something"
 117         self.polling_iter -= 1
 118         return None
 119
 120     def test_get_cmd_output(self):
 121         proc = mock.Mock()
 122         attrs = {'poll.side_effect': self.polling,
 123                  'stdout.readline.return_value': 'line'}
 124         proc.configure_mock(**attrs)
 125         self.assertEqual(self.rally_base.get_cmd_output(proc),
 126                          'lineline')
 127
 128     def test_excl_scenario_default(self):
 129         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 130         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 131         dic = {'scenario': [{'scenarios': ['test_scenario'],
 132                              'installers': ['test_installer'],
 133                              'tests': ['test']},
 134                             {'scenarios': ['other_scenario'],
 135                              'installers': ['test_installer'],
 136                              'tests': ['other_test']}]}
 137         with mock.patch('__builtin__.open', mock.mock_open()), \
 138             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 139                        'yaml.safe_load',
 140                        return_value=dic):
 141                 self.assertEqual(self.rally_base.excl_scenario(),
 142                                  ['test'])
 143
 144     def test_excl_scenario_regex(self):
 145         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 146         CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
 147         dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'],
 148                              'installers': ['test_installer'],
 149                              'tests': ['test1']},
 150                             {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
 151                              'installers': ['test_installer'],
 152                              'tests': ['test2']},
 153                             {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
 154                              'installers': ['test_installer'],
 155                              'tests': ['test3']},
 156                             {'scenarios': ['^os-'],
 157                              'installers': ['test_installer'],
 158                              'tests': ['test4']},
 159                             {'scenarios': ['other_scenario'],
 160                              'installers': ['test_installer'],
 161                              'tests': ['test0a']},
 162                             {'scenarios': [''],  # empty scenario
 163                              'installers': ['test_installer'],
 164                              'tests': ['test0b']}]}
 165         with mock.patch('__builtin__.open', mock.mock_open()), \
 166             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 167                        'yaml.safe_load',
 168                        return_value=dic):
 169             self.assertEqual(self.rally_base.excl_scenario(),
 170                              ['test1', 'test2', 'test3', 'test4'])
 171
 172     def test_excl_scenario_exception(self):
 173         with mock.patch('__builtin__.open', side_effect=Exception):
 174                 self.assertEqual(self.rally_base.excl_scenario(),
 175                                  [])
 176
 177     def test_excl_func_default(self):
 178         CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
 179         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
 180         dic = {'functionality': [{'functions': ['no_live_migration'],
 181                                   'tests': ['test']}]}
 182         with mock.patch('__builtin__.open', mock.mock_open()), \
 183             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 184                        'yaml.safe_load',
 185                        return_value=dic), \
 186             mock.patch.object(self.rally_base, 'live_migration_supported',
 187                               return_value=False):
 188                 self.assertEqual(self.rally_base.excl_func(),
 189                                  ['test'])
 190
 191     def test_excl_func_exception(self):
 192         with mock.patch('__builtin__.open', side_effect=Exception):
 193                 self.assertEqual(self.rally_base.excl_func(),
 194                                  [])
 195
 196     def test_file_is_empty_default(self):
 197         mock_obj = mock.Mock()
 198         attrs = {'st_size': 10}
 199         mock_obj.configure_mock(**attrs)
 200         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 201                         'os.stat',
 202                         return_value=mock_obj):
 203             self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 204                              False)
 205
 206     def test_file_is_empty_exception(self):
 207         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 208                         'os.stat',
 209                         side_effect=Exception):
 210             self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 211                              True)
 212
 213     def test_run_task_missing_task_file(self):
 214         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 215                         'os.path.exists',
 216                         return_value=False), \
 217                 self.assertRaises(Exception):
 218             self.rally_base._run_task('test_name')
 219
 220     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info')
 221     def test_run_task_no_tests_for_scenario(self, mock_logger_info):
 222         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 223                         'os.path.exists',
 224                         return_value=True), \
 225             mock.patch.object(self.rally_base, '_prepare_test_list',
 226                               return_value='test_file_name'), \
 227             mock.patch.object(self.rally_base, 'file_is_empty',
 228                               return_value=True):
 229             self.rally_base._run_task('test_name')
 230             str = 'No tests for scenario "test_name"'
 231             mock_logger_info.assert_any_call(str)
 232
 233     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error')
 234     def test_run_task_taskid_missing(self, mock_logger_error):
 235         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 236                         'os.path.exists',
 237                         return_value=True), \
 238             mock.patch.object(self.rally_base, '_prepare_test_list',
 239                               return_value='test_file_name'), \
 240             mock.patch.object(self.rally_base, 'file_is_empty',
 241                               return_value=False), \
 242             mock.patch.object(self.rally_base, '_build_task_args',
 243                               return_value={}), \
 244             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 245                        'subprocess.Popen'), \
 246             mock.patch.object(self.rally_base, '_get_output',
 247                               return_value=mock.Mock()), \
 248             mock.patch.object(self.rally_base, 'get_task_id',
 249                               return_value=None), \
 250             mock.patch.object(self.rally_base, 'get_cmd_output',
 251                               return_value=''):
 252             self.rally_base._run_task('test_name')
 253             str = 'Failed to retrieve task_id, validating task...'
 254             mock_logger_error.assert_any_call(str)
 255
 256     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info')
 257     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error')
 258     def test_run_task_default(self, mock_logger_error,
 259                               mock_logger_info):
 260         popen = mock.Mock()
 261         attrs = {'read.return_value': 'json_result'}
 262         popen.configure_mock(**attrs)
 263
 264         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 265                         'os.path.exists',
 266                         return_value=True), \
 267             mock.patch.object(self.rally_base, '_prepare_test_list',
 268                               return_value='test_file_name'), \
 269             mock.patch.object(self.rally_base, 'file_is_empty',
 270                               return_value=False), \
 271             mock.patch.object(self.rally_base, '_build_task_args',
 272                               return_value={}), \
 273             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 274                        'subprocess.Popen'), \
 275             mock.patch.object(self.rally_base, '_get_output',
 276                               return_value=mock.Mock()), \
 277             mock.patch.object(self.rally_base, 'get_task_id',
 278                               return_value='1'), \
 279             mock.patch.object(self.rally_base, 'get_cmd_output',
 280                               return_value=''), \
 281             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 282                        'os.makedirs'), \
 283             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 284                        'os.popen',
 285                        return_value=popen), \
 286             mock.patch('__builtin__.open', mock.mock_open()), \
 287             mock.patch.object(self.rally_base, 'task_succeed',
 288                               return_value=True):
 289             self.rally_base._run_task('test_name')
 290             str = 'Test scenario: "test_name" OK.\n'
 291             mock_logger_info.assert_any_call(str)
 292
 293     def test_prepare_env_testname_invalid(self):
 294         self.rally_base.TESTS = ['test1', 'test2']
 295         self.rally_base.test_name = 'test'
 296         with self.assertRaises(Exception):
 297             self.rally_base._prepare_env()
 298
 299     def test_prepare_env_volume_creation_failed(self):
 300         self.rally_base.TESTS = ['test1', 'test2']
 301         self.rally_base.test_name = 'test1'
 302         volume_type = None
 303         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 304                         'os_utils.list_volume_types',
 305                         return_value=None), \
 306             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 307                        'os_utils.create_volume_type',
 308                        return_value=volume_type), \
 309                 self.assertRaises(Exception):
 310             self.rally_base._prepare_env()
 311
 312     def test_prepare_env_image_missing(self):
 313         self.rally_base.TESTS = ['test1', 'test2']
 314         self.rally_base.test_name = 'test1'
 315         volume_type = mock.Mock()
 316         image_id = None
 317         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 318                         'os_utils.list_volume_types',
 319                         return_value=None), \
 320             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 321                        'os_utils.create_volume_type',
 322                        return_value=volume_type), \
 323             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 324                        'os_utils.get_or_create_image',
 325                        return_value=(True, image_id)), \
 326                 self.assertRaises(Exception):
 327             self.rally_base._prepare_env()
 328
 329     def test_prepare_env_image_shared_network_creation_failed(self):
 330         self.rally_base.TESTS = ['test1', 'test2']
 331         self.rally_base.test_name = 'test1'
 332         volume_type = mock.Mock()
 333         image_id = 'image_id'
 334         network_dict = None
 335         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 336                         'os_utils.list_volume_types',
 337                         return_value=None), \
 338             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 339                        'os_utils.create_volume_type',
 340                        return_value=volume_type), \
 341             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 342                        'os_utils.get_or_create_image',
 343                        return_value=(True, image_id)), \
 344             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 345                        'os_utils.create_shared_network_full',
 346                        return_value=network_dict), \
 347                 self.assertRaises(Exception):
 348             self.rally_base._prepare_env()
 349
 350     def test_run_tests_all(self):
 351         self.rally_base.TESTS = ['test1', 'test2']
 352         self.rally_base.test_name = 'all'
 353         with mock.patch.object(self.rally_base, '_run_task',
 354                                return_value=mock.Mock()):
 355             self.rally_base._run_tests()
 356             self.rally_base._run_task.assert_any_call('test1')
 357             self.rally_base._run_task.assert_any_call('test2')
 358
 359     def test_run_tests_default(self):
 360         self.rally_base.TESTS = ['test1', 'test2']
 361         self.rally_base.test_name = 'test1'
 362         with mock.patch.object(self.rally_base, '_run_task',
 363                                return_value=mock.Mock()):
 364             self.rally_base._run_tests()
 365             self.rally_base._run_task.assert_any_call('test1')
 366
 367     def test_clean_up_default(self):
 368         self.rally_base.volume_type = mock.Mock()
 369         self.rally_base.cinder_client = mock.Mock()
 370         self.rally_base.image_exists = False
 371         self.rally_base.image_id = 1
 372         self.rally_base.nova_client = mock.Mock()
 373         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 374                         'os_utils.delete_volume_type') as mock_vol_method, \
 375             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 376                        'os_utils.delete_glance_image') as mock_glance_method:
 377             self.rally_base._clean_up()
 378             mock_vol_method.assert_any_call(self.rally_base.cinder_client,
 379                                             self.rally_base.volume_type)
 380             mock_glance_method.assert_any_call(self.rally_base.nova_client,
 381                                                1)
 382
 383     def test_run_default(self):
 384         with mock.patch.object(self.rally_base, '_prepare_env'), \
 385             mock.patch.object(self.rally_base, '_run_tests'), \
 386             mock.patch.object(self.rally_base, '_generate_report'), \
 387                 mock.patch.object(self.rally_base, '_clean_up'):
 388             self.assertEqual(self.rally_base.run(),
 389                              testcase.TestCase.EX_OK)
 390
 391     def test_run_exception(self):
 392         with mock.patch.object(self.rally_base, '_prepare_env',
 393                                side_effect=Exception):
 394             self.assertEqual(self.rally_base.run(),
 395                              testcase.TestCase.EX_RUN_ERROR)
 396
 397
 398 if __name__ == "__main__":
 399     logging.disable(logging.CRITICAL)
 400     unittest.main(verbosity=2)