functest/tests/unit/openstack/rally/test_rally.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 import json
   9 import logging
  10 import os
  11 import unittest
  12
  13 import mock
  14
  15 from functest.core import testcase
  16 from functest.opnfv_tests.openstack.rally import rally
  17 from functest.utils.constants import CONST
  18
  19
  20 class OSRallyTesting(unittest.TestCase):
  21
  22     logging.disable(logging.CRITICAL)
  23
  24     def setUp(self):
  25         self.nova_client = mock.Mock()
  26         self.neutron_client = mock.Mock()
  27         self.cinder_client = mock.Mock()
  28         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  29                         'os_utils.get_nova_client',
  30                         return_value=self.nova_client), \
  31             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  32                        'os_utils.get_neutron_client',
  33                        return_value=self.neutron_client), \
  34             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  35                        'os_utils.get_cinder_client',
  36                        return_value=self.cinder_client):
  37             self.rally_base = rally.RallyBase()
  38             self.rally_base.network_dict['net_id'] = 'test_net_id'
  39             self.polling_iter = 2
  40
  41     def test_build_task_args_missing_floating_network(self):
  42         CONST.OS_AUTH_URL = None
  43         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  44                         'os_utils.get_external_net',
  45                         return_value=None):
  46             task_args = self.rally_base._build_task_args('test_file_name')
  47             self.assertEqual(task_args['floating_network'], '')
  48
  49     def test_build_task_args_missing_net_id(self):
  50         CONST.OS_AUTH_URL = None
  51         self.rally_base.network_dict['net_id'] = ''
  52         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  53                         'os_utils.get_external_net',
  54                         return_value='test_floating_network'):
  55             task_args = self.rally_base._build_task_args('test_file_name')
  56             self.assertEqual(task_args['netid'], '')
  57
  58     def test_build_task_args_missing_auth_url(self):
  59         CONST.OS_AUTH_URL = None
  60         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  61                         'os_utils.get_external_net',
  62                         return_value='test_floating_network'):
  63             task_args = self.rally_base._build_task_args('test_file_name')
  64             self.assertEqual(task_args['request_url'], '')
  65
  66     def check_scenario_file(self, value):
  67         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  68         if yaml_file in value:
  69             return False
  70         return True
  71
  72     def test_prepare_test_list_missing_scenario_file(self):
  73         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  74                         'os.path.exists',
  75                         side_effect=self.check_scenario_file), \
  76                 self.assertRaises(Exception):
  77             self.rally_base._prepare_test_list('test_file_name')
  78
  79     def check_temp_dir(self, value):
  80         yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  81         if yaml_file in value:
  82             return True
  83         return False
  84
  85     def test_prepare_test_list_missing_temp_dir(self):
  86         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  87                         'os.path.exists',
  88                         side_effect=self.check_temp_dir), \
  89             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
  90                        'os.makedirs') as mock_os_makedir, \
  91             mock.patch.object(self.rally_base, 'apply_blacklist',
  92                               return_value=mock.Mock()) as mock_method:
  93             yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
  94             ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
  95             self.assertEqual(self.rally_base.
  96                              _prepare_test_list('test_file_name'),
  97                              ret_val)
  98             self.assertTrue(mock_method.called)
  99             self.assertTrue(mock_os_makedir.called)
 100
 101     def test_get_task_id_default(self):
 102         cmd_raw = 'Task 1: started'
 103         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
 104                          '1')
 105
 106     def test_get_task_id_missing_id(self):
 107         cmd_raw = ''
 108         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
 109                          None)
 110
 111     def test_task_succeed_fail(self):
 112         json_raw = json.dumps([None])
 113         self.assertEqual(self.rally_base.task_succeed(json_raw),
 114                          False)
 115         json_raw = json.dumps([{'result': [{'error': ['test_error']}]}])
 116         self.assertEqual(self.rally_base.task_succeed(json_raw),
 117                          False)
 118
 119     def test_task_succeed_success(self):
 120         json_raw = json.dumps('')
 121         self.assertEqual(self.rally_base.task_succeed(json_raw),
 122                          True)
 123
 124     def polling(self):
 125         if self.polling_iter == 0:
 126             return "something"
 127         self.polling_iter -= 1
 128         return None
 129
 130     def test_get_cmd_output(self):
 131         proc = mock.Mock()
 132         attrs = {'poll.side_effect': self.polling,
 133                  'stdout.readline.return_value': 'line'}
 134         proc.configure_mock(**attrs)
 135         self.assertEqual(self.rally_base.get_cmd_output(proc),
 136                          'lineline')
 137
 138     def test_excl_scenario_default(self):
 139         CONST.INSTALLER_TYPE = 'test_installer'
 140         CONST.DEPLOY_SCENARIO = 'test_scenario'
 141         dic = {'scenario': [{'scenarios': ['test_scenario'],
 142                              'installers': ['test_installer'],
 143                              'tests': ['test']}]}
 144         with mock.patch('__builtin__.open', mock.mock_open()), \
 145             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 146                        'yaml.safe_load',
 147                        return_value=dic):
 148                 self.assertEqual(self.rally_base.excl_scenario(),
 149                                  ['test'])
 150
 151     def test_excl_scenario_exception(self):
 152         with mock.patch('__builtin__.open', side_effect=Exception):
 153                 self.assertEqual(self.rally_base.excl_scenario(),
 154                                  [])
 155
 156     def test_excl_func_default(self):
 157         CONST.INSTALLER_TYPE = 'test_installer'
 158         CONST.DEPLOY_SCENARIO = 'test_scenario'
 159         dic = {'functionality': [{'functions': ['no_live_migration'],
 160                                   'tests': ['test']}]}
 161         with mock.patch('__builtin__.open', mock.mock_open()), \
 162             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 163                        'yaml.safe_load',
 164                        return_value=dic), \
 165             mock.patch.object(self.rally_base, 'live_migration_supported',
 166                               return_value=False):
 167                 self.assertEqual(self.rally_base.excl_func(),
 168                                  ['test'])
 169
 170     def test_excl_func_exception(self):
 171         with mock.patch('__builtin__.open', side_effect=Exception):
 172                 self.assertEqual(self.rally_base.excl_func(),
 173                                  [])
 174
 175     def test_file_is_empty_default(self):
 176         mock_obj = mock.Mock()
 177         attrs = {'st_size': 10}
 178         mock_obj.configure_mock(**attrs)
 179         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 180                         'os.stat',
 181                         return_value=mock_obj):
 182             self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 183                              False)
 184
 185     def test_file_is_empty_exception(self):
 186         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 187                         'os.stat',
 188                         side_effect=Exception):
 189             self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
 190                              True)
 191
 192     def test_run_task_missing_task_file(self):
 193         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 194                         'os.path.exists',
 195                         return_value=False), \
 196                 self.assertRaises(Exception):
 197             self.rally_base._run_task('test_name')
 198
 199     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info')
 200     def test_run_task_no_tests_for_scenario(self, mock_logger_info):
 201         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 202                         'os.path.exists',
 203                         return_value=True), \
 204             mock.patch.object(self.rally_base, '_prepare_test_list',
 205                               return_value='test_file_name'), \
 206             mock.patch.object(self.rally_base, 'file_is_empty',
 207                               return_value=True):
 208             self.rally_base._run_task('test_name')
 209             str = 'No tests for scenario "test_name"'
 210             mock_logger_info.assert_any_call(str)
 211
 212     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error')
 213     def test_run_task_taskid_missing(self, mock_logger_error):
 214         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 215                         'os.path.exists',
 216                         return_value=True), \
 217             mock.patch.object(self.rally_base, '_prepare_test_list',
 218                               return_value='test_file_name'), \
 219             mock.patch.object(self.rally_base, 'file_is_empty',
 220                               return_value=False), \
 221             mock.patch.object(self.rally_base, '_build_task_args',
 222                               return_value={}), \
 223             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 224                        'subprocess.Popen'), \
 225             mock.patch.object(self.rally_base, '_get_output',
 226                               return_value=mock.Mock()), \
 227             mock.patch.object(self.rally_base, 'get_task_id',
 228                               return_value=None), \
 229             mock.patch.object(self.rally_base, 'get_cmd_output',
 230                               return_value=''):
 231             self.rally_base._run_task('test_name')
 232             str = 'Failed to retrieve task_id, validating task...'
 233             mock_logger_error.assert_any_call(str)
 234
 235     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info')
 236     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.error')
 237     def test_run_task_default(self, mock_logger_error,
 238                               mock_logger_info):
 239         popen = mock.Mock()
 240         attrs = {'read.return_value': 'json_result'}
 241         popen.configure_mock(**attrs)
 242
 243         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 244                         'os.path.exists',
 245                         return_value=True), \
 246             mock.patch.object(self.rally_base, '_prepare_test_list',
 247                               return_value='test_file_name'), \
 248             mock.patch.object(self.rally_base, 'file_is_empty',
 249                               return_value=False), \
 250             mock.patch.object(self.rally_base, '_build_task_args',
 251                               return_value={}), \
 252             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 253                        'subprocess.Popen'), \
 254             mock.patch.object(self.rally_base, '_get_output',
 255                               return_value=mock.Mock()), \
 256             mock.patch.object(self.rally_base, 'get_task_id',
 257                               return_value='1'), \
 258             mock.patch.object(self.rally_base, 'get_cmd_output',
 259                               return_value=''), \
 260             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 261                        'os.makedirs'), \
 262             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 263                        'os.popen',
 264                        return_value=popen), \
 265             mock.patch('__builtin__.open', mock.mock_open()), \
 266             mock.patch.object(self.rally_base, 'task_succeed',
 267                               return_value=True):
 268             self.rally_base._run_task('test_name')
 269             str = 'Test scenario: "test_name" OK.\n'
 270             mock_logger_info.assert_any_call(str)
 271
 272     def test_prepare_env_testname_invalid(self):
 273         self.rally_base.TESTS = ['test1', 'test2']
 274         self.rally_base.test_name = 'test'
 275         with self.assertRaises(Exception):
 276             self.rally_base._prepare_env()
 277
 278     def test_prepare_env_volume_creation_failed(self):
 279         self.rally_base.TESTS = ['test1', 'test2']
 280         self.rally_base.test_name = 'test1'
 281         volume_type = None
 282         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 283                         'os_utils.list_volume_types',
 284                         return_value=None), \
 285             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 286                        'os_utils.create_volume_type',
 287                        return_value=volume_type), \
 288                 self.assertRaises(Exception):
 289             self.rally_base._prepare_env()
 290
 291     def test_prepare_env_image_missing(self):
 292         self.rally_base.TESTS = ['test1', 'test2']
 293         self.rally_base.test_name = 'test1'
 294         volume_type = mock.Mock()
 295         image_id = None
 296         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 297                         'os_utils.list_volume_types',
 298                         return_value=None), \
 299             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 300                        'os_utils.create_volume_type',
 301                        return_value=volume_type), \
 302             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 303                        'os_utils.get_or_create_image',
 304                        return_value=(True, image_id)), \
 305                 self.assertRaises(Exception):
 306             self.rally_base._prepare_env()
 307
 308     def test_prepare_env_image_shared_network_creation_failed(self):
 309         self.rally_base.TESTS = ['test1', 'test2']
 310         self.rally_base.test_name = 'test1'
 311         volume_type = mock.Mock()
 312         image_id = 'image_id'
 313         network_dict = None
 314         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 315                         'os_utils.list_volume_types',
 316                         return_value=None), \
 317             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 318                        'os_utils.create_volume_type',
 319                        return_value=volume_type), \
 320             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 321                        'os_utils.get_or_create_image',
 322                        return_value=(True, image_id)), \
 323             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 324                        'os_utils.create_shared_network_full',
 325                        return_value=network_dict), \
 326                 self.assertRaises(Exception):
 327             self.rally_base._prepare_env()
 328
 329     def test_run_tests_all(self):
 330         self.rally_base.TESTS = ['test1', 'test2']
 331         self.rally_base.test_name = 'all'
 332         with mock.patch.object(self.rally_base, '_run_task',
 333                                return_value=mock.Mock()):
 334             self.rally_base._run_tests()
 335             self.rally_base._run_task.assert_any_call('test1')
 336             self.rally_base._run_task.assert_any_call('test2')
 337
 338     def test_run_tests_default(self):
 339         self.rally_base.TESTS = ['test1', 'test2']
 340         self.rally_base.test_name = 'test1'
 341         with mock.patch.object(self.rally_base, '_run_task',
 342                                return_value=mock.Mock()):
 343             self.rally_base._run_tests()
 344             self.rally_base._run_task.assert_any_call('test1')
 345
 346     @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info')
 347     def test_generate_report(self, mock_logger_info):
 348         summary = [{'test_name': 'test_name',
 349                     'overall_duration': 5,
 350                     'nb_tests': 3,
 351                     'success': 5}]
 352         self.rally_base.summary = summary
 353         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 354                         'ft_utils.check_success_rate',
 355                         return_value='criteria'):
 356             self.rally_base._generate_report()
 357             self.assertTrue(mock_logger_info.called)
 358
 359     def test_clean_up_default(self):
 360         self.rally_base.volume_type = mock.Mock()
 361         self.rally_base.cinder_client = mock.Mock()
 362         self.rally_base.image_exists = False
 363         self.rally_base.image_id = 1
 364         self.rally_base.nova_client = mock.Mock()
 365         with mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 366                         'os_utils.delete_volume_type') as mock_vol_method, \
 367             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
 368                        'os_utils.delete_glance_image') as mock_glance_method:
 369             self.rally_base._clean_up()
 370             mock_vol_method.assert_any_call(self.rally_base.cinder_client,
 371                                             self.rally_base.volume_type)
 372             mock_glance_method.assert_any_call(self.rally_base.nova_client,
 373                                                1)
 374
 375     def test_run_default(self):
 376         with mock.patch.object(self.rally_base, '_prepare_env'), \
 377             mock.patch.object(self.rally_base, '_run_tests'), \
 378             mock.patch.object(self.rally_base, '_generate_report'), \
 379                 mock.patch.object(self.rally_base, '_clean_up'):
 380             self.assertEqual(self.rally_base.run(),
 381                              testcase.TestCase.EX_OK)
 382
 383     def test_run_exception(self):
 384         with mock.patch.object(self.rally_base, '_prepare_env',
 385                                side_effect=Exception):
 386             self.assertEqual(self.rally_base.run(),
 387                              testcase.TestCase.EX_RUN_ERROR)
 388
 389
 390 if __name__ == "__main__":
 391     unittest.main(verbosity=2)