functest/tests/unit/ci/test_run_tests.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 import logging
   9 import unittest
  10
  11 import mock
  12
  13 from functest.ci import run_tests
  14 from functest.utils.constants import CONST
  15 from functest.core.testcase import TestCase
  16
  17
  18 class FakeModule(TestCase):
  19
  20     def run(self):
  21         return TestCase.EX_OK
  22
  23     def push_to_db(self):
  24         return TestCase.EX_OK
  25
  26     def is_successful(self):
  27         return TestCase.EX_OK
  28
  29
  30 class RunTestsTesting(unittest.TestCase):
  31
  32     def setUp(self):
  33         self.runner = run_tests.Runner()
  34         mock_test_case = mock.Mock()
  35         mock_test_case.is_successful.return_value = TestCase.EX_OK
  36         self.runner.executed_test_cases['test1'] = mock_test_case
  37         self.runner.executed_test_cases['test2'] = mock_test_case
  38         self.sep = 'test_sep'
  39         self.creds = {'OS_AUTH_URL': 'http://test_ip:test_port/v2.0',
  40                       'OS_USERNAME': 'test_os_username',
  41                       'OS_TENANT_NAME': 'test_tenant',
  42                       'OS_PASSWORD': 'test_password'}
  43         self.test = {'test_name': 'test_name'}
  44         self.tier = mock.Mock()
  45         test1 = mock.Mock()
  46         test1.get_name.return_value = 'test1'
  47         test2 = mock.Mock()
  48         test2.get_name.return_value = 'test2'
  49         attrs = {'get_name.return_value': 'test_tier',
  50                  'get_tests.return_value': [test1, test2],
  51                  'get_ci_loop.return_value': 'test_ci_loop',
  52                  'get_test_names.return_value': ['test1', 'test2']}
  53         self.tier.configure_mock(**attrs)
  54
  55         self.tiers = mock.Mock()
  56         attrs = {'get_tiers.return_value': [self.tier]}
  57         self.tiers.configure_mock(**attrs)
  58
  59         self.run_tests_parser = run_tests.RunTestsParser()
  60
  61         self.config_file_yaml = {'general': {
  62                                      'openstack': {
  63                                          'image_name': 'test_image_name'}},
  64                                  'results': {
  65                                      'test_db_url': 'url1'}}
  66         self.config_file_patch_yaml = {'fdio': {'general': {
  67                                        'openstack': {
  68                                            'image_name':
  69                                            'test_image_name_2'}}}}
  70         self.config_file_aarcg64_patch_yaml = {'os': {'general': {
  71                     'openstack': {'image_name': 'test_image_name_3'}}}}
  72
  73     @mock.patch('functest.ci.run_tests.Runner.patch_file')
  74     def test_update_config_file_default(self, mock_patch):
  75         self.runner.update_config_file()
  76         mock_patch.assert_called()
  77
  78     def test_patch_file_missing_file(self):
  79         patch_file_path = "unexisting_file"
  80         with self.assertRaises(IOError):
  81             self.runner.patch_file(patch_file_path)
  82
  83     @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
  84     @mock.patch('functest.ci.run_tests.ft_utils.get_functest_yaml')
  85     def test_patch_file_default(self, *mock_methods):
  86         CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-nofeature-noha')
  87         with mock.patch(
  88                'six.moves.builtins.open', mock.mock_open()), mock.patch(
  89                'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
  90                'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
  91             yaml1.return_value = self.config_file_patch_yaml
  92             yaml2.return_value = self.config_file_yaml
  93             self.runner.patch_file(yaml1)
  94             mock_methods[1].assert_not_called()
  95             mock_methods[0].assert_not_called()
  96
  97     @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
  98     @mock.patch('functest.ci.run_tests.os.remove')
  99     def test_patch_file_match_scenario(self, *mock_methods):
 100         CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-fdio-noha')
 101         with mock.patch(
 102                'six.moves.builtins.open', mock.mock_open()), mock.patch(
 103                'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
 104                'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
 105             yaml1.return_value = self.config_file_patch_yaml
 106             yaml2.return_value = self.config_file_yaml
 107             self.runner.patch_file(yaml2)
 108             mock_methods[1].assert_called()
 109             mock_methods[0].assert_called()
 110
 111     @mock.patch('functest.ci.run_tests.logger.error')
 112     def test_source_rc_file_missing_file(self, mock_logger_error):
 113         with mock.patch('functest.ci.run_tests.os.path.isfile',
 114                         return_value=False), \
 115                 self.assertRaises(Exception):
 116             self.runner.source_rc_file()
 117
 118     @mock.patch('functest.ci.run_tests.logger.debug')
 119     @mock.patch('functest.ci.run_tests.os.path.isfile',
 120                 return_value=True)
 121     def test_source_rc_file_default(self, *args):
 122         with mock.patch('functest.ci.run_tests.os_utils.source_credentials',
 123                         return_value=self.creds):
 124             self.runner.source_rc_file()
 125
 126     def test_get_run_dict_if_defined_default(self):
 127         mock_obj = mock.Mock()
 128         with mock.patch('functest.ci.run_tests.'
 129                         'ft_utils.get_dict_by_test',
 130                         return_value={'run': mock_obj}):
 131             self.assertEqual(self.runner.get_run_dict('test_name'),
 132                              mock_obj)
 133
 134     @mock.patch('functest.ci.run_tests.logger.error')
 135     def test_get_run_dict_if_defined_missing_config_option(self,
 136                                                            mock_logger_error):
 137         with mock.patch('functest.ci.run_tests.'
 138                         'ft_utils.get_dict_by_test',
 139                         return_value=None):
 140             testname = 'test_name'
 141             self.assertEqual(self.runner.get_run_dict(testname),
 142                              None)
 143             mock_logger_error.assert_called_once_with("Cannot get {}'s config "
 144                                                       "options"
 145                                                       .format(testname))
 146
 147         with mock.patch('functest.ci.run_tests.'
 148                         'ft_utils.get_dict_by_test',
 149                         return_value={}):
 150             testname = 'test_name'
 151             self.assertEqual(self.runner.get_run_dict(testname),
 152                              None)
 153
 154     @mock.patch('functest.ci.run_tests.logger.exception')
 155     def test_get_run_dict_if_defined_exception(self,
 156                                                mock_logger_except):
 157         with mock.patch('functest.ci.run_tests.'
 158                         'ft_utils.get_dict_by_test',
 159                         side_effect=Exception):
 160             testname = 'test_name'
 161             self.assertEqual(self.runner.get_run_dict(testname),
 162                              None)
 163             mock_logger_except.assert_called_once_with("Cannot get {}'s config"
 164                                                        " options"
 165                                                        .format(testname))
 166
 167     def test_run_tests_import_test_class_exception(self):
 168         mock_test = mock.Mock()
 169         args = {'get_name.return_value': 'test_name',
 170                 'needs_clean.return_value': False}
 171         mock_test.configure_mock(**args)
 172         with mock.patch('functest.ci.run_tests.Runner.source_rc_file'), \
 173             mock.patch('functest.ci.run_tests.Runner.get_run_dict',
 174                        return_value=None), \
 175                 self.assertRaises(Exception) as context:
 176             self.runner(mock_test, 'tier_name')
 177             msg = "Cannot import the class for the test case."
 178             self.assertTrue(msg in context)
 179
 180     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 181     @mock.patch('importlib.import_module', name="module",
 182                 return_value=mock.Mock(test_class=mock.Mock(
 183                     side_effect=FakeModule)))
 184     @mock.patch('functest.utils.functest_utils.get_dict_by_test')
 185     def test_run_tests_default(self, *args):
 186         mock_test = mock.Mock()
 187         kwargs = {'get_name.return_value': 'test_name',
 188                   'needs_clean.return_value': True}
 189         mock_test.configure_mock(**kwargs)
 190         test_run_dict = {'module': 'test_module',
 191                          'class': 'test_class'}
 192         with mock.patch('functest.ci.run_tests.Runner.get_run_dict',
 193                         return_value=test_run_dict):
 194             self.runner.clean_flag = True
 195             self.runner.run_test(mock_test)
 196         self.assertEqual(self.runner.overall_result,
 197                          run_tests.Result.EX_OK)
 198
 199     @mock.patch('functest.ci.run_tests.Runner.run_test',
 200                 return_value=TestCase.EX_OK)
 201     def test_run_tier_default(self, *mock_methods):
 202         self.assertEqual(self.runner.run_tier(self.tier),
 203                          run_tests.Result.EX_OK)
 204         mock_methods[0].assert_called_with(mock.ANY)
 205
 206     @mock.patch('functest.ci.run_tests.logger.info')
 207     def test_run_tier_missing_test(self, mock_logger_info):
 208         self.tier.get_tests.return_value = None
 209         self.assertEqual(self.runner.run_tier(self.tier),
 210                          run_tests.Result.EX_ERROR)
 211         self.assertTrue(mock_logger_info.called)
 212
 213     @mock.patch('functest.ci.run_tests.logger.info')
 214     @mock.patch('functest.ci.run_tests.Runner.run_tier')
 215     @mock.patch('functest.ci.run_tests.Runner.summary')
 216     def test_run_all_default(self, *mock_methods):
 217         CONST.__setattr__('CI_LOOP', 'test_ci_loop')
 218         self.runner.run_all()
 219         mock_methods[1].assert_not_called()
 220         self.assertTrue(mock_methods[2].called)
 221
 222     @mock.patch('functest.ci.run_tests.logger.info')
 223     @mock.patch('functest.ci.run_tests.Runner.summary')
 224     def test_run_all_missing_tier(self, *mock_methods):
 225         CONST.__setattr__('CI_LOOP', 'loop_re_not_available')
 226         self.runner.run_all()
 227         self.assertTrue(mock_methods[1].called)
 228
 229     @mock.patch('functest.ci.run_tests.Runner.source_rc_file',
 230                 side_effect=Exception)
 231     @mock.patch('functest.ci.run_tests.Runner.summary')
 232     def test_main_failed(self, *mock_methods):
 233         kwargs = {'test': 'test_name', 'noclean': True, 'report': True}
 234         args = {'get_tier.return_value': False,
 235                 'get_test.return_value': False}
 236         self.runner._tiers = mock.Mock()
 237         self.runner._tiers.configure_mock(**args)
 238         self.assertEqual(self.runner.main(**kwargs),
 239                          run_tests.Result.EX_ERROR)
 240         mock_methods[1].assert_called_once_with()
 241
 242     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 243     @mock.patch('functest.ci.run_tests.Runner.run_test',
 244                 return_value=TestCase.EX_OK)
 245     @mock.patch('functest.ci.run_tests.Runner.summary')
 246     def test_main_tier(self, *mock_methods):
 247         mock_tier = mock.Mock()
 248         test_mock = mock.Mock()
 249         test_mock.get_name.return_value = 'test1'
 250         args = {'get_name.return_value': 'tier_name',
 251                 'get_tests.return_value': [test_mock]}
 252         mock_tier.configure_mock(**args)
 253         kwargs = {'test': 'tier_name', 'noclean': True, 'report': True}
 254         args = {'get_tier.return_value': mock_tier,
 255                 'get_test.return_value': None}
 256         self.runner._tiers = mock.Mock()
 257         self.runner._tiers.configure_mock(**args)
 258         self.assertEqual(self.runner.main(**kwargs),
 259                          run_tests.Result.EX_OK)
 260         mock_methods[1].assert_called()
 261
 262     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 263     @mock.patch('functest.ci.run_tests.Runner.run_test',
 264                 return_value=TestCase.EX_OK)
 265     def test_main_test(self, *mock_methods):
 266         kwargs = {'test': 'test_name', 'noclean': True, 'report': True}
 267         args = {'get_tier.return_value': None,
 268                 'get_test.return_value': 'test_name'}
 269         self.runner._tiers = mock.Mock()
 270         self.runner._tiers.configure_mock(**args)
 271         self.assertEqual(self.runner.main(**kwargs),
 272                          run_tests.Result.EX_OK)
 273         mock_methods[0].assert_called_once_with('test_name')
 274
 275     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 276     @mock.patch('functest.ci.run_tests.Runner.run_all')
 277     @mock.patch('functest.ci.run_tests.Runner.summary')
 278     def test_main_all_tier(self, *mock_methods):
 279         kwargs = {'test': 'all', 'noclean': True, 'report': True}
 280         args = {'get_tier.return_value': None,
 281                 'get_test.return_value': None}
 282         self.runner._tiers = mock.Mock()
 283         self.runner._tiers.configure_mock(**args)
 284         self.assertEqual(self.runner.main(**kwargs),
 285                          run_tests.Result.EX_OK)
 286         mock_methods[1].assert_called_once_with()
 287
 288     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 289     @mock.patch('functest.ci.run_tests.Runner.summary')
 290     def test_main_any_tier_test_ko(self, *mock_methods):
 291         kwargs = {'test': 'any', 'noclean': True, 'report': True}
 292         args = {'get_tier.return_value': None,
 293                 'get_test.return_value': None}
 294         self.runner._tiers = mock.Mock()
 295         self.runner._tiers.configure_mock(**args)
 296         self.assertEqual(self.runner.main(**kwargs),
 297                          run_tests.Result.EX_ERROR)
 298
 299
 300 if __name__ == "__main__":
 301     logging.disable(logging.CRITICAL)
 302     unittest.main(verbosity=2)