functest/tests/unit/ci/test_run_tests.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 import logging
   9 import unittest
  10 import os
  11
  12 import mock
  13
  14 from functest.ci import run_tests
  15 from functest.utils.constants import CONST
  16 from functest.core.testcase import TestCase
  17
  18
  19 class FakeModule(TestCase):
  20
  21     def run(self):
  22         return TestCase.EX_OK
  23
  24     def push_to_db(self):
  25         return TestCase.EX_OK
  26
  27     def is_successful(self):
  28         return TestCase.EX_OK
  29
  30
  31 class RunTestsTesting(unittest.TestCase):
  32
  33     def setUp(self):
  34         self.runner = run_tests.Runner()
  35         mock_test_case = mock.Mock()
  36         mock_test_case.is_successful.return_value = TestCase.EX_OK
  37         self.runner.executed_test_cases['test1'] = mock_test_case
  38         self.runner.executed_test_cases['test2'] = mock_test_case
  39         self.sep = 'test_sep'
  40         self.creds = {'OS_AUTH_URL': 'http://test_ip:test_port/v2.0',
  41                       'OS_USERNAME': 'test_os_username',
  42                       'OS_TENANT_NAME': 'test_tenant',
  43                       'OS_PASSWORD': 'test_password'}
  44         self.test = {'test_name': 'test_name'}
  45         self.tier = mock.Mock()
  46         test1 = mock.Mock()
  47         test1.get_name.return_value = 'test1'
  48         test2 = mock.Mock()
  49         test2.get_name.return_value = 'test2'
  50         attrs = {'get_name.return_value': 'test_tier',
  51                  'get_tests.return_value': [test1, test2],
  52                  'get_ci_loop.return_value': 'test_ci_loop',
  53                  'get_test_names.return_value': ['test1', 'test2']}
  54         self.tier.configure_mock(**attrs)
  55
  56         self.tiers = mock.Mock()
  57         attrs = {'get_tiers.return_value': [self.tier]}
  58         self.tiers.configure_mock(**attrs)
  59
  60         self.run_tests_parser = run_tests.RunTestsParser()
  61
  62         self.config_file_yaml = {'general': {
  63                                      'openstack': {
  64                                          'image_name': 'test_image_name'}},
  65                                  'results': {
  66                                      'test_db_url': 'url1'}}
  67         self.config_file_patch_yaml = {'fdio': {'general': {
  68                                        'openstack': {
  69                                            'image_name':
  70                                            'test_image_name_2'}}}}
  71         self.config_file_aarcg64_patch_yaml = {'os': {'general': {
  72                     'openstack': {'image_name': 'test_image_name_3'}}}}
  73
  74     @mock.patch('functest.ci.run_tests.Runner.patch_file')
  75     @mock.patch('functest.ci.run_tests.Runner.update_db_url')
  76     def test_update_config_file_default(self, *mock_methods):
  77         self.runner.update_config_file()
  78         mock_methods[1].assert_called()
  79         mock_methods[0].assert_not_called()
  80
  81     @mock.patch('functest.ci.run_tests.Runner.patch_file')
  82     @mock.patch('functest.ci.run_tests.Runner.update_db_url')
  83     @mock.patch.dict(os.environ, {'TEST_DB_URL': 'somevalue'})
  84     def test_update_config_file_update_db(self, *mock_methods):
  85         self.runner.update_config_file()
  86         mock_methods[1].assert_called()
  87         mock_methods[0].assert_called()
  88
  89     def test_patch_file_missing_file(self):
  90         patch_file_path = "unexisting_file"
  91         with self.assertRaises(IOError):
  92             self.runner.patch_file(patch_file_path)
  93
  94     @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
  95     @mock.patch('functest.ci.run_tests.ft_utils.get_functest_yaml')
  96     def test_patch_file_default(self, *mock_methods):
  97         CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-nofeature-noha')
  98         with mock.patch(
  99                'six.moves.builtins.open', mock.mock_open()), mock.patch(
 100                'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
 101                'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
 102             yaml1.return_value = self.config_file_patch_yaml
 103             yaml2.return_value = self.config_file_yaml
 104             self.runner.patch_file(yaml1)
 105             mock_methods[1].assert_not_called()
 106             mock_methods[0].assert_not_called()
 107
 108     @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
 109     @mock.patch('functest.ci.run_tests.os.remove')
 110     def test_patch_file_match_scenario(self, *mock_methods):
 111         CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-fdio-noha')
 112         with mock.patch(
 113                'six.moves.builtins.open', mock.mock_open()), mock.patch(
 114                'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
 115                'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
 116             yaml1.return_value = self.config_file_patch_yaml
 117             yaml2.return_value = self.config_file_yaml
 118             self.runner.patch_file(yaml2)
 119             mock_methods[1].assert_called()
 120             mock_methods[0].assert_called()
 121
 122     def test_update_db_url_missing_file(self):
 123         run_tests.CONFIG_FUNCTEST_PATH = "unexisting_file"
 124         with self.assertRaises(IOError):
 125             self.runner.update_db_url()
 126
 127     @mock.patch('functest.ci.run_tests.yaml.safe_load')
 128     @mock.patch('functest.ci.run_tests.ft_utils.get_functest_yaml')
 129     @mock.patch.dict(os.environ, {'TEST_DB_URL': 'url2'})
 130     def test_update_db_url_default(self, *mock_methods):
 131         with mock.patch(
 132                'six.moves.builtins.open', mock.mock_open()), mock.patch(
 133                'functest.ci.run_tests.yaml.safe_load') as yaml1:
 134             yaml1.return_value = self.config_file_yaml
 135             self.runner.update_db_url()
 136             self.assertEqual(
 137                 yaml1.return_value['results']['test_db_url'], 'url2')
 138             mock_methods[0].assert_not_called()
 139             mock_methods[1].assert_not_called()
 140
 141     @mock.patch('functest.ci.run_tests.logger.error')
 142     def test_source_rc_file_missing_file(self, mock_logger_error):
 143         with mock.patch('functest.ci.run_tests.os.path.isfile',
 144                         return_value=False), \
 145                 self.assertRaises(Exception):
 146             self.runner.source_rc_file()
 147
 148     @mock.patch('functest.ci.run_tests.logger.debug')
 149     @mock.patch('functest.ci.run_tests.os.path.isfile',
 150                 return_value=True)
 151     def test_source_rc_file_default(self, *args):
 152         with mock.patch('functest.ci.run_tests.os_utils.source_credentials',
 153                         return_value=self.creds):
 154             self.runner.source_rc_file()
 155
 156     def test_get_run_dict_if_defined_default(self):
 157         mock_obj = mock.Mock()
 158         with mock.patch('functest.ci.run_tests.'
 159                         'ft_utils.get_dict_by_test',
 160                         return_value={'run': mock_obj}):
 161             self.assertEqual(self.runner.get_run_dict('test_name'),
 162                              mock_obj)
 163
 164     @mock.patch('functest.ci.run_tests.logger.error')
 165     def test_get_run_dict_if_defined_missing_config_option(self,
 166                                                            mock_logger_error):
 167         with mock.patch('functest.ci.run_tests.'
 168                         'ft_utils.get_dict_by_test',
 169                         return_value=None):
 170             testname = 'test_name'
 171             self.assertEqual(self.runner.get_run_dict(testname),
 172                              None)
 173             mock_logger_error.assert_called_once_with("Cannot get {}'s config "
 174                                                       "options"
 175                                                       .format(testname))
 176
 177         with mock.patch('functest.ci.run_tests.'
 178                         'ft_utils.get_dict_by_test',
 179                         return_value={}):
 180             testname = 'test_name'
 181             self.assertEqual(self.runner.get_run_dict(testname),
 182                              None)
 183
 184     @mock.patch('functest.ci.run_tests.logger.exception')
 185     def test_get_run_dict_if_defined_exception(self,
 186                                                mock_logger_except):
 187         with mock.patch('functest.ci.run_tests.'
 188                         'ft_utils.get_dict_by_test',
 189                         side_effect=Exception):
 190             testname = 'test_name'
 191             self.assertEqual(self.runner.get_run_dict(testname),
 192                              None)
 193             mock_logger_except.assert_called_once_with("Cannot get {}'s config"
 194                                                        " options"
 195                                                        .format(testname))
 196
 197     def test_run_tests_import_test_class_exception(self):
 198         mock_test = mock.Mock()
 199         args = {'get_name.return_value': 'test_name',
 200                 'needs_clean.return_value': False}
 201         mock_test.configure_mock(**args)
 202         with mock.patch('functest.ci.run_tests.Runner.source_rc_file'), \
 203             mock.patch('functest.ci.run_tests.Runner.get_run_dict',
 204                        return_value=None), \
 205                 self.assertRaises(Exception) as context:
 206             self.runner(mock_test, 'tier_name')
 207             msg = "Cannot import the class for the test case."
 208             self.assertTrue(msg in context)
 209
 210     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 211     @mock.patch('importlib.import_module', name="module",
 212                 return_value=mock.Mock(test_class=mock.Mock(
 213                     side_effect=FakeModule)))
 214     @mock.patch('functest.utils.functest_utils.get_dict_by_test')
 215     def test_run_tests_default(self, *args):
 216         mock_test = mock.Mock()
 217         kwargs = {'get_name.return_value': 'test_name',
 218                   'needs_clean.return_value': True}
 219         mock_test.configure_mock(**kwargs)
 220         test_run_dict = {'module': 'test_module',
 221                          'class': 'test_class'}
 222         with mock.patch('functest.ci.run_tests.Runner.get_run_dict',
 223                         return_value=test_run_dict):
 224             self.runner.clean_flag = True
 225             self.runner.run_test(mock_test)
 226         self.assertEqual(self.runner.overall_result,
 227                          run_tests.Result.EX_OK)
 228
 229     @mock.patch('functest.ci.run_tests.Runner.run_test',
 230                 return_value=TestCase.EX_OK)
 231     def test_run_tier_default(self, *mock_methods):
 232         self.assertEqual(self.runner.run_tier(self.tier),
 233                          run_tests.Result.EX_OK)
 234         mock_methods[0].assert_called_with(mock.ANY)
 235
 236     @mock.patch('functest.ci.run_tests.logger.info')
 237     def test_run_tier_missing_test(self, mock_logger_info):
 238         self.tier.get_tests.return_value = None
 239         self.assertEqual(self.runner.run_tier(self.tier),
 240                          run_tests.Result.EX_ERROR)
 241         self.assertTrue(mock_logger_info.called)
 242
 243     @mock.patch('functest.ci.run_tests.logger.info')
 244     @mock.patch('functest.ci.run_tests.Runner.run_tier')
 245     @mock.patch('functest.ci.run_tests.Runner.summary')
 246     def test_run_all_default(self, *mock_methods):
 247         CONST.__setattr__('CI_LOOP', 'test_ci_loop')
 248         self.runner.run_all()
 249         mock_methods[1].assert_not_called()
 250         self.assertTrue(mock_methods[2].called)
 251
 252     @mock.patch('functest.ci.run_tests.logger.info')
 253     @mock.patch('functest.ci.run_tests.Runner.summary')
 254     def test_run_all_missing_tier(self, *mock_methods):
 255         CONST.__setattr__('CI_LOOP', 'loop_re_not_available')
 256         self.runner.run_all()
 257         self.assertTrue(mock_methods[1].called)
 258
 259     @mock.patch('functest.ci.run_tests.Runner.source_rc_file',
 260                 side_effect=Exception)
 261     @mock.patch('functest.ci.run_tests.Runner.summary')
 262     def test_main_failed(self, *mock_methods):
 263         kwargs = {'test': 'test_name', 'noclean': True, 'report': True}
 264         args = {'get_tier.return_value': False,
 265                 'get_test.return_value': False}
 266         self.runner._tiers = mock.Mock()
 267         self.runner._tiers.configure_mock(**args)
 268         self.assertEqual(self.runner.main(**kwargs),
 269                          run_tests.Result.EX_ERROR)
 270         mock_methods[1].assert_called_once_with()
 271
 272     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 273     @mock.patch('functest.ci.run_tests.Runner.run_test',
 274                 return_value=TestCase.EX_OK)
 275     @mock.patch('functest.ci.run_tests.Runner.summary')
 276     def test_main_tier(self, *mock_methods):
 277         mock_tier = mock.Mock()
 278         test_mock = mock.Mock()
 279         test_mock.get_name.return_value = 'test1'
 280         args = {'get_name.return_value': 'tier_name',
 281                 'get_tests.return_value': [test_mock]}
 282         mock_tier.configure_mock(**args)
 283         kwargs = {'test': 'tier_name', 'noclean': True, 'report': True}
 284         args = {'get_tier.return_value': mock_tier,
 285                 'get_test.return_value': None}
 286         self.runner._tiers = mock.Mock()
 287         self.runner._tiers.configure_mock(**args)
 288         self.assertEqual(self.runner.main(**kwargs),
 289                          run_tests.Result.EX_OK)
 290         mock_methods[1].assert_called()
 291
 292     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 293     @mock.patch('functest.ci.run_tests.Runner.run_test',
 294                 return_value=TestCase.EX_OK)
 295     def test_main_test(self, *mock_methods):
 296         kwargs = {'test': 'test_name', 'noclean': True, 'report': True}
 297         args = {'get_tier.return_value': None,
 298                 'get_test.return_value': 'test_name'}
 299         self.runner._tiers = mock.Mock()
 300         self.runner._tiers.configure_mock(**args)
 301         self.assertEqual(self.runner.main(**kwargs),
 302                          run_tests.Result.EX_OK)
 303         mock_methods[0].assert_called_once_with('test_name')
 304
 305     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 306     @mock.patch('functest.ci.run_tests.Runner.run_all')
 307     @mock.patch('functest.ci.run_tests.Runner.summary')
 308     def test_main_all_tier(self, *mock_methods):
 309         kwargs = {'test': 'all', 'noclean': True, 'report': True}
 310         args = {'get_tier.return_value': None,
 311                 'get_test.return_value': None}
 312         self.runner._tiers = mock.Mock()
 313         self.runner._tiers.configure_mock(**args)
 314         self.assertEqual(self.runner.main(**kwargs),
 315                          run_tests.Result.EX_OK)
 316         mock_methods[1].assert_called_once_with()
 317
 318     @mock.patch('functest.ci.run_tests.Runner.source_rc_file')
 319     @mock.patch('functest.ci.run_tests.Runner.summary')
 320     def test_main_any_tier_test_ko(self, *mock_methods):
 321         kwargs = {'test': 'any', 'noclean': True, 'report': True}
 322         args = {'get_tier.return_value': None,
 323                 'get_test.return_value': None}
 324         self.runner._tiers = mock.Mock()
 325         self.runner._tiers.configure_mock(**args)
 326         self.assertEqual(self.runner.main(**kwargs),
 327                          run_tests.Result.EX_ERROR)
 328
 329
 330 if __name__ == "__main__":
 331     logging.disable(logging.CRITICAL)
 332     unittest.main(verbosity=2)