functest/tests/unit/opnfv_tests/openstack/tempest/test_tempest.py

   1 #!/usr/bin/env python
   2
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7
   8 import logging
   9 import unittest
  10
  11 import mock
  12
  13 from functest.core import testcase
  14 from functest.opnfv_tests.openstack.tempest import tempest
  15 from functest.opnfv_tests.openstack.tempest import conf_utils
  16
  17
  18 class OSTempestTesting(unittest.TestCase):
  19
  20     logging.disable(logging.CRITICAL)
  21
  22     def setUp(self):
  23         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  24                         'conf_utils.get_verifier_id',
  25                         return_value='test_deploy_id'), \
  26             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  27                        'conf_utils.get_verifier_deployment_id',
  28                        return_value='test_deploy_id'), \
  29             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  30                        'conf_utils.get_verifier_repo_dir',
  31                        return_value='test_verifier_repo_dir'), \
  32             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  33                        'conf_utils.get_verifier_deployment_dir',
  34                        return_value='test_verifier_deploy_dir'):
  35             self.tempestcommon = tempest.TempestCommon()
  36
  37     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.debug')
  38     def test_generate_test_list_defcore_mode(self, mock_logger_debug):
  39         self.tempestcommon.MODE = 'defcore'
  40         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  41                         'shutil.copyfile') as m:
  42             self.tempestcommon.generate_test_list('test_verifier_repo_dir')
  43             self.assertTrue(m.called)
  44
  45     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.error')
  46     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.debug')
  47     def test_generate_test_list_custom_mode_missing_file(self,
  48                                                          mock_logger_debug,
  49                                                          mock_logger_error):
  50         self.tempestcommon.MODE = 'custom'
  51         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  52                         'os.path.isfile', return_value=False), \
  53                 self.assertRaises(Exception) as context:
  54             msg = "Tempest test list file %s NOT found."
  55             self.tempestcommon.generate_test_list('test_verifier_repo_dir')
  56             self.assertTrue((msg % conf_utils.TEMPEST_CUSTOM) in context)
  57
  58     def test_generate_test_list_custom_mode_default(self):
  59         self.tempestcommon.MODE = 'custom'
  60         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  61                         'shutil.copyfile') as m, \
  62             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  63                        'os.path.isfile', return_value=True):
  64             self.tempestcommon.generate_test_list('test_verifier_repo_dir')
  65             self.assertTrue(m.called)
  66
  67     def _test_generate_test_list_mode_default(self, mode):
  68         self.tempestcommon.MODE = mode
  69         if self.tempestcommon.MODE == 'smoke':
  70             testr_mode = "smoke"
  71         elif self.tempestcommon.MODE == 'feature_multisite':
  72             testr_mode = "'[Kk]ingbird'"
  73         elif self.tempestcommon.MODE == 'full':
  74             testr_mode = ""
  75         else:
  76             testr_mode = 'tempest.api.' + self.tempestcommon.MODE
  77         conf_utils.TEMPEST_RAW_LIST = 'raw_list'
  78         verifier_repo_dir = 'test_verifier_repo_dir'
  79         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
  80                         'ft_utils.execute_command') as m:
  81             cmd = ("cd {0};"
  82                    "testr list-tests {1} > {2};"
  83                    "cd -;".format(verifier_repo_dir,
  84                                   testr_mode,
  85                                   conf_utils.TEMPEST_RAW_LIST))
  86             self.tempestcommon.generate_test_list('test_verifier_repo_dir')
  87             m.assert_any_call(cmd)
  88
  89     def test_generate_test_list_smoke_mode(self):
  90         self._test_generate_test_list_mode_default('smoke')
  91
  92     def test_generate_test_list_feature_multisite_mode(self):
  93         self._test_generate_test_list_mode_default('feature_multisite')
  94
  95     def test_generate_test_list_full_mode(self):
  96         self._test_generate_test_list_mode_default('full')
  97
  98     def test_parse_verifier_result_missing_verification_uuid(self):
  99         self.tempestcommon.VERIFICATION_ID = ''
 100         with self.assertRaises(Exception):
 101             self.tempestcommon.parse_verifier_result()
 102
 103     @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.info')
 104     def test_parse_verifier_result_default(self, mock_logger_info):
 105         self.tempestcommon.VERIFICATION_ID = 'test_uuid'
 106         self.tempestcommon.case_name = 'test_case_name'
 107         stdout = ['Testscount||2', 'Success||2', 'Skipped||0', 'Failures||0']
 108         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 109                         'subprocess.Popen') as mock_popen, \
 110             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 111                        'ft_utils.check_success_rate') as mock_method, \
 112                 mock.patch('__builtin__.open', mock.mock_open()):
 113             mock_stdout = mock.Mock()
 114             attrs = {'stdout': stdout}
 115             mock_stdout.configure_mock(**attrs)
 116             mock_popen.return_value = mock_stdout
 117
 118             self.tempestcommon.parse_verifier_result()
 119             mock_method.assert_any_call('test_case_name', 100)
 120
 121     def test_run_missing_create_tempest_dir(self):
 122         ret = testcase.TestCase.EX_RUN_ERROR
 123         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 124                         'os.path.exists', return_value=False), \
 125             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 126                        'os.makedirs') as mock_os_makedirs, \
 127             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 128                        'conf_utils.create_tempest_resources',
 129                        return_value="image_and_flavor"):
 130             self.assertEqual(self.tempestcommon.run(),
 131                              ret)
 132             self.assertTrue(mock_os_makedirs.called)
 133
 134     def test_run_missing_configure_tempest(self):
 135         ret = testcase.TestCase.EX_RUN_ERROR
 136         ret_ok = testcase.TestCase.EX_OK
 137         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 138                         'os.path.exists', return_value=False), \
 139             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 140                        'os.makedirs') as mock_os_makedirs, \
 141             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 142                        'conf_utils.create_tempest_resources',
 143                        return_value=ret_ok), \
 144             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 145                        'conf_utils.configure_tempest',
 146                        return_value=ret):
 147             self.assertEqual(self.tempestcommon.run(),
 148                              ret)
 149             self.assertTrue(mock_os_makedirs.called)
 150
 151     def test_run_missing_generate_test_list(self):
 152         ret = testcase.TestCase.EX_RUN_ERROR
 153         ret_ok = testcase.TestCase.EX_OK
 154         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 155                         'os.path.exists', return_value=False), \
 156             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 157                        'os.makedirs') as mock_os_makedirs, \
 158             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 159                        'conf_utils.create_tempest_resources',
 160                        return_value=ret_ok), \
 161             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 162                        'conf_utils.configure_tempest',
 163                        return_value=ret_ok), \
 164             mock.patch.object(self.tempestcommon, 'generate_test_list',
 165                               return_value=ret):
 166             self.assertEqual(self.tempestcommon.run(),
 167                              ret)
 168             self.assertTrue(mock_os_makedirs.called)
 169
 170     def test_run_missing_apply_tempest_blacklist(self):
 171         ret = testcase.TestCase.EX_RUN_ERROR
 172         ret_ok = testcase.TestCase.EX_OK
 173         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 174                         'os.path.exists', return_value=False), \
 175             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 176                        'os.makedirs') as mock_os_makedirs, \
 177             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 178                        'conf_utils.create_tempest_resources',
 179                        return_value=ret_ok), \
 180             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 181                        'conf_utils.configure_tempest',
 182                        return_value=ret_ok), \
 183             mock.patch.object(self.tempestcommon, 'generate_test_list',
 184                               return_value=ret_ok), \
 185             mock.patch.object(self.tempestcommon, 'apply_tempest_blacklist',
 186                               return_value=ret):
 187             self.assertEqual(self.tempestcommon.run(),
 188                              ret)
 189             self.assertTrue(mock_os_makedirs.called)
 190
 191     def test_run_missing_parse_verifier_result(self):
 192         ret = testcase.TestCase.EX_RUN_ERROR
 193         ret_ok = testcase.TestCase.EX_OK
 194         with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 195                         'os.path.exists', return_value=False), \
 196             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 197                        'os.makedirs') as mock_os_makedirs, \
 198             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 199                        'conf_utils.create_tempest_resources',
 200                        return_value=ret_ok), \
 201             mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
 202                        'conf_utils.configure_tempest',
 203                        return_value=ret_ok), \
 204             mock.patch.object(self.tempestcommon, 'generate_test_list',
 205                               return_value=ret_ok), \
 206             mock.patch.object(self.tempestcommon, 'apply_tempest_blacklist',
 207                               return_value=ret_ok), \
 208             mock.patch.object(self.tempestcommon, 'run_verifier_tests',
 209                               return_value=ret_ok), \
 210             mock.patch.object(self.tempestcommon, 'parse_verifier_result',
 211                               return_value=ret):
 212             self.assertEqual(self.tempestcommon.run(),
 213                              ret)
 214             self.assertTrue(mock_os_makedirs.called)
 215
 216
 217 if __name__ == "__main__":
 218     unittest.main(verbosity=2)