3 # All rights reserved. This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # pylint: disable=missing-docstring,protected-access,invalid-name
18 from xtesting.core import testcase
20 from functest.opnfv_tests.openstack.rally import rally
21 from functest.utils import config
24 class OSRallyTesting(unittest.TestCase):
25 # pylint: disable=too-many-public-methods
27 with mock.patch('os_client_config.get_config') as mock_get_config, \
28 mock.patch('shade.OpenStackCloud') as mock_shade, \
29 mock.patch('functest.core.tenantnetwork.NewProject') \
31 self.rally_base = rally.RallyBase()
32 self.rally_base.image = munch.Munch(name='foo')
33 self.rally_base.flavor = munch.Munch(name='foo')
34 self.rally_base.flavor_alt = munch.Munch(name='bar')
35 self.assertTrue(mock_get_config.called)
36 self.assertTrue(mock_shade.called)
37 self.assertTrue(mock_new_project.called)
39 def test_build_task_args_missing_floating_network(self):
40 os.environ['OS_AUTH_URL'] = ''
41 self.rally_base.ext_net = None
42 task_args = self.rally_base.build_task_args('test_name')
43 self.assertEqual(task_args['floating_network'], '')
45 def test_build_task_args_missing_net_id(self):
46 os.environ['OS_AUTH_URL'] = ''
47 self.rally_base.network = None
48 task_args = self.rally_base.build_task_args('test_name')
49 self.assertEqual(task_args['netid'], '')
52 def check_scenario_file(value):
53 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
54 if yaml_file in value:
58 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
59 def test_prepare_test_list_missing_scenario_file(self, mock_func):
60 mock_func.side_effect = self.check_scenario_file
61 with self.assertRaises(Exception):
62 self.rally_base._prepare_test_list('test_file_name')
63 mock_func.assert_called()
66 def check_temp_dir(value):
67 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
68 if yaml_file in value:
72 @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
73 'RallyBase.get_verifier_deployment_id', return_value='foo')
74 @mock.patch('subprocess.check_output')
75 def test_create_rally_deployment(self, mock_exec, mock_get_id):
76 # pylint: disable=unused-argument
77 self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
79 mock.call(['rally', 'deployment', 'destroy', '--deployment',
80 str(getattr(config.CONF, 'rally_deployment_name'))]),
81 mock.call().decode("utf-8"),
82 mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
83 str(getattr(config.CONF, 'rally_deployment_name'))],
85 mock.call().decode("utf-8"),
86 mock.call(['rally', 'deployment', 'check']),
87 mock.call().decode("utf-8")]
88 mock_exec.assert_has_calls(calls)
90 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
91 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
92 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
94 def test_prepare_test_list_missing_temp_dir(
95 self, mock_method, mock_os_makedirs, mock_path_exists):
96 mock_path_exists.side_effect = self.check_temp_dir
98 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
99 ret_val = os.path.join(self.rally_base.TEMP_DIR, yaml_file)
100 self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
102 mock_path_exists.assert_called()
103 mock_method.assert_called()
104 mock_os_makedirs.assert_called()
106 def test_get_task_id_default(self):
107 cmd_raw = b'Task 1: started'
108 self.assertEqual(self.rally_base.get_task_id(cmd_raw),
111 def test_get_task_id_missing_id(self):
113 self.assertEqual(self.rally_base.get_task_id(cmd_raw),
116 def test_task_succeed_fail(self):
117 json_raw = json.dumps({})
118 self.assertEqual(self.rally_base.task_succeed(json_raw),
120 json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
121 self.assertEqual(self.rally_base.task_succeed(json_raw),
124 def test_task_succeed_success(self):
125 json_raw = json.dumps({'tasks': [{'status': 'finished',
127 self.assertEqual(self.rally_base.task_succeed(json_raw),
130 @mock.patch('six.moves.builtins.open', mock.mock_open())
131 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
132 return_value={'scenario': [
133 {'scenarios': ['test_scenario'],
135 {'scenarios': ['other_scenario'],
136 'tests': ['other_test']}]})
137 def test_excl_scenario_default(self, mock_func):
138 os.environ['INSTALLER_TYPE'] = 'test_installer'
139 os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
140 self.assertEqual(self.rally_base.excl_scenario(), ['test'])
141 mock_func.assert_called()
143 @mock.patch('six.moves.builtins.open', mock.mock_open())
144 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
145 return_value={'scenario': [
146 {'scenarios': ['^os-[^-]+-featT-modeT$'],
148 {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
150 {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
152 {'scenarios': ['^os-'],
154 {'scenarios': ['other_scenario'],
155 'tests': ['test0a']},
156 {'scenarios': [''], # empty scenario
157 'tests': ['test0b']}]})
158 def test_excl_scenario_regex(self, mock_func):
159 os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
160 self.assertEqual(self.rally_base.excl_scenario(),
161 ['test1', 'test2', 'test3', 'test4'])
162 mock_func.assert_called()
164 @mock.patch('six.moves.builtins.open', side_effect=Exception)
165 def test_excl_scenario_exception(self, mock_open):
166 self.assertEqual(self.rally_base.excl_scenario(), [])
167 mock_open.assert_called()
169 @mock.patch('six.moves.builtins.open', mock.mock_open())
170 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
171 return_value={'functionality': [
172 {'functions': ['no_migration'], 'tests': ['test']}]})
173 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
174 '_migration_supported', return_value=False)
175 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
176 '_network_trunk_supported', return_value=False)
177 def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
178 os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
179 self.assertEqual(self.rally_base.excl_func(), ['test'])
180 mock_func.assert_called()
181 mock_trunk.assert_called()
182 mock_yaml_load.assert_called()
184 @mock.patch('six.moves.builtins.open', side_effect=Exception)
185 def test_excl_func_exception(self, mock_open):
186 self.assertEqual(self.rally_base.excl_func(), [])
187 mock_open.assert_called()
189 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
190 def test_file_is_empty_default(self, mock_os_stat):
191 attrs = {'st_size': 10}
192 mock_os_stat.return_value.configure_mock(**attrs)
193 self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
195 mock_os_stat.assert_called()
197 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
198 side_effect=Exception)
199 def test_file_is_empty_exception(self, mock_os_stat):
200 self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
201 mock_os_stat.assert_called()
203 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
205 def test_run_task_missing_task_file(self, mock_path_exists):
206 with self.assertRaises(Exception):
207 self.rally_base.prepare_run()
208 mock_path_exists.assert_called()
210 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
211 '_prepare_test_list', return_value='test_file_name')
212 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
213 'file_is_empty', return_value=True)
214 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
215 def test_prepare_task_no_tests_for_scenario(
216 self, mock_logger_info, mock_file_empty, mock_prep_list):
217 self.rally_base.prepare_task('test_name')
218 mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
220 mock_file_empty.assert_called()
221 mock_prep_list.assert_called()
223 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
224 '_prepare_test_list', return_value='test_file_name')
225 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
226 'file_is_empty', return_value=False)
227 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
228 'build_task_args', return_value={})
229 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
230 'get_task_id', return_value=None)
231 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
233 @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
234 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
235 def test_run_task_taskid_missing(self, mock_logger_error, *args):
236 # pylint: disable=unused-argument
237 with self.assertRaises(Exception):
238 self.rally_base.run_task('test_name')
239 text = 'Failed to retrieve task_id'
240 mock_logger_error.assert_any_call(text)
242 @mock.patch('six.moves.builtins.open', mock.mock_open())
243 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
244 '_prepare_test_list', return_value='test_file_name')
245 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
246 'file_is_empty', return_value=False)
247 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
248 'build_task_args', return_value={})
249 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
250 'get_task_id', return_value='1')
251 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
252 'task_succeed', return_value=True)
253 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
255 @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
256 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
257 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
258 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
259 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
261 def test_run_task_default(self, mock_save_res, *args):
262 # pylint: disable=unused-argument
263 self.rally_base.run_task('test_name')
264 mock_save_res.assert_called()
266 @mock.patch('six.moves.builtins.open', mock.mock_open())
267 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
268 'task_succeed', return_value=True)
269 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
271 @mock.patch('subprocess.check_output')
272 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
273 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
274 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
275 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
277 def test_save_results(self, mock_summary, *args):
278 # pylint: disable=unused-argument
279 self.rally_base._save_results('test_name', '1234')
280 mock_summary.assert_called()
282 def test_prepare_run_testname_invalid(self):
283 self.rally_base.TESTS = ['test1', 'test2']
284 with self.assertRaises(Exception):
285 self.rally_base.prepare_run(tests=['test'])
287 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
288 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
289 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copytree')
290 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.rmtree')
291 def test_prepare_run_flavor_alt_creation_failed(self, *args):
292 # pylint: disable=unused-argument
293 self.rally_base.TESTS = ['test1', 'test2']
294 with mock.patch.object(self.rally_base.cloud,
295 'list_hypervisors') as mock_list_hyperv, \
296 mock.patch.object(self.rally_base, 'create_flavor_alt',
297 side_effect=Exception) \
298 as mock_create_flavor:
299 with self.assertRaises(Exception):
300 self.rally_base.prepare_run(tests=['test1'])
301 mock_list_hyperv.assert_called_once()
302 mock_create_flavor.assert_called_once()
304 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
305 'prepare_task', return_value=True)
306 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
308 def test_run_tests_all(self, mock_run_task, mock_prepare_task):
309 self.rally_base.tests = ['test1', 'test2']
310 self.rally_base.run_tests()
311 mock_prepare_task.assert_any_call('test1')
312 mock_prepare_task.assert_any_call('test2')
313 mock_run_task.assert_any_call('test1')
314 mock_run_task.assert_any_call('test2')
316 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
317 'prepare_task', return_value=True)
318 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
320 def test_run_tests_default(self, mock_run_task, mock_prepare_task):
321 self.rally_base.tests = ['test1', 'test2']
322 self.rally_base.run_tests()
323 mock_prepare_task.assert_any_call('test1')
324 mock_prepare_task.assert_any_call('test2')
325 mock_run_task.assert_any_call('test1')
326 mock_run_task.assert_any_call('test2')
328 def test_clean_up_default(self):
329 with mock.patch.object(self.rally_base.orig_cloud,
330 'delete_flavor') as mock_delete_flavor:
331 self.rally_base.flavor_alt = mock.Mock()
332 self.rally_base.clean()
333 self.assertEqual(mock_delete_flavor.call_count, 1)
335 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
336 'create_rally_deployment')
337 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
339 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
341 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
343 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
345 def test_run_default(self, *args):
346 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
350 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
351 'create_rally_deployment', side_effect=Exception)
352 def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
353 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
354 mock_create_rally_dep.assert_called()
356 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
357 'create_rally_deployment', return_value=mock.Mock())
358 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
359 'prepare_run', side_effect=Exception)
360 def test_run_exception_prepare_run(self, mock_prep_env, *args):
361 # pylint: disable=unused-argument
362 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
363 mock_prep_env.assert_called()
365 def test_append_summary(self):
371 'full_duration': 1.23,
376 'full_duration': 2.78,
384 self.rally_base._append_summary(json.dumps(json_dict), "foo_test")
385 self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
386 self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
387 self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
388 self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
389 self.assertEqual(self.rally_base.summary[0]['success'], [])
390 self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task'])
392 def test_is_successful_false(self):
393 with mock.patch('six.moves.builtins.super') as mock_super:
394 self.rally_base.summary = [{"task_status": True},
395 {"task_status": False}]
396 self.assertEqual(self.rally_base.is_successful(),
397 testcase.TestCase.EX_TESTCASE_FAILED)
398 mock_super(rally.RallyBase, self).is_successful.assert_not_called()
400 def test_is_successful_true(self):
401 with mock.patch('six.moves.builtins.super') as mock_super:
402 mock_super(rally.RallyBase, self).is_successful.return_value = 424
403 self.rally_base.summary = [{"task_status": True},
404 {"task_status": True}]
405 self.assertEqual(self.rally_base.is_successful(), 424)
406 mock_super(rally.RallyBase, self).is_successful.assert_called()
408 @mock.patch('subprocess.check_output',
409 side_effect=subprocess.CalledProcessError('', ''))
410 def test_export_task_ko(self, *args):
411 file_name = "{}/{}.html".format(
412 self.rally_base.results_dir, self.rally_base.case_name)
413 with self.assertRaises(subprocess.CalledProcessError):
414 self.rally_base.export_task(file_name)
415 cmd = ["rally", "task", "export", "--type", "html", "--deployment",
416 str(getattr(config.CONF, 'rally_deployment_name')),
418 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
420 @mock.patch('subprocess.check_output', return_value=b'')
421 def test_export_task(self, *args):
422 file_name = "{}/{}.html".format(
423 self.rally_base.results_dir, self.rally_base.case_name)
424 self.assertEqual(self.rally_base.export_task(file_name), None)
425 cmd = ["rally", "task", "export", "--type", "html", "--deployment",
426 str(getattr(config.CONF, 'rally_deployment_name')),
428 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
430 @mock.patch('subprocess.check_output',
431 side_effect=subprocess.CalledProcessError('', ''))
432 def test_verify_report_ko(self, *args):
433 file_name = "{}/{}.html".format(
434 self.rally_base.results_dir, self.rally_base.case_name)
435 with self.assertRaises(subprocess.CalledProcessError):
436 self.rally_base.verify_report(file_name, "1")
437 cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
439 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
441 @mock.patch('subprocess.check_output', return_value=b'')
442 def test_verify_report(self, *args):
443 file_name = "{}/{}.html".format(
444 self.rally_base.results_dir, self.rally_base.case_name)
445 self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
446 cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
448 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
451 if __name__ == "__main__":
452 logging.disable(logging.CRITICAL)
453 unittest.main(verbosity=2)