3 # All rights reserved. This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # pylint: disable=missing-docstring,protected-access,invalid-name
18 from xtesting.core import testcase
20 from functest.opnfv_tests.openstack.rally import rally
21 from functest.utils import config
24 class OSRallyTesting(unittest.TestCase):
25 # pylint: disable=too-many-public-methods
27 with mock.patch('os_client_config.get_config') as mock_get_config, \
28 mock.patch('shade.OpenStackCloud') as mock_shade, \
29 mock.patch('functest.core.tenantnetwork.NewProject') \
31 self.rally_base = rally.RallyBase()
32 self.rally_base.image = munch.Munch(name='foo')
33 self.rally_base.flavor = munch.Munch(name='foo')
34 self.rally_base.flavor_alt = munch.Munch(name='bar')
35 self.assertTrue(mock_get_config.called)
36 self.assertTrue(mock_shade.called)
37 self.assertTrue(mock_new_project.called)
39 def test_build_task_args_missing_floating_network(self):
40 os.environ['OS_AUTH_URL'] = ''
41 self.rally_base.ext_net = None
42 task_args = self.rally_base.build_task_args('test_name')
43 self.assertEqual(task_args['floating_network'], '')
45 def test_build_task_args_missing_net_id(self):
46 os.environ['OS_AUTH_URL'] = ''
47 self.rally_base.network = None
48 task_args = self.rally_base.build_task_args('test_name')
49 self.assertEqual(task_args['netid'], '')
52 def check_scenario_file(value):
53 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
54 if yaml_file in value:
58 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
59 def test_prepare_test_list_missing_scenario_file(self, mock_func):
60 mock_func.side_effect = self.check_scenario_file
61 with self.assertRaises(Exception):
62 self.rally_base._prepare_test_list('test_file_name')
63 mock_func.assert_called()
66 def check_temp_dir(value):
67 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
68 if yaml_file in value:
72 @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
73 'RallyBase.get_verifier_deployment_id', return_value='foo')
74 @mock.patch('subprocess.check_output')
75 def test_create_rally_deployment(self, mock_exec, mock_get_id):
76 # pylint: disable=unused-argument
77 self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
79 mock.call(['rally', 'deployment', 'destroy', '--deployment',
80 str(getattr(config.CONF, 'rally_deployment_name'))]),
81 mock.call().decode("utf-8"),
82 mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
83 str(getattr(config.CONF, 'rally_deployment_name'))],
85 mock.call().decode("utf-8"),
86 mock.call(['rally', 'deployment', 'check']),
87 mock.call().decode("utf-8")]
88 mock_exec.assert_has_calls(calls)
90 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
91 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
92 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
94 def test_prepare_test_list_missing_temp_dir(
95 self, mock_method, mock_os_makedirs, mock_path_exists):
96 mock_path_exists.side_effect = self.check_temp_dir
98 yaml_file = 'opnfv-{}.yaml'.format('test_file_name')
99 ret_val = os.path.join(self.rally_base.temp_dir, yaml_file)
100 self.assertEqual(self.rally_base._prepare_test_list('test_file_name'),
102 mock_path_exists.assert_called()
103 mock_method.assert_called()
104 mock_os_makedirs.assert_called()
106 def test_get_task_id_default(self):
107 cmd_raw = b'Task 1: started'
108 self.assertEqual(self.rally_base.get_task_id(cmd_raw),
111 def test_get_task_id_missing_id(self):
113 self.assertEqual(self.rally_base.get_task_id(cmd_raw),
116 def test_task_succeed_fail(self):
117 json_raw = json.dumps({})
118 self.assertEqual(self.rally_base.task_succeed(json_raw),
120 json_raw = json.dumps({'tasks': [{'status': 'crashed'}]})
121 self.assertEqual(self.rally_base.task_succeed(json_raw),
124 def test_task_succeed_success(self):
125 json_raw = json.dumps({'tasks': [{'status': 'finished',
127 self.assertEqual(self.rally_base.task_succeed(json_raw),
130 @mock.patch('six.moves.builtins.open', mock.mock_open())
131 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
132 return_value={'scenario': [
133 {'scenarios': ['test_scenario'],
135 {'scenarios': ['other_scenario'],
136 'tests': ['other_test']}]})
137 def test_excl_scenario_default(self, mock_func):
138 os.environ['INSTALLER_TYPE'] = 'test_installer'
139 os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
140 self.assertEqual(self.rally_base.excl_scenario(), ['test'])
141 mock_func.assert_called()
143 @mock.patch('six.moves.builtins.open', mock.mock_open())
144 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
145 return_value={'scenario': [
146 {'scenarios': ['^os-[^-]+-featT-modeT$'],
148 {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
150 {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
152 {'scenarios': ['^os-'],
154 {'scenarios': ['other_scenario'],
155 'tests': ['test0a']},
156 {'scenarios': [''], # empty scenario
157 'tests': ['test0b']}]})
158 def test_excl_scenario_regex(self, mock_func):
159 os.environ['DEPLOY_SCENARIO'] = 'os-ctrlT-featT-modeT'
160 self.assertEqual(self.rally_base.excl_scenario(),
161 ['test1', 'test2', 'test3', 'test4'])
162 mock_func.assert_called()
164 @mock.patch('six.moves.builtins.open', side_effect=Exception)
165 def test_excl_scenario_exception(self, mock_open):
166 self.assertEqual(self.rally_base.excl_scenario(), [])
167 mock_open.assert_called()
169 @mock.patch('six.moves.builtins.open', mock.mock_open())
170 @mock.patch('functest.opnfv_tests.openstack.rally.rally.yaml.safe_load',
171 return_value={'functionality': [
172 {'functions': ['no_migration'], 'tests': ['test']}]})
173 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
174 '_migration_supported', return_value=False)
175 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
176 '_network_trunk_supported', return_value=False)
177 def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load):
178 os.environ['DEPLOY_SCENARIO'] = 'test_scenario'
179 self.assertEqual(self.rally_base.excl_func(), ['test'])
180 mock_func.assert_called()
181 mock_trunk.assert_called()
182 mock_yaml_load.assert_called()
184 @mock.patch('six.moves.builtins.open', side_effect=Exception)
185 def test_excl_func_exception(self, mock_open):
186 self.assertEqual(self.rally_base.excl_func(), [])
187 mock_open.assert_called()
189 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat')
190 def test_file_is_empty_default(self, mock_os_stat):
191 attrs = {'st_size': 10}
192 mock_os_stat.return_value.configure_mock(**attrs)
193 self.assertEqual(self.rally_base.file_is_empty('test_file_name'),
195 mock_os_stat.assert_called()
197 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.stat',
198 side_effect=Exception)
199 def test_file_is_empty_exception(self, mock_os_stat):
200 self.assertEqual(self.rally_base.file_is_empty('test_file_name'), True)
201 mock_os_stat.assert_called()
203 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
205 def test_run_task_missing_task_file(self, mock_path_exists):
206 with self.assertRaises(Exception):
207 self.rally_base.prepare_run()
208 mock_path_exists.assert_called()
210 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
211 '_prepare_test_list', return_value='test_file_name')
212 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
213 'file_is_empty', return_value=True)
214 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
215 def test_prepare_task_no_tests_for_scenario(
216 self, mock_logger_info, mock_file_empty, mock_prep_list):
217 self.rally_base.prepare_task('test_name')
218 mock_logger_info.assert_any_call('No tests for scenario \"%s\"',
220 mock_file_empty.assert_called()
221 mock_prep_list.assert_called()
223 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
224 '_prepare_test_list', return_value='test_file_name')
225 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
226 'file_is_empty', return_value=False)
227 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
228 'build_task_args', return_value={})
229 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
230 'get_task_id', return_value=None)
231 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
233 @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
234 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
235 def test_run_task_taskid_missing(self, mock_logger_error, *args):
236 # pylint: disable=unused-argument
237 with self.assertRaises(Exception):
238 self.rally_base.run_task('test_name')
239 text = 'Failed to retrieve task_id'
240 mock_logger_error.assert_any_call(text)
242 @mock.patch('six.moves.builtins.open', mock.mock_open())
243 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
244 '_prepare_test_list', return_value='test_file_name')
245 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
246 'file_is_empty', return_value=False)
247 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
248 'build_task_args', return_value={})
249 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
250 'get_task_id', return_value='1')
251 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
252 'task_succeed', return_value=True)
253 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
255 @mock.patch('functest.opnfv_tests.openstack.rally.rally.subprocess.Popen')
256 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
257 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
258 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.error')
259 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
261 def test_run_task_default(self, mock_save_res, *args):
262 # pylint: disable=unused-argument
263 self.rally_base.run_task('test_name')
264 mock_save_res.assert_called()
266 @mock.patch('six.moves.builtins.open', mock.mock_open())
267 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
268 'task_succeed', return_value=True)
269 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists',
271 @mock.patch('subprocess.check_output')
272 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
273 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.info')
274 @mock.patch('functest.opnfv_tests.openstack.rally.rally.LOGGER.debug')
275 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
277 def test_save_results(self, mock_summary, *args):
278 # pylint: disable=unused-argument
279 self.rally_base._save_results('test_name', '1234')
280 mock_summary.assert_called()
282 def test_prepare_run_testname_invalid(self):
283 self.rally_base.stests = ['test1', 'test2']
284 with self.assertRaises(Exception):
285 self.rally_base.prepare_run(tests=['test'])
287 @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
288 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
289 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copytree')
290 @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.rmtree')
291 def test_prepare_run_flavor_alt_creation_failed(self, *args):
292 # pylint: disable=unused-argument
293 self.rally_base.stests = ['test1', 'test2']
294 with mock.patch.object(self.rally_base, 'count_active_hypervisors') \
295 as mock_list_hyperv, \
296 mock.patch.object(self.rally_base, 'create_flavor_alt',
297 side_effect=Exception) \
298 as mock_create_flavor:
299 with self.assertRaises(Exception):
300 self.rally_base.prepare_run(tests=['test1'])
301 mock_list_hyperv.assert_called_once()
302 mock_create_flavor.assert_called_once()
304 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
305 'prepare_task', return_value=True)
306 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
308 def test_run_tests_all(self, mock_run_task, mock_prepare_task):
309 self.rally_base.tests = ['test1', 'test2']
310 self.rally_base.run_tests()
311 mock_prepare_task.assert_any_call('test1')
312 mock_prepare_task.assert_any_call('test2')
313 mock_run_task.assert_any_call('test1')
314 mock_run_task.assert_any_call('test2')
316 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
317 'prepare_task', return_value=True)
318 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
320 def test_run_tests_default(self, mock_run_task, mock_prepare_task):
321 self.rally_base.tests = ['test1', 'test2']
322 self.rally_base.run_tests()
323 mock_prepare_task.assert_any_call('test1')
324 mock_prepare_task.assert_any_call('test2')
325 mock_run_task.assert_any_call('test1')
326 mock_run_task.assert_any_call('test2')
328 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
330 def test_clean_up_default(self, *args):
331 with mock.patch.object(self.rally_base.orig_cloud,
332 'delete_flavor') as mock_delete_flavor:
333 self.rally_base.flavor_alt = mock.Mock()
334 self.rally_base.clean()
335 self.assertEqual(mock_delete_flavor.call_count, 1)
336 args[0].assert_called_once_with()
338 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
340 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
341 'create_rally_deployment')
342 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
344 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
346 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
348 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
350 def test_run_default(self, *args):
351 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
355 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
357 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
358 'create_rally_deployment', side_effect=Exception)
359 def test_run_exception_create_rally_dep(self, *args):
360 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
361 args[0].assert_called()
362 args[1].assert_called_once_with(self.rally_base.res_dir)
364 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
366 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
367 'create_rally_deployment', return_value=mock.Mock())
368 @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
369 'prepare_run', side_effect=Exception)
370 def test_run_exception_prepare_run(self, mock_prep_env, *args):
371 # pylint: disable=unused-argument
372 self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
373 mock_prep_env.assert_called()
374 args[1].assert_called_once_with(self.rally_base.res_dir)
376 def test_append_summary(self):
382 'full_duration': 1.23,
387 'full_duration': 2.78,
395 self.rally_base._append_summary(json.dumps(json_dict), "foo_test")
396 self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
397 self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
398 self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
399 self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
400 self.assertEqual(self.rally_base.summary[0]['success'], [])
401 self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task'])
403 def test_is_successful_false(self):
404 with mock.patch('six.moves.builtins.super') as mock_super:
405 self.rally_base.summary = [{"task_status": True},
406 {"task_status": False}]
407 self.assertEqual(self.rally_base.is_successful(),
408 testcase.TestCase.EX_TESTCASE_FAILED)
409 mock_super(rally.RallyBase, self).is_successful.assert_not_called()
411 def test_is_successful_true(self):
412 with mock.patch('six.moves.builtins.super') as mock_super:
413 mock_super(rally.RallyBase, self).is_successful.return_value = 424
414 self.rally_base.summary = [{"task_status": True},
415 {"task_status": True}]
416 self.assertEqual(self.rally_base.is_successful(), 424)
417 mock_super(rally.RallyBase, self).is_successful.assert_called()
419 @mock.patch('subprocess.check_output',
420 side_effect=subprocess.CalledProcessError('', ''))
421 def test_export_task_ko(self, *args):
422 file_name = "{}/{}.html".format(
423 self.rally_base.results_dir, self.rally_base.case_name)
424 with self.assertRaises(subprocess.CalledProcessError):
425 self.rally_base.export_task(file_name)
426 cmd = ["rally", "task", "export", "--type", "html", "--deployment",
427 str(getattr(config.CONF, 'rally_deployment_name')),
429 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
431 @mock.patch('subprocess.check_output', return_value=b'')
432 def test_export_task(self, *args):
433 file_name = "{}/{}.html".format(
434 self.rally_base.results_dir, self.rally_base.case_name)
435 self.assertEqual(self.rally_base.export_task(file_name), None)
436 cmd = ["rally", "task", "export", "--type", "html", "--deployment",
437 str(getattr(config.CONF, 'rally_deployment_name')),
439 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
441 @mock.patch('subprocess.check_output',
442 side_effect=subprocess.CalledProcessError('', ''))
443 def test_verify_report_ko(self, *args):
444 file_name = "{}/{}.html".format(
445 self.rally_base.results_dir, self.rally_base.case_name)
446 with self.assertRaises(subprocess.CalledProcessError):
447 self.rally_base.verify_report(file_name, "1")
448 cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
450 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
452 @mock.patch('subprocess.check_output', return_value=b'')
453 def test_verify_report(self, *args):
454 file_name = "{}/{}.html".format(
455 self.rally_base.results_dir, self.rally_base.case_name)
456 self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
457 cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
459 args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
462 if __name__ == "__main__":
463 logging.disable(logging.CRITICAL)
464 unittest.main(verbosity=2)