3 # Copyright (c) 2016 Orange and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 """Define the classes required to fully cover odl."""
17 from keystoneauth1.exceptions import auth_plugins
19 from robot.errors import DataError, RobotError
20 from robot.result import model
21 from robot.utils.robottime import timestamp_to_secs
24 from functest.core import testcase
25 from functest.opnfv_tests.sdn.odl import odl
27 __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
30 class ODLVisitorTesting(unittest.TestCase):
32 """The class testing ODLResultVisitor."""
33 # pylint: disable=missing-docstring
36 self.visitor = odl.ODLResultVisitor()
39 self.assertFalse(self.visitor.get_data())
42 data = {'name': 'foo',
45 'starttime': "20161216 16:00:00.000",
46 'endtime': "20161216 16:00:01.000",
48 'text': 'Hello, World!',
50 test = model.TestCase(
51 name=data['name'], status=data['status'], message=data['text'],
52 starttime=data['starttime'], endtime=data['endtime'])
53 test.parent = mock.Mock()
54 config = {'name': data['parent'],
55 'criticality.test_is_critical.return_value': data[
57 test.parent.configure_mock(**config)
58 self.visitor.visit_test(test)
59 self.assertEqual(self.visitor.get_data(), [data])
62 class ODLTesting(unittest.TestCase):
64 """The super class which testing classes could inherit."""
65 # pylint: disable=missing-docstring
67 logging.disable(logging.CRITICAL)
69 _keystone_ip = "127.0.0.1"
70 _neutron_ip = "127.0.0.2"
71 _sdn_controller_ip = "127.0.0.3"
72 _os_auth_url = "http://{}:5000/v2.0".format(_keystone_ip)
73 _os_tenantname = "admin"
74 _os_username = "admin"
75 _os_password = "admin"
77 _odl_restconfport = "8181"
78 _odl_username = "admin"
79 _odl_password = "admin"
82 for var in ("INSTALLER_TYPE", "SDN_CONTROLLER", "SDN_CONTROLLER_IP"):
85 os.environ["OS_AUTH_URL"] = self._os_auth_url
86 os.environ["OS_USERNAME"] = self._os_username
87 os.environ["OS_PASSWORD"] = self._os_password
88 os.environ["OS_TENANT_NAME"] = self._os_tenantname
89 self.test = odl.ODLTests(case_name='odl', project_name='functest')
90 self.defaultargs = {'odlusername': self._odl_username,
91 'odlpassword': self._odl_password,
92 'neutronip': self._keystone_ip,
93 'osauthurl': self._os_auth_url,
94 'osusername': self._os_username,
95 'ostenantname': self._os_tenantname,
96 'ospassword': self._os_password,
97 'odlip': self._keystone_ip,
98 'odlwebport': self._odl_webport,
99 'odlrestconfport': self._odl_restconfport,
103 class ODLParseResultTesting(ODLTesting):
105 """The class testing ODLTests.parse_results()."""
106 # pylint: disable=missing-docstring
108 _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000',
109 'endtime': '20161216 16:00:01.000'}
111 @mock.patch('robot.api.ExecutionResult', side_effect=DataError)
112 def test_raises_exc(self, mock_method):
113 with self.assertRaises(DataError):
114 self.test.parse_results()
115 mock_method.assert_called_once_with(
116 os.path.join(odl.ODLTests.res_dir, 'output.xml'))
118 def _test_result(self, config, result):
120 suite.configure_mock(**config)
121 with mock.patch('robot.api.ExecutionResult',
122 return_value=mock.Mock(suite=suite)):
123 self.test.parse_results()
124 self.assertEqual(self.test.result, result)
125 self.assertEqual(self.test.start_time,
126 timestamp_to_secs(config['starttime']))
127 self.assertEqual(self.test.stop_time,
128 timestamp_to_secs(config['endtime']))
129 self.assertEqual(self.test.details,
130 {'description': config['name'], 'tests': []})
132 def test_null_passed(self):
133 self._config.update({'statistics.critical.passed': 0,
134 'statistics.critical.total': 20})
135 self._test_result(self._config, 0)
137 def test_no_test(self):
138 self._config.update({'statistics.critical.passed': 20,
139 'statistics.critical.total': 0})
140 self._test_result(self._config, 0)
142 def test_half_success(self):
143 self._config.update({'statistics.critical.passed': 10,
144 'statistics.critical.total': 20})
145 self._test_result(self._config, 50)
147 def test_success(self):
148 self._config.update({'statistics.critical.passed': 20,
149 'statistics.critical.total': 20})
150 self._test_result(self._config, 100)
153 class ODLRobotTesting(ODLTesting):
155 """The class testing ODLTests.set_robotframework_vars()."""
156 # pylint: disable=missing-docstring
158 @mock.patch('fileinput.input', side_effect=Exception())
159 def test_set_vars_ko(self, mock_method):
160 self.assertFalse(self.test.set_robotframework_vars())
161 mock_method.assert_called_once_with(
162 os.path.join(odl.ODLTests.odl_test_repo,
163 'csit/variables/Variables.py'), inplace=True)
165 @mock.patch('fileinput.input', return_value=[])
166 def test_set_vars_empty(self, mock_method):
167 self.assertTrue(self.test.set_robotframework_vars())
168 mock_method.assert_called_once_with(
169 os.path.join(odl.ODLTests.odl_test_repo,
170 'csit/variables/Variables.py'), inplace=True)
172 @mock.patch('sys.stdout', new_callable=six.StringIO)
173 def _test_set_vars(self, msg1, msg2, *args):
174 line = mock.MagicMock()
175 line.__iter__.return_value = [msg1]
176 with mock.patch('fileinput.input', return_value=line) as mock_method:
177 self.assertTrue(self.test.set_robotframework_vars())
178 mock_method.assert_called_once_with(
179 os.path.join(odl.ODLTests.odl_test_repo,
180 'csit/variables/Variables.py'), inplace=True)
181 self.assertEqual(args[0].getvalue(), "{}\n".format(msg2))
183 def test_set_vars_auth_default(self):
184 self._test_set_vars("AUTH = []",
185 "AUTH = [u'admin', u'admin']")
187 def test_set_vars_auth1(self):
188 self._test_set_vars("AUTH1 = []", "AUTH1 = []")
190 @mock.patch('sys.stdout', new_callable=six.StringIO)
191 def test_set_vars_auth_foo(self, *args):
192 line = mock.MagicMock()
193 line.__iter__.return_value = ["AUTH = []"]
194 with mock.patch('fileinput.input', return_value=line) as mock_method:
195 self.assertTrue(self.test.set_robotframework_vars('foo', 'bar'))
196 mock_method.assert_called_once_with(
197 os.path.join(odl.ODLTests.odl_test_repo,
198 'csit/variables/Variables.py'), inplace=True)
199 self.assertEqual(args[0].getvalue(),
200 "AUTH = [u'{}', u'{}']\n".format('foo', 'bar'))
203 class ODLMainTesting(ODLTesting):
205 """The class testing ODLTests.main()."""
206 # pylint: disable=missing-docstring
208 def _get_main_kwargs(self, key=None):
209 kwargs = {'odlusername': self._odl_username,
210 'odlpassword': self._odl_password,
211 'neutronip': self._neutron_ip,
212 'osauthurl': self._os_auth_url,
213 'osusername': self._os_username,
214 'ostenantname': self._os_tenantname,
215 'ospassword': self._os_password,
216 'odlip': self._sdn_controller_ip,
217 'odlwebport': self._odl_webport,
218 'odlrestconfport': self._odl_restconfport}
223 def _test_main(self, status, *args):
224 kwargs = self._get_main_kwargs()
225 self.assertEqual(self.test.main(**kwargs), status)
227 args[0].assert_called_once_with(
228 odl.ODLTests.res_dir)
230 variable = ['KEYSTONE:{}'.format(self._keystone_ip),
231 'NEUTRON:{}'.format(self._neutron_ip),
232 'OS_AUTH_URL:"{}"'.format(self._os_auth_url),
233 'OSUSERNAME:"{}"'.format(self._os_username),
234 'OSTENANTNAME:"{}"'.format(self._os_tenantname),
235 'OSPASSWORD:"{}"'.format(self._os_password),
236 'ODL_SYSTEM_IP:{}'.format(self._sdn_controller_ip),
237 'PORT:{}'.format(self._odl_webport),
238 'RESTCONFPORT:{}'.format(self._odl_restconfport)]
239 args[1].assert_called_once_with(
240 odl.ODLTests.basic_suite_dir,
241 odl.ODLTests.neutron_suite_dir,
243 output=os.path.join(odl.ODLTests.res_dir, 'output.xml'),
248 args[2].assert_called_with(
249 os.path.join(odl.ODLTests.res_dir, 'stdout.txt'))
251 def _test_no_keyword(self, key):
252 kwargs = self._get_main_kwargs(key)
253 self.assertEqual(self.test.main(**kwargs),
254 testcase.TestCase.EX_RUN_ERROR)
256 def test_no_odlusername(self):
257 self._test_no_keyword('odlusername')
259 def test_no_odlpassword(self):
260 self._test_no_keyword('odlpassword')
262 def test_no_neutronip(self):
263 self._test_no_keyword('neutronip')
265 def test_no_osauthurl(self):
266 self._test_no_keyword('osauthurl')
268 def test_no_osusername(self):
269 self._test_no_keyword('osusername')
271 def test_no_ostenantname(self):
272 self._test_no_keyword('ostenantname')
274 def test_no_ospassword(self):
275 self._test_no_keyword('ospassword')
277 def test_no_odlip(self):
278 self._test_no_keyword('odlip')
280 def test_no_odlwebport(self):
281 self._test_no_keyword('odlwebport')
283 def test_no_odlrestconfport(self):
284 self._test_no_keyword('odlrestconfport')
286 def test_set_vars_ko(self):
287 with mock.patch.object(self.test, 'set_robotframework_vars',
288 return_value=False) as mock_object:
289 self._test_main(testcase.TestCase.EX_RUN_ERROR)
290 mock_object.assert_called_once_with(
291 self._odl_username, self._odl_password)
293 @mock.patch('os.makedirs', side_effect=Exception)
294 def test_makedirs_exc(self, mock_method):
295 with mock.patch.object(self.test, 'set_robotframework_vars',
296 return_value=True), \
297 self.assertRaises(Exception):
298 self._test_main(testcase.TestCase.EX_RUN_ERROR,
301 @mock.patch('os.makedirs', side_effect=OSError)
302 def test_makedirs_oserror(self, mock_method):
303 with mock.patch.object(self.test, 'set_robotframework_vars',
305 self._test_main(testcase.TestCase.EX_RUN_ERROR,
308 @mock.patch('robot.run', side_effect=RobotError)
309 @mock.patch('os.makedirs')
310 def test_run_ko(self, *args):
311 with mock.patch.object(self.test, 'set_robotframework_vars',
312 return_value=True), \
313 mock.patch.object(odl, 'open', mock.mock_open(),
315 self.assertRaises(RobotError):
316 self._test_main(testcase.TestCase.EX_RUN_ERROR, *args)
318 @mock.patch('robot.run')
319 @mock.patch('os.makedirs')
320 def test_parse_results_ko(self, *args):
321 with mock.patch.object(self.test, 'set_robotframework_vars',
322 return_value=True), \
323 mock.patch.object(odl, 'open', mock.mock_open(),
325 mock.patch.object(self.test, 'parse_results',
326 side_effect=RobotError):
327 self._test_main(testcase.TestCase.EX_RUN_ERROR, *args)
329 @mock.patch('os.remove', side_effect=Exception)
330 @mock.patch('robot.run')
331 @mock.patch('os.makedirs')
332 def test_remove_exc(self, *args):
333 with mock.patch.object(self.test, 'set_robotframework_vars',
334 return_value=True), \
335 mock.patch.object(self.test, 'parse_results'), \
336 self.assertRaises(Exception):
337 self._test_main(testcase.TestCase.EX_OK, *args)
339 @mock.patch('os.remove')
340 @mock.patch('robot.run')
341 @mock.patch('os.makedirs')
342 def test_ok(self, *args):
343 with mock.patch.object(self.test, 'set_robotframework_vars',
344 return_value=True), \
345 mock.patch.object(odl, 'open', mock.mock_open(),
347 mock.patch.object(self.test, 'parse_results'):
348 self._test_main(testcase.TestCase.EX_OK, *args)
350 @mock.patch('os.remove')
351 @mock.patch('robot.run')
352 @mock.patch('os.makedirs', side_effect=OSError(errno.EEXIST, ''))
353 def test_makedirs_oserror17(self, *args):
354 with mock.patch.object(self.test, 'set_robotframework_vars',
355 return_value=True), \
356 mock.patch.object(odl, 'open', mock.mock_open(),
357 create=True) as mock_open, \
358 mock.patch.object(self.test, 'parse_results'):
359 self._test_main(testcase.TestCase.EX_OK, *args)
360 mock_open.assert_called_once_with(
361 os.path.join(odl.ODLTests.res_dir, 'stdout.txt'), 'w+')
363 @mock.patch('os.remove')
364 @mock.patch('robot.run', return_value=1)
365 @mock.patch('os.makedirs')
366 def test_testcases_in_failure(self, *args):
367 with mock.patch.object(self.test, 'set_robotframework_vars',
368 return_value=True), \
369 mock.patch.object(odl, 'open', mock.mock_open(),
370 create=True) as mock_open, \
371 mock.patch.object(self.test, 'parse_results'):
372 self._test_main(testcase.TestCase.EX_OK, *args)
373 mock_open.assert_called_once_with(
374 os.path.join(odl.ODLTests.res_dir, 'stdout.txt'), 'w+')
376 @mock.patch('os.remove', side_effect=OSError)
377 @mock.patch('robot.run')
378 @mock.patch('os.makedirs')
379 def test_remove_oserror(self, *args):
380 with mock.patch.object(self.test, 'set_robotframework_vars',
381 return_value=True), \
382 mock.patch.object(odl, 'open', mock.mock_open(),
383 create=True) as mock_open, \
384 mock.patch.object(self.test, 'parse_results'):
385 self._test_main(testcase.TestCase.EX_OK, *args)
386 mock_open.assert_called_once_with(
387 os.path.join(odl.ODLTests.res_dir, 'stdout.txt'), 'w+')
390 class ODLRunTesting(ODLTesting):
392 """The class testing ODLTests.run()."""
393 # pylint: disable=missing-docstring
395 def _test_no_env_var(self, var):
396 with mock.patch('functest.utils.openstack_utils.get_endpoint',
397 return_value="http://{}:9696".format(
398 ODLTesting._neutron_ip)):
400 self.assertEqual(self.test.run(),
401 testcase.TestCase.EX_RUN_ERROR)
403 def _test_run(self, status=testcase.TestCase.EX_OK,
404 exception=None, **kwargs):
405 odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3'
406 odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080'
407 odlrestconfport = (kwargs['odlrestconfport']
408 if 'odlrestconfport' in kwargs else '8181')
410 with mock.patch('functest.utils.openstack_utils.get_endpoint',
411 return_value="http://{}:9696".format(
412 ODLTesting._neutron_ip)):
414 self.test.main = mock.Mock(side_effect=exception)
416 self.test.main = mock.Mock(return_value=status)
417 self.assertEqual(self.test.run(), status)
418 self.test.main.assert_called_once_with(
419 odl.ODLTests.default_suites,
420 neutronip=self._neutron_ip,
421 odlip=odlip, odlpassword=self._odl_password,
422 odlrestconfport=odlrestconfport,
423 odlusername=self._odl_username, odlwebport=odlwebport,
424 osauthurl=self._os_auth_url,
425 ospassword=self._os_password, ostenantname=self._os_tenantname,
426 osusername=self._os_username)
428 def _test_multiple_suites(self, suites,
429 status=testcase.TestCase.EX_OK, **kwargs):
430 odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3'
431 odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080'
432 odlrestconfport = (kwargs['odlrestconfport']
433 if 'odlrestconfport' in kwargs else '8181')
434 with mock.patch('functest.utils.openstack_utils.get_endpoint',
435 return_value="http://{}:9696".format(
436 ODLTesting._neutron_ip)):
437 self.test.main = mock.Mock(return_value=status)
438 self.assertEqual(self.test.run(suites=suites), status)
439 self.test.main.assert_called_once_with(
441 neutronip=self._neutron_ip,
442 odlip=odlip, odlpassword=self._odl_password,
443 odlrestconfport=odlrestconfport,
444 odlusername=self._odl_username, odlwebport=odlwebport,
445 osauthurl=self._os_auth_url,
446 ospassword=self._os_password, ostenantname=self._os_tenantname,
447 osusername=self._os_username)
450 with mock.patch('functest.utils.openstack_utils.get_endpoint',
451 side_effect=auth_plugins.MissingAuthPlugin()):
452 self.assertEqual(self.test.run(),
453 testcase.TestCase.EX_RUN_ERROR)
455 def test_no_os_auth_url(self):
456 self._test_no_env_var("OS_AUTH_URL")
458 def test_no_os_username(self):
459 self._test_no_env_var("OS_USERNAME")
461 def test_no_os_password(self):
462 self._test_no_env_var("OS_PASSWORD")
464 def test_no_os_tenant_name(self):
465 self._test_no_env_var("OS_TENANT_NAME")
467 def test_main_false(self):
468 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
469 self._test_run(testcase.TestCase.EX_RUN_ERROR,
470 odlip=self._sdn_controller_ip,
471 odlwebport=self._odl_webport)
473 def test_main_exc(self):
474 with self.assertRaises(Exception):
475 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
476 self._test_run(status=testcase.TestCase.EX_RUN_ERROR,
477 exception=Exception(),
478 odlip=self._sdn_controller_ip,
479 odlwebport=self._odl_webport)
481 def test_no_sdn_controller_ip(self):
482 with mock.patch('functest.utils.openstack_utils.get_endpoint',
483 return_value="http://{}:9696".format(
484 ODLTesting._neutron_ip)):
485 self.assertEqual(self.test.run(),
486 testcase.TestCase.EX_RUN_ERROR)
488 def test_without_installer_type(self):
489 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
490 self._test_run(testcase.TestCase.EX_OK,
491 odlip=self._sdn_controller_ip,
492 odlwebport=self._odl_webport)
494 def test_suites(self):
495 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
496 self._test_multiple_suites(
497 [odl.ODLTests.basic_suite_dir],
498 testcase.TestCase.EX_OK,
499 odlip=self._sdn_controller_ip,
500 odlwebport=self._odl_webport)
503 os.environ["INSTALLER_TYPE"] = "fuel"
504 self._test_run(testcase.TestCase.EX_OK,
505 odlip=self._neutron_ip, odlwebport='8282')
507 def test_apex_no_controller_ip(self):
508 with mock.patch('functest.utils.openstack_utils.get_endpoint',
509 return_value="http://{}:9696".format(
510 ODLTesting._neutron_ip)):
511 os.environ["INSTALLER_TYPE"] = "apex"
512 self.assertEqual(self.test.run(),
513 testcase.TestCase.EX_RUN_ERROR)
516 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
517 os.environ["INSTALLER_TYPE"] = "apex"
518 self._test_run(testcase.TestCase.EX_OK,
519 odlip=self._sdn_controller_ip, odlwebport='8081',
520 odlrestconfport='8081')
522 def test_netvirt_no_controller_ip(self):
523 with mock.patch('functest.utils.openstack_utils.get_endpoint',
524 return_value="http://{}:9696".format(
525 ODLTesting._neutron_ip)):
526 os.environ["INSTALLER_TYPE"] = "netvirt"
527 self.assertEqual(self.test.run(),
528 testcase.TestCase.EX_RUN_ERROR)
530 def test_netvirt(self):
531 os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
532 os.environ["INSTALLER_TYPE"] = "netvirt"
533 self._test_run(testcase.TestCase.EX_OK,
534 odlip=self._sdn_controller_ip, odlwebport='8081',
535 odlrestconfport='8081')
537 def test_joid_no_controller_ip(self):
538 with mock.patch('functest.utils.openstack_utils.get_endpoint',
539 return_value="http://{}:9696".format(
540 ODLTesting._neutron_ip)):
541 os.environ["INSTALLER_TYPE"] = "joid"
542 self.assertEqual(self.test.run(),
543 testcase.TestCase.EX_RUN_ERROR)
546 os.environ["SDN_CONTROLLER"] = self._sdn_controller_ip
547 os.environ["INSTALLER_TYPE"] = "joid"
548 self._test_run(testcase.TestCase.EX_OK,
549 odlip=self._sdn_controller_ip, odlwebport='8080')
551 def test_compass(self):
552 os.environ["INSTALLER_TYPE"] = "compass"
553 self._test_run(testcase.TestCase.EX_OK,
554 odlip=self._neutron_ip, odlwebport='8181')
557 class ODLArgParserTesting(ODLTesting):
559 """The class testing ODLParser."""
560 # pylint: disable=missing-docstring
563 self.parser = odl.ODLParser()
564 super(ODLArgParserTesting, self).setUp()
566 def test_default(self):
567 self.assertEqual(self.parser.parse_args(), self.defaultargs)
569 def test_basic(self):
570 self.defaultargs['neutronip'] = self._neutron_ip
571 self.defaultargs['odlip'] = self._sdn_controller_ip
573 self.parser.parse_args(
574 ["--neutronip={}".format(self._neutron_ip),
575 "--odlip={}".format(self._sdn_controller_ip)]),
578 @mock.patch('sys.stderr', new_callable=six.StringIO)
579 def test_fail(self, mock_method):
580 self.defaultargs['foo'] = 'bar'
581 with self.assertRaises(SystemExit):
582 self.parser.parse_args(["--foo=bar"])
583 self.assertTrue(mock_method.getvalue().startswith("usage:"))
585 def _test_arg(self, arg, value):
586 self.defaultargs[arg] = value
588 self.parser.parse_args(["--{}={}".format(arg, value)]),
591 def test_odlusername(self):
592 self._test_arg('odlusername', 'foo')
594 def test_odlpassword(self):
595 self._test_arg('odlpassword', 'foo')
597 def test_osauthurl(self):
598 self._test_arg('osauthurl', 'http://127.0.0.4:5000/v2')
600 def test_neutronip(self):
601 self._test_arg('neutronip', '127.0.0.4')
603 def test_osusername(self):
604 self._test_arg('osusername', 'foo')
606 def test_ostenantname(self):
607 self._test_arg('ostenantname', 'foo')
609 def test_ospassword(self):
610 self._test_arg('ospassword', 'foo')
612 def test_odlip(self):
613 self._test_arg('odlip', '127.0.0.4')
615 def test_odlwebport(self):
616 self._test_arg('odlwebport', '80')
618 def test_odlrestconfport(self):
619 self._test_arg('odlrestconfport', '80')
621 def test_pushtodb(self):
622 self.defaultargs['pushtodb'] = True
623 self.assertEqual(self.parser.parse_args(["--{}".format('pushtodb')]),
626 def test_multiple_args(self):
627 self.defaultargs['neutronip'] = self._neutron_ip
628 self.defaultargs['odlip'] = self._sdn_controller_ip
630 self.parser.parse_args(
631 ["--neutronip={}".format(self._neutron_ip),
632 "--odlip={}".format(self._sdn_controller_ip)]),
636 if __name__ == "__main__":
637 logging.disable(logging.CRITICAL)
638 unittest.main(verbosity=2)