### methods
-| Method | Purpose |
-|-------------------|--------------------------------------------|
-| run(**kwargs) | run the test case |
-| is_successful() | interpret the results of the test case |
-| get_duration() | return the duration of the test case |
-| push_to_db() | push the results of the test case to the DB|
+| Method | Purpose |
+|-------------------|-------------------------------|
+| run(**kwargs) | run the test case |
+| is_successful() | interpret the results |
+| get_duration() | return the duration |
+| push_to_db() | push the results to the DB |
+| create_snapshot() | save the testing environement |
+| clean() | clean the resources |
### run(**kwargs)
project_name: functest
criteria: 100
blocking: true
-clean_flag: false
description: ''
dependencies:
installer: ''
project_name: functest
criteria: 100
blocking: true
-clean_flag: false
description: ''
dependencies:
installer: ''
project_name: functest
criteria: 100
blocking: true
-clean_flag: false
description: ''
dependencies:
installer: ''
project_name: functest
criteria: 100
blocking: true
-clean_flag: false
description: ''
dependencies:
installer: ''
all the OpenStack resources (images, networks, volumes, security groups, tenants,
users) so that an eventual cleanup does not remove any of these defaults.
-It is also called before running a test except if it is disabled by configuration
-in the testcases.yaml file (clean_flag=false). This flag has been added as some
-upstream tests already include their own cleaning mechanism (e.g. Rally).
-
The script **openstack_clean.py** which is located in
-*$REPOS_DIR/functest/functest/utils/* is normally called after a test execution. It is
-in charge of cleaning the OpenStack resources that are not specified in the
-defaults file generated previously which is stored in
+*$REPOS_DIR/functest/functest/utils/* is in charge of cleaning the OpenStack resources
+that are not specified in the defaults file generated previously which is stored in
*/home/opnfv/functest/conf/openstack_snapshot.yaml* in the Functest docker container.
It is important to mention that if there are new OpenStack resources created
* the name of the test case
* the criteria (experimental): a criteria used to declare the test case as PASS or FAIL
* blocking: if set to true, if the test is failed, the execution of the following tests is canceled
- * clean_flag: shall the functect internal mechanism be invoked after the test
* the description of the test case
* the dependencies: a combination of 2 regex on the scenario and the installer name
* run: In Danube we introduced the notion of abstract class in order to harmonize the way to run internal, feature or vnf tests
import functest.ci.tier_builder as tb
import functest.core.testcase as testcase
import functest.utils.functest_utils as ft_utils
-import functest.utils.openstack_clean as os_clean
-import functest.utils.openstack_snapshot as os_snapshot
import functest.utils.openstack_utils as os_utils
from functest.utils.constants import CONST
elif key == 'OS_PASSWORD':
CONST.__setattr__('OS_PASSWORD', value)
- @staticmethod
- def generate_os_snapshot():
- os_snapshot.main()
-
- @staticmethod
- def cleanup():
- os_clean.main()
-
@staticmethod
def get_run_dict(testname):
try:
"The test case {} is not enabled".format(test.get_name()))
logger.info("\n") # blank line
self.print_separator("=")
- logger.info("Running test case '%s'..." % test.get_name())
+ logger.info("Running test case '%s'...", test.get_name())
self.print_separator("=")
logger.debug("\n%s" % test)
self.source_rc_file()
- if test.needs_clean() and self.clean_flag:
- self.generate_os_snapshot()
-
flags = " -t %s" % test.get_name()
if self.report_flag:
flags += " -r"
test_dict = ft_utils.get_dict_by_test(test.get_name())
test_case = cls(**test_dict)
self.executed_test_cases.append(test_case)
+ if self.clean_flag:
+ if test_case.create_snapshot() != test_case.EX_OK:
+ return result
try:
kwargs = run_dict['args']
result = test_case.run(**kwargs)
test_case.push_to_db()
result = test_case.is_successful()
logger.info("Test result:\n\n%s\n", test_case)
+ if self.clean_flag:
+ test_case.clean()
except ImportError:
logger.exception("Cannot import module {}".format(
run_dict['module']))
else:
raise Exception("Cannot import the class for the test case.")
- if test.needs_clean() and self.clean_flag:
- self.cleanup()
- if result != testcase.TestCase.EX_OK:
- logger.error("The test case '%s' failed. " % test.get_name())
- self.overall_result = Result.EX_ERROR
- if test.is_blocking():
- raise BlockingTestFailed(
- "The test case {} failed and is blocking".format(
- test.get_name()))
+ return result
def run_tier(self, tier):
tier_name = tier.get_name()
self.print_separator("#")
logger.debug("\n%s" % tier)
for test in tests:
- self.run_test(test, tier_name)
+ result = self.run_test(test, tier_name)
+ if result != testcase.TestCase.EX_OK:
+ logger.error("The test case '%s' failed.", test.get_name())
+ self.overall_result = Result.EX_ERROR
+ if test.is_blocking():
+ raise BlockingTestFailed(
+ "The test case {} failed and is blocking".format(
+ test.get_name()))
def run_all(self, tiers):
summary = ""
if _tiers.get_tier(kwargs['test']):
self.run_tier(_tiers.get_tier(kwargs['test']))
elif _tiers.get_test(kwargs['test']):
- self.run_test(_tiers.get_test(kwargs['test']),
- _tiers.get_tier_name(kwargs['test']),
- kwargs['test'])
+ result = self.run_test(
+ _tiers.get_test(kwargs['test']),
+ _tiers.get_tier_name(kwargs['test']),
+ kwargs['test'])
+ if result != testcase.TestCase.EX_OK:
+ logger.error("The test case '%s' failed.",
+ kwargs['test'])
+ self.overall_result = Result.EX_ERROR
elif kwargs['test'] == "all":
self.run_all(_tiers)
else:
project_name: functest
criteria: 100
blocking: true
- clean_flag: false
description: >-
This test case verifies the retrieval of OpenStack clients:
Keystone, Glance, Neutron and Nova and may perform some
project_name: functest
criteria: 100
blocking: true
- clean_flag: false
description: >-
This test case verifies the retrieval of OpenStack clients:
Keystone, Glance, Neutron and Nova and may perform some
project_name: functest
criteria: 100
blocking: true
- clean_flag: false
description: >-
This test case creates executes the SimpleHealthCheck
Python test class which creates an, image, flavor, network,
project_name: functest
criteria: 100
blocking: true
- clean_flag: true
description: >-
This test case verifies: 1) SSH to an instance using floating
IPs over the public network. 2) Connectivity between 2 instances
project_name: functest
criteria: 100
blocking: true
- clean_flag: true
description: >-
This test case verifies: 1) Boot a VM with given userdata.
2) Connectivity between 2 instances over a private network.
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
This test case runs the smoke subset of the OpenStack
Tempest suite. The list of test cases is generated by
project_name: functest
criteria: 100
blocking: false
- clean_flag: false
description: >-
This test case runs a sub group of tests of the OpenStack
Rally suite in smoke mode.
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
This test case runs a sub group of tests of the OpenStack
Defcore testcases by using refstack client.
project_name: functest
criteria: 100
blocking: true
- clean_flag: false
description: >-
Test Suite for the OpenDaylight SDN Controller. It
integrates some test suites from upstream using
project_name: functest
criteria: 100
blocking: false
- clean_flag: false
description: >-
Test Suite for the OpenDaylight SDN Controller when
the NetVirt features are installed. It integrates
project_name: functest
criteria: 100
blocking: false
- clean_flag: false
description: >-
Test Suite for the OpenDaylight SDN Controller when GBP features are
installed. It integrates some test suites from upstream using
project_name: functest
criteria: 100
blocking: true
- clean_flag: true
description: >-
Test Suite for the ONOS SDN Controller. It integrates
some test suites from upstream using TestON as the test
project_name: functest
criteria: 100
blocking: false
- clean_flag: false
description: >-
This test case contains tests that setup and destroy
environments with VMs with and without Floating IPs
project_name: promise
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Promise project.
dependencies:
project_name: doctor
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Doctor project.
dependencies:
project_name: sdnvpn
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from SDNVPN project.
dependencies:
project_name: securityscanning
criteria: 100
blocking: false
- clean_flag: true
description: >-
Simple Security Scan
dependencies:
project_name: copper
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite for policy management based on OpenStack Congress
dependencies:
project_name: multisite
criteria: 100
blocking: false
- clean_flag: false
description: >-
Test suite from kingbird
dependencies:
project_name: sfc
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite for odl-sfc to test two chains and two SFs
dependencies:
project_name: functest
criteria: 100
blocking: true
- clean_flag: true
description: >-
Test Suite for onos-sfc to test sfc function.
dependencies:
project_name: parser
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Parser project.
dependencies:
project_name: domino
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Domino project.
dependencies:
project_name: netready
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Netready project.
dependencies:
project_name: barometer
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite for the Barometer project. Separate tests verify the
proper configuration and functionality of the following
project_name: functest
criteria: 80
blocking: false
- clean_flag: true
description: >-
The list of test cases is generated by
Tempest automatically and depends on the parameters of
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
The test case allows running a customized list of tempest
test cases defined in a file under
project_name: functest
criteria: 90
blocking: false
- clean_flag: false
description: >-
This test case runs the full suite of scenarios of the OpenStack
Rally suite using several threads and iterations.
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
This test case deploys an OpenSource vIMS solution from Clearwater
using the Cloudify orchestrator. It also runs some signaling traffic.
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
Test suite from Parser project.
dependencies:
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
VNF deployment with OpenBaton (Orchestra)
dependencies:
project_name: opera
criteria: 100
blocking: false
- clean_flag: true
description: >-
VNF deployment with OPEN-O
dependencies:
project_name: functest
criteria: 100
blocking: false
- clean_flag: true
description: >-
This test case is vRouter testing.
dependencies:
dependency=dep,
criteria=dic_testcase['criteria'],
blocking=dic_testcase['blocking'],
- clean_flag=dic_testcase['clean_flag'],
description=dic_testcase['description'])
if (testcase.is_compatible(self.ci_installer,
self.ci_scenario) and
dependency,
criteria,
blocking,
- clean_flag,
description=""):
self.name = name
self.enabled = enabled
self.dependency = dependency
self.criteria = criteria
self.blocking = blocking
- self.clean_flag = clean_flag
self.description = description
@staticmethod
def is_blocking(self):
return self.blocking
- def needs_clean(self):
- return self.clean_flag
-
def __str__(self):
lines = split_text(self.description, LINE_LENGTH - 6)
import prettytable
import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot
__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>"
except Exception: # pylint: disable=broad-except
self.__logger.exception("The results cannot be pushed to DB")
return TestCase.EX_PUSH_TO_DB_ERROR
+
+ def create_snapshot(self): # pylint: disable=no-self-use
+ """Save the testing environement before running test.
+
+ It can be overriden if resources must be listed running the
+ test case.
+
+ Returns:
+ TestCase.EX_OK
+ """
+ return TestCase.EX_OK
+
+ def clean(self):
+ """Clean the resources.
+
+ It can be overriden if resources must be deleted after
+ running the test case.
+ """
+
+
+class OSGCTestCase(TestCase):
+ """Model for single test case which requires an OpenStack Garbage
+ Collector."""
+
+ __logger = logging.getLogger(__name__)
+
+ def create_snapshot(self):
+ """Create a snapshot listing the OpenStack resources.
+
+ Returns:
+ TestCase.EX_OK if os_snapshot.main() returns 0.
+ TestCase.EX_RUN_ERROR otherwise.
+ """
+ try:
+ assert os_snapshot.main() == 0
+ self.__logger.info("OpenStack resources snapshot created")
+ return TestCase.EX_OK
+ except Exception: # pylint: disable=broad-except
+ self.__logger.exception("Cannot create the snapshot")
+ return TestCase.EX_RUN_ERROR
+
+ def clean(self):
+ """Clean the OpenStack resources."""
+ try:
+ assert os_clean.main() == 0
+ self.__logger.info("OpenStack resources cleaned")
+ except Exception: # pylint: disable=broad-except
+ self.__logger.exception("Cannot clean the OpenStack resources")
logger = logging.getLogger(__name__)
-class RallyBase(testcase.TestCase):
+class RallyBase(testcase.OSGCTestCase):
TESTS = ['authenticate', 'glance', 'cinder', 'heat', 'keystone',
'neutron', 'nova', 'quotas', 'requests', 'vm', 'all']
GLANCE_IMAGE_NAME = CONST.__getattribute__('openstack_image_name')
logger = logging.getLogger(__name__)
-class RefstackClient(testcase.TestCase):
+class RefstackClient(testcase.OSGCTestCase):
def __init__(self, **kwargs):
if "case_name" not in kwargs:
logger = logging.getLogger(__name__)
-class TempestCommon(testcase.TestCase):
+class TempestCommon(testcase.OSGCTestCase):
def __init__(self, **kwargs):
super(TempestCommon, self).__init__(**kwargs)
import time
import uuid
-from functest.core.testcase import TestCase
+from functest.core import testcase
from functest.utils import functest_utils
from functest.utils.constants import CONST
from snaps.openstack.utils import deploy_utils, nova_utils
-class VPingBase(TestCase):
+class VPingBase(testcase.OSGCTestCase):
"""
Base class for vPing tests that check connectivity between two VMs shared
else:
raise Exception('VMs never became active')
- if result == TestCase.EX_RUN_ERROR:
- return TestCase.EX_RUN_ERROR
+ if result == testcase.TestCase.EX_RUN_ERROR:
+ return testcase.TestCase.EX_RUN_ERROR
self.stop_time = time.time()
self.result = 100
- return TestCase.EX_OK
+ return testcase.TestCase.EX_OK
def _cleanup(self):
"""
'OS_PASSWORD': 'test_password'}
self.test = {'test_name': 'test_name'}
self.tier = mock.Mock()
+ test1 = mock.Mock()
+ test1.get_name.return_value = 'test1'
+ test2 = mock.Mock()
+ test2.get_name.return_value = 'test2'
attrs = {'get_name.return_value': 'test_tier',
- 'get_tests.return_value': ['test1', 'test2'],
+ 'get_tests.return_value': [test1, test2],
'get_ci_loop.return_value': 'test_ci_loop',
'get_test_names.return_value': ['test1', 'test2']}
self.tier.configure_mock(**attrs)
return_value=self.creds):
self.runner.source_rc_file()
- @mock.patch('functest.ci.run_tests.os_snapshot.main')
- def test_generate_os_snapshot(self, mock_os_snap):
- self.runner.generate_os_snapshot()
- self.assertTrue(mock_os_snap.called)
-
- @mock.patch('functest.ci.run_tests.os_clean.main')
- def test_cleanup(self, mock_os_clean):
- self.runner.cleanup()
- self.assertTrue(mock_os_clean.called)
-
def test_get_run_dict_if_defined_default(self):
mock_obj = mock.Mock()
with mock.patch('functest.ci.run_tests.'
@mock.patch('functest.ci.run_tests.Runner.print_separator')
@mock.patch('functest.ci.run_tests.Runner.source_rc_file')
- @mock.patch('functest.ci.run_tests.Runner.generate_os_snapshot')
- @mock.patch('functest.ci.run_tests.Runner.cleanup')
@mock.patch('importlib.import_module', name="module",
return_value=mock.Mock(test_class=mock.Mock(
side_effect=FakeModule)))
def test_run_tier_default(self, mock_logger_info):
with mock.patch('functest.ci.run_tests.Runner.print_separator'), \
mock.patch(
- 'functest.ci.run_tests.Runner.run_test') as mock_method:
+ 'functest.ci.run_tests.Runner.run_test',
+ return_value=TestCase.EX_OK) as mock_method:
self.runner.run_tier(self.tier)
- mock_method.assert_any_call('test1', 'test_tier')
- mock_method.assert_any_call('test2', 'test_tier')
+ mock_method.assert_any_call(mock.ANY, 'test_tier')
self.assertTrue(mock_logger_info.called)
@mock.patch('functest.ci.run_tests.logger.info')
with mock.patch('functest.ci.run_tests.tb.TierBuilder',
return_value=mock_obj), \
mock.patch('functest.ci.run_tests.Runner.source_rc_file'), \
- mock.patch('functest.ci.run_tests.Runner.run_test') as m:
+ mock.patch('functest.ci.run_tests.Runner.run_test',
+ return_value=TestCase.EX_OK) as m:
self.assertEqual(self.runner.main(**kwargs),
run_tests.Result.EX_OK)
self.assertTrue(m.called)
'case_name': 'test_name',
'criteria': 'test_criteria',
'blocking': 'test_blocking',
- 'clean_flag': 'test_clean_flag',
'description': 'test_desc'}
self.dic_tier = {'name': 'test_tier',
self.mock_depend,
'test_criteria',
'test_blocking',
- 'test_clean_flag',
description='test_desc')
self.dependency = tier_handler.Dependency('test_installer',
self.assertIn(duration, message)
self.assertIn('FAIL', message)
+ def test_create_snapshot(self):
+ self.assertEqual(self.test.create_snapshot(),
+ testcase.TestCase.EX_OK)
+
+ def test_clean(self):
+ self.assertEqual(self.test.clean(), None)
+
+
+class OSGCTestCaseTesting(unittest.TestCase):
+ """The class testing OSGCTestCase."""
+ # pylint: disable=missing-docstring
+
+ def setUp(self):
+ self.test = testcase.OSGCTestCase()
+
+ @mock.patch('functest.utils.openstack_snapshot.main',
+ side_effect=Exception)
+ def test_create_snapshot_exc(self, mock_method=None):
+ self.assertEqual(self.test.create_snapshot(),
+ testcase.TestCase.EX_RUN_ERROR)
+ mock_method.assert_called_once_with()
+
+ @mock.patch('functest.utils.openstack_snapshot.main', return_value=-1)
+ def test_create_snapshot_ko(self, mock_method=None):
+ self.assertEqual(self.test.create_snapshot(),
+ testcase.TestCase.EX_RUN_ERROR)
+ mock_method.assert_called_once_with()
+
+ @mock.patch('functest.utils.openstack_snapshot.main', return_value=0)
+ def test_create_snapshot_env(self, mock_method=None):
+ self.assertEqual(self.test.create_snapshot(),
+ testcase.TestCase.EX_OK)
+ mock_method.assert_called_once_with()
+
+ @mock.patch('functest.utils.openstack_clean.main', side_effect=Exception)
+ def test_clean_exc(self, mock_method=None):
+ self.assertEqual(self.test.clean(), None)
+ mock_method.assert_called_once_with()
+
+ @mock.patch('functest.utils.openstack_clean.main', return_value=-1)
+ def test_clean_ko(self, mock_method=None):
+ self.assertEqual(self.test.clean(), None)
+ mock_method.assert_called_once_with()
+
+ @mock.patch('functest.utils.openstack_clean.main', return_value=0)
+ def test_clean(self, mock_method=None):
+ self.assertEqual(self.test.clean(), None)
+ mock_method.assert_called_once_with()
+
if __name__ == "__main__":
logging.disable(logging.CRITICAL)
# http://www.apache.org/licenses/LICENSE-2.0
#
-import time
-
import logging
+import sys
+import time
import yaml
import functest.utils.openstack_utils as os_utils
except Exception:
logger.info("The file %s does not exist. The OpenStack snapshot must"
" be created first. Aborting cleanup." % OS_SNAPSHOT_FILE)
- exit(0)
+ return 0
default_images = snapshot_yaml.get('images')
default_instances = snapshot_yaml.get('instances')
if not os_utils.check_credentials():
logger.error("Please source the openrc credentials and run "
"the script again.")
- exit(-1)
+ return -1
remove_instances(nova_client, default_instances)
separator()
if __name__ == '__main__':
logging.basicConfig()
- main()
+ sys.exit(main())
import logging
import yaml
+import sys
import functest.utils.openstack_utils as os_utils
from functest.utils.constants import CONST
if not os_utils.check_credentials():
logger.error("Please source the openrc credentials and run the" +
"script again.")
- exit(-1)
+ return -1
snapshot = {}
snapshot.update(get_instances(nova_client))
if __name__ == '__main__':
logging.basicConfig()
- main()
+ sys.exit(main())
Code Review / functest-xtesting.git / commitdiff