Add clean_flag to test cases

Some test cases such as tempest or those using SNAPS
have a built-in cleaup or teardown mechanism.
This patch helps to save some time by bypassing the
openstack snapshot and cleanup when running those
kind of tests cases.

Change-Id: I50e7a17937a813532b3756474158416d3e85b922
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index f920e70..2706a80 100755 (executable)
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -143,7 +143,7 @@ def run_test(test, tier_name, testcases=None):
     logger.debug("\n%s" % test)
     source_rc_file()
 
-    if GlobalVariables.CLEAN_FLAG:
+    if test.needs_clean() and GlobalVariables.CLEAN_FLAG:
         generate_os_snapshot()
 
     flags = (" -t %s" % (test_name))
@@ -157,6 +157,7 @@ def run_test(test, tier_name, testcases=None):
             module = importlib.import_module(run_dict['module'])
             cls = getattr(module, run_dict['class'])
             test_case = cls()
+
             try:
                 kwargs = run_dict['args']
                 result = test_case.run(**kwargs)
@@ -179,8 +180,9 @@ def run_test(test, tier_name, testcases=None):
                         cmd, test_name))
         result = ft_utils.execute_command(cmd)
 
-    if GlobalVariables.CLEAN_FLAG:
+    if test.needs_clean() and GlobalVariables.CLEAN_FLAG:
         cleanup()
+
     end = datetime.datetime.now()
     duration = (end - start).seconds
     duration_str = ("%02d:%02d" % divmod(duration, 60))

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index eea1f51..f8bbe9b 100755 (executable)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -11,6 +11,7 @@ tiers:
                 name: snaps_health_check
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: false
                 description: >-
                     This test case creates executes the SimpleHealthCheck
                     Python test class which creates an, image, flavor, network,
@@ -26,6 +27,7 @@ tiers:
                 name: connection_check
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: false
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
@@ -44,6 +46,7 @@ tiers:
                 name: api_check
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: false
                 description: >-
                     This test case verifies the retrieval of OpenStack clients:
                     Keystone, Glance, Neutron and Nova and may perform some
@@ -68,6 +71,7 @@ tiers:
                 name: vping_ssh
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: true
                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
@@ -83,6 +87,7 @@ tiers:
                 name: vping_userdata
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: true
                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.
@@ -97,6 +102,7 @@ tiers:
                 name: tempest_smoke_serial
                 criteria: 'success_rate == 100%'
                 blocking: false
+                clean_flag: false
                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by
@@ -113,6 +119,7 @@ tiers:
                 name: rally_sanity
                 criteria: 'success_rate == 100%'
                 blocking: false
+                clean_flag: false
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
@@ -127,6 +134,7 @@ tiers:
                 name: odl
                 criteria: 'success_rate == 100%'
                 blocking: true
+                clean_flag: false
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It
                     integrates some test suites from upstream using
@@ -146,6 +154,7 @@ tiers:
                 name: odl_netvirt
                 criteria: 'success_rate == 100%'
                 blocking: true
+                clean_flag: false
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller when
                     the NetVirt features are installed. It integrates
@@ -167,6 +176,7 @@ tiers:
                 name: onos
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: true
                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test
@@ -180,6 +190,7 @@ tiers:
                 name: snaps_smoke
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: false
                 description: >-
                     This test case contains tests that setup and destroy
                     environments with VMs with and without Floating IPs
@@ -208,6 +219,7 @@ tiers:
                 name: promise
                 criteria: 'success_rate == 100%'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Promise project.
                 dependencies:
@@ -221,6 +233,7 @@ tiers:
                 name: doctor
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Doctor project.
                 dependencies:
@@ -234,6 +247,7 @@ tiers:
                 name: bgpvpn
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:
@@ -247,6 +261,7 @@ tiers:
                 name: security_scan
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Simple Security Scan
                 dependencies:
@@ -259,6 +274,7 @@ tiers:
 #                name: copper
 #                criteria: 'status == "PASS"'
 #                blocking: false
+#                clean_flag: true
 #                description: >-
 #                    Test suite for policy management based on OpenStack Congress
 #                dependencies:
@@ -271,6 +287,7 @@ tiers:
                 name: multisite
                 criteria: 'success_rate == 100%'
                 blocking: false
+                clean_flag: false
                 description: >-
                     Test suite from kingbird
                 dependencies:
@@ -283,6 +300,7 @@ tiers:
                 name: odl-sfc
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite for odl-sfc to test two chains and two SFs
                 dependencies:
@@ -295,6 +313,7 @@ tiers:
                 name: onos_sfc
                 criteria: 'status == "PASS"'
                 blocking: true
+                clean_flag: true
                 description: >-
                     Test Suite for onos-sfc to test sfc function.
                 dependencies:
@@ -304,6 +323,7 @@ tiers:
                 name: parser
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Parser project.
                 dependencies:
@@ -316,6 +336,7 @@ tiers:
                 name: domino
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Domino project.
                 dependencies:
@@ -328,6 +349,7 @@ tiers:
                 name: orchestra
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test OpenBaton (Orchestra) stack
                 dependencies:
@@ -340,6 +362,7 @@ tiers:
                 name: netready
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Netready project.
                 dependencies:
@@ -352,6 +375,7 @@ tiers:
                 name: barometer
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite for the Barometer project. Separate tests verify the
                     proper configuration and functionality of the following
@@ -374,6 +398,7 @@ tiers:
 #                name: tempest_full_parallel
 #                criteria: 'success_rate >= 80%'
 #                blocking: false
+#                clean_flag: false
 #                description: >-
 #                    The list of test cases is generated by
 #                    Tempest automatically and depends on the parameters of
@@ -388,6 +413,7 @@ tiers:
                 name: tempest_defcore
                 criteria: 'success_rate == 100%'
                 blocking: false
+                clean_flag: false
                 description: >-
                     This is the set of Tempest test cases created by OpenStack
                     Interop Working Group for certification purposes.
@@ -401,6 +427,7 @@ tiers:
 #                name: rally_full
 #                criteria: 'success_rate >= 90%'
 #                blocking: false
+#                clean_flag: false
 #                description: >-
 #                    This test case runs the full suite of scenarios of the OpenStack
 #                    Rally suite using several threads and iterations.
@@ -422,6 +449,7 @@ tiers:
                 name: cloudify_ims
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.
@@ -435,6 +463,7 @@ tiers:
                 name: aaa
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from Parser project.
                 dependencies:
@@ -448,6 +477,7 @@ tiers:
                 name: juju_epc
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Test suite from OAI project, vEPC deployed with Juju.
                 dependencies:
@@ -461,6 +491,7 @@ tiers:
                 name: orchestra_ims
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     VNF deployment with OpenBaton (Orchestra)
                 dependencies:
@@ -474,6 +505,7 @@ tiers:
                 name: opera_ims
                 criteria: 'ret == 0'
                 blocking: false
+                clean_flag: true
                 description: >-
                     Evolution of vIMS
                 dependencies:
@@ -487,6 +519,7 @@ tiers:
                 name: vyos_vrouter
                 criteria: 'status == "PASS"'
                 blocking: false
+                clean_flag: true
                 description: >-
                     This test case is vRouter testing.
                 dependencies:

diff --git a/functest/ci/tier_builder.py b/functest/ci/tier_builder.py
index dae7c73..f4c6f70 100755 (executable)
--- a/functest/ci/tier_builder.py
+++ b/functest/ci/tier_builder.py
@@ -50,6 +50,7 @@ class TierBuilder(object):
                                        dependency=dep,
                                        criteria=dic_testcase['criteria'],
                                        blocking=dic_testcase['blocking'],
+                                       clean_flag=dic_testcase['clean_flag'],
                                        description=dic_testcase['description'])
                 if testcase.is_compatible(self.ci_installer, self.ci_scenario):
                     tier.add_test(testcase)

diff --git a/functest/ci/tier_handler.py b/functest/ci/tier_handler.py
index 127986b..6b4864b 100755 (executable)
--- a/functest/ci/tier_handler.py
+++ b/functest/ci/tier_handler.py
@@ -104,12 +104,18 @@ class Tier(object):
 
 class TestCase(object):
 
-    def __init__(self, name, dependency, criteria, blocking, description=""):
+    def __init__(self, name,
+                 dependency,
+                 criteria,
+                 blocking,
+                 clean_flag,
+                 description=""):
         self.name = name
         self.dependency = dependency
-        self.description = description
         self.criteria = criteria
         self.blocking = blocking
+        self.clean_flag = clean_flag
+        self.description = description
 
     @staticmethod
     def is_none(item):
@@ -138,6 +144,9 @@ class TestCase(object):
     def is_blocking(self):
         return self.blocking
 
+    def needs_clean(self):
+        return self.clean_flag
+
     def __str__(self):
         lines = split_text(self.description, LINE_LENGTH - 6)