Support to run mandatory or optional test cases separately

1. Currently, dovetail can run a test suite or a test area in that test suite.
2. The test areas in one test suite are not divided into optional and mandatory.
3. Split them and support to run just mandatory or optional.
4. Support to run multiple test areas.

JIRA: DOVETAIL-505

Change-Id: I42cd7b4e11c3e3674c806e9bc999b782bf5c85c6
Signed-off-by: xudan <xudan16@huawei.com>

diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 83b97d5..15fa7ba 100644 (file)
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -2,11 +2,22 @@
 proposed_tests:
   name: proposed_tests
   testcases_list:
+  # mandatory test cases
     # osinterop
     - dovetail.osinterop.tc001
     # vping
     - dovetail.vping.tc001
     - dovetail.vping.tc002
+    # HA
+    - dovetail.ha.tc001
+    - dovetail.ha.tc002
+    - dovetail.ha.tc003
+    - dovetail.ha.tc004
+    - dovetail.ha.tc005
+    - dovetail.ha.tc006
+    - dovetail.ha.tc007
+    - dovetail.ha.tc008
+  # optional test cases
     # ipv6
     - dovetail.ipv6.tc001
     - dovetail.ipv6.tc002
@@ -33,15 +44,6 @@ proposed_tests:
     - dovetail.ipv6.tc023
     - dovetail.ipv6.tc024
     - dovetail.ipv6.tc025
-    # HA
-    - dovetail.ha.tc001
-    - dovetail.ha.tc002
-    - dovetail.ha.tc003
-    - dovetail.ha.tc004
-    - dovetail.ha.tc005
-    - dovetail.ha.tc006
-    - dovetail.ha.tc007
-    - dovetail.ha.tc008
     # sdnvpn
     - dovetail.sdnvpn.tc001
     - dovetail.sdnvpn.tc002
@@ -53,5 +55,6 @@ proposed_tests:
     - dovetail.tempest.tc003
     - dovetail.tempest.tc004
     - dovetail.tempest.tc005
+  # other test cases
     # resiliency
     # - dovetail.resiliency.tc001

diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 58409f2..470c710 100644 (file)
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -45,7 +45,7 @@ cli:
       testarea:
         flags:
           - '--testarea'
-        default: 'full'
+        multiple: 'True'
         help: 'compliance testarea within testsuite'
       debug:
         flags:

diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index e93bd64..b95c6d3 100644 (file)
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -37,6 +37,9 @@ testarea_supported:
   - vping
   - resiliency
   - tempest
+  - optional
+  - mandatory
+  - full
 
 functest_testsuite:
   - refstack_defcore
@@ -93,3 +96,13 @@ validate_input:
     - 'cvp.0.4.0'
     - 'cvp.0.5.0'
     - 'cvp.0.6.0'
+
+mandatory:
+  - osinterop
+  - ha
+  - vping
+
+optional:
+  - ipv6
+  - tempest
+  - sdnvpn

diff --git a/dovetail/report.py b/dovetail/report.py
index fa6a0ba..bd74ad1 100644 (file)
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -54,11 +54,8 @@ class Report(object):
         report_obj['duration'] = duration
 
         report_obj['testcases_list'] = []
-        testarea_list = []
-        for value in testsuite_yaml['testcases_list']:
-            if value is not None and (testarea == 'full' or testarea in value):
-                testarea_list.append(value)
-        for testcase_name in testarea_list:
+        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        for testcase_name in testcase_list:
             testcase = Testcase.get(testcase_name)
             testcase_inreport = {}
             testcase_inreport['name'] = testcase_name

diff --git a/dovetail/run.py b/dovetail/run.py
index 1133b86..eb2cc96 100755 (executable)
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -38,14 +38,10 @@ def load_testcase():
 
 
 def run_test(testsuite, testarea, logger):
-    testarea_list = []
-    for value in testsuite['testcases_list']:
-        if value is not None and (testarea == 'full' or testarea in value):
-            testarea_list.append(value)
-
+    testcase_list = Testcase.get_testcase_list(testsuite, testarea)
     duration = 0
     start_time = time.time()
-    for testcase_name in testarea_list:
+    for testcase_name in testcase_list:
         logger.info('>>[testcase]: {}'.format(testcase_name))
         testcase = Testcase.get(testcase_name)
         if testcase is None:
@@ -275,14 +271,11 @@ def main(*args, **kwargs):
     else:
         dt_cfg.dovetail_config['offline'] = False
 
-    testarea = kwargs['testarea']
+    origin_testarea = kwargs['testarea']
     testsuite_validation = False
-    testarea_validation = False
-    if (testarea == 'full') or \
-       (testarea in dt_cfg.dovetail_config['testarea_supported']):
-        testarea_validation = True
     if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']:
         testsuite_validation = True
+    testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
     if testsuite_validation and testarea_validation:
         testsuite_yaml = load_testsuite(kwargs['testsuite'])
         load_testcase()
@@ -292,7 +285,7 @@ def main(*args, **kwargs):
         Report.save_logs()
     else:
         logger.error('Invalid input commands, testsuite {} testarea {}'
-                     .format(kwargs['testsuite'], testarea))
+                     .format(kwargs['testsuite'], origin_testarea))
 
 
 dt_cfg.load_config_files()

diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index b681996..9984548 100644 (file)
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -243,6 +243,44 @@ class Testcase(object):
             return cls.testcase_list[testcase_name]
         return None
 
+    @classmethod
+    def check_testarea(cls, testarea):
+        area_no_duplicate = []
+        area_full = ['full']
+
+        # testarea is empty, run full testsuite
+        if not testarea:
+            return True, area_full
+
+        mandatory_list = dt_cfg.dovetail_config['mandatory']
+        optional_list = dt_cfg.dovetail_config['optional']
+        for area in testarea:
+            if area not in dt_cfg.dovetail_config['testarea_supported']:
+                return False, None
+            if area == 'full':
+                return True, area_full
+            if area == 'mandatory':
+                for mandatory_area in mandatory_list:
+                    area_no_duplicate.append(mandatory_area)
+                continue
+            if area == 'optional':
+                for optional_area in optional_list:
+                    area_no_duplicate.append(optional_area)
+                continue
+            area_no_duplicate.append(area)
+        area_no_duplicate = list(set(area_no_duplicate))
+        return True, area_no_duplicate
+
+    @classmethod
+    def get_testcase_list(cls, testsuite, testarea):
+        testcase_list = []
+        for value in testsuite['testcases_list']:
+            for area in testarea:
+                if value is not None and (area == 'full' or area in value):
+                    testcase_list.append(value)
+                    break
+        return testcase_list
+
 
 class FunctestTestcase(Testcase):