dovetail tool: bugfix: test cases are erroneously judged

JIRA: DOVETAIL-375
JIRA: DOVETAIL-390

1. defcore test cases are erroneously judged because of:
   a) defcore results json data lack some keys
   b) some sub-testcases' names are the same as the prefix of another sub-testcases.
2. this patch removes the extra keys and judge sub-testcases accurately.
3. remove "store_type" in functest_config.yml and yardstick_config.yml.
4. change some log levels from info to debug.

Change-Id: I5f126e70309409cac66ef9a0909f1573d325098b
Signed-off-by: xudan <xudan16@huawei.com>

diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index f636c20..e3800ce 100644 (file)
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -12,6 +12,5 @@ functest:
     - 'echo test for postcondition in functest'
   result:
     dir: '/home/opnfv/functest/results'
-    store_type: 'file'
     file_path: 'dump.txt'
   openrc: '/home/opnfv/functest/conf/openstack.creds'

diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml
index cb4b1c7..e9a674a 100644 (file)
--- a/dovetail/conf/yardstick_config.yml
+++ b/dovetail/conf/yardstick_config.yml
@@ -18,6 +18,5 @@ yardstick:
   result:
     dir: '/home/opnfv/yardstick/results'
     log: '/tmp/yardstick'
-    store_type: 'file'
     file_path: 'yardstick.log'
   openrc: '/etc/yardstick/openstack.creds'

diff --git a/dovetail/report.py b/dovetail/report.py
index 6c45e00..dc53d30 100644 (file)
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -19,6 +19,7 @@ from pbr import version
 import utils.dovetail_logger as dt_logger
 
 from utils.dovetail_config import DovetailConfig as dt_cfg
+import utils.dovetail_utils as dt_utils
 from testcase import Testcase
 
 
@@ -208,76 +209,62 @@ class FunctestCrawler(object):
             dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger()
 
     def crawl(self, testcase=None):
-        store_type = \
-            dt_cfg.dovetail_config[self.type]['result']['store_type']
-        if store_type == 'file':
+        report_dest = dt_cfg.dovetail_config['report_dest']
+        if report_dest.lower() == 'file':
             return self.crawl_from_file(testcase)
 
-        if store_type == 'url':
+        if report_dest.lower().startswith('http'):
             return self.crawl_from_url(testcase)
 
     def crawl_from_file(self, testcase=None):
         dovetail_config = dt_cfg.dovetail_config
         criteria = 'FAIL'
+        details = {}
         timestart = 0
-        testcase_duration = 0
+        timestop = 0
+        duration = 0
         testcase_name = testcase.validate_testcase()
-        json_results = {}
+        file_path = \
+            os.path.join(dovetail_config['result_dir'],
+                         dovetail_config[self.type]['result']['file_path'])
+        if not os.path.exists(file_path):
+            self.logger.info('result file not found: %s', file_path)
+            return None
         if testcase_name in dt_cfg.dovetail_config['functest_testcase']:
-            file_path = \
-                os.path.join(dovetail_config['result_dir'],
-                             dovetail_config[self.type]['result']['file_path'])
-            if not os.path.exists(file_path):
-                self.logger.info('result file not found: %s', file_path)
-                return None
-            with open(file_path, 'r') as f:
-                for jsonfile in f:
-                    try:
-                        data = json.loads(jsonfile)
-                        if testcase_name == data['case_name']:
-                            criteria = data['details']['status']
-                            timestart = data['details']['timestart']
-                            testcase_duration = data['details']['duration']
-                    except Exception:
-                        continue
-
-            json_results = {'criteria': criteria,
-                            'details': {"timestart": timestart,
-                                        "duration": testcase_duration,
-                                        "tests": '', "failures": ''}}
+            complex_testcase = False
         elif testcase_name in dt_cfg.dovetail_config['functest_testsuite']:
-            file_path = \
-                os.path.join(dovetail_config['result_dir'],
-                             dovetail_config[self.type]['result']['file_path'])
-            if not os.path.exists(file_path):
-                self.logger.info('result file not found: %s', file_path)
-                return None
-            tests = 0
-            failed_num = 0
-            error_case = ''
-            skipped_case = ''
-            with open(file_path, 'r') as f:
-                for jsonfile in f:
-                    try:
-                        data = json.loads(jsonfile)
-                        if testcase_name == data['case_name']:
-                            criteria = data['details']['status']
-                            timestart = data['details']['timestart']
-                            testcase_duration = data['details']['duration']
+            complex_testcase = True
+        else:
+            self.logger.error("Wrong Functest test case %s.", testcase_name)
+            return None
+        with open(file_path, 'r') as f:
+            for jsonfile in f:
+                try:
+                    data = json.loads(jsonfile)
+                    if testcase_name == data['case_name']:
+                        criteria = data['criteria']
+                        timestart = data['start_date']
+                        timestop = data['stop_date']
+                        duration = dt_utils.get_duration(timestart, timestop,
+                                                         self.logger)
+                        if complex_testcase:
                             tests = data['details']['tests']
                             failed_num = data['details']['failures']
                             error_case = data['details']['errors']
                             skipped_case = data['details']['skipped']
-                    except Exception:
-                        continue
-
-            json_results = {'criteria': criteria,
-                            'details': {"timestart": timestart,
-                                        "duration": testcase_duration,
-                                        "tests": tests,
-                                        "failures": failed_num,
-                                        "errors": error_case,
-                                        "skipped": skipped_case}}
+                            details = {"tests": tests,
+                                       "failures": failed_num,
+                                       "errors": error_case,
+                                       "skipped": skipped_case}
+                except KeyError as e:
+                    self.logger.error("Key error, exception: %s", e)
+                    return None
+                except ValueError:
+                    continue
+
+        json_results = {'criteria': criteria, 'timestart': timestart,
+                        'timestop': timestop, 'duration': duration,
+                        'details': details}
 
         self.logger.debug('Results: %s', str(json_results))
         return json_results
@@ -310,12 +297,11 @@ class YardstickCrawler(object):
             dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger()
 
     def crawl(self, testcase=None):
-        store_type = \
-            dt_cfg.dovetail_config[self.type]['result']['store_type']
-        if store_type == 'file':
+        report_dest = dt_cfg.dovetail_config['report_dest']
+        if report_dest.lower() == 'file':
             return self.crawl_from_file(testcase)
 
-        if store_type == 'url':
+        if report_dest.lower().startswith('http'):
             return self.crawl_from_url(testcase)
 
     def crawl_from_file(self, testcase=None):
@@ -389,6 +375,15 @@ class FunctestChecker(object):
         cls.logger = \
             dt_logger.Logger(__name__ + '.FunctestChecker').getLogger()
 
+    @staticmethod
+    def get_sub_testcase(sub_testcase, result):
+        reg = sub_testcase + '[\s+\d+]'
+        find_reg = re.compile(reg)
+        match = find_reg.findall(result)
+        if match:
+            return True
+        return False
+
     def check(self, testcase, db_result):
         sub_testcase_list = testcase.sub_testcase()
 
@@ -406,10 +401,13 @@ class FunctestChecker(object):
         testcase_passed = 'SKIP'
         for sub_testcase in sub_testcase_list:
             self.logger.debug('check sub_testcase:%s', sub_testcase)
-            if sub_testcase in db_result['details']['errors']:
+            if self.get_sub_testcase(sub_testcase,
+                                     db_result['details']['errors']):
                 testcase.sub_testcase_passed(sub_testcase, 'FAIL')
                 testcase_passed = 'FAIL'
-            elif sub_testcase in db_result['details']['skipped']:
+                continue
+            if self.get_sub_testcase(sub_testcase,
+                                     db_result['details']['skipped']):
                 testcase.sub_testcase_passed(sub_testcase, 'SKIP')
             else:
                 testcase.sub_testcase_passed(sub_testcase, 'PASS')

diff --git a/dovetail/run.py b/dovetail/run.py
index aae35e1..492046d 100755 (executable)
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -71,9 +71,14 @@ def run_test(testsuite, testarea, logger):
 
 
 def check_tc_result(testcase, logger):
+    result_dir = dt_cfg.dovetail_config['result_dir']
+    validate_type = testcase.validate_type()
+    functest_result = dt_cfg.dovetail_config['functest']['result']['file_path']
     if dt_cfg.dovetail_config['report_dest'].startswith("http"):
-        if testcase.validate_type() == 'yardstick':
-            logger.info("Results have been stored with files.")
+        if validate_type.lower() == 'yardstick':
+            logger.info("Results have been stored with file %s.",
+                        os.path.join(result_dir,
+                                     testcase.validate_testcase() + '.out'))
         else:
             if dt_utils.check_db_results(dt_cfg.dovetail_config['report_dest'],
                                          dt_cfg.dovetail_config['build_tag'],
@@ -83,9 +88,15 @@ def check_tc_result(testcase, logger):
             else:
                 logger.error("Fail to push results to database.")
     if dt_cfg.dovetail_config['report_dest'] == "file":
-        logger.info("Results have been stored with files.")
-        result = Report.get_result(testcase)
-        Report.check_result(testcase, result)
+        if validate_type.lower() == 'yardstick':
+            logger.info("Results have been stored with file %s.",
+                        os.path.join(result_dir,
+                                     testcase.validate_testcase() + '.out'))
+        if validate_type.lower() == 'functest':
+            logger.info("Results have been stored with file %s.",
+                        os.path.join(result_dir, functest_result))
+        # result = Report.get_result(testcase)
+        # Report.check_result(testcase, result)
 
 
 def validate_input(input_dict, check_dict, logger):
@@ -211,9 +222,9 @@ def main(*args, **kwargs):
     if testsuite_validation and testarea_validation:
         testsuite_yaml = load_testsuite(kwargs['testsuite'])
         load_testcase()
-        duration = run_test(testsuite_yaml, testarea, logger)
-        if dt_cfg.dovetail_config['report_dest'] == "file":
-            Report.generate(testsuite_yaml, testarea, duration)
+        run_test(testsuite_yaml, testarea, logger)
+        # if dt_cfg.dovetail_config['report_dest'] == "file":
+        #    Report.generate(testsuite_yaml, testarea, duration)
     else:
         logger.error('invalid input commands, testsuite %s testarea %s',
                      kwargs['testsuite'], testarea)

diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 21d655e..13a31fd 100644 (file)
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -150,13 +150,14 @@ class Testcase(object):
     def mk_src_file(self):
         testcase_src_file = self.testcase['validate']['pre_copy']['src_file']
         try:
-            with open(os.path.join(dt_cfg.dovetail_config['result_dir'],
-                      testcase_src_file), 'w+') as src_file:
+            file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
+                                     testcase_src_file)
+            with open(file_path, 'w+') as src_file:
                 if self.sub_testcase() is not None:
                     for sub_test in self.sub_testcase():
-                        self.logger.info('save testcases %s', sub_test)
+                        self.logger.debug('save testcases %s', sub_test)
                         src_file.write(sub_test + '\n')
-            self.logger.info('save testcases to %s', src_file)
+            self.logger.debug('save testcases to %s', file_path)
         except Exception:
             self.logger.error('Failed to save: %s', src_file)
 

diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 766bb64..fa99e00 100644 (file)
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -16,6 +16,7 @@ import subprocess
 from collections import Mapping, Set, Sequence
 import json
 import urllib2
+from datetime import datetime
 
 
 def exec_log(verbose, logger, msg, level, flush=False):
@@ -144,6 +145,19 @@ def check_db_results(db_url, build_tag, testcase, logger):
         return False
 
 
+def get_duration(start_date, stop_date, logger):
+    fmt = '%Y-%m-%d %H:%M:%S'
+    try:
+        datetime_start = datetime.strptime(start_date, fmt)
+        datetime_stop = datetime.strptime(stop_date, fmt)
+        delta = (datetime_stop - datetime_start).seconds
+        res = "%sm%ss" % (delta / 60, delta % 60)
+        return res
+    except ValueError as e:
+        logger.error("ValueError: %s", e)
+        return None
+
+
 def show_progress_bar(length):
     max_len = 50
     length %= max_len