Tempest and SNAPS not pushing results to DB in case of FAIL

this problem was due to the fact that we add exit condition
EX_ERROR when the result was FAIL

EX_ERROR shall be used as exit condition when there is an
execution error, i.e the test case cannot be run

if it can be run, there is no execution error
the result can be PASS or FAIL, but from the CI, execution is OK

JIRA: FUNCTEST-714

Change-Id: Id434bb9aa88b50277c40c717259304f26f7b3966
Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>

diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py
index 1eed92b..c168d7d 100644 (file)
--- a/functest/core/pytest_suite_runner.py
+++ b/functest/core/pytest_suite_runner.py
@@ -41,14 +41,18 @@ class PyTestSuiteRunner(base.TestcaseBase):
             for test, message in result.failures:
                 self.logger.error(str(test) + " FAILED with " + message)
 
+        # a result can be PASS or FAIL
+        # But in this case it means that the Execution was OK
+        # we shall distinguish Execution Error from FAIL results
+        # TestcaseBase.EX_RUN_ERROR means that the test case was not run
+        # not that it was run but the result was FAIL
+        exit_code = base.TestcaseBase.EX_OK
         if ((result.errors and len(result.errors) > 0)
                 or (result.failures and len(result.failures) > 0)):
             self.logger.info("%s FAILED" % self.case_name)
             self.criteria = 'FAIL'
-            exit_code = base.TestcaseBase.EX_RUN_ERROR
         else:
             self.logger.info("%s OK" % self.case_name)
-            exit_code = base.TestcaseBase.EX_OK
             self.criteria = 'PASS'
 
         self.details = {}

diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index de55311..e7cac7a 100644 (file)
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -533,11 +533,6 @@ class RallyBase(testcase_base.TestcaseBase):
             return testcase_base.TestcaseBase.EX_RUN_ERROR
         self.stop_time = time.time()
 
-        if self.criteria == "PASS":
-            return testcase_base.TestcaseBase.EX_OK
-        else:
-            return testcase_base.TestcaseBase.EX_TESTCASE_FAILED
-
 
 class RallySanity(RallyBase):
     def __init__(self):

diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 0014b71..39df766 100644 (file)
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -341,11 +341,6 @@ class TempestCommon(testcase_base.TestcaseBase):
 
         self.stop_time = time.time()
 
-        if self.criteria == "PASS":
-            return testcase_base.TestcaseBase.EX_OK
-        else:
-            return testcase_base.TestcaseBase.EX_TESTCASE_FAILED
-
 
 class TempestSmokeSerial(TempestCommon):