unify testcases.yaml obtain process

[functest.git] / ci / run_tests.py

diff --git a/ci/run_tests.py b/ci/run_tests.py
index 0fb8501..d8f4883 100755 (executable)
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python -u
 #
 # Author: Jose Lausuch (jose.lausuch@ericsson.com)
 #
@@ -14,7 +14,9 @@ import os
 import re
 import sys
 
+import functest.ci.generate_report as generate_report
 import functest.ci.tier_builder as tb
+from functest.testcases.Controllers.ODL.OpenDaylightTesting import ODLTestCases
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_clean as os_clean
@@ -44,6 +46,11 @@ FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 EXEC_SCRIPT = ("%sci/exec_test.sh" % FUNCTEST_REPO)
 CLEAN_FLAG = True
 REPORT_FLAG = False
+EXECUTED_TEST_CASES = []
+
+# This will be the return code of this script. If any of the tests fails,
+# this variable will change to -1
+OVERALL_RESULT = 0
 
 
 def print_separator(str, count=45):
@@ -70,7 +77,16 @@ def cleanup():
     os_clean.main()
 
 
-def run_test(test):
+def update_test_info(test_name, result, duration):
+    for test in EXECUTED_TEST_CASES:
+        if test['test_name'] == test_name:
+            test.update({"result": result,
+                         "duration": duration})
+
+
+def run_test(test, tier_name):
+    global OVERALL_RESULT, EXECUTED_TEST_CASES
+    result_str = "PASS"
     start = datetime.datetime.now()
     test_name = test.get_name()
     logger.info("\n")  # blank line
@@ -86,27 +102,43 @@ def run_test(test):
     if REPORT_FLAG:
         flags += " -r"
 
-    cmd = ("%s%s" % (EXEC_SCRIPT, flags))
-    logger.debug("Executing command '%s'" % cmd)
-
-    result = ft_utils.execute_command(cmd, logger, exit_on_error=False)
-
-    if result != 0:
-        logger.error("The test case '%s' failed. Cleaning and exiting."
-                     % test_name)
-        if CLEAN_FLAG:
-            cleanup()
-        sys.exit(1)
+    if test_name == 'odl':
+        result = ODLTestCases.functest_run()
+        if result and REPORT_FLAG:
+            result = ODLTestCases.push_to_db()
+        result = not result
+    else:
+        cmd = ("%s%s" % (EXEC_SCRIPT, flags))
+        logger.debug("Executing command '%s'" % cmd)
+        result = ft_utils.execute_command(cmd, exit_on_error=False)
 
     if CLEAN_FLAG:
         cleanup()
     end = datetime.datetime.now()
     duration = (end - start).seconds
-    str = ("%02d:%02d" % divmod(duration, 60))
-    logger.info("Test execution time: %s" % str)
+    duration_str = ("%02d:%02d" % divmod(duration, 60))
+    logger.info("Test execution time: %s" % duration_str)
+
+    if result != 0:
+        logger.error("The test case '%s' failed. " % test_name)
+        OVERALL_RESULT = -1
+        result_str = "FAIL"
+
+        if test.is_blocking():
+            if not args.test or args.test == "all":
+                logger.info("This test case is blocking. Aborting overall "
+                            "execution.")
+                # if it is a single test we don't print the whole results table
+                update_test_info(test_name, result_str, duration_str)
+                generate_report.main(EXECUTED_TEST_CASES)
+            logger.info("Execution exit value: %s" % OVERALL_RESULT)
+            sys.exit(OVERALL_RESULT)
+
+    update_test_info(test_name, result_str, duration_str)
 
 
 def run_tier(tier):
+    tier_name = tier.get_name()
     tests = tier.get_tests()
     if tests is None or len(tests) == 0:
         logger.info("There are no supported test cases in this tier "
@@ -114,14 +146,15 @@ def run_tier(tier):
         return 0
     logger.info("\n\n")  # blank line
     print_separator("#")
-    logger.info("Running tier '%s'" % tier.get_name())
+    logger.info("Running tier '%s'" % tier_name)
     print_separator("#")
     logger.debug("\n%s" % tier)
     for test in tests:
-        run_test(test)
+        run_test(test, tier_name)
 
 
 def run_all(tiers):
+    global EXECUTED_TEST_CASES
     summary = ""
     BUILD_TAG = os.getenv('BUILD_TAG')
     if BUILD_TAG is not None and re.search("daily", BUILD_TAG) is not None:
@@ -140,10 +173,12 @@ def run_all(tiers):
                            tier.get_test_names()))
 
     logger.info("Tests to be executed:%s" % summary)
-
+    EXECUTED_TEST_CASES = generate_report.init(tiers_to_run)
     for tier in tiers_to_run:
         run_tier(tier)
 
+    generate_report.main(EXECUTED_TEST_CASES)
+
 
 def main():
     global CLEAN_FLAG
@@ -152,7 +187,7 @@ def main():
     CI_INSTALLER_TYPE = os.getenv('INSTALLER_TYPE')
     CI_SCENARIO = os.getenv('DEPLOY_SCENARIO')
 
-    file = FUNCTEST_REPO + "/ci/testcases.yaml"
+    file = ft_utils.get_testcases_file()
     _tiers = tb.TierBuilder(CI_INSTALLER_TYPE, CI_SCENARIO, file)
 
     if args.noclean:
@@ -167,7 +202,7 @@ def main():
             run_tier(_tiers.get_tier(args.test))
 
         elif _tiers.get_test(args.test):
-            run_test(_tiers.get_test(args.test))
+            run_test(_tiers.get_test(args.test), _tiers.get_tier(args.test))
 
         elif args.test == "all":
             run_all(_tiers)
@@ -181,7 +216,8 @@ def main():
     else:
         run_all(_tiers)
 
-    sys.exit(0)
+    logger.info("Execution exit value: %s" % OVERALL_RESULT)
+    sys.exit(OVERALL_RESULT)
 
 if __name__ == '__main__':
     main()