Bad status on rally_sanity

[functest.git] / ci / run_tests.py

diff --git a/ci/run_tests.py b/ci/run_tests.py
old mode 100644 (file)
new mode 100755 (executable)
index 1062144..cb05d43
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env python
 #
 # Author: Jose Lausuch (jose.lausuch@ericsson.com)
 #
@@ -9,17 +9,19 @@
 #
 
 import argparse
+import datetime
 import os
-import subprocess
+import re
 import sys
 
 import functest.ci.tier_builder as tb
-import functest.utils.clean_openstack as clean_os
 import functest.utils.functest_logger as ft_logger
+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot
 import functest.utils.openstack_utils as os_utils
 
 
-""" arguments """
 parser = argparse.ArgumentParser()
 parser.add_argument("-t", "--test", dest="test", action='store',
                     help="Test case or tier (group of tests) to be executed. "
@@ -33,7 +35,7 @@ args = parser.parse_args()
 
 
 """ logging configuration """
-logger = ft_logger.Logger("run_test").getLogger()
+logger = ft_logger.Logger("run_tests").getLogger()
 
 
 """ global variables """
@@ -43,12 +45,15 @@ EXEC_SCRIPT = ("%sci/exec_test.sh" % FUNCTEST_REPO)
 CLEAN_FLAG = True
 REPORT_FLAG = False
 
+# This will be the return code of this script. If any of the tests fails,
+# this variable will change to -1
+OVERALL_RESULT = 0
+
 
 def print_separator(str, count=45):
     line = ""
     for i in range(0, count - 1):
         line += str
-
     logger.info("%s" % line)
 
 
@@ -57,66 +62,95 @@ def source_rc_file():
     if not os.path.isfile(rc_file):
         logger.error("RC file %s does not exist..." % rc_file)
         sys.exit(1)
-    logger.info("Sourcing the OpenStack RC file...")
+    logger.debug("Sourcing the OpenStack RC file...")
     os_utils.source_credentials(rc_file)
 
 
+def generate_os_snapshot():
+    os_snapshot.main()
+
+
 def cleanup():
-    print_separator("+")
-    logger.info("Cleaning OpenStack resources...")
-    print_separator("+")
-    clean_os.main()
-    print_separator("")
+    os_clean.main()
 
 
 def run_test(test):
+    global OVERALL_RESULT
+    start = datetime.datetime.now()
     test_name = test.get_name()
-    print_separator("")
+    logger.info("\n")  # blank line
     print_separator("=")
     logger.info("Running test case '%s'..." % test_name)
     print_separator("=")
     logger.debug("\n%s" % test)
+
+    if CLEAN_FLAG:
+        generate_os_snapshot()
+
     flags = (" -t %s" % (test_name))
     if REPORT_FLAG:
         flags += " -r"
 
     cmd = ("%s%s" % (EXEC_SCRIPT, flags))
     logger.debug("Executing command '%s'" % cmd)
-    print_separator("")
-
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
-
-    while p.poll() is None:
-        line = p.stdout.readline().rstrip()
-        logger.debug(line)
 
-    if p != 0:
-        logger.error("The command '%s' failed. Cleaning and exiting." % cmd)
-        if CLEAN_FLAG:
-            cleanup()
-        sys.exit(1)
+    result = ft_utils.execute_command(cmd, logger, exit_on_error=False)
 
     if CLEAN_FLAG:
         cleanup()
 
+    end = datetime.datetime.now()
+    duration = (end - start).seconds
+    str = ("%02d:%02d" % divmod(duration, 60))
+    logger.info("Test execution time: %s" % str)
+
+    if result != 0:
+        logger.error("The test case '%s' failed. " % test_name)
+        OVERALL_RESULT = -1
+
+        if test.get_blocking():
+            logger.info("This test case is blocking. Exiting...")
+            sys.exit(OVERALL_RESULT)
+
+    return result
 
 
 def run_tier(tier):
+    tests = tier.get_tests()
+    if tests is None or len(tests) == 0:
+        logger.info("There are no supported test cases in this tier "
+                    "for the given scenario")
+        return 0
+    logger.info("\n\n")  # blank line
     print_separator("#")
     logger.info("Running tier '%s'" % tier.get_name())
     print_separator("#")
     logger.debug("\n%s" % tier)
-    for test in tier.get_tests():
+    for test in tests:
         run_test(test)
 
 
 def run_all(tiers):
-    logger.debug("Tiers to be executed:")
-    for tier in tiers.get_tiers():
-        logger.info("\n    - %s. %s:\n\t%s"
-                    % (tier.get_order(), tier.get_name(), tier.get_tests()))
+    summary = ""
+    BUILD_TAG = os.getenv('BUILD_TAG')
+    if BUILD_TAG is not None and re.search("daily", BUILD_TAG) is not None:
+        CI_LOOP = "daily"
+    else:
+        CI_LOOP = "weekly"
+
+    tiers_to_run = []
 
     for tier in tiers.get_tiers():
+        if (len(tier.get_tests()) != 0 and
+                re.search(CI_LOOP, tier.get_ci_loop()) is not None):
+            tiers_to_run.append(tier)
+            summary += ("\n    - %s:\n\t   %s"
+                        % (tier.get_name(),
+                           tier.get_test_names()))
+
+    logger.info("Tests to be executed:%s" % summary)
+
+    for tier in tiers_to_run:
         run_tier(tier)
 
 
@@ -156,7 +190,7 @@ def main():
     else:
         run_all(_tiers)
 
-    sys.exit(0)
+    sys.exit(OVERALL_RESULT)
 
 if __name__ == '__main__':
     main()