Extend rally test case exclusion feature

[functest-xtesting.git] / testcases / OpenStack / rally / run_rally-cert.py

diff --git a/testcases/OpenStack/rally/run_rally-cert.py b/testcases/OpenStack/rally/run_rally-cert.py
index d35639c..8b8adce 100755 (executable)
--- a/testcases/OpenStack/rally/run_rally-cert.py
+++ b/testcases/OpenStack/rally/run_rally-cert.py
@@ -187,7 +187,6 @@ def build_task_args(test_file_name):
 
     net_id = network_dict['net_id']
     task_args['netid'] = str(net_id)
-    task_args['live_migration'] = live_migration_supported()
 
     auth_url = os.getenv('OS_AUTH_URL')
     if auth_url is not None:
@@ -273,36 +272,66 @@ def get_cmd_output(proc):
     return result
 
 
-def apply_blacklist(case_file_name, result_file_name):
-    logger.debug("Applying blacklist...")
-    cases_file = open(case_file_name, 'r')
-    result_file = open(result_file_name, 'w')
+def excl_scenario():
     black_tests = []
 
     try:
+        with open(BLACKLIST_FILE, 'r') as black_list_file:
+            black_list_yaml = yaml.safe_load(black_list_file)
+
         installer_type = os.getenv('INSTALLER_TYPE')
         deploy_scenario = os.getenv('DEPLOY_SCENARIO')
         if (bool(installer_type) * bool(deploy_scenario)):
-            # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the file
-            with open(BLACKLIST_FILE, 'r') as black_list_file:
-                black_list_yaml = yaml.safe_load(black_list_file)
-
-            for item in black_list_yaml:
-                scenarios = item['scenarios']
-                installers = item['installers']
-                if (deploy_scenario in scenarios and
-                        installer_type in installers):
-                    tests = item['tests']
-                    black_tests.extend(tests)
+            if 'scenario' in black_list_yaml.keys():
+                for item in black_list_yaml['scenario']:
+                    scenarios = item['scenarios']
+                    installers = item['installers']
+                    if (deploy_scenario in scenarios and
+                            installer_type in installers):
+                        tests = item['tests']
+                        black_tests.extend(tests)
     except:
-        black_tests = []
-        logger.debug("Blacklisting not applied.")
+        logger.debug("Scenario exclusion not applied.")
+
+    return black_tests
+
+
+def excl_func():
+    black_tests = []
+    func_list = []
+
+    try:
+        with open(BLACKLIST_FILE, 'r') as black_list_file:
+            black_list_yaml = yaml.safe_load(black_list_file)
+
+        if not live_migration_supported():
+            func_list.append("no_live_migration")
+
+        if 'functionality' in black_list_yaml.keys():
+            for item in black_list_yaml['functionality']:
+                functions = item['functions']
+                for func in func_list:
+                    if func in functions:
+                        tests = item['tests']
+                        black_tests.extend(tests)
+    except:
+        logger.debug("Functionality exclusion not applied.")
+
+    return black_tests
+
+
+def apply_blacklist(case_file_name, result_file_name):
+    logger.debug("Applying blacklist...")
+    cases_file = open(case_file_name, 'r')
+    result_file = open(result_file_name, 'w')
+
+    black_tests = list(set(excl_func() + excl_scenario()))
 
     include = True
     for cases_line in cases_file:
         if include:
             for black_tests_line in black_tests:
-                if black_tests_line == cases_line.strip().rstrip(':'):
+                if re.search(black_tests_line, cases_line.strip().rstrip(':')):
                     include = False
                     break
             else:
@@ -337,6 +366,17 @@ def prepare_test_list(test_name):
         os.makedirs(TEMP_DIR)
 
     apply_blacklist(scenario_file_name, test_file_name)
+    return test_file_name
+
+
+def file_is_empty(file_name):
+    try:
+        if os.stat(file_name).st_size > 0:
+            return False
+    except:
+        pass
+
+    return True
 
 
 def run_task(test_name):
@@ -354,7 +394,10 @@ def run_task(test_name):
         logger.error("Task file '%s' does not exist." % task_file)
         exit(-1)
 
-    prepare_test_list(test_name)
+    file_name = prepare_test_list(test_name)
+    if file_is_empty(file_name):
+        logger.info('No tests for scenario "{}"'.format(test_name))
+        return
 
     cmd_line = ("rally task start --abort-on-sla-failure " +
                 "--task {} ".format(task_file) +
@@ -381,7 +424,7 @@ def run_task(test_name):
 
     # check for result directory and create it otherwise
     if not os.path.exists(RESULTS_DIR):
-        logger.debug('%s does not exist, we create it.'.format(RESULTS_DIR))
+        logger.debug('{} does not exist, we create it.'.format(RESULTS_DIR))
         os.makedirs(RESULTS_DIR)
 
     # write html report file
@@ -521,7 +564,12 @@ def main():
     total_duration_str = time.strftime("%H:%M:%S", time.gmtime(total_duration))
     total_duration_str2 = "{0:<10}".format(total_duration_str)
     total_nb_tests_str = "{0:<13}".format(total_nb_tests)
-    success_rate = "{:0.2f}".format(total_success / len(SUMMARY))
+
+    if len(SUMMARY):
+        success_rate = total_success / len(SUMMARY)
+    else:
+        success_rate = 100
+    success_rate = "{:0.2f}".format(success_rate)
     success_rate_str = "{0:<10}".format(str(success_rate) + '%')
     report += "+===================+============+===============+===========+"
     report += "\n"