X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=functest%2Fopnfv_tests%2Fopenstack%2Ftempest%2Ftempest.py;h=c3184e352a6aff9a1028ff3cc985d4690ab76c3a;hb=457321c2c8ed7e8dcb1daccb6d2fd9814e6a98ca;hp=20b1ebb4c6436a742c8ab8d809c474675b3a56ca;hpb=254a7e643bccd48bf562f8362745783dfb9542b4;p=functest.git diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 20b1ebb4c..c3184e352 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -16,103 +16,36 @@ import time import yaml -import conf_utils -import functest.core.testcase_base as testcase_base +from functest.core import testcase +from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils -import functest.utils.openstack_utils as os_utils """ logging configuration """ logger = ft_logger.Logger("Tempest").getLogger() -class TempestCommon(testcase_base.TestcaseBase): +class TempestCommon(testcase.TestCase): - def __init__(self): - super(TempestCommon, self).__init__() + def __init__(self, case_name=''): + super(TempestCommon, self).__init__(case_name) self.MODE = "" self.OPTION = "" - self.FLAVOR_ID = None - self.IMAGE_ID = None - self.DEPLOYMENT_DIR = self.get_deployment_dir() - - @staticmethod - def get_deployment_dir(): - """ - Returns current Rally deployment directory - """ - cmd = ("rally deployment list | awk '/" + - CONST.rally_deployment_name + - "/ {print $2}'") - p = subprocess.Popen(cmd, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - deployment_uuid = p.stdout.readline().rstrip() - if deployment_uuid == "": - logger.error("Rally deployment not found.") - exit(-1) - return os.path.join(CONST.dir_rally_inst, - "tempest/for-deployment-" + deployment_uuid) + self.VERIFIER_ID = conf_utils.get_verifier_id() + self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir( + self.VERIFIER_ID) + self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id() + self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir( + self.VERIFIER_ID, self.DEPLOYMENT_ID) + self.VERIFICATION_ID = None @staticmethod def read_file(filename): with open(filename) as src: return [line.strip() for line in src.readlines()] - def create_tempest_resources(self): - keystone_client = os_utils.get_keystone_client() - - logger.debug("Creating tenant and user for Tempest suite") - tenant_id = os_utils.create_tenant( - keystone_client, - CONST.tempest_identity_tenant_name, - CONST.tempest_identity_tenant_description) - if not tenant_id: - logger.error("Error : Failed to create %s tenant" - % CONST.tempest_identity_tenant_name) - - user_id = os_utils.create_user(keystone_client, - CONST.tempest_identity_user_name, - CONST.tempest_identity_user_password, - None, tenant_id) - if not user_id: - logger.error("Error : Failed to create %s user" % - CONST.tempest_identity_user_name) - - logger.debug("Creating private network for Tempest suite") - network_dic = \ - os_utils.create_shared_network_full( - CONST.tempest_private_net_name, - CONST.tempest_private_subnet_name, - CONST.tempest_router_name, - CONST.tempest_private_subnet_cidr) - if not network_dic: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - if CONST.tempest_use_custom_images: - # adding alternative image should be trivial should we need it - logger.debug("Creating image for Tempest suite") - _, self.IMAGE_ID = os_utils.get_or_create_image( - CONST.openstack_image_name, conf_utils.GLANCE_IMAGE_PATH, - CONST.openstack_image_disk_format) - if not self.IMAGE_ID: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - if CONST.tempest_use_custom_flavors: - # adding alternative flavor should be trivial should we need it - logger.debug("Creating flavor for Tempest suite") - _, self.FLAVOR_ID = os_utils.get_or_create_flavor( - CONST.openstack_flavor_name, - CONST.openstack_flavor_ram, - CONST.openstack_flavor_disk, - CONST.openstack_flavor_vcpus) - if not self.FLAVOR_ID: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - return testcase_base.TestcaseBase.EX_OK - - def generate_test_list(self, DEPLOYMENT_DIR): + def generate_test_list(self, verifier_repo_dir): logger.debug("Generating test case list...") if self.MODE == 'defcore': shutil.copyfile( @@ -122,24 +55,24 @@ class TempestCommon(testcase_base.TestcaseBase): shutil.copyfile( conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST) else: - logger.error("Tempest test list file %s NOT found." - % conf_utils.TEMPEST_CUSTOM) - return testcase_base.TestcaseBase.EX_RUN_ERROR + raise Exception("Tempest test list file %s NOT found." + % conf_utils.TEMPEST_CUSTOM) else: if self.MODE == 'smoke': testr_mode = "smoke" elif self.MODE == 'feature_multisite': - testr_mode = " | grep -i kingbird " + testr_mode = "'[Kk]ingbird'" elif self.MODE == 'full': testr_mode = "" else: testr_mode = 'tempest.api.' + self.MODE - cmd = ("cd " + DEPLOYMENT_DIR + ";" + "testr list-tests " + - testr_mode + ">" + conf_utils.TEMPEST_RAW_LIST + ";cd") + cmd = ("cd {0};" + "testr list-tests {1} > {2};" + "cd -;".format(verifier_repo_dir, + testr_mode, + conf_utils.TEMPEST_RAW_LIST)) ft_utils.execute_command(cmd) - return testcase_base.TestcaseBase.EX_OK - def apply_tempest_blacklist(self): logger.debug("Applying tempest blacklist...") cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST) @@ -163,7 +96,7 @@ class TempestCommon(testcase_base.TestcaseBase): for test in tests: black_tests.append(test) break - except: + except Exception: black_tests = [] logger.debug("Tempest blacklist file does not exist.") @@ -174,55 +107,30 @@ class TempestCommon(testcase_base.TestcaseBase): else: result_file.write(str(cases_line) + '\n') result_file.close() - return testcase_base.TestcaseBase.EX_OK - def run(self): - - self.start_time = time.time() + def run_verifier_tests(self): + self.OPTION += (" --load-list {} --detailed" + .format(conf_utils.TEMPEST_LIST)) - if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR): - os.makedirs(conf_utils.TEMPEST_RESULTS_DIR) - - # Pre-configuration - res = self.create_tempest_resources() - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = conf_utils.configure_tempest(logger, - self.DEPLOYMENT_DIR, - self.IMAGE_ID, - self.FLAVOR_ID) - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = self.generate_test_list(self.DEPLOYMENT_DIR) - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = self.apply_tempest_blacklist() - if res != testcase_base.TestcaseBase.EX_OK: - return res - - self.OPTION += (" --tests-file %s " % conf_utils.TEMPEST_LIST) - - cmd_line = "rally verify start " + self.OPTION + " --system-wide" + cmd_line = "rally verify start " + self.OPTION logger.info("Starting Tempest test suite: '%s'." % cmd_line) header = ("Tempest environment:\n" - " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % + " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % (CONST.INSTALLER_TYPE, CONST.DEPLOY_SCENARIO, CONST.NODE_NAME, time.strftime("%a %b %d %H:%M:%S %Z %Y"))) - f_stdout = open(conf_utils.TEMPEST_RESULTS_DIR + "/tempest.log", 'w+') + f_stdout = open( + os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+') f_stderr = open( - conf_utils.TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+') - f_env = open(conf_utils.TEMPEST_RESULTS_DIR + "/environment.log", 'w+') + os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "tempest-error.log"), 'w+') + f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "environment.log"), 'w+') f_env.write(header) - # subprocess.call(cmd_line, shell=True, - # stdout=f_stdout, stderr=f_stderr) p = subprocess.Popen( cmd_line, shell=True, stdout=subprocess.PIPE, @@ -233,6 +141,12 @@ class TempestCommon(testcase_base.TestcaseBase): for line in iter(p.stdout.readline, b''): if re.search("\} tempest\.", line): logger.info(line.replace('\n', '')) + elif re.search('Starting verification', line): + logger.info(line.replace('\n', '')) + first_pos = line.index("UUID=") + len("UUID=") + last_pos = line.index(") for deployment") + self.VERIFICATION_ID = line[first_pos:last_pos] + logger.debug('Verication UUID: %s' % self.VERIFICATION_ID) f_stdout.write(line) p.wait() @@ -240,37 +154,50 @@ class TempestCommon(testcase_base.TestcaseBase): f_stderr.close() f_env.close() - cmd_line = "rally verify show" - output = "" + def parse_verifier_result(self): + if self.VERIFICATION_ID is None: + raise Exception('Verification UUID not found') + + cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID) + logger.info("Showing result for a verification: '%s'." % cmd_line) p = subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.STDOUT) for line in p.stdout: - if re.search("Tests\:", line): + new_line = line.replace(' ', '').split('|') + if 'Tests' in new_line: break - output += line - logger.info(output) - - cmd_line = "rally verify list" - cmd = os.popen(cmd_line) - output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|") - # Format: - # | UUID | Deployment UUID | smoke | tests | failures | Created at | - # Duration | Status | - num_tests = output[4] - num_failures = output[5] - duration = output[7] - # Compute duration (lets assume it does not take more than 60 min) - dur_min = int(duration.split(':')[1]) - dur_sec_float = float(duration.split(':')[2]) - dur_sec_int = int(round(dur_sec_float, 0)) - dur_sec_int = dur_sec_int + 60 * dur_min + + logger.info(line) + if 'Testscount' in new_line: + num_tests = new_line[2] + elif 'Success' in new_line: + num_success = new_line[2] + elif 'Skipped' in new_line: + num_skipped = new_line[2] + elif 'Failures' in new_line: + num_failures = new_line[2] try: - diff = (int(num_tests) - int(num_failures)) - success_rate = 100 * diff / int(num_tests) - except: + num_executed = int(num_tests) - int(num_skipped) + success_rate = 100 * int(num_success) / int(num_executed) + with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "tempest.log"), 'r') as logfile: + output = logfile.read() + + error_logs = "" + for match in re.findall('(.*?)[. ]*fail ', output): + error_logs += match + skipped_testcase = "" + for match in re.findall('(.*?)[. ]*skip:', output): + skipped_testcase += match + + self.details = {"tests": int(num_tests), + "failures": int(num_failures), + "errors": error_logs, + "skipped": skipped_testcase} + except Exception: success_rate = 0 self.criteria = ft_utils.check_success_rate( @@ -278,54 +205,76 @@ class TempestCommon(testcase_base.TestcaseBase): logger.info("Tempest %s success_rate is %s%%, is marked as %s" % (self.case_name, success_rate, self.criteria)) - self.stop_time = time.time() + def run(self): - if self.criteria == "PASS": - return testcase_base.TestcaseBase.EX_OK - else: - return testcase_base.TestcaseBase.EX_TESTCASE_FAILED + self.start_time = time.time() + + if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR): + os.makedirs(conf_utils.TEMPEST_RESULTS_DIR) + + try: + image_and_flavor = conf_utils.create_tempest_resources() + conf_utils.configure_tempest( + self.DEPLOYMENT_DIR, + IMAGE_ID=image_and_flavor.get("image_id"), + FLAVOR_ID=image_and_flavor.get("flavor_id"), + MODE=self.MODE) + self.generate_test_list(self.VERIFIER_REPO_DIR) + self.apply_tempest_blacklist() + self.run_verifier_tests() + self.parse_verifier_result() + res = testcase.TestCase.EX_OK + except Exception as e: + logger.error('Error with run: %s' % e) + res = testcase.TestCase.EX_RUN_ERROR + + self.stop_time = time.time() + return res class TempestSmokeSerial(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_smoke_serial" + def __init__(self, case_name='tempest_smoke_serial'): + TempestCommon.__init__(self, case_name) self.MODE = "smoke" - self.OPTION = "--concur 1" + self.OPTION = "--concurrency 1" class TempestSmokeParallel(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_smoke_parallel" + def __init__(self, case_name='tempest_smoke_parallel'): + TempestCommon.__init__(self, case_name) self.MODE = "smoke" self.OPTION = "" class TempestFullParallel(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_full_parallel" + def __init__(self, case_name="tempest_full_parallel"): + TempestCommon.__init__(self, case_name) self.MODE = "full" class TempestMultisite(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "multisite" + def __init__(self, case_name="multisite"): + TempestCommon.__init__(self, case_name) self.MODE = "feature_multisite" - self.OPTION = "--concur 1" - conf_utils.configure_tempest_multisite(logger, self.DEPLOYMENT_DIR) + self.OPTION = "--concurrency 1" + conf_utils.install_verifier_ext(CONST.dir_repo_kingbird) class TempestCustom(TempestCommon): - def __init__(self, mode, option): - TempestCommon.__init__(self) - self.case_name = "tempest_custom" - self.MODE = mode - self.OPTION = option + def __init__(self, case_name="tempest_custom"): + TempestCommon.__init__(self, case_name) + self.MODE = "custom" + self.OPTION = "--concurrency 1" + + +class TempestDefcore(TempestCommon): + + def __init__(self, case_name="tempest_defcore"): + TempestCommon.__init__(self, case_name) + self.MODE = "defcore" + self.OPTION = "--concurrency 1"