X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=functest%2Fopnfv_tests%2Fopenstack%2Ftempest%2Ftempest.py;h=c3184e352a6aff9a1028ff3cc985d4690ab76c3a;hb=457321c2c8ed7e8dcb1daccb6d2fd9814e6a98ca;hp=69337cbe25cbbb4f92691aa1fbb732defaac080d;hpb=92e7d8cee4addb8ea66516a2d5fd295500dcb132;p=functest.git diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 69337cbe2..c3184e352 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -16,25 +16,22 @@ import time import yaml -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils -import functest.utils.openstack_utils as os_utils """ logging configuration """ logger = ft_logger.Logger("Tempest").getLogger() -class TempestCommon(testcase_base.TestcaseBase): +class TempestCommon(testcase.TestCase): - def __init__(self): - super(TempestCommon, self).__init__() + def __init__(self, case_name=''): + super(TempestCommon, self).__init__(case_name) self.MODE = "" self.OPTION = "" - self.FLAVOR_ID = None - self.IMAGE_ID = None self.VERIFIER_ID = conf_utils.get_verifier_id() self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir( self.VERIFIER_ID) @@ -48,57 +45,6 @@ class TempestCommon(testcase_base.TestcaseBase): with open(filename) as src: return [line.strip() for line in src.readlines()] - def create_tempest_resources(self): - keystone_client = os_utils.get_keystone_client() - - logger.debug("Creating tenant and user for Tempest suite") - tenant_id = os_utils.create_tenant( - keystone_client, - CONST.tempest_identity_tenant_name, - CONST.tempest_identity_tenant_description) - if not tenant_id: - logger.error("Error : Failed to create %s tenant" - % CONST.tempest_identity_tenant_name) - - user_id = os_utils.create_user(keystone_client, - CONST.tempest_identity_user_name, - CONST.tempest_identity_user_password, - None, tenant_id) - if not user_id: - logger.error("Error : Failed to create %s user" % - CONST.tempest_identity_user_name) - - logger.debug("Creating private network for Tempest suite") - network_dic = os_utils.create_shared_network_full( - CONST.tempest_private_net_name, - CONST.tempest_private_subnet_name, - CONST.tempest_router_name, - CONST.tempest_private_subnet_cidr) - if not network_dic: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - if CONST.tempest_use_custom_images: - # adding alternative image should be trivial should we need it - logger.debug("Creating image for Tempest suite") - _, self.IMAGE_ID = os_utils.get_or_create_image( - CONST.openstack_image_name, conf_utils.GLANCE_IMAGE_PATH, - CONST.openstack_image_disk_format) - if not self.IMAGE_ID: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - if CONST.tempest_use_custom_flavors: - # adding alternative flavor should be trivial should we need it - logger.debug("Creating flavor for Tempest suite") - _, self.FLAVOR_ID = os_utils.get_or_create_flavor( - CONST.openstack_flavor_name, - CONST.openstack_flavor_ram, - CONST.openstack_flavor_disk, - CONST.openstack_flavor_vcpus) - if not self.FLAVOR_ID: - return testcase_base.TestcaseBase.EX_RUN_ERROR - - return testcase_base.TestcaseBase.EX_OK - def generate_test_list(self, verifier_repo_dir): logger.debug("Generating test case list...") if self.MODE == 'defcore': @@ -109,14 +55,13 @@ class TempestCommon(testcase_base.TestcaseBase): shutil.copyfile( conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST) else: - logger.error("Tempest test list file %s NOT found." - % conf_utils.TEMPEST_CUSTOM) - return testcase_base.TestcaseBase.EX_RUN_ERROR + raise Exception("Tempest test list file %s NOT found." + % conf_utils.TEMPEST_CUSTOM) else: if self.MODE == 'smoke': testr_mode = "smoke" elif self.MODE == 'feature_multisite': - testr_mode = " | grep -i kingbird " + testr_mode = "'[Kk]ingbird'" elif self.MODE == 'full': testr_mode = "" else: @@ -128,8 +73,6 @@ class TempestCommon(testcase_base.TestcaseBase): conf_utils.TEMPEST_RAW_LIST)) ft_utils.execute_command(cmd) - return testcase_base.TestcaseBase.EX_OK - def apply_tempest_blacklist(self): logger.debug("Applying tempest blacklist...") cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST) @@ -164,21 +107,16 @@ class TempestCommon(testcase_base.TestcaseBase): else: result_file.write(str(cases_line) + '\n') result_file.close() - return testcase_base.TestcaseBase.EX_OK - - def _parse_verification_id(line): - first_pos = line.index("UUID=") + len("UUID=") - last_pos = line.index(") for deployment") - return line[first_pos:last_pos] def run_verifier_tests(self): - self.OPTION += (" --load-list {}".format(conf_utils.TEMPEST_LIST)) + self.OPTION += (" --load-list {} --detailed" + .format(conf_utils.TEMPEST_LIST)) cmd_line = "rally verify start " + self.OPTION logger.info("Starting Tempest test suite: '%s'." % cmd_line) header = ("Tempest environment:\n" - " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % + " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % (CONST.INSTALLER_TYPE, CONST.DEPLOY_SCENARIO, CONST.NODE_NAME, @@ -217,7 +155,7 @@ class TempestCommon(testcase_base.TestcaseBase): f_env.close() def parse_verifier_result(self): - if not self.VERIFICATION_ID: + if self.VERIFICATION_ID is None: raise Exception('Verification UUID not found') cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID) @@ -238,10 +176,27 @@ class TempestCommon(testcase_base.TestcaseBase): num_success = new_line[2] elif 'Skipped' in new_line: num_skipped = new_line[2] + elif 'Failures' in new_line: + num_failures = new_line[2] try: num_executed = int(num_tests) - int(num_skipped) success_rate = 100 * int(num_success) / int(num_executed) + with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "tempest.log"), 'r') as logfile: + output = logfile.read() + + error_logs = "" + for match in re.findall('(.*?)[. ]*fail ', output): + error_logs += match + skipped_testcase = "" + for match in re.findall('(.*?)[. ]*skip:', output): + skipped_testcase += match + + self.details = {"tests": int(num_tests), + "failures": int(num_failures), + "errors": error_logs, + "skipped": skipped_testcase} except Exception: success_rate = 0 @@ -257,76 +212,69 @@ class TempestCommon(testcase_base.TestcaseBase): if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR): os.makedirs(conf_utils.TEMPEST_RESULTS_DIR) - # Pre-configuration - res = self.create_tempest_resources() - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = conf_utils.configure_tempest(self.DEPLOYMENT_DIR, - self.IMAGE_ID, - self.FLAVOR_ID) - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = self.generate_test_list(self.VERIFIER_REPO_DIR) - if res != testcase_base.TestcaseBase.EX_OK: - return res - - res = self.apply_tempest_blacklist() - if res != testcase_base.TestcaseBase.EX_OK: - return res - - self.run_verifier_tests() - self.parse_verifier_result() + try: + image_and_flavor = conf_utils.create_tempest_resources() + conf_utils.configure_tempest( + self.DEPLOYMENT_DIR, + IMAGE_ID=image_and_flavor.get("image_id"), + FLAVOR_ID=image_and_flavor.get("flavor_id"), + MODE=self.MODE) + self.generate_test_list(self.VERIFIER_REPO_DIR) + self.apply_tempest_blacklist() + self.run_verifier_tests() + self.parse_verifier_result() + res = testcase.TestCase.EX_OK + except Exception as e: + logger.error('Error with run: %s' % e) + res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() - - if self.criteria == "PASS": - return testcase_base.TestcaseBase.EX_OK - else: - return testcase_base.TestcaseBase.EX_TESTCASE_FAILED + return res class TempestSmokeSerial(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_smoke_serial" + def __init__(self, case_name='tempest_smoke_serial'): + TempestCommon.__init__(self, case_name) self.MODE = "smoke" self.OPTION = "--concurrency 1" class TempestSmokeParallel(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_smoke_parallel" + def __init__(self, case_name='tempest_smoke_parallel'): + TempestCommon.__init__(self, case_name) self.MODE = "smoke" self.OPTION = "" class TempestFullParallel(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "tempest_full_parallel" + def __init__(self, case_name="tempest_full_parallel"): + TempestCommon.__init__(self, case_name) self.MODE = "full" class TempestMultisite(TempestCommon): - def __init__(self): - TempestCommon.__init__(self) - self.case_name = "multisite" + def __init__(self, case_name="multisite"): + TempestCommon.__init__(self, case_name) self.MODE = "feature_multisite" self.OPTION = "--concurrency 1" - conf_utils.configure_tempest_multisite(self.DEPLOYMENT_DIR) + conf_utils.install_verifier_ext(CONST.dir_repo_kingbird) class TempestCustom(TempestCommon): - def __init__(self, mode, option): - TempestCommon.__init__(self) - self.case_name = "tempest_custom" - self.MODE = mode - self.OPTION = option + def __init__(self, case_name="tempest_custom"): + TempestCommon.__init__(self, case_name) + self.MODE = "custom" + self.OPTION = "--concurrency 1" + + +class TempestDefcore(TempestCommon): + + def __init__(self, case_name="tempest_defcore"): + TempestCommon.__init__(self, case_name) + self.MODE = "defcore" + self.OPTION = "--concurrency 1"