X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=functest%2Fopnfv_tests%2Fopenstack%2Ftempest%2Ftempest.py;h=f30bbc552d333bd14cf66568c30d725bf1e8a07d;hb=59cf95974b2b9c0271c1ff9c9604b960c4bcfa26;hp=e6dc77fcbfc04c7f4d1f27deba5227aea67015dd;hpb=518ad1e201389ecef17c0770a94793315d0d0fd7;p=functest.git diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index e6dc77fcb..f30bbc552 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -18,13 +18,12 @@ import re import shutil import subprocess import time -import uuid -import os_client_config from six.moves import configparser from xtesting.core import testcase import yaml +from functest.core import singlevm from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config from functest.utils import env @@ -32,18 +31,39 @@ from functest.utils import env LOGGER = logging.getLogger(__name__) -class TempestCommon(testcase.TestCase): +class TempestCommon(singlevm.VmReady2): # pylint: disable=too-many-instance-attributes """TempestCommon testcases implementation class.""" - TEMPEST_RESULTS_DIR = os.path.join( - getattr(config.CONF, 'dir_results'), 'tempest') + visibility = 'public' + shared_network = True + filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img' def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs["case_name"] = 'tempest' super(TempestCommon, self).__init__(**kwargs) - self.resources = TempestResourcesManager() - self.mode = "" - self.option = [] + assert self.orig_cloud + assert self.cloud + assert self.project + if self.orig_cloud.get_role("admin"): + role_name = "admin" + elif self.orig_cloud.get_role("Admin"): + role_name = "Admin" + else: + raise Exception("Cannot detect neither admin nor Admin") + self.orig_cloud.grant_role( + role_name, user=self.project.user.id, + project=self.project.project.id, + domain=self.project.domain.id) + environ = dict( + os.environ, + OS_USERNAME=self.project.user.name, + OS_PROJECT_NAME=self.project.project.name, + OS_PROJECT_ID=self.project.project.id, + OS_PASSWORD=self.project.password) + conf_utils.create_rally_deployment(environ=environ) + conf_utils.create_verifier() self.verifier_id = conf_utils.get_verifier_id() self.verifier_repo_dir = conf_utils.get_verifier_repo_dir( self.verifier_id) @@ -51,10 +71,47 @@ class TempestCommon(testcase.TestCase): self.deployment_dir = conf_utils.get_verifier_deployment_dir( self.verifier_id, self.deployment_id) self.verification_id = None - self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR + self.res_dir = os.path.join( + getattr(config.CONF, 'dir_results'), self.case_name) self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt') self.list = os.path.join(self.res_dir, 'test_list.txt') self.conf_file = None + self.image_alt = None + self.flavor_alt = None + self.services = [] + try: + self.services = kwargs['run']['args']['services'] + except Exception: # pylint: disable=broad-except + pass + self.neutron_extensions = [] + try: + self.neutron_extensions = kwargs['run']['args'][ + 'neutron_extensions'] + except Exception: # pylint: disable=broad-except + pass + + def check_services(self): + """Check the mandatory services.""" + for service in self.services: + try: + self.cloud.search_services(service)[0] + except Exception: # pylint: disable=broad-except + self.is_skipped = True + break + + def check_extensions(self): + """Check the mandatory network extensions.""" + extensions = self.cloud.get_network_extensions() + for network_extension in self.neutron_extensions: + if network_extension not in extensions: + LOGGER.warning( + "Cannot find Neutron extension: %s", network_extension) + self.is_skipped = True + break + + def check_requirements(self): + self.check_services() + self.check_extensions() @staticmethod def read_file(filename): @@ -77,10 +134,10 @@ class TempestCommon(testcase.TestCase): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in proc.stdout: + LOGGER.info(line.rstrip()) new_line = line.replace(' ', '').split('|') if 'Tests' in new_line: break - LOGGER.info(line) if 'Testscount' in new_line: result['num_tests'] = int(new_line[2]) elif 'Success' in new_line: @@ -101,10 +158,11 @@ class TempestCommon(testcase.TestCase): shutil.copyfile(conf_file, os.path.join(res_dir, 'tempest.conf')) - def generate_test_list(self): + def generate_test_list(self, **kwargs): """Generate test list based on the test mode.""" LOGGER.debug("Generating test case list...") - if self.mode == 'custom': + self.backup_tempest_config(self.conf_file, '/etc') + if kwargs.get('mode') == 'custom': if os.path.isfile(conf_utils.TEMPEST_CUSTOM): shutil.copyfile( conf_utils.TEMPEST_CUSTOM, self.list) @@ -112,16 +170,13 @@ class TempestCommon(testcase.TestCase): raise Exception("Tempest test list file %s NOT found." % conf_utils.TEMPEST_CUSTOM) else: - if self.mode == 'smoke': - testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'" - elif self.mode == 'full': - testr_mode = r"'^tempest\.'" - else: - testr_mode = self.mode - cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format( + testr_mode = kwargs.get( + 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$') + cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format( self.verifier_repo_dir, testr_mode, self.list) output = subprocess.check_output(cmd, shell=True) LOGGER.info("%s\n%s", cmd, output) + os.remove('/etc/tempest.conf') def apply_tempest_blacklist(self): """Exclude blacklisted test cases.""" @@ -133,19 +188,15 @@ class TempestCommon(testcase.TestCase): result_file = open(self.list, 'w') black_tests = [] try: - installer_type = env.get('INSTALLER_TYPE') deploy_scenario = env.get('DEPLOY_SCENARIO') - if bool(installer_type) * bool(deploy_scenario): - # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the - # file + if bool(deploy_scenario): + # if DEPLOY_SCENARIO is set we read the file black_list_file = open(conf_utils.TEMPEST_BLACKLIST) black_list_yaml = yaml.safe_load(black_list_file) black_list_file.close() for item in black_list_yaml: scenarios = item['scenarios'] - installers = item['installers'] - if (deploy_scenario in scenarios and - installer_type in installers): + if deploy_scenario in scenarios: tests = item['tests'] for test in tests: black_tests.append(test) @@ -162,41 +213,40 @@ class TempestCommon(testcase.TestCase): result_file.write(str(cases_line) + '\n') result_file.close() - def run_verifier_tests(self): + def run_verifier_tests(self, **kwargs): """Execute tempest test cases.""" cmd = ["rally", "verify", "start", "--load-list", self.list] - cmd.extend(self.option) + cmd.extend(kwargs.get('option', [])) LOGGER.info("Starting Tempest test suite: '%s'.", cmd) f_stdout = open( os.path.join(self.res_dir, "tempest.log"), 'w+') - f_stderr = open( - os.path.join(self.res_dir, - "tempest-error.log"), 'w+') proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, - stderr=f_stderr, + stderr=subprocess.STDOUT, bufsize=1) with proc.stdout: for line in iter(proc.stdout.readline, b''): if re.search(r"\} tempest\.", line): - LOGGER.info(line.replace('\n', '')) + LOGGER.info(line.rstrip()) elif re.search(r'(?=\(UUID=(.*)\))', line): self.verification_id = re.search( r'(?=\(UUID=(.*)\))', line).group(1) - LOGGER.info('Verification UUID: %s', self.verification_id) f_stdout.write(line) proc.wait() - f_stdout.close() - f_stderr.close() if self.verification_id is None: raise Exception('Verification UUID not found') + LOGGER.info('Verification UUID: %s', self.verification_id) + + shutil.copy( + "{}/tempest.log".format(self.deployment_dir), + "{}/tempest.debug.log".format(self.res_dir)) def parse_verifier_result(self): """Parse and save test results.""" @@ -214,7 +264,7 @@ class TempestCommon(testcase.TestCase): return with open(os.path.join(self.res_dir, - "tempest-error.log"), 'r') as logfile: + "tempest.log"), 'r') as logfile: output = logfile.read() success_testcases = [] @@ -252,6 +302,80 @@ class TempestCommon(testcase.TestCase): subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'): + """Set image name as tempest img_name_regex""" + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + if not rconfig.has_section('openstack'): + rconfig.add_section('openstack') + rconfig.set('openstack', 'img_name_regex', '^{}$'.format( + self.image.name)) + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + def update_default_role(self, rally_conf='/etc/rally/rally.conf'): + """Detect and update the default role if required""" + role = self.get_default_role(self.cloud) + if not role: + return + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + if not rconfig.has_section('openstack'): + rconfig.add_section('openstack') + rconfig.set('openstack', 'swift_operator_role', role.name) + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'): + """Print rally logs in res dir""" + if not os.path.exists(self.res_dir): + os.makedirs(self.res_dir) + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + rconfig.set('DEFAULT', 'log-file', 'rally.log') + rconfig.set('DEFAULT', 'log_dir', self.res_dir) + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + @staticmethod + def clean_rally_conf(rally_conf='/etc/rally/rally.conf'): + """Clean Rally config""" + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + if rconfig.has_option('openstack', 'img_name_regex'): + rconfig.remove_option('openstack', 'img_name_regex') + if rconfig.has_option('openstack', 'swift_operator_role'): + rconfig.remove_option('openstack', 'swift_operator_role') + if rconfig.has_option('DEFAULT', 'log-file'): + rconfig.remove_option('DEFAULT', 'log-file') + if rconfig.has_option('DEFAULT', 'log_dir'): + rconfig.remove_option('DEFAULT', 'log_dir') + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + def update_scenario_section(self): + """Update scenario section in tempest.conf""" + rconfig = configparser.RawConfigParser() + rconfig.read(self.conf_file) + filename = getattr( + config.CONF, '{}_image'.format(self.case_name), self.filename) + if not rconfig.has_section('scenario'): + rconfig.add_section('scenario') + rconfig.set('scenario', 'img_file', os.path.basename(filename)) + rconfig.set('scenario', 'img_dir', os.path.dirname(filename)) + rconfig.set('scenario', 'img_disk_format', getattr( + config.CONF, '{}_image_format'.format(self.case_name), + self.image_format)) + extra_properties = self.extra_properties.copy() + extra_properties.update( + getattr(config.CONF, '{}_extra_properties'.format( + self.case_name), {})) + rconfig.set( + 'scenario', 'img_properties', + conf_utils.convert_dict_to_ini(extra_properties)) + with open(self.conf_file, 'wb') as config_file: + rconfig.write(config_file) + def configure(self, **kwargs): # pylint: disable=unused-argument """ Create all openstack resources for tempest-based testcases and write @@ -259,242 +383,56 @@ class TempestCommon(testcase.TestCase): """ if not os.path.exists(self.res_dir): os.makedirs(self.res_dir) - self.resources.create() - compute_cnt = len(self.resources.cloud.list_hypervisors()) + compute_cnt = len(self.cloud.list_hypervisors()) + + self.image_alt = self.publish_image_alt() + self.flavor_alt = self.create_flavor_alt() + LOGGER.debug("flavor: %s", self.flavor_alt) + self.conf_file = conf_utils.configure_verifier(self.deployment_dir) conf_utils.configure_tempest_update_params( - self.conf_file, network_name=self.resources.network.id, - image_id=self.resources.image.id, - flavor_id=self.resources.flavor.id, + self.conf_file, network_name=self.network.name, + image_id=self.image.id, + flavor_id=self.flavor.id, compute_cnt=compute_cnt, - image_alt_id=self.resources.image_alt.id, - flavor_alt_id=self.resources.flavor_alt.id) + image_alt_id=self.image_alt.id, + flavor_alt_id=self.flavor_alt.id, + domain_name=self.cloud.auth.get("project_domain_name", "Default")) + self.update_scenario_section() self.backup_tempest_config(self.conf_file, self.res_dir) def run(self, **kwargs): self.start_time = time.time() try: + assert super(TempestCommon, self).run( + **kwargs) == testcase.TestCase.EX_OK + if not os.path.exists(self.res_dir): + os.makedirs(self.res_dir) + self.update_rally_regex() + self.update_default_role() + self.update_rally_logs() + shutil.copy("/etc/rally/rally.conf", self.res_dir) self.configure(**kwargs) - self.generate_test_list() + self.generate_test_list(**kwargs) self.apply_tempest_blacklist() - self.run_verifier_tests() + self.run_verifier_tests(**kwargs) self.parse_verifier_result() self.generate_report() res = testcase.TestCase.EX_OK except Exception: # pylint: disable=broad-except LOGGER.exception('Error with run') + self.result = 0 res = testcase.TestCase.EX_RUN_ERROR - finally: - self.resources.cleanup() self.stop_time = time.time() return res - -class TempestSmokeSerial(TempestCommon): - """Tempest smoke serial testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'tempest_smoke_serial' - TempestCommon.__init__(self, **kwargs) - self.mode = "smoke" - self.option = ["--concurrency", "1"] - - -class TempestNeutronTrunk(TempestCommon): - """Tempest neutron trunk testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'neutron_trunk' - TempestCommon.__init__(self, **kwargs) - self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'" - self.res_dir = os.path.join( - getattr(config.CONF, 'dir_results'), 'neutron_trunk') - self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt') - self.list = os.path.join(self.res_dir, 'test_list.txt') - - def configure(self, **kwargs): - super(TempestNeutronTrunk, self).configure(**kwargs) - rconfig = configparser.RawConfigParser() - rconfig.read(self.conf_file) - rconfig.set('network-feature-enabled', 'api_extensions', 'all') - with open(self.conf_file, 'wb') as config_file: - rconfig.write(config_file) - - -class TempestBarbican(TempestCommon): - """Tempest Barbican testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'barbican' - TempestCommon.__init__(self, **kwargs) - self.mode = "'barbican_tempest_plugin.tests.(api|scenario)'" - self.res_dir = os.path.join( - getattr(config.CONF, 'dir_results'), 'barbican') - self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt') - self.list = os.path.join(self.res_dir, 'test_list.txt') - - def generate_test_list(self): - self.backup_tempest_config(self.conf_file, '/etc') - super(TempestBarbican, self).generate_test_list() - os.remove('/etc/tempest.conf') - - -class TempestSmokeParallel(TempestCommon): - """Tempest smoke parallel testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'tempest_smoke_parallel' - TempestCommon.__init__(self, **kwargs) - self.mode = "smoke" - - -class TempestFullParallel(TempestCommon): - """Tempest full parallel testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'tempest_full_parallel' - TempestCommon.__init__(self, **kwargs) - self.mode = "full" - - -class TempestCustom(TempestCommon): - """Tempest custom testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'tempest_custom' - TempestCommon.__init__(self, **kwargs) - self.mode = "custom" - self.option = ["--concurrency", "1"] - - -class TempestDefcore(TempestCommon): - """Tempest Defcore testcase implementation.""" - def __init__(self, **kwargs): - if "case_name" not in kwargs: - kwargs["case_name"] = 'tempest_defcore' - TempestCommon.__init__(self, **kwargs) - self.mode = "defcore" - self.option = ["--concurrency", "1"] - - -class TempestResourcesManager(object): - # pylint: disable=too-many-instance-attributes - """Tempest resource manager.""" - def __init__(self): - self.guid = '-' + str(uuid.uuid4()) - self.cloud = os_client_config.make_shade() - LOGGER.debug("cloud: %s", self.cloud) - self.domain = self.cloud.get_domain( - name_or_id=self.cloud.auth.get( - "project_domain_name", "Default")) - LOGGER.debug("domain: %s", self.domain) - self.project = None - self.user = None - self.network = None - self.subnet = None - self.image = None - self.image_alt = None - self.flavor = None - self.flavor_alt = None - - def _create_project(self): - """Create project for tests.""" - self.project = self.cloud.create_project( - getattr(config.CONF, 'tempest_identity_tenant_name') + self.guid, - description=getattr( - config.CONF, 'tempest_identity_tenant_description'), - domain_id=self.domain.id) - LOGGER.debug("project: %s", self.project) - - def _create_user(self): - """Create user for tests.""" - self.user = self.cloud.create_user( - name=getattr( - config.CONF, 'tempest_identity_user_name') + self.guid, - password=getattr(config.CONF, 'tempest_identity_user_password'), - default_project=getattr( - config.CONF, 'tempest_identity_tenant_name') + self.guid, - domain_id=self.domain.id) - LOGGER.debug("user: %s", self.user) - - def _create_network(self): - """Create network for tests.""" - tempest_net_name = getattr( - config.CONF, 'tempest_private_net_name') + self.guid - provider = {} - if hasattr(config.CONF, 'tempest_network_type'): - provider["network_type"] = getattr( - config.CONF, 'tempest_network_type') - if hasattr(config.CONF, 'tempest_physical_network'): - provider["physical_network"] = getattr( - config.CONF, 'tempest_physical_network') - if hasattr(config.CONF, 'tempest_segmentation_id'): - provider["segmentation_id"] = getattr( - config.CONF, 'tempest_segmentation_id') - LOGGER.info( - "Creating network with name: '%s'", tempest_net_name) - self.network = self.cloud.create_network( - tempest_net_name, provider=provider) - LOGGER.debug("network: %s", self.network) - - self.subnet = self.cloud.create_subnet( - self.network.id, - subnet_name=getattr( - config.CONF, 'tempest_private_subnet_name') + self.guid, - cidr=getattr(config.CONF, 'tempest_private_subnet_cidr'), - enable_dhcp=True, - dns_nameservers=[env.get('NAMESERVER')]) - LOGGER.debug("subnet: %s", self.subnet) - - def _create_image(self, name): - """Create image for tests""" - LOGGER.info("Creating image with name: '%s'", name) - meta = getattr(config.CONF, 'openstack_extra_properties', None) - image = self.cloud.create_image( - name, filename=getattr(config.CONF, 'openstack_image_url'), - is_public=True, meta=meta) - LOGGER.debug("image: %s", image) - return image - - def _create_flavor(self, name): - """Create flavor for tests.""" - flavor = self.cloud.create_flavor( - name, getattr(config.CONF, 'openstack_flavor_ram'), - getattr(config.CONF, 'openstack_flavor_vcpus'), - getattr(config.CONF, 'openstack_flavor_disk')) - self.cloud.set_flavor_specs( - flavor.id, getattr(config.CONF, 'flavor_extra_specs', {})) - LOGGER.debug("flavor: %s", flavor) - return flavor - - def create(self, create_project=False): - """Create resources for Tempest test suite.""" - if create_project: - self._create_project() - self._create_user() - self._create_network() - - LOGGER.debug("Creating two images for Tempest suite") - self.image = self._create_image( - getattr(config.CONF, 'openstack_image_name') + self.guid) - self.image_alt = self._create_image( - getattr(config.CONF, 'openstack_image_name_alt') + self.guid) - - LOGGER.info("Creating two flavors for Tempest suite") - self.flavor = self._create_flavor( - getattr(config.CONF, 'openstack_flavor_name') + self.guid) - self.flavor_alt = self._create_flavor( - getattr(config.CONF, 'openstack_flavor_name_alt') + self.guid) - - def cleanup(self): + def clean(self): """ Cleanup all OpenStack objects. Should be called on completion. """ - self.cloud.delete_image(self.image) - self.cloud.delete_image(self.image_alt) - self.cloud.delete_network(self.network.id) - self.cloud.delete_flavor(self.flavor.id) - self.cloud.delete_flavor(self.flavor_alt.id) - if self.project: - self.cloud.delete_user(self.user.id) - self.cloud.delete_project(self.project.id) + self.clean_rally_conf() + if self.image_alt: + self.cloud.delete_image(self.image_alt) + if self.flavor_alt: + self.orig_cloud.delete_flavor(self.flavor_alt.id) + super(TempestCommon, self).clean()