functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 from six.moves import configparser
  25 from xtesting.core import testcase
  26 import yaml
  27
  28 from functest.core import singlevm
  29 from functest.opnfv_tests.openstack.rally import rally
  30 from functest.utils import config
  31 from functest.utils import env
  32 from functest.utils import functest_utils
  33
  34 LOGGER = logging.getLogger(__name__)
  35
  36
  37 class TempestCommon(singlevm.VmReady2):
  38     # pylint: disable=too-many-instance-attributes,too-many-public-methods
  39     """TempestCommon testcases implementation class."""
  40
  41     visibility = 'public'
  42     filename_alt = '/home/opnfv/functest/images/cirros-0.6.1-x86_64-disk.img'
  43     shared_network = True
  44     tempest_conf_yaml = pkg_resources.resource_filename(
  45         'functest',
  46         'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
  47     tempest_custom = pkg_resources.resource_filename(
  48         'functest',
  49         'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  50     tempest_blacklist = pkg_resources.resource_filename(
  51         'functest',
  52         'opnfv_tests/openstack/tempest/custom_tests/blacklist.yaml')
  53     tempest_public_blacklist = pkg_resources.resource_filename(
  54         'functest',
  55         'opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml')
  56
  57     def __init__(self, **kwargs):
  58         if "case_name" not in kwargs:
  59             kwargs["case_name"] = 'tempest'
  60         super().__init__(**kwargs)
  61         assert self.orig_cloud
  62         assert self.cloud
  63         assert self.project
  64         if self.orig_cloud.get_role("admin"):
  65             self.role_name = "admin"
  66         elif self.orig_cloud.get_role("Admin"):
  67             self.role_name = "Admin"
  68         else:
  69             raise Exception("Cannot detect neither admin nor Admin")
  70         self.orig_cloud.grant_role(
  71             self.role_name, user=self.project.user.id,
  72             project=self.project.project.id,
  73             domain=self.project.domain.id)
  74         self.orig_cloud.grant_role(
  75             self.role_name, user=self.project.user.id,
  76             domain=self.project.domain.id)
  77         self.deployment_id = None
  78         self.verifier_id = None
  79         self.verifier_repo_dir = None
  80         self.deployment_dir = None
  81         self.verification_id = None
  82         self.res_dir = os.path.join(
  83             getattr(config.CONF, 'dir_results'), self.case_name)
  84         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  85         self.list = os.path.join(self.res_dir, 'test_list.txt')
  86         self.conf_file = None
  87         self.image_alt = None
  88         self.flavor_alt = None
  89         self.services = []
  90         try:
  91             self.services = kwargs['run']['args']['services']
  92         except Exception:  # pylint: disable=broad-except
  93             pass
  94         self.neutron_extensions = []
  95         try:
  96             self.neutron_extensions = kwargs['run']['args'][
  97                 'neutron_extensions']
  98         except Exception:  # pylint: disable=broad-except
  99             pass
 100         self.deny_skipping = kwargs.get("deny_skipping", False)
 101         self.tests_count = kwargs.get("tests_count", 0)
 102
 103     def check_services(self):
 104         """Check the mandatory services."""
 105         for service in self.services:
 106             try:
 107                 self.cloud.search_services(service)[0]
 108             except Exception:  # pylint: disable=broad-except
 109                 self.is_skipped = True
 110                 break
 111
 112     def check_extensions(self):
 113         """Check the mandatory network extensions."""
 114         extensions = self.cloud.get_network_extensions()
 115         for network_extension in self.neutron_extensions:
 116             if network_extension not in extensions:
 117                 LOGGER.warning(
 118                     "Cannot find Neutron extension: %s", network_extension)
 119                 self.is_skipped = True
 120                 break
 121
 122     def check_requirements(self):
 123         self.check_services()
 124         self.check_extensions()
 125         if self.is_skipped:
 126             self.project.clean()
 127
 128     @staticmethod
 129     def read_file(filename):
 130         """Read file and return content as a stripped list."""
 131         with open(filename, encoding='utf-8') as src:
 132             return [line.strip() for line in src.readlines()]
 133
 134     @staticmethod
 135     def get_verifier_result(verif_id):
 136         """Retrieve verification results."""
 137         result = {
 138             'num_tests': 0,
 139             'num_success': 0,
 140             'num_failures': 0,
 141             'num_skipped': 0
 142         }
 143         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 144         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 145         with subprocess.Popen(
 146                 cmd, stdout=subprocess.PIPE,
 147                 stderr=subprocess.STDOUT) as proc:
 148             for line in proc.stdout:
 149                 LOGGER.info(line.decode("utf-8").rstrip())
 150                 new_line = line.decode("utf-8").replace(' ', '').split('|')
 151                 if 'Tests' in new_line:
 152                     break
 153                 if 'Testscount' in new_line:
 154                     result['num_tests'] = int(new_line[2])
 155                 elif 'Success' in new_line:
 156                     result['num_success'] = int(new_line[2])
 157                 elif 'Skipped' in new_line:
 158                     result['num_skipped'] = int(new_line[2])
 159                 elif 'Failures' in new_line:
 160                     result['num_failures'] = int(new_line[2])
 161         return result
 162
 163     @staticmethod
 164     def backup_tempest_config(conf_file, res_dir):
 165         """
 166         Copy config file to tempest results directory
 167         """
 168         if not os.path.exists(res_dir):
 169             os.makedirs(res_dir)
 170         shutil.copyfile(conf_file,
 171                         os.path.join(res_dir, 'tempest.conf'))
 172
 173     @staticmethod
 174     def create_verifier():
 175         """Create new verifier"""
 176         LOGGER.info("Create verifier from existing repo...")
 177         cmd = ['rally', 'verify', 'delete-verifier',
 178                '--id', str(getattr(config.CONF, 'tempest_verifier_name')),
 179                '--force']
 180         try:
 181             output = subprocess.check_output(cmd)
 182             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 183         except subprocess.CalledProcessError:
 184             pass
 185
 186         cmd = ['rally', 'verify', 'create-verifier',
 187                '--source', str(getattr(config.CONF, 'dir_repo_tempest')),
 188                '--name', str(getattr(config.CONF, 'tempest_verifier_name')),
 189                '--type', 'tempest', '--system-wide']
 190         output = subprocess.check_output(cmd)
 191         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 192         return TempestCommon.get_verifier_id()
 193
 194     @staticmethod
 195     def get_verifier_id():
 196         """
 197         Returns verifier id for current Tempest
 198         """
 199         cmd = ("rally verify list-verifiers | awk '/" +
 200                getattr(config.CONF, 'tempest_verifier_name') +
 201                "/ {print $2}'")
 202         with subprocess.Popen(
 203                 cmd, shell=True, stdout=subprocess.PIPE,
 204                 stderr=subprocess.DEVNULL) as proc:
 205             verifier_uuid = proc.stdout.readline().rstrip()
 206         return verifier_uuid.decode("utf-8")
 207
 208     @staticmethod
 209     def get_verifier_repo_dir(verifier_id):
 210         """
 211         Returns installed verifier repo directory for Tempest
 212         """
 213         return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
 214                             'verification',
 215                             f'verifier-{verifier_id}',
 216                             'repo')
 217
 218     @staticmethod
 219     def get_verifier_deployment_dir(verifier_id, deployment_id):
 220         """
 221         Returns Rally deployment directory for current verifier
 222         """
 223         return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
 224                             'verification',
 225                             f'verifier-{verifier_id}',
 226                             f'for-deployment-{deployment_id}')
 227
 228     @staticmethod
 229     def update_tempest_conf_file(conf_file, rconfig):
 230         """Update defined paramters into tempest config file"""
 231         with open(TempestCommon.tempest_conf_yaml, encoding='utf-8') as yfile:
 232             conf_yaml = yaml.safe_load(yfile)
 233         if conf_yaml:
 234             sections = rconfig.sections()
 235             for section in conf_yaml:
 236                 if section not in sections:
 237                     rconfig.add_section(section)
 238                 sub_conf = conf_yaml.get(section)
 239                 for key, value in sub_conf.items():
 240                     rconfig.set(section, key, value)
 241
 242         with open(conf_file, 'w', encoding='utf-8') as config_file:
 243             rconfig.write(config_file)
 244
 245     @staticmethod
 246     def configure_tempest_update_params(
 247             tempest_conf_file, image_id=None, flavor_id=None,
 248             compute_cnt=1, image_alt_id=None, flavor_alt_id=None,
 249             admin_role_name='admin', cidr='192.168.120.0/24',
 250             domain_id='default'):
 251         # pylint: disable=too-many-branches,too-many-arguments
 252         # pylint: disable=too-many-statements,too-many-locals
 253         """
 254         Add/update needed parameters into tempest.conf file
 255         """
 256         LOGGER.debug("Updating selected tempest.conf parameters...")
 257         rconfig = configparser.RawConfigParser()
 258         rconfig.read(tempest_conf_file)
 259         rconfig.set(
 260             'compute', 'volume_device_name', env.get('VOLUME_DEVICE_NAME'))
 261         if image_id is not None:
 262             rconfig.set('compute', 'image_ref', image_id)
 263         if image_alt_id is not None:
 264             rconfig.set('compute', 'image_ref_alt', image_alt_id)
 265         if flavor_id is not None:
 266             rconfig.set('compute', 'flavor_ref', flavor_id)
 267         if flavor_alt_id is not None:
 268             rconfig.set('compute', 'flavor_ref_alt', flavor_alt_id)
 269         if compute_cnt > 1:
 270             # enable multinode tests
 271             rconfig.set('compute', 'min_compute_nodes', compute_cnt)
 272             rconfig.set('compute-feature-enabled', 'live_migration', True)
 273         if os.environ.get('OS_REGION_NAME'):
 274             rconfig.set('identity', 'region', os.environ.get('OS_REGION_NAME'))
 275         rconfig.set('identity', 'admin_role', admin_role_name)
 276         rconfig.set('identity', 'default_domain_id', domain_id)
 277         if not rconfig.has_section('network'):
 278             rconfig.add_section('network')
 279         rconfig.set('network', 'default_network', cidr)
 280         rconfig.set('network', 'project_network_cidr', cidr)
 281         rconfig.set('network', 'project_networks_reachable', False)
 282         rconfig.set(
 283             'identity', 'v3_endpoint_type',
 284             os.environ.get('OS_INTERFACE', 'public'))
 285
 286         sections = rconfig.sections()
 287         services_list = [
 288             'compute', 'volume', 'image', 'network', 'data-processing',
 289             'object-storage', 'orchestration']
 290         for service in services_list:
 291             if service not in sections:
 292                 rconfig.add_section(service)
 293             rconfig.set(service, 'endpoint_type',
 294                         os.environ.get('OS_INTERFACE', 'public'))
 295
 296         LOGGER.debug('Add/Update required params defined in tempest_conf.yaml '
 297                      'into tempest.conf file')
 298         TempestCommon.update_tempest_conf_file(tempest_conf_file, rconfig)
 299
 300     @staticmethod
 301     def configure_verifier(deployment_dir):
 302         """
 303         Execute rally verify configure-verifier, which generates tempest.conf
 304         """
 305         cmd = ['rally', 'verify', 'configure-verifier', '--reconfigure',
 306                '--id', str(getattr(config.CONF, 'tempest_verifier_name'))]
 307         output = subprocess.check_output(cmd)
 308         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 309
 310         LOGGER.debug("Looking for tempest.conf file...")
 311         tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 312         if not os.path.isfile(tempest_conf_file):
 313             LOGGER.error("Tempest configuration file %s NOT found.",
 314                          tempest_conf_file)
 315             return None
 316         return tempest_conf_file
 317
 318     def generate_test_list(self, **kwargs):
 319         """Generate test list based on the test mode."""
 320         LOGGER.debug("Generating test case list...")
 321         self.backup_tempest_config(self.conf_file, '/etc')
 322         if kwargs.get('mode') == 'custom':
 323             if os.path.isfile(self.tempest_custom):
 324                 shutil.copyfile(
 325                     self.tempest_custom, self.list)
 326             else:
 327                 raise Exception(
 328                     f"Tempest test list file {self.tempest_custom} NOT found.")
 329         else:
 330             testr_mode = kwargs.get(
 331                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 332             cmd = (f"(cd {self.verifier_repo_dir}; "
 333                    f"stestr list '{testr_mode}' > {self.list} 2>/dev/null)")
 334             output = subprocess.check_output(cmd, shell=True)
 335             LOGGER.info("%s\n%s", cmd, output.decode("utf-8"))
 336         os.remove('/etc/tempest.conf')
 337
 338     def apply_tempest_blacklist(self, black_list):
 339         """Exclude blacklisted test cases."""
 340         LOGGER.debug("Applying tempest blacklist...")
 341         if os.path.exists(self.raw_list):
 342             os.remove(self.raw_list)
 343         os.rename(self.list, self.raw_list)
 344         cases_file = self.read_file(self.raw_list)
 345         with open(self.list, 'w', encoding='utf-8') as result_file:
 346             black_tests = []
 347             try:
 348                 deploy_scenario = env.get('DEPLOY_SCENARIO')
 349                 if bool(deploy_scenario):
 350                     # if DEPLOY_SCENARIO is set we read the file
 351                     with open(black_list, encoding='utf-8') as black_list_file:
 352                         black_list_yaml = yaml.safe_load(black_list_file)
 353                         black_list_file.close()
 354                         for item in black_list_yaml:
 355                             scenarios = item['scenarios']
 356                             in_it = rally.RallyBase.in_iterable_re
 357                             if in_it(deploy_scenario, scenarios):
 358                                 tests = item['tests']
 359                                 black_tests.extend(tests)
 360             except Exception:  # pylint: disable=broad-except
 361                 black_tests = []
 362                 LOGGER.debug("Tempest blacklist file does not exist.")
 363
 364             for cases_line in cases_file:
 365                 for black_tests_line in black_tests:
 366                     if re.search(black_tests_line, cases_line):
 367                         break
 368                 else:
 369                     result_file.write(str(cases_line) + '\n')
 370
 371     def run_verifier_tests(self, **kwargs):
 372         """Execute tempest test cases."""
 373         cmd = ["rally", "verify", "start", "--load-list",
 374                self.list]
 375         cmd.extend(kwargs.get('option', []))
 376         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 377
 378         with open(
 379                 os.path.join(self.res_dir, "tempest.log"), 'w+',
 380                 encoding='utf-8') as f_stdout:
 381             with subprocess.Popen(
 382                     cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 383                     bufsize=1) as proc:
 384                 with proc.stdout:
 385                     for line in iter(proc.stdout.readline, b''):
 386                         if re.search(r"\} tempest\.", line.decode("utf-8")):
 387                             LOGGER.info(line.rstrip())
 388                         elif re.search(r'(?=\(UUID=(.*)\))',
 389                                        line.decode("utf-8")):
 390                             self.verification_id = re.search(
 391                                 r'(?=\(UUID=(.*)\))',
 392                                 line.decode("utf-8")).group(1)
 393                         f_stdout.write(line.decode("utf-8"))
 394                     proc.wait()
 395
 396         if self.verification_id is None:
 397             raise Exception('Verification UUID not found')
 398         LOGGER.info('Verification UUID: %s', self.verification_id)
 399
 400         shutil.copy(
 401             f"{self.deployment_dir}/tempest.log",
 402             f"{self.res_dir}/tempest.debug.log")
 403
 404     def parse_verifier_result(self):
 405         """Parse and save test results."""
 406         stat = self.get_verifier_result(self.verification_id)
 407         try:
 408             num_executed = stat['num_tests'] - stat['num_skipped']
 409             try:
 410                 self.result = 100 * stat['num_success'] / num_executed
 411             except ZeroDivisionError:
 412                 self.result = 0
 413                 if stat['num_tests'] > 0:
 414                     LOGGER.info("All tests have been skipped")
 415                 else:
 416                     LOGGER.error("No test has been executed")
 417                     return
 418
 419             with open(os.path.join(self.res_dir, "rally.log"),
 420                       'r', encoding='utf-8') as logfile:
 421                 output = logfile.read()
 422
 423             success_testcases = []
 424             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 425                                     output):
 426                 success_testcases.append(match)
 427             failed_testcases = []
 428             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 429                                     output):
 430                 failed_testcases.append(match)
 431             skipped_testcases = []
 432             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',
 433                                     output):
 434                 skipped_testcases.append(match)
 435
 436             self.details = {"tests_number": stat['num_tests'],
 437                             "success_number": stat['num_success'],
 438                             "skipped_number": stat['num_skipped'],
 439                             "failures_number": stat['num_failures'],
 440                             "success": success_testcases,
 441                             "skipped": skipped_testcases,
 442                             "failures": failed_testcases}
 443         except Exception:  # pylint: disable=broad-except
 444             self.result = 0
 445
 446         LOGGER.info("Tempest %s success_rate is %s%%",
 447                     self.case_name, self.result)
 448
 449     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 450         """Set image name as tempest img_name_regex"""
 451         rconfig = configparser.RawConfigParser()
 452         rconfig.read(rally_conf)
 453         if not rconfig.has_section('openstack'):
 454             rconfig.add_section('openstack')
 455         rconfig.set('openstack', 'img_name_regex', f'^{self.image.name}$')
 456         with open(rally_conf, 'w', encoding='utf-8') as config_file:
 457             rconfig.write(config_file)
 458
 459     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 460         """Detect and update the default role if required"""
 461         role = self.get_default_role(self.cloud)
 462         if not role:
 463             return
 464         rconfig = configparser.RawConfigParser()
 465         rconfig.read(rally_conf)
 466         if not rconfig.has_section('openstack'):
 467             rconfig.add_section('openstack')
 468         rconfig.set('openstack', 'swift_operator_role', role.name)
 469         with open(rally_conf, 'w', encoding='utf-8') as config_file:
 470             rconfig.write(config_file)
 471
 472     @staticmethod
 473     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 474         """Clean Rally config"""
 475         rconfig = configparser.RawConfigParser()
 476         rconfig.read(rally_conf)
 477         if rconfig.has_option('openstack', 'img_name_regex'):
 478             rconfig.remove_option('openstack', 'img_name_regex')
 479         if rconfig.has_option('openstack', 'swift_operator_role'):
 480             rconfig.remove_option('openstack', 'swift_operator_role')
 481         with open(rally_conf, 'w', encoding='utf-8') as config_file:
 482             rconfig.write(config_file)
 483
 484     def update_auth_section(self):
 485         """Update auth section in tempest.conf"""
 486         rconfig = configparser.RawConfigParser()
 487         rconfig.read(self.conf_file)
 488         if not rconfig.has_section("auth"):
 489             rconfig.add_section("auth")
 490         if env.get("NEW_USER_ROLE").lower() != "member":
 491             tempest_roles = []
 492             if rconfig.has_option("auth", "tempest_roles"):
 493                 tempest_roles = functest_utils.convert_ini_to_list(
 494                     rconfig.get("auth", "tempest_roles"))
 495             rconfig.set(
 496                 'auth', 'tempest_roles',
 497                 functest_utils.convert_list_to_ini(
 498                     [env.get("NEW_USER_ROLE")] + tempest_roles))
 499         if not json.loads(env.get("USE_DYNAMIC_CREDENTIALS").lower()):
 500             rconfig.set('auth', 'use_dynamic_credentials', False)
 501             account_file = os.path.join(
 502                 getattr(config.CONF, 'dir_functest_data'), 'accounts.yaml')
 503             assert os.path.exists(
 504                 account_file), f"{account_file} doesn't exist"
 505             rconfig.set('auth', 'test_accounts_file', account_file)
 506         if env.get('NO_TENANT_NETWORK').lower() == 'true':
 507             rconfig.set('auth', 'create_isolated_networks', False)
 508         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 509             rconfig.write(config_file)
 510
 511     def update_network_section(self):
 512         """Update network section in tempest.conf"""
 513         rconfig = configparser.RawConfigParser()
 514         rconfig.read(self.conf_file)
 515         if self.ext_net:
 516             if not rconfig.has_section('network'):
 517                 rconfig.add_section('network')
 518             rconfig.set('network', 'public_network_id', self.ext_net.id)
 519             rconfig.set('network', 'floating_network_name', self.ext_net.name)
 520             rconfig.set('network-feature-enabled', 'floating_ips', True)
 521         else:
 522             if not rconfig.has_section('network-feature-enabled'):
 523                 rconfig.add_section('network-feature-enabled')
 524             rconfig.set('network-feature-enabled', 'floating_ips', False)
 525         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 526             rconfig.write(config_file)
 527
 528     def update_compute_section(self):
 529         """Update compute section in tempest.conf"""
 530         rconfig = configparser.RawConfigParser()
 531         rconfig.read(self.conf_file)
 532         if not rconfig.has_section('compute'):
 533             rconfig.add_section('compute')
 534         rconfig.set(
 535             'compute', 'fixed_network_name',
 536             self.network.name if self.network else env.get("EXTERNAL_NETWORK"))
 537         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 538             rconfig.write(config_file)
 539
 540     def update_validation_section(self):
 541         """Update validation section in tempest.conf"""
 542         rconfig = configparser.RawConfigParser()
 543         rconfig.read(self.conf_file)
 544         if not rconfig.has_section('validation'):
 545             rconfig.add_section('validation')
 546         rconfig.set(
 547             'validation', 'connect_method',
 548             'floating' if self.ext_net else 'fixed')
 549         rconfig.set(
 550             'validation', 'network_for_ssh',
 551             self.network.name if self.network else env.get("EXTERNAL_NETWORK"))
 552         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 553             rconfig.write(config_file)
 554
 555     def update_scenario_section(self):
 556         """Update scenario section in tempest.conf"""
 557         rconfig = configparser.RawConfigParser()
 558         rconfig.read(self.conf_file)
 559         filename = getattr(
 560             config.CONF, f'{self.case_name}_image', self.filename)
 561         if not rconfig.has_section('scenario'):
 562             rconfig.add_section('scenario')
 563         rconfig.set('scenario', 'img_file', filename)
 564         rconfig.set('scenario', 'img_disk_format', getattr(
 565             config.CONF, f'{self.case_name}_image_format',
 566             self.image_format))
 567         extra_properties = self.extra_properties.copy()
 568         if env.get('IMAGE_PROPERTIES'):
 569             extra_properties.update(
 570                 functest_utils.convert_ini_to_dict(
 571                     env.get('IMAGE_PROPERTIES')))
 572         extra_properties.update(
 573             getattr(config.CONF, f'{self.case_name}_extra_properties', {}))
 574         rconfig.set(
 575             'scenario', 'img_properties',
 576             functest_utils.convert_dict_to_ini(extra_properties))
 577         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 578             rconfig.write(config_file)
 579
 580     def update_dashboard_section(self):
 581         """Update dashboard section in tempest.conf"""
 582         rconfig = configparser.RawConfigParser()
 583         rconfig.read(self.conf_file)
 584         if env.get('DASHBOARD_URL'):
 585             if not rconfig.has_section('dashboard'):
 586                 rconfig.add_section('dashboard')
 587             rconfig.set('dashboard', 'dashboard_url', env.get('DASHBOARD_URL'))
 588         else:
 589             rconfig.set('service_available', 'horizon', False)
 590         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 591             rconfig.write(config_file)
 592
 593     def configure(self, **kwargs):  # pylint: disable=unused-argument
 594         """
 595         Create all openstack resources for tempest-based testcases and write
 596         tempest.conf.
 597         """
 598         if not os.path.exists(self.res_dir):
 599             os.makedirs(self.res_dir)
 600         self.deployment_id = rally.RallyBase.create_rally_deployment(
 601             environ=self.project.get_environ())
 602         if not self.deployment_id:
 603             raise Exception("Deployment create failed")
 604         self.verifier_id = self.create_verifier()
 605         if not self.verifier_id:
 606             raise Exception("Verifier create failed")
 607         self.verifier_repo_dir = self.get_verifier_repo_dir(
 608             self.verifier_id)
 609         self.deployment_dir = self.get_verifier_deployment_dir(
 610             self.verifier_id, self.deployment_id)
 611
 612         compute_cnt = self.count_hypervisors() if self.count_hypervisors(
 613             ) <= 10 else 10
 614         self.image_alt = self.publish_image_alt()
 615         self.flavor_alt = self.create_flavor_alt()
 616         LOGGER.debug("flavor: %s", self.flavor_alt)
 617
 618         self.conf_file = self.configure_verifier(self.deployment_dir)
 619         if not self.conf_file:
 620             raise Exception("Tempest verifier configuring failed")
 621         self.configure_tempest_update_params(
 622             self.conf_file,
 623             image_id=self.image.id,
 624             flavor_id=self.flavor.id,
 625             compute_cnt=compute_cnt,
 626             image_alt_id=self.image_alt.id,
 627             flavor_alt_id=self.flavor_alt.id,
 628             admin_role_name=self.role_name, cidr=self.cidr,
 629             domain_id=self.project.domain.id)
 630         self.update_auth_section()
 631         self.update_network_section()
 632         self.update_compute_section()
 633         self.update_validation_section()
 634         self.update_scenario_section()
 635         self.update_dashboard_section()
 636         self.backup_tempest_config(self.conf_file, self.res_dir)
 637
 638     def run(self, **kwargs):
 639         self.start_time = time.time()
 640         try:
 641             assert super().run(
 642                 **kwargs) == testcase.TestCase.EX_OK
 643             if not os.path.exists(self.res_dir):
 644                 os.makedirs(self.res_dir)
 645             self.update_rally_regex()
 646             self.update_default_role()
 647             rally.RallyBase.update_rally_logs(self.res_dir)
 648             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 649             self.configure(**kwargs)
 650             self.generate_test_list(**kwargs)
 651             self.apply_tempest_blacklist(TempestCommon.tempest_blacklist)
 652             if env.get('PUBLIC_ENDPOINT_ONLY').lower() == 'true':
 653                 self.apply_tempest_blacklist(
 654                     TempestCommon.tempest_public_blacklist)
 655             self.run_verifier_tests(**kwargs)
 656             self.parse_verifier_result()
 657             rally.RallyBase.verify_report(
 658                 os.path.join(self.res_dir, "tempest-report.html"),
 659                 self.verification_id)
 660             rally.RallyBase.verify_report(
 661                 os.path.join(self.res_dir, "tempest-report.xml"),
 662                 self.verification_id, "junit-xml")
 663             res = testcase.TestCase.EX_OK
 664         except Exception:  # pylint: disable=broad-except
 665             LOGGER.exception('Error with run')
 666             self.result = 0
 667             res = testcase.TestCase.EX_RUN_ERROR
 668         self.stop_time = time.time()
 669         return res
 670
 671     def clean(self):
 672         """
 673         Cleanup all OpenStack objects. Should be called on completion.
 674         """
 675         self.clean_rally_conf()
 676         rally.RallyBase.clean_rally_logs()
 677         if self.image_alt:
 678             self.cloud.delete_image(self.image_alt)
 679         if self.flavor_alt:
 680             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 681         super().clean()
 682
 683     def is_successful(self):
 684         """The overall result of the test."""
 685         skips = self.details.get("skipped_number", 0)
 686         if skips > 0 and self.deny_skipping:
 687             return testcase.TestCase.EX_TESTCASE_FAILED
 688         if self.tests_count and (
 689                 self.details.get("tests_number", 0) != self.tests_count):
 690             return testcase.TestCase.EX_TESTCASE_FAILED
 691         return super().is_successful()
 692
 693
 694 class TempestHeat(TempestCommon):
 695     """Tempest Heat testcase implementation class."""
 696
 697     filename_alt = ('/home/opnfv/functest/images/'
 698                     'Fedora-Cloud-Base-30-1.2.x86_64.qcow2')
 699     flavor_alt_ram = 512
 700     flavor_alt_vcpus = 1
 701     flavor_alt_disk = 4
 702
 703     def __init__(self, **kwargs):
 704         super().__init__(**kwargs)
 705         self.user2 = self.orig_cloud.create_user(
 706             name=f'{self.case_name}-user2_{self.project.guid}',
 707             password=self.project.password,
 708             domain_id=self.project.domain.id)
 709         self.orig_cloud.grant_role(
 710             self.role_name, user=self.user2.id,
 711             project=self.project.project.id, domain=self.project.domain.id)
 712         if not self.orig_cloud.get_role("heat_stack_owner"):
 713             self.role = self.orig_cloud.create_role("heat_stack_owner")
 714         self.orig_cloud.grant_role(
 715             "heat_stack_owner", user=self.user2.id,
 716             project=self.project.project.id,
 717             domain=self.project.domain.id)
 718
 719     def configure(self, **kwargs):
 720         assert self.user2
 721         super().configure(**kwargs)
 722         rconfig = configparser.RawConfigParser()
 723         rconfig.read(self.conf_file)
 724         if not rconfig.has_section('heat_plugin'):
 725             rconfig.add_section('heat_plugin')
 726         # It fails if region and domain ids are unset
 727         rconfig.set(
 728             'heat_plugin', 'region',
 729             os.environ.get('OS_REGION_NAME', 'RegionOne'))
 730         rconfig.set('heat_plugin', 'auth_url', os.environ["OS_AUTH_URL"])
 731         rconfig.set('heat_plugin', 'project_domain_id', self.project.domain.id)
 732         rconfig.set('heat_plugin', 'user_domain_id', self.project.domain.id)
 733         rconfig.set(
 734             'heat_plugin', 'project_domain_name', self.project.domain.name)
 735         rconfig.set(
 736             'heat_plugin', 'user_domain_name', self.project.domain.name)
 737         rconfig.set('heat_plugin', 'username', self.user2.name)
 738         rconfig.set('heat_plugin', 'password', self.project.password)
 739         rconfig.set('heat_plugin', 'project_name', self.project.project.name)
 740         rconfig.set('heat_plugin', 'admin_username', self.project.user.name)
 741         rconfig.set('heat_plugin', 'admin_password', self.project.password)
 742         rconfig.set(
 743             'heat_plugin', 'admin_project_name', self.project.project.name)
 744         rconfig.set('heat_plugin', 'image_ref', self.image_alt.id)
 745         rconfig.set('heat_plugin', 'instance_type', self.flavor_alt.id)
 746         rconfig.set('heat_plugin', 'minimal_image_ref', self.image.id)
 747         rconfig.set('heat_plugin', 'minimal_instance_type', self.flavor.id)
 748         if self.ext_net:
 749             rconfig.set(
 750                 'heat_plugin', 'floating_network_name', self.ext_net.name)
 751         if self.network:
 752             rconfig.set('heat_plugin', 'fixed_network_name', self.network.name)
 753             rconfig.set('heat_plugin', 'fixed_subnet_name', self.subnet.name)
 754             rconfig.set('heat_plugin', 'network_for_ssh', self.network.name)
 755         else:
 756             LOGGER.warning(
 757                 'No tenant network created. '
 758                 'Trying EXTERNAL_NETWORK as a fallback')
 759             rconfig.set(
 760                 'heat_plugin', 'fixed_network_name',
 761                 env.get("EXTERNAL_NETWORK"))
 762             rconfig.set(
 763                 'heat_plugin', 'network_for_ssh', env.get("EXTERNAL_NETWORK"))
 764         with open(self.conf_file, 'w', encoding='utf-8') as config_file:
 765             rconfig.write(config_file)
 766         self.backup_tempest_config(self.conf_file, self.res_dir)
 767
 768     def clean(self):
 769         """
 770         Cleanup all OpenStack objects. Should be called on completion.
 771         """
 772         super().clean()
 773         if self.user2:
 774             self.orig_cloud.delete_user(self.user2.id)