functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 from six.moves import configparser
  25 from xtesting.core import testcase
  26 import yaml
  27
  28 from functest.core import singlevm
  29 from functest.opnfv_tests.openstack.rally import rally
  30 from functest.utils import config
  31 from functest.utils import env
  32 from functest.utils import functest_utils
  33
  34 LOGGER = logging.getLogger(__name__)
  35
  36
  37 class TempestCommon(singlevm.VmReady2):
  38     # pylint: disable=too-many-instance-attributes,too-many-public-methods
  39     """TempestCommon testcases implementation class."""
  40
  41     visibility = 'public'
  42     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  43     shared_network = True
  44     TEMPEST_CONF_YAML = pkg_resources.resource_filename(
  45         'functest',
  46         'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
  47     TEMPEST_CUSTOM = pkg_resources.resource_filename(
  48         'functest',
  49         'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  50     TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  51         'functest',
  52         'opnfv_tests/openstack/tempest/custom_tests/blacklist.yaml')
  53
  54     def __init__(self, **kwargs):
  55         if "case_name" not in kwargs:
  56             kwargs["case_name"] = 'tempest'
  57         super(TempestCommon, self).__init__(**kwargs)
  58         assert self.orig_cloud
  59         assert self.cloud
  60         assert self.project
  61         if self.orig_cloud.get_role("admin"):
  62             self.role_name = "admin"
  63         elif self.orig_cloud.get_role("Admin"):
  64             self.role_name = "Admin"
  65         else:
  66             raise Exception("Cannot detect neither admin nor Admin")
  67         self.orig_cloud.grant_role(
  68             self.role_name, user=self.project.user.id,
  69             project=self.project.project.id,
  70             domain=self.project.domain.id)
  71         self.orig_cloud.grant_role(
  72             self.role_name, user=self.project.user.id,
  73             domain=self.project.domain.id)
  74         self.deployment_id = None
  75         self.verifier_id = None
  76         self.verifier_repo_dir = None
  77         self.deployment_dir = None
  78         self.verification_id = None
  79         self.res_dir = os.path.join(
  80             getattr(config.CONF, 'dir_results'), self.case_name)
  81         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  82         self.list = os.path.join(self.res_dir, 'test_list.txt')
  83         self.conf_file = None
  84         self.image_alt = None
  85         self.flavor_alt = None
  86         self.services = []
  87         try:
  88             self.services = kwargs['run']['args']['services']
  89         except Exception:  # pylint: disable=broad-except
  90             pass
  91         self.neutron_extensions = []
  92         try:
  93             self.neutron_extensions = kwargs['run']['args'][
  94                 'neutron_extensions']
  95         except Exception:  # pylint: disable=broad-except
  96             pass
  97         self.deny_skipping = kwargs.get("deny_skipping", False)
  98
  99     def check_services(self):
 100         """Check the mandatory services."""
 101         for service in self.services:
 102             try:
 103                 self.cloud.search_services(service)[0]
 104             except Exception:  # pylint: disable=broad-except
 105                 self.is_skipped = True
 106                 break
 107
 108     def check_extensions(self):
 109         """Check the mandatory network extensions."""
 110         extensions = self.cloud.get_network_extensions()
 111         for network_extension in self.neutron_extensions:
 112             if network_extension not in extensions:
 113                 LOGGER.warning(
 114                     "Cannot find Neutron extension: %s", network_extension)
 115                 self.is_skipped = True
 116                 break
 117
 118     def check_requirements(self):
 119         self.check_services()
 120         self.check_extensions()
 121         if self.is_skipped:
 122             self.project.clean()
 123
 124     @staticmethod
 125     def read_file(filename):
 126         """Read file and return content as a stripped list."""
 127         with open(filename) as src:
 128             return [line.strip() for line in src.readlines()]
 129
 130     @staticmethod
 131     def get_verifier_result(verif_id):
 132         """Retrieve verification results."""
 133         result = {
 134             'num_tests': 0,
 135             'num_success': 0,
 136             'num_failures': 0,
 137             'num_skipped': 0
 138         }
 139         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 140         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 141         proc = subprocess.Popen(cmd,
 142                                 stdout=subprocess.PIPE,
 143                                 stderr=subprocess.STDOUT)
 144         for line in proc.stdout:
 145             LOGGER.info(line.rstrip())
 146             new_line = line.replace(' ', '').split('|')
 147             if 'Tests' in new_line:
 148                 break
 149             if 'Testscount' in new_line:
 150                 result['num_tests'] = int(new_line[2])
 151             elif 'Success' in new_line:
 152                 result['num_success'] = int(new_line[2])
 153             elif 'Skipped' in new_line:
 154                 result['num_skipped'] = int(new_line[2])
 155             elif 'Failures' in new_line:
 156                 result['num_failures'] = int(new_line[2])
 157         return result
 158
 159     @staticmethod
 160     def backup_tempest_config(conf_file, res_dir):
 161         """
 162         Copy config file to tempest results directory
 163         """
 164         if not os.path.exists(res_dir):
 165             os.makedirs(res_dir)
 166         shutil.copyfile(conf_file,
 167                         os.path.join(res_dir, 'tempest.conf'))
 168
 169     @staticmethod
 170     def create_verifier():
 171         """Create new verifier"""
 172         LOGGER.info("Create verifier from existing repo...")
 173         cmd = ['rally', 'verify', 'delete-verifier',
 174                '--id', str(getattr(config.CONF, 'tempest_verifier_name')),
 175                '--force']
 176         try:
 177             output = subprocess.check_output(cmd)
 178             LOGGER.info("%s\n%s", " ".join(cmd), output)
 179         except subprocess.CalledProcessError:
 180             pass
 181
 182         cmd = ['rally', 'verify', 'create-verifier',
 183                '--source', str(getattr(config.CONF, 'dir_repo_tempest')),
 184                '--name', str(getattr(config.CONF, 'tempest_verifier_name')),
 185                '--type', 'tempest', '--system-wide']
 186         output = subprocess.check_output(cmd)
 187         LOGGER.info("%s\n%s", " ".join(cmd), output)
 188         return TempestCommon.get_verifier_id()
 189
 190     @staticmethod
 191     def get_verifier_id():
 192         """
 193         Returns verifier id for current Tempest
 194         """
 195         cmd = ("rally verify list-verifiers | awk '/" +
 196                getattr(config.CONF, 'tempest_verifier_name') +
 197                "/ {print $2}'")
 198         proc = subprocess.Popen(cmd, shell=True,
 199                                 stdout=subprocess.PIPE,
 200                                 stderr=subprocess.STDOUT)
 201         verifier_uuid = proc.stdout.readline().rstrip()
 202         return verifier_uuid
 203
 204     @staticmethod
 205     def get_verifier_repo_dir(verifier_id):
 206         """
 207         Returns installed verifier repo directory for Tempest
 208         """
 209         return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
 210                             'verification',
 211                             'verifier-{}'.format(verifier_id),
 212                             'repo')
 213
 214     @staticmethod
 215     def get_verifier_deployment_dir(verifier_id, deployment_id):
 216         """
 217         Returns Rally deployment directory for current verifier
 218         """
 219         return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
 220                             'verification',
 221                             'verifier-{}'.format(verifier_id),
 222                             'for-deployment-{}'.format(deployment_id))
 223
 224     @staticmethod
 225     def update_tempest_conf_file(conf_file, rconfig):
 226         """Update defined paramters into tempest config file"""
 227         with open(TempestCommon.TEMPEST_CONF_YAML) as yfile:
 228             conf_yaml = yaml.safe_load(yfile)
 229         if conf_yaml:
 230             sections = rconfig.sections()
 231             for section in conf_yaml:
 232                 if section not in sections:
 233                     rconfig.add_section(section)
 234                 sub_conf = conf_yaml.get(section)
 235                 for key, value in sub_conf.items():
 236                     rconfig.set(section, key, value)
 237
 238         with open(conf_file, 'wb') as config_file:
 239             rconfig.write(config_file)
 240
 241     @staticmethod
 242     def configure_tempest_update_params(
 243             tempest_conf_file, image_id=None, flavor_id=None,
 244             compute_cnt=1, image_alt_id=None, flavor_alt_id=None,
 245             admin_role_name='admin', cidr='192.168.120.0/24',
 246             domain_id='default'):
 247         # pylint: disable=too-many-branches,too-many-arguments,
 248         # too-many-statements
 249         """
 250         Add/update needed parameters into tempest.conf file
 251         """
 252         LOGGER.debug("Updating selected tempest.conf parameters...")
 253         rconfig = configparser.RawConfigParser()
 254         rconfig.read(tempest_conf_file)
 255         rconfig.set(
 256             'compute', 'volume_device_name', env.get('VOLUME_DEVICE_NAME'))
 257         if image_id is not None:
 258             rconfig.set('compute', 'image_ref', image_id)
 259         if image_alt_id is not None:
 260             rconfig.set('compute', 'image_ref_alt', image_alt_id)
 261         if flavor_id is not None:
 262             rconfig.set('compute', 'flavor_ref', flavor_id)
 263         if flavor_alt_id is not None:
 264             rconfig.set('compute', 'flavor_ref_alt', flavor_alt_id)
 265         if compute_cnt > 1:
 266             # enable multinode tests
 267             rconfig.set('compute', 'min_compute_nodes', compute_cnt)
 268             rconfig.set('compute-feature-enabled', 'live_migration', True)
 269         filters = ['RetryFilter', 'AvailabilityZoneFilter', 'ComputeFilter',
 270                    'ComputeCapabilitiesFilter', 'ImagePropertiesFilter',
 271                    'ServerGroupAntiAffinityFilter',
 272                    'ServerGroupAffinityFilter']
 273         rconfig.set(
 274             'compute-feature-enabled', 'scheduler_available_filters',
 275             functest_utils.convert_list_to_ini(filters))
 276         if os.environ.get('OS_REGION_NAME'):
 277             rconfig.set('identity', 'region', os.environ.get('OS_REGION_NAME'))
 278         if env.get("NEW_USER_ROLE").lower() != "member":
 279             rconfig.set(
 280                 'auth', 'tempest_roles',
 281                 functest_utils.convert_list_to_ini([env.get("NEW_USER_ROLE")]))
 282         if not json.loads(env.get("USE_DYNAMIC_CREDENTIALS").lower()):
 283             rconfig.set('auth', 'use_dynamic_credentials', False)
 284             account_file = os.path.join(
 285                 getattr(config.CONF, 'dir_functest_data'), 'accounts.yaml')
 286             assert os.path.exists(
 287                 account_file), "{} doesn't exist".format(account_file)
 288             rconfig.set('auth', 'test_accounts_file', account_file)
 289         rconfig.set('identity', 'auth_version', 'v3')
 290         rconfig.set('identity', 'admin_role', admin_role_name)
 291         rconfig.set('identity', 'admin_domain_scope', True)
 292         rconfig.set('identity', 'default_domain_id', domain_id)
 293         if not rconfig.has_section('network'):
 294             rconfig.add_section('network')
 295         rconfig.set('network', 'default_network', cidr)
 296         rconfig.set('network', 'project_network_cidr', cidr)
 297         rconfig.set('network', 'project_networks_reachable', False)
 298         rconfig.set(
 299             'identity', 'v3_endpoint_type',
 300             os.environ.get('OS_INTERFACE', 'public'))
 301
 302         sections = rconfig.sections()
 303         services_list = [
 304             'compute', 'volume', 'image', 'network', 'data-processing',
 305             'object-storage', 'orchestration']
 306         for service in services_list:
 307             if service not in sections:
 308                 rconfig.add_section(service)
 309             rconfig.set(service, 'endpoint_type',
 310                         os.environ.get('OS_INTERFACE', 'public'))
 311
 312         LOGGER.debug('Add/Update required params defined in tempest_conf.yaml '
 313                      'into tempest.conf file')
 314         TempestCommon.update_tempest_conf_file(tempest_conf_file, rconfig)
 315
 316     @staticmethod
 317     def configure_verifier(deployment_dir):
 318         """
 319         Execute rally verify configure-verifier, which generates tempest.conf
 320         """
 321         cmd = ['rally', 'verify', 'configure-verifier', '--reconfigure',
 322                '--id', str(getattr(config.CONF, 'tempest_verifier_name'))]
 323         output = subprocess.check_output(cmd)
 324         LOGGER.info("%s\n%s", " ".join(cmd), output)
 325
 326         LOGGER.debug("Looking for tempest.conf file...")
 327         tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 328         if not os.path.isfile(tempest_conf_file):
 329             LOGGER.error("Tempest configuration file %s NOT found.",
 330                          tempest_conf_file)
 331             return None
 332         return tempest_conf_file
 333
 334     def generate_test_list(self, **kwargs):
 335         """Generate test list based on the test mode."""
 336         LOGGER.debug("Generating test case list...")
 337         self.backup_tempest_config(self.conf_file, '/etc')
 338         if kwargs.get('mode') == 'custom':
 339             if os.path.isfile(self.TEMPEST_CUSTOM):
 340                 shutil.copyfile(
 341                     self.TEMPEST_CUSTOM, self.list)
 342             else:
 343                 raise Exception("Tempest test list file %s NOT found."
 344                                 % self.TEMPEST_CUSTOM)
 345         else:
 346             testr_mode = kwargs.get(
 347                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 348             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 349                 self.verifier_repo_dir, testr_mode, self.list)
 350             output = subprocess.check_output(cmd, shell=True)
 351             LOGGER.info("%s\n%s", cmd, output)
 352         os.remove('/etc/tempest.conf')
 353
 354     def apply_tempest_blacklist(self):
 355         """Exclude blacklisted test cases."""
 356         LOGGER.debug("Applying tempest blacklist...")
 357         if os.path.exists(self.raw_list):
 358             os.remove(self.raw_list)
 359         os.rename(self.list, self.raw_list)
 360         cases_file = self.read_file(self.raw_list)
 361         result_file = open(self.list, 'w')
 362         black_tests = []
 363         try:
 364             deploy_scenario = env.get('DEPLOY_SCENARIO')
 365             if bool(deploy_scenario):
 366                 # if DEPLOY_SCENARIO is set we read the file
 367                 black_list_file = open(self.TEMPEST_BLACKLIST)
 368                 black_list_yaml = yaml.safe_load(black_list_file)
 369                 black_list_file.close()
 370                 for item in black_list_yaml:
 371                     scenarios = item['scenarios']
 372                     if deploy_scenario in scenarios:
 373                         tests = item['tests']
 374                         for test in tests:
 375                             black_tests.append(test)
 376                         break
 377         except Exception:  # pylint: disable=broad-except
 378             black_tests = []
 379             LOGGER.debug("Tempest blacklist file does not exist.")
 380
 381         for cases_line in cases_file:
 382             for black_tests_line in black_tests:
 383                 if black_tests_line in cases_line:
 384                     break
 385             else:
 386                 result_file.write(str(cases_line) + '\n')
 387         result_file.close()
 388
 389     def run_verifier_tests(self, **kwargs):
 390         """Execute tempest test cases."""
 391         cmd = ["rally", "verify", "start", "--load-list",
 392                self.list]
 393         cmd.extend(kwargs.get('option', []))
 394         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 395
 396         f_stdout = open(
 397             os.path.join(self.res_dir, "tempest.log"), 'w+')
 398
 399         proc = subprocess.Popen(
 400             cmd,
 401             stdout=subprocess.PIPE,
 402             stderr=subprocess.STDOUT,
 403             bufsize=1)
 404
 405         with proc.stdout:
 406             for line in iter(proc.stdout.readline, b''):
 407                 if re.search(r"\} tempest\.", line):
 408                     LOGGER.info(line.rstrip())
 409                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 410                     self.verification_id = re.search(
 411                         r'(?=\(UUID=(.*)\))', line).group(1)
 412                 f_stdout.write(line)
 413         proc.wait()
 414         f_stdout.close()
 415
 416         if self.verification_id is None:
 417             raise Exception('Verification UUID not found')
 418         LOGGER.info('Verification UUID: %s', self.verification_id)
 419
 420         shutil.copy(
 421             "{}/tempest.log".format(self.deployment_dir),
 422             "{}/tempest.debug.log".format(self.res_dir))
 423
 424     def parse_verifier_result(self):
 425         """Parse and save test results."""
 426         stat = self.get_verifier_result(self.verification_id)
 427         try:
 428             num_executed = stat['num_tests'] - stat['num_skipped']
 429             try:
 430                 self.result = 100 * stat['num_success'] / num_executed
 431             except ZeroDivisionError:
 432                 self.result = 0
 433                 if stat['num_tests'] > 0:
 434                     LOGGER.info("All tests have been skipped")
 435                 else:
 436                     LOGGER.error("No test has been executed")
 437                     return
 438
 439             with open(os.path.join(self.res_dir,
 440                                    "rally.log"), 'r') as logfile:
 441                 output = logfile.read()
 442
 443             success_testcases = []
 444             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 445                                     output):
 446                 success_testcases.append(match)
 447             failed_testcases = []
 448             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 449                                     output):
 450                 failed_testcases.append(match)
 451             skipped_testcases = []
 452             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',
 453                                     output):
 454                 skipped_testcases.append(match)
 455
 456             self.details = {"tests_number": stat['num_tests'],
 457                             "success_number": stat['num_success'],
 458                             "skipped_number": stat['num_skipped'],
 459                             "failures_number": stat['num_failures'],
 460                             "success": success_testcases,
 461                             "skipped": skipped_testcases,
 462                             "failures": failed_testcases}
 463         except Exception:  # pylint: disable=broad-except
 464             self.result = 0
 465
 466         LOGGER.info("Tempest %s success_rate is %s%%",
 467                     self.case_name, self.result)
 468
 469     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 470         """Set image name as tempest img_name_regex"""
 471         rconfig = configparser.RawConfigParser()
 472         rconfig.read(rally_conf)
 473         if not rconfig.has_section('openstack'):
 474             rconfig.add_section('openstack')
 475         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 476             self.image.name))
 477         with open(rally_conf, 'wb') as config_file:
 478             rconfig.write(config_file)
 479
 480     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 481         """Detect and update the default role if required"""
 482         role = self.get_default_role(self.cloud)
 483         if not role:
 484             return
 485         rconfig = configparser.RawConfigParser()
 486         rconfig.read(rally_conf)
 487         if not rconfig.has_section('openstack'):
 488             rconfig.add_section('openstack')
 489         rconfig.set('openstack', 'swift_operator_role', role.name)
 490         with open(rally_conf, 'wb') as config_file:
 491             rconfig.write(config_file)
 492
 493     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 494         """Print rally logs in res dir"""
 495         if not os.path.exists(self.res_dir):
 496             os.makedirs(self.res_dir)
 497         rconfig = configparser.RawConfigParser()
 498         rconfig.read(rally_conf)
 499         rconfig.set('DEFAULT', 'debug', True)
 500         rconfig.set('DEFAULT', 'use_stderr', False)
 501         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 502         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 503         with open(rally_conf, 'wb') as config_file:
 504             rconfig.write(config_file)
 505
 506     @staticmethod
 507     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 508         """Clean Rally config"""
 509         rconfig = configparser.RawConfigParser()
 510         rconfig.read(rally_conf)
 511         if rconfig.has_option('openstack', 'img_name_regex'):
 512             rconfig.remove_option('openstack', 'img_name_regex')
 513         if rconfig.has_option('openstack', 'swift_operator_role'):
 514             rconfig.remove_option('openstack', 'swift_operator_role')
 515         if rconfig.has_option('DEFAULT', 'use_stderr'):
 516             rconfig.remove_option('DEFAULT', 'use_stderr')
 517         if rconfig.has_option('DEFAULT', 'debug'):
 518             rconfig.remove_option('DEFAULT', 'debug')
 519         if rconfig.has_option('DEFAULT', 'log-file'):
 520             rconfig.remove_option('DEFAULT', 'log-file')
 521         if rconfig.has_option('DEFAULT', 'log_dir'):
 522             rconfig.remove_option('DEFAULT', 'log_dir')
 523         with open(rally_conf, 'wb') as config_file:
 524             rconfig.write(config_file)
 525
 526     def update_network_section(self):
 527         """Update network section in tempest.conf"""
 528         rconfig = configparser.RawConfigParser()
 529         rconfig.read(self.conf_file)
 530         if not rconfig.has_section('network'):
 531             rconfig.add_section('network')
 532         rconfig.set('network', 'public_network_id', self.ext_net.id)
 533         rconfig.set('network', 'floating_network_name', self.ext_net.name)
 534         with open(self.conf_file, 'wb') as config_file:
 535             rconfig.write(config_file)
 536
 537     def update_compute_section(self):
 538         """Update compute section in tempest.conf"""
 539         rconfig = configparser.RawConfigParser()
 540         rconfig.read(self.conf_file)
 541         if not rconfig.has_section('compute'):
 542             rconfig.add_section('compute')
 543         rconfig.set('compute', 'fixed_network_name', self.network.name)
 544         with open(self.conf_file, 'wb') as config_file:
 545             rconfig.write(config_file)
 546
 547     def update_scenario_section(self):
 548         """Update scenario section in tempest.conf"""
 549         rconfig = configparser.RawConfigParser()
 550         rconfig.read(self.conf_file)
 551         filename = getattr(
 552             config.CONF, '{}_image'.format(self.case_name), self.filename)
 553         if not rconfig.has_section('scenario'):
 554             rconfig.add_section('scenario')
 555         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 556         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 557         rconfig.set('scenario', 'img_disk_format', getattr(
 558             config.CONF, '{}_image_format'.format(self.case_name),
 559             self.image_format))
 560         extra_properties = self.extra_properties.copy()
 561         if env.get('IMAGE_PROPERTIES'):
 562             extra_properties.update(
 563                 functest_utils.convert_ini_to_dict(
 564                     env.get('IMAGE_PROPERTIES')))
 565         extra_properties.update(
 566             getattr(config.CONF, '{}_extra_properties'.format(
 567                 self.case_name), {}))
 568         rconfig.set(
 569             'scenario', 'img_properties',
 570             functest_utils.convert_dict_to_ini(extra_properties))
 571         with open(self.conf_file, 'wb') as config_file:
 572             rconfig.write(config_file)
 573
 574     def configure(self, **kwargs):  # pylint: disable=unused-argument
 575         """
 576         Create all openstack resources for tempest-based testcases and write
 577         tempest.conf.
 578         """
 579         if not os.path.exists(self.res_dir):
 580             os.makedirs(self.res_dir)
 581         environ = dict(
 582             os.environ,
 583             OS_USERNAME=self.project.user.name,
 584             OS_PROJECT_NAME=self.project.project.name,
 585             OS_PROJECT_ID=self.project.project.id,
 586             OS_PASSWORD=self.project.password)
 587         try:
 588             del environ['OS_TENANT_NAME']
 589             del environ['OS_TENANT_ID']
 590         except Exception:  # pylint: disable=broad-except
 591             pass
 592         self.deployment_id = rally.RallyBase.create_rally_deployment(
 593             environ=environ)
 594         if not self.deployment_id:
 595             raise Exception("Deployment create failed")
 596         self.verifier_id = self.create_verifier()
 597         if not self.verifier_id:
 598             raise Exception("Verifier create failed")
 599         self.verifier_repo_dir = self.get_verifier_repo_dir(
 600             self.verifier_id)
 601         self.deployment_dir = self.get_verifier_deployment_dir(
 602             self.verifier_id, self.deployment_id)
 603
 604         compute_cnt = len(self.orig_cloud.list_hypervisors())
 605
 606         self.image_alt = self.publish_image_alt()
 607         self.flavor_alt = self.create_flavor_alt()
 608         LOGGER.debug("flavor: %s", self.flavor_alt)
 609
 610         self.conf_file = self.configure_verifier(self.deployment_dir)
 611         if not self.conf_file:
 612             raise Exception("Tempest verifier configuring failed")
 613         self.configure_tempest_update_params(
 614             self.conf_file,
 615             image_id=self.image.id,
 616             flavor_id=self.flavor.id,
 617             compute_cnt=compute_cnt,
 618             image_alt_id=self.image_alt.id,
 619             flavor_alt_id=self.flavor_alt.id,
 620             admin_role_name=self.role_name, cidr=self.cidr,
 621             domain_id=self.project.domain.id)
 622         self.update_network_section()
 623         self.update_compute_section()
 624         self.update_scenario_section()
 625         self.backup_tempest_config(self.conf_file, self.res_dir)
 626
 627     def run(self, **kwargs):
 628         self.start_time = time.time()
 629         try:
 630             assert super(TempestCommon, self).run(
 631                 **kwargs) == testcase.TestCase.EX_OK
 632             if not os.path.exists(self.res_dir):
 633                 os.makedirs(self.res_dir)
 634             self.update_rally_regex()
 635             self.update_default_role()
 636             self.update_rally_logs()
 637             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 638             self.configure(**kwargs)
 639             self.generate_test_list(**kwargs)
 640             self.apply_tempest_blacklist()
 641             self.run_verifier_tests(**kwargs)
 642             self.parse_verifier_result()
 643             rally.RallyBase.verify_report(
 644                 os.path.join(self.res_dir, "tempest-report.html"),
 645                 self.verification_id)
 646             rally.RallyBase.verify_report(
 647                 os.path.join(self.res_dir, "tempest-report.xml"),
 648                 self.verification_id, "junit-xml")
 649             res = testcase.TestCase.EX_OK
 650         except Exception:  # pylint: disable=broad-except
 651             LOGGER.exception('Error with run')
 652             self.result = 0
 653             res = testcase.TestCase.EX_RUN_ERROR
 654         self.stop_time = time.time()
 655         return res
 656
 657     def clean(self):
 658         """
 659         Cleanup all OpenStack objects. Should be called on completion.
 660         """
 661         self.clean_rally_conf()
 662         if self.image_alt:
 663             self.cloud.delete_image(self.image_alt)
 664         if self.flavor_alt:
 665             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 666         super(TempestCommon, self).clean()
 667
 668     def is_successful(self):
 669         """The overall result of the test."""
 670         skips = self.details.get("skipped_number", 0)
 671         if skips > 0 and self.deny_skipping:
 672             return testcase.TestCase.EX_TESTCASE_FAILED
 673         return super(TempestCommon, self).is_successful()