functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         environ = dict(
  60             os.environ,
  61             OS_USERNAME=self.project.user.name,
  62             OS_PROJECT_NAME=self.project.project.name,
  63             OS_PROJECT_ID=self.project.project.id,
  64             OS_PASSWORD=self.project.password)
  65         self.deployment_id = conf_utils.create_rally_deployment(
  66             environ=environ)
  67         if not self.deployment_id:
  68             raise Exception("Deployment create failed")
  69         self.verifier_id = conf_utils.create_verifier()
  70         if not self.verifier_id:
  71             raise Exception("Verifier create failed")
  72         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  73             self.verifier_id)
  74         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  75             self.verifier_id, self.deployment_id)
  76         self.verification_id = None
  77         self.res_dir = os.path.join(
  78             getattr(config.CONF, 'dir_results'), self.case_name)
  79         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  80         self.list = os.path.join(self.res_dir, 'test_list.txt')
  81         self.conf_file = None
  82         self.image_alt = None
  83         self.flavor_alt = None
  84         self.services = []
  85         try:
  86             self.services = kwargs['run']['args']['services']
  87         except Exception:  # pylint: disable=broad-except
  88             pass
  89         self.neutron_extensions = []
  90         try:
  91             self.neutron_extensions = kwargs['run']['args'][
  92                 'neutron_extensions']
  93         except Exception:  # pylint: disable=broad-except
  94             pass
  95
  96     def check_services(self):
  97         """Check the mandatory services."""
  98         for service in self.services:
  99             try:
 100                 self.cloud.search_services(service)[0]
 101             except Exception:  # pylint: disable=broad-except
 102                 self.is_skipped = True
 103                 break
 104
 105     def check_extensions(self):
 106         """Check the mandatory network extensions."""
 107         extensions = self.cloud.get_network_extensions()
 108         for network_extension in self.neutron_extensions:
 109             if network_extension not in extensions:
 110                 LOGGER.warning(
 111                     "Cannot find Neutron extension: %s", network_extension)
 112                 self.is_skipped = True
 113                 break
 114
 115     def check_requirements(self):
 116         self.check_services()
 117         self.check_extensions()
 118
 119     @staticmethod
 120     def read_file(filename):
 121         """Read file and return content as a stripped list."""
 122         with open(filename) as src:
 123             return [line.strip() for line in src.readlines()]
 124
 125     @staticmethod
 126     def get_verifier_result(verif_id):
 127         """Retrieve verification results."""
 128         result = {
 129             'num_tests': 0,
 130             'num_success': 0,
 131             'num_failures': 0,
 132             'num_skipped': 0
 133         }
 134         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 135         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 136         proc = subprocess.Popen(cmd,
 137                                 stdout=subprocess.PIPE,
 138                                 stderr=subprocess.STDOUT)
 139         for line in proc.stdout:
 140             LOGGER.info(line.rstrip())
 141             new_line = line.replace(' ', '').split('|')
 142             if 'Tests' in new_line:
 143                 break
 144             if 'Testscount' in new_line:
 145                 result['num_tests'] = int(new_line[2])
 146             elif 'Success' in new_line:
 147                 result['num_success'] = int(new_line[2])
 148             elif 'Skipped' in new_line:
 149                 result['num_skipped'] = int(new_line[2])
 150             elif 'Failures' in new_line:
 151                 result['num_failures'] = int(new_line[2])
 152         return result
 153
 154     @staticmethod
 155     def backup_tempest_config(conf_file, res_dir):
 156         """
 157         Copy config file to tempest results directory
 158         """
 159         if not os.path.exists(res_dir):
 160             os.makedirs(res_dir)
 161         shutil.copyfile(conf_file,
 162                         os.path.join(res_dir, 'tempest.conf'))
 163
 164     def generate_test_list(self, **kwargs):
 165         """Generate test list based on the test mode."""
 166         LOGGER.debug("Generating test case list...")
 167         self.backup_tempest_config(self.conf_file, '/etc')
 168         if kwargs.get('mode') == 'custom':
 169             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 170                 shutil.copyfile(
 171                     conf_utils.TEMPEST_CUSTOM, self.list)
 172             else:
 173                 raise Exception("Tempest test list file %s NOT found."
 174                                 % conf_utils.TEMPEST_CUSTOM)
 175         else:
 176             testr_mode = kwargs.get(
 177                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 178             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 179                 self.verifier_repo_dir, testr_mode, self.list)
 180             output = subprocess.check_output(cmd, shell=True)
 181             LOGGER.info("%s\n%s", cmd, output)
 182         os.remove('/etc/tempest.conf')
 183
 184     def apply_tempest_blacklist(self):
 185         """Exclude blacklisted test cases."""
 186         LOGGER.debug("Applying tempest blacklist...")
 187         if os.path.exists(self.raw_list):
 188             os.remove(self.raw_list)
 189         os.rename(self.list, self.raw_list)
 190         cases_file = self.read_file(self.raw_list)
 191         result_file = open(self.list, 'w')
 192         black_tests = []
 193         try:
 194             deploy_scenario = env.get('DEPLOY_SCENARIO')
 195             if bool(deploy_scenario):
 196                 # if DEPLOY_SCENARIO is set we read the file
 197                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 198                 black_list_yaml = yaml.safe_load(black_list_file)
 199                 black_list_file.close()
 200                 for item in black_list_yaml:
 201                     scenarios = item['scenarios']
 202                     if deploy_scenario in scenarios:
 203                         tests = item['tests']
 204                         for test in tests:
 205                             black_tests.append(test)
 206                         break
 207         except Exception:  # pylint: disable=broad-except
 208             black_tests = []
 209             LOGGER.debug("Tempest blacklist file does not exist.")
 210
 211         for cases_line in cases_file:
 212             for black_tests_line in black_tests:
 213                 if black_tests_line in cases_line:
 214                     break
 215             else:
 216                 result_file.write(str(cases_line) + '\n')
 217         result_file.close()
 218
 219     def run_verifier_tests(self, **kwargs):
 220         """Execute tempest test cases."""
 221         cmd = ["rally", "verify", "start", "--load-list",
 222                self.list]
 223         cmd.extend(kwargs.get('option', []))
 224         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 225
 226         f_stdout = open(
 227             os.path.join(self.res_dir, "tempest.log"), 'w+')
 228
 229         proc = subprocess.Popen(
 230             cmd,
 231             stdout=subprocess.PIPE,
 232             stderr=subprocess.STDOUT,
 233             bufsize=1)
 234
 235         with proc.stdout:
 236             for line in iter(proc.stdout.readline, b''):
 237                 if re.search(r"\} tempest\.", line):
 238                     LOGGER.info(line.rstrip())
 239                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 240                     self.verification_id = re.search(
 241                         r'(?=\(UUID=(.*)\))', line).group(1)
 242                 f_stdout.write(line)
 243         proc.wait()
 244         f_stdout.close()
 245
 246         if self.verification_id is None:
 247             raise Exception('Verification UUID not found')
 248         LOGGER.info('Verification UUID: %s', self.verification_id)
 249
 250         shutil.copy(
 251             "{}/tempest.log".format(self.deployment_dir),
 252             "{}/tempest.debug.log".format(self.res_dir))
 253
 254     def parse_verifier_result(self):
 255         """Parse and save test results."""
 256         stat = self.get_verifier_result(self.verification_id)
 257         try:
 258             num_executed = stat['num_tests'] - stat['num_skipped']
 259             try:
 260                 self.result = 100 * stat['num_success'] / num_executed
 261             except ZeroDivisionError:
 262                 self.result = 0
 263                 if stat['num_tests'] > 0:
 264                     LOGGER.info("All tests have been skipped")
 265                 else:
 266                     LOGGER.error("No test has been executed")
 267                     return
 268
 269             with open(os.path.join(self.res_dir,
 270                                    "tempest.log"), 'r') as logfile:
 271                 output = logfile.read()
 272
 273             success_testcases = []
 274             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 275                                     output):
 276                 success_testcases.append(match)
 277             failed_testcases = []
 278             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 279                                     output):
 280                 failed_testcases.append(match)
 281             skipped_testcases = []
 282             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 283                                     output):
 284                 skipped_testcases.append(match)
 285
 286             self.details = {"tests_number": stat['num_tests'],
 287                             "success_number": stat['num_success'],
 288                             "skipped_number": stat['num_skipped'],
 289                             "failures_number": stat['num_failures'],
 290                             "success": success_testcases,
 291                             "skipped": skipped_testcases,
 292                             "failures": failed_testcases}
 293         except Exception:  # pylint: disable=broad-except
 294             self.result = 0
 295
 296         LOGGER.info("Tempest %s success_rate is %s%%",
 297                     self.case_name, self.result)
 298
 299     def generate_report(self):
 300         """Generate verification report."""
 301         html_file = os.path.join(self.res_dir,
 302                                  "tempest-report.html")
 303         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 304                self.verification_id, "--to", html_file]
 305         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 306                          stderr=subprocess.STDOUT)
 307
 308     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 309         """Set image name as tempest img_name_regex"""
 310         rconfig = configparser.RawConfigParser()
 311         rconfig.read(rally_conf)
 312         if not rconfig.has_section('openstack'):
 313             rconfig.add_section('openstack')
 314         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 315             self.image.name))
 316         with open(rally_conf, 'wb') as config_file:
 317             rconfig.write(config_file)
 318
 319     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 320         """Detect and update the default role if required"""
 321         role = self.get_default_role(self.cloud)
 322         if not role:
 323             return
 324         rconfig = configparser.RawConfigParser()
 325         rconfig.read(rally_conf)
 326         if not rconfig.has_section('openstack'):
 327             rconfig.add_section('openstack')
 328         rconfig.set('openstack', 'swift_operator_role', role.name)
 329         with open(rally_conf, 'wb') as config_file:
 330             rconfig.write(config_file)
 331
 332     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 333         """Print rally logs in res dir"""
 334         if not os.path.exists(self.res_dir):
 335             os.makedirs(self.res_dir)
 336         rconfig = configparser.RawConfigParser()
 337         rconfig.read(rally_conf)
 338         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 339         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 340         with open(rally_conf, 'wb') as config_file:
 341             rconfig.write(config_file)
 342
 343     @staticmethod
 344     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 345         """Clean Rally config"""
 346         rconfig = configparser.RawConfigParser()
 347         rconfig.read(rally_conf)
 348         if rconfig.has_option('openstack', 'img_name_regex'):
 349             rconfig.remove_option('openstack', 'img_name_regex')
 350         if rconfig.has_option('openstack', 'swift_operator_role'):
 351             rconfig.remove_option('openstack', 'swift_operator_role')
 352         if rconfig.has_option('DEFAULT', 'log-file'):
 353             rconfig.remove_option('DEFAULT', 'log-file')
 354         if rconfig.has_option('DEFAULT', 'log_dir'):
 355             rconfig.remove_option('DEFAULT', 'log_dir')
 356         with open(rally_conf, 'wb') as config_file:
 357             rconfig.write(config_file)
 358
 359     def update_scenario_section(self):
 360         """Update scenario section in tempest.conf"""
 361         rconfig = configparser.RawConfigParser()
 362         rconfig.read(self.conf_file)
 363         filename = getattr(
 364             config.CONF, '{}_image'.format(self.case_name), self.filename)
 365         if not rconfig.has_section('scenario'):
 366             rconfig.add_section('scenario')
 367         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 368         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 369         rconfig.set('scenario', 'img_disk_format', getattr(
 370             config.CONF, '{}_image_format'.format(self.case_name),
 371             self.image_format))
 372         extra_properties = self.extra_properties.copy()
 373         extra_properties.update(
 374             getattr(config.CONF, '{}_extra_properties'.format(
 375                 self.case_name), {}))
 376         rconfig.set(
 377             'scenario', 'img_properties',
 378             conf_utils.convert_dict_to_ini(extra_properties))
 379         with open(self.conf_file, 'wb') as config_file:
 380             rconfig.write(config_file)
 381
 382     def configure(self, **kwargs):  # pylint: disable=unused-argument
 383         """
 384         Create all openstack resources for tempest-based testcases and write
 385         tempest.conf.
 386         """
 387         if not os.path.exists(self.res_dir):
 388             os.makedirs(self.res_dir)
 389         compute_cnt = len(self.cloud.list_hypervisors())
 390
 391         self.image_alt = self.publish_image_alt()
 392         self.flavor_alt = self.create_flavor_alt()
 393         LOGGER.debug("flavor: %s", self.flavor_alt)
 394
 395         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 396         if not self.conf_file:
 397             raise Exception("Tempest verifier configuring failed")
 398         conf_utils.configure_tempest_update_params(
 399             self.conf_file, network_name=self.network.name,
 400             image_id=self.image.id,
 401             flavor_id=self.flavor.id,
 402             compute_cnt=compute_cnt,
 403             image_alt_id=self.image_alt.id,
 404             flavor_alt_id=self.flavor_alt.id,
 405             domain_name=self.cloud.auth.get("project_domain_name", "Default"))
 406         self.update_scenario_section()
 407         self.backup_tempest_config(self.conf_file, self.res_dir)
 408
 409     def run(self, **kwargs):
 410         self.start_time = time.time()
 411         try:
 412             assert super(TempestCommon, self).run(
 413                 **kwargs) == testcase.TestCase.EX_OK
 414             if not os.path.exists(self.res_dir):
 415                 os.makedirs(self.res_dir)
 416             self.update_rally_regex()
 417             self.update_default_role()
 418             self.update_rally_logs()
 419             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 420             self.configure(**kwargs)
 421             self.generate_test_list(**kwargs)
 422             self.apply_tempest_blacklist()
 423             self.run_verifier_tests(**kwargs)
 424             self.parse_verifier_result()
 425             self.generate_report()
 426             res = testcase.TestCase.EX_OK
 427         except Exception:  # pylint: disable=broad-except
 428             LOGGER.exception('Error with run')
 429             self.result = 0
 430             res = testcase.TestCase.EX_RUN_ERROR
 431         self.stop_time = time.time()
 432         return res
 433
 434     def clean(self):
 435         """
 436         Cleanup all OpenStack objects. Should be called on completion.
 437         """
 438         self.clean_rally_conf()
 439         if self.image_alt:
 440             self.cloud.delete_image(self.image_alt)
 441         if self.flavor_alt:
 442             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 443         super(TempestCommon, self).clean()