functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30 from functest.utils import functest_utils
  31
  32 LOGGER = logging.getLogger(__name__)
  33
  34
  35 class TempestCommon(singlevm.VmReady2):
  36     # pylint: disable=too-many-instance-attributes
  37     """TempestCommon testcases implementation class."""
  38
  39     visibility = 'public'
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             self.role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             self.role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             self.role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         self.orig_cloud.grant_role(
  60             self.role_name, user=self.project.user.id,
  61             domain=self.project.domain.id)
  62         self.deployment_id = None
  63         self.verifier_id = None
  64         self.verifier_repo_dir = None
  65         self.deployment_dir = None
  66         self.verification_id = None
  67         self.res_dir = os.path.join(
  68             getattr(config.CONF, 'dir_results'), self.case_name)
  69         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  70         self.list = os.path.join(self.res_dir, 'test_list.txt')
  71         self.conf_file = None
  72         self.image_alt = None
  73         self.flavor_alt = None
  74         self.services = []
  75         try:
  76             self.services = kwargs['run']['args']['services']
  77         except Exception:  # pylint: disable=broad-except
  78             pass
  79         self.neutron_extensions = []
  80         try:
  81             self.neutron_extensions = kwargs['run']['args'][
  82                 'neutron_extensions']
  83         except Exception:  # pylint: disable=broad-except
  84             pass
  85
  86     def create_network_resources(self):
  87         pass
  88
  89     def check_services(self):
  90         """Check the mandatory services."""
  91         for service in self.services:
  92             try:
  93                 self.cloud.search_services(service)[0]
  94             except Exception:  # pylint: disable=broad-except
  95                 self.is_skipped = True
  96                 break
  97
  98     def check_extensions(self):
  99         """Check the mandatory network extensions."""
 100         extensions = self.cloud.get_network_extensions()
 101         for network_extension in self.neutron_extensions:
 102             if network_extension not in extensions:
 103                 LOGGER.warning(
 104                     "Cannot find Neutron extension: %s", network_extension)
 105                 self.is_skipped = True
 106                 break
 107
 108     def check_requirements(self):
 109         self.check_services()
 110         self.check_extensions()
 111         if self.is_skipped:
 112             self.project.clean()
 113
 114     @staticmethod
 115     def read_file(filename):
 116         """Read file and return content as a stripped list."""
 117         with open(filename) as src:
 118             return [line.strip() for line in src.readlines()]
 119
 120     @staticmethod
 121     def get_verifier_result(verif_id):
 122         """Retrieve verification results."""
 123         result = {
 124             'num_tests': 0,
 125             'num_success': 0,
 126             'num_failures': 0,
 127             'num_skipped': 0
 128         }
 129         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 130         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 131         proc = subprocess.Popen(cmd,
 132                                 stdout=subprocess.PIPE,
 133                                 stderr=subprocess.STDOUT)
 134         for line in proc.stdout:
 135             LOGGER.info(line.rstrip())
 136             new_line = line.replace(' ', '').split('|')
 137             if 'Tests' in new_line:
 138                 break
 139             if 'Testscount' in new_line:
 140                 result['num_tests'] = int(new_line[2])
 141             elif 'Success' in new_line:
 142                 result['num_success'] = int(new_line[2])
 143             elif 'Skipped' in new_line:
 144                 result['num_skipped'] = int(new_line[2])
 145             elif 'Failures' in new_line:
 146                 result['num_failures'] = int(new_line[2])
 147         return result
 148
 149     @staticmethod
 150     def backup_tempest_config(conf_file, res_dir):
 151         """
 152         Copy config file to tempest results directory
 153         """
 154         if not os.path.exists(res_dir):
 155             os.makedirs(res_dir)
 156         shutil.copyfile(conf_file,
 157                         os.path.join(res_dir, 'tempest.conf'))
 158
 159     def generate_test_list(self, **kwargs):
 160         """Generate test list based on the test mode."""
 161         LOGGER.debug("Generating test case list...")
 162         self.backup_tempest_config(self.conf_file, '/etc')
 163         if kwargs.get('mode') == 'custom':
 164             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 165                 shutil.copyfile(
 166                     conf_utils.TEMPEST_CUSTOM, self.list)
 167             else:
 168                 raise Exception("Tempest test list file %s NOT found."
 169                                 % conf_utils.TEMPEST_CUSTOM)
 170         else:
 171             testr_mode = kwargs.get(
 172                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 173             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 174                 self.verifier_repo_dir, testr_mode, self.list)
 175             output = subprocess.check_output(cmd, shell=True)
 176             LOGGER.info("%s\n%s", cmd, output)
 177         os.remove('/etc/tempest.conf')
 178
 179     def apply_tempest_blacklist(self):
 180         """Exclude blacklisted test cases."""
 181         LOGGER.debug("Applying tempest blacklist...")
 182         if os.path.exists(self.raw_list):
 183             os.remove(self.raw_list)
 184         os.rename(self.list, self.raw_list)
 185         cases_file = self.read_file(self.raw_list)
 186         result_file = open(self.list, 'w')
 187         black_tests = []
 188         try:
 189             deploy_scenario = env.get('DEPLOY_SCENARIO')
 190             if bool(deploy_scenario):
 191                 # if DEPLOY_SCENARIO is set we read the file
 192                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 193                 black_list_yaml = yaml.safe_load(black_list_file)
 194                 black_list_file.close()
 195                 for item in black_list_yaml:
 196                     scenarios = item['scenarios']
 197                     if deploy_scenario in scenarios:
 198                         tests = item['tests']
 199                         for test in tests:
 200                             black_tests.append(test)
 201                         break
 202         except Exception:  # pylint: disable=broad-except
 203             black_tests = []
 204             LOGGER.debug("Tempest blacklist file does not exist.")
 205
 206         for cases_line in cases_file:
 207             for black_tests_line in black_tests:
 208                 if black_tests_line in cases_line:
 209                     break
 210             else:
 211                 result_file.write(str(cases_line) + '\n')
 212         result_file.close()
 213
 214     def run_verifier_tests(self, **kwargs):
 215         """Execute tempest test cases."""
 216         cmd = ["rally", "verify", "start", "--load-list",
 217                self.list]
 218         cmd.extend(kwargs.get('option', []))
 219         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 220
 221         f_stdout = open(
 222             os.path.join(self.res_dir, "tempest.log"), 'w+')
 223
 224         proc = subprocess.Popen(
 225             cmd,
 226             stdout=subprocess.PIPE,
 227             stderr=subprocess.STDOUT,
 228             bufsize=1)
 229
 230         with proc.stdout:
 231             for line in iter(proc.stdout.readline, b''):
 232                 if re.search(r"\} tempest\.", line):
 233                     LOGGER.info(line.rstrip())
 234                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 235                     self.verification_id = re.search(
 236                         r'(?=\(UUID=(.*)\))', line).group(1)
 237                 f_stdout.write(line)
 238         proc.wait()
 239         f_stdout.close()
 240
 241         if self.verification_id is None:
 242             raise Exception('Verification UUID not found')
 243         LOGGER.info('Verification UUID: %s', self.verification_id)
 244
 245         shutil.copy(
 246             "{}/tempest.log".format(self.deployment_dir),
 247             "{}/tempest.debug.log".format(self.res_dir))
 248
 249     def parse_verifier_result(self):
 250         """Parse and save test results."""
 251         stat = self.get_verifier_result(self.verification_id)
 252         try:
 253             num_executed = stat['num_tests'] - stat['num_skipped']
 254             try:
 255                 self.result = 100 * stat['num_success'] / num_executed
 256             except ZeroDivisionError:
 257                 self.result = 0
 258                 if stat['num_tests'] > 0:
 259                     LOGGER.info("All tests have been skipped")
 260                 else:
 261                     LOGGER.error("No test has been executed")
 262                     return
 263
 264             with open(os.path.join(self.res_dir,
 265                                    "tempest.log"), 'r') as logfile:
 266                 output = logfile.read()
 267
 268             success_testcases = []
 269             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 270                                     output):
 271                 success_testcases.append(match)
 272             failed_testcases = []
 273             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 274                                     output):
 275                 failed_testcases.append(match)
 276             skipped_testcases = []
 277             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 278                                     output):
 279                 skipped_testcases.append(match)
 280
 281             self.details = {"tests_number": stat['num_tests'],
 282                             "success_number": stat['num_success'],
 283                             "skipped_number": stat['num_skipped'],
 284                             "failures_number": stat['num_failures'],
 285                             "success": success_testcases,
 286                             "skipped": skipped_testcases,
 287                             "failures": failed_testcases}
 288         except Exception:  # pylint: disable=broad-except
 289             self.result = 0
 290
 291         LOGGER.info("Tempest %s success_rate is %s%%",
 292                     self.case_name, self.result)
 293
 294     def generate_report(self):
 295         """Generate verification report."""
 296         html_file = os.path.join(self.res_dir,
 297                                  "tempest-report.html")
 298         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 299                self.verification_id, "--to", str(html_file)]
 300         subprocess.check_output(cmd)
 301
 302     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 303         """Set image name as tempest img_name_regex"""
 304         rconfig = configparser.RawConfigParser()
 305         rconfig.read(rally_conf)
 306         if not rconfig.has_section('openstack'):
 307             rconfig.add_section('openstack')
 308         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 309             self.image.name))
 310         with open(rally_conf, 'wb') as config_file:
 311             rconfig.write(config_file)
 312
 313     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 314         """Detect and update the default role if required"""
 315         role = self.get_default_role(self.cloud)
 316         if not role:
 317             return
 318         rconfig = configparser.RawConfigParser()
 319         rconfig.read(rally_conf)
 320         if not rconfig.has_section('openstack'):
 321             rconfig.add_section('openstack')
 322         rconfig.set('openstack', 'swift_operator_role', role.name)
 323         with open(rally_conf, 'wb') as config_file:
 324             rconfig.write(config_file)
 325
 326     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 327         """Print rally logs in res dir"""
 328         if not os.path.exists(self.res_dir):
 329             os.makedirs(self.res_dir)
 330         rconfig = configparser.RawConfigParser()
 331         rconfig.read(rally_conf)
 332         rconfig.set('DEFAULT', 'debug', True)
 333         rconfig.set('DEFAULT', 'use_stderr', False)
 334         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 335         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 336         with open(rally_conf, 'wb') as config_file:
 337             rconfig.write(config_file)
 338
 339     @staticmethod
 340     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 341         """Clean Rally config"""
 342         rconfig = configparser.RawConfigParser()
 343         rconfig.read(rally_conf)
 344         if rconfig.has_option('openstack', 'img_name_regex'):
 345             rconfig.remove_option('openstack', 'img_name_regex')
 346         if rconfig.has_option('openstack', 'swift_operator_role'):
 347             rconfig.remove_option('openstack', 'swift_operator_role')
 348         if rconfig.has_option('DEFAULT', 'use_stderr'):
 349             rconfig.remove_option('DEFAULT', 'use_stderr')
 350         if rconfig.has_option('DEFAULT', 'debug'):
 351             rconfig.remove_option('DEFAULT', 'debug')
 352         if rconfig.has_option('DEFAULT', 'log-file'):
 353             rconfig.remove_option('DEFAULT', 'log-file')
 354         if rconfig.has_option('DEFAULT', 'log_dir'):
 355             rconfig.remove_option('DEFAULT', 'log_dir')
 356         with open(rally_conf, 'wb') as config_file:
 357             rconfig.write(config_file)
 358
 359     def update_scenario_section(self):
 360         """Update scenario section in tempest.conf"""
 361         rconfig = configparser.RawConfigParser()
 362         rconfig.read(self.conf_file)
 363         filename = getattr(
 364             config.CONF, '{}_image'.format(self.case_name), self.filename)
 365         if not rconfig.has_section('scenario'):
 366             rconfig.add_section('scenario')
 367         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 368         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 369         rconfig.set('scenario', 'img_disk_format', getattr(
 370             config.CONF, '{}_image_format'.format(self.case_name),
 371             self.image_format))
 372         extra_properties = self.extra_properties.copy()
 373         if env.get('IMAGE_PROPERTIES'):
 374             extra_properties.update(
 375                 functest_utils.convert_ini_to_dict(
 376                     env.get('IMAGE_PROPERTIES')))
 377         extra_properties.update(
 378             getattr(config.CONF, '{}_extra_properties'.format(
 379                 self.case_name), {}))
 380         rconfig.set(
 381             'scenario', 'img_properties',
 382             functest_utils.convert_dict_to_ini(extra_properties))
 383         with open(self.conf_file, 'wb') as config_file:
 384             rconfig.write(config_file)
 385
 386     def configure(self, **kwargs):  # pylint: disable=unused-argument
 387         """
 388         Create all openstack resources for tempest-based testcases and write
 389         tempest.conf.
 390         """
 391         if not os.path.exists(self.res_dir):
 392             os.makedirs(self.res_dir)
 393         environ = dict(
 394             os.environ,
 395             OS_USERNAME=self.project.user.name,
 396             OS_PROJECT_NAME=self.project.project.name,
 397             OS_PROJECT_ID=self.project.project.id,
 398             OS_PASSWORD=self.project.password)
 399         try:
 400             del environ['OS_TENANT_NAME']
 401             del environ['OS_TENANT_ID']
 402         except Exception:  # pylint: disable=broad-except
 403             pass
 404         self.deployment_id = conf_utils.create_rally_deployment(
 405             environ=environ)
 406         if not self.deployment_id:
 407             raise Exception("Deployment create failed")
 408         self.verifier_id = conf_utils.create_verifier()
 409         if not self.verifier_id:
 410             raise Exception("Verifier create failed")
 411         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
 412             self.verifier_id)
 413         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
 414             self.verifier_id, self.deployment_id)
 415
 416         compute_cnt = len(self.orig_cloud.list_hypervisors())
 417
 418         self.image_alt = self.publish_image_alt()
 419         self.flavor_alt = self.create_flavor_alt()
 420         LOGGER.debug("flavor: %s", self.flavor_alt)
 421
 422         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 423         if not self.conf_file:
 424             raise Exception("Tempest verifier configuring failed")
 425         conf_utils.configure_tempest_update_params(
 426             self.conf_file,
 427             image_id=self.image.id,
 428             flavor_id=self.flavor.id,
 429             compute_cnt=compute_cnt,
 430             image_alt_id=self.image_alt.id,
 431             flavor_alt_id=self.flavor_alt.id,
 432             admin_role_name=self.role_name, cidr=self.cidr,
 433             domain_id=self.project.domain.id)
 434         self.update_scenario_section()
 435         self.backup_tempest_config(self.conf_file, self.res_dir)
 436
 437     def run(self, **kwargs):
 438         self.start_time = time.time()
 439         try:
 440             assert super(TempestCommon, self).run(
 441                 **kwargs) == testcase.TestCase.EX_OK
 442             if not os.path.exists(self.res_dir):
 443                 os.makedirs(self.res_dir)
 444             self.update_rally_regex()
 445             self.update_default_role()
 446             self.update_rally_logs()
 447             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 448             self.configure(**kwargs)
 449             self.generate_test_list(**kwargs)
 450             self.apply_tempest_blacklist()
 451             self.run_verifier_tests(**kwargs)
 452             self.parse_verifier_result()
 453             self.generate_report()
 454             res = testcase.TestCase.EX_OK
 455         except Exception:  # pylint: disable=broad-except
 456             LOGGER.exception('Error with run')
 457             self.result = 0
 458             res = testcase.TestCase.EX_RUN_ERROR
 459         self.stop_time = time.time()
 460         return res
 461
 462     def clean(self):
 463         """
 464         Cleanup all OpenStack objects. Should be called on completion.
 465         """
 466         self.clean_rally_conf()
 467         if self.image_alt:
 468             self.cloud.delete_image(self.image_alt)
 469         if self.flavor_alt:
 470             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 471         super(TempestCommon, self).clean()