functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  40
  41     def __init__(self, **kwargs):
  42         if "case_name" not in kwargs:
  43             kwargs["case_name"] = 'tempest'
  44         super(TempestCommon, self).__init__(**kwargs)
  45         assert self.orig_cloud
  46         assert self.cloud
  47         assert self.project
  48         if self.orig_cloud.get_role("admin"):
  49             role_name = "admin"
  50         elif self.orig_cloud.get_role("Admin"):
  51             role_name = "Admin"
  52         else:
  53             raise Exception("Cannot detect neither admin nor Admin")
  54         self.orig_cloud.grant_role(
  55             role_name, user=self.project.user.id,
  56             project=self.project.project.id,
  57             domain=self.project.domain.id)
  58         self.orig_cloud.grant_role(
  59             role_name, user=self.project.user.id,
  60             domain=self.project.domain.id)
  61         environ = dict(
  62             os.environ,
  63             OS_USERNAME=self.project.user.name,
  64             OS_PROJECT_NAME=self.project.project.name,
  65             OS_PROJECT_ID=self.project.project.id,
  66             OS_PASSWORD=self.project.password)
  67         self.deployment_id = conf_utils.create_rally_deployment(
  68             environ=environ)
  69         if not self.deployment_id:
  70             raise Exception("Deployment create failed")
  71         self.verifier_id = conf_utils.create_verifier()
  72         if not self.verifier_id:
  73             raise Exception("Verifier create failed")
  74         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  75             self.verifier_id)
  76         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  77             self.verifier_id, self.deployment_id)
  78         self.verification_id = None
  79         self.res_dir = os.path.join(
  80             getattr(config.CONF, 'dir_results'), self.case_name)
  81         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  82         self.list = os.path.join(self.res_dir, 'test_list.txt')
  83         self.conf_file = None
  84         self.image_alt = None
  85         self.flavor_alt = None
  86         self.services = []
  87         try:
  88             self.services = kwargs['run']['args']['services']
  89         except Exception:  # pylint: disable=broad-except
  90             pass
  91         self.neutron_extensions = []
  92         try:
  93             self.neutron_extensions = kwargs['run']['args'][
  94                 'neutron_extensions']
  95         except Exception:  # pylint: disable=broad-except
  96             pass
  97
  98     def create_network_resources(self):
  99         pass
 100
 101     def check_services(self):
 102         """Check the mandatory services."""
 103         for service in self.services:
 104             try:
 105                 self.cloud.search_services(service)[0]
 106             except Exception:  # pylint: disable=broad-except
 107                 self.is_skipped = True
 108                 break
 109
 110     def check_extensions(self):
 111         """Check the mandatory network extensions."""
 112         extensions = self.cloud.get_network_extensions()
 113         for network_extension in self.neutron_extensions:
 114             if network_extension not in extensions:
 115                 LOGGER.warning(
 116                     "Cannot find Neutron extension: %s", network_extension)
 117                 self.is_skipped = True
 118                 break
 119
 120     def check_requirements(self):
 121         self.check_services()
 122         self.check_extensions()
 123         if self.is_skipped:
 124             self.project.clean()
 125
 126     @staticmethod
 127     def read_file(filename):
 128         """Read file and return content as a stripped list."""
 129         with open(filename) as src:
 130             return [line.strip() for line in src.readlines()]
 131
 132     @staticmethod
 133     def get_verifier_result(verif_id):
 134         """Retrieve verification results."""
 135         result = {
 136             'num_tests': 0,
 137             'num_success': 0,
 138             'num_failures': 0,
 139             'num_skipped': 0
 140         }
 141         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 142         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 143         proc = subprocess.Popen(cmd,
 144                                 stdout=subprocess.PIPE,
 145                                 stderr=subprocess.STDOUT)
 146         for line in proc.stdout:
 147             LOGGER.info(line.rstrip())
 148             new_line = line.replace(' ', '').split('|')
 149             if 'Tests' in new_line:
 150                 break
 151             if 'Testscount' in new_line:
 152                 result['num_tests'] = int(new_line[2])
 153             elif 'Success' in new_line:
 154                 result['num_success'] = int(new_line[2])
 155             elif 'Skipped' in new_line:
 156                 result['num_skipped'] = int(new_line[2])
 157             elif 'Failures' in new_line:
 158                 result['num_failures'] = int(new_line[2])
 159         return result
 160
 161     @staticmethod
 162     def backup_tempest_config(conf_file, res_dir):
 163         """
 164         Copy config file to tempest results directory
 165         """
 166         if not os.path.exists(res_dir):
 167             os.makedirs(res_dir)
 168         shutil.copyfile(conf_file,
 169                         os.path.join(res_dir, 'tempest.conf'))
 170
 171     def generate_test_list(self, **kwargs):
 172         """Generate test list based on the test mode."""
 173         LOGGER.debug("Generating test case list...")
 174         self.backup_tempest_config(self.conf_file, '/etc')
 175         if kwargs.get('mode') == 'custom':
 176             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 177                 shutil.copyfile(
 178                     conf_utils.TEMPEST_CUSTOM, self.list)
 179             else:
 180                 raise Exception("Tempest test list file %s NOT found."
 181                                 % conf_utils.TEMPEST_CUSTOM)
 182         else:
 183             testr_mode = kwargs.get(
 184                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 185             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 186                 self.verifier_repo_dir, testr_mode, self.list)
 187             output = subprocess.check_output(cmd, shell=True)
 188             LOGGER.info("%s\n%s", cmd, output)
 189         os.remove('/etc/tempest.conf')
 190
 191     def apply_tempest_blacklist(self):
 192         """Exclude blacklisted test cases."""
 193         LOGGER.debug("Applying tempest blacklist...")
 194         if os.path.exists(self.raw_list):
 195             os.remove(self.raw_list)
 196         os.rename(self.list, self.raw_list)
 197         cases_file = self.read_file(self.raw_list)
 198         result_file = open(self.list, 'w')
 199         black_tests = []
 200         try:
 201             deploy_scenario = env.get('DEPLOY_SCENARIO')
 202             if bool(deploy_scenario):
 203                 # if DEPLOY_SCENARIO is set we read the file
 204                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 205                 black_list_yaml = yaml.safe_load(black_list_file)
 206                 black_list_file.close()
 207                 for item in black_list_yaml:
 208                     scenarios = item['scenarios']
 209                     if deploy_scenario in scenarios:
 210                         tests = item['tests']
 211                         for test in tests:
 212                             black_tests.append(test)
 213                         break
 214         except Exception:  # pylint: disable=broad-except
 215             black_tests = []
 216             LOGGER.debug("Tempest blacklist file does not exist.")
 217
 218         for cases_line in cases_file:
 219             for black_tests_line in black_tests:
 220                 if black_tests_line in cases_line:
 221                     break
 222             else:
 223                 result_file.write(str(cases_line) + '\n')
 224         result_file.close()
 225
 226     def run_verifier_tests(self, **kwargs):
 227         """Execute tempest test cases."""
 228         cmd = ["rally", "verify", "start", "--load-list",
 229                self.list]
 230         cmd.extend(kwargs.get('option', []))
 231         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 232
 233         f_stdout = open(
 234             os.path.join(self.res_dir, "tempest.log"), 'w+')
 235
 236         proc = subprocess.Popen(
 237             cmd,
 238             stdout=subprocess.PIPE,
 239             stderr=subprocess.STDOUT,
 240             bufsize=1)
 241
 242         with proc.stdout:
 243             for line in iter(proc.stdout.readline, b''):
 244                 if re.search(r"\} tempest\.", line):
 245                     LOGGER.info(line.rstrip())
 246                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 247                     self.verification_id = re.search(
 248                         r'(?=\(UUID=(.*)\))', line).group(1)
 249                 f_stdout.write(line)
 250         proc.wait()
 251         f_stdout.close()
 252
 253         if self.verification_id is None:
 254             raise Exception('Verification UUID not found')
 255         LOGGER.info('Verification UUID: %s', self.verification_id)
 256
 257         shutil.copy(
 258             "{}/tempest.log".format(self.deployment_dir),
 259             "{}/tempest.debug.log".format(self.res_dir))
 260
 261     def parse_verifier_result(self):
 262         """Parse and save test results."""
 263         stat = self.get_verifier_result(self.verification_id)
 264         try:
 265             num_executed = stat['num_tests'] - stat['num_skipped']
 266             try:
 267                 self.result = 100 * stat['num_success'] / num_executed
 268             except ZeroDivisionError:
 269                 self.result = 0
 270                 if stat['num_tests'] > 0:
 271                     LOGGER.info("All tests have been skipped")
 272                 else:
 273                     LOGGER.error("No test has been executed")
 274                     return
 275
 276             with open(os.path.join(self.res_dir,
 277                                    "tempest.log"), 'r') as logfile:
 278                 output = logfile.read()
 279
 280             success_testcases = []
 281             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 282                                     output):
 283                 success_testcases.append(match)
 284             failed_testcases = []
 285             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 286                                     output):
 287                 failed_testcases.append(match)
 288             skipped_testcases = []
 289             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 290                                     output):
 291                 skipped_testcases.append(match)
 292
 293             self.details = {"tests_number": stat['num_tests'],
 294                             "success_number": stat['num_success'],
 295                             "skipped_number": stat['num_skipped'],
 296                             "failures_number": stat['num_failures'],
 297                             "success": success_testcases,
 298                             "skipped": skipped_testcases,
 299                             "failures": failed_testcases}
 300         except Exception:  # pylint: disable=broad-except
 301             self.result = 0
 302
 303         LOGGER.info("Tempest %s success_rate is %s%%",
 304                     self.case_name, self.result)
 305
 306     def generate_report(self):
 307         """Generate verification report."""
 308         html_file = os.path.join(self.res_dir,
 309                                  "tempest-report.html")
 310         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 311                self.verification_id, "--to", html_file]
 312         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 313                          stderr=subprocess.STDOUT)
 314
 315     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 316         """Set image name as tempest img_name_regex"""
 317         rconfig = configparser.RawConfigParser()
 318         rconfig.read(rally_conf)
 319         if not rconfig.has_section('openstack'):
 320             rconfig.add_section('openstack')
 321         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 322             self.image.name))
 323         with open(rally_conf, 'wb') as config_file:
 324             rconfig.write(config_file)
 325
 326     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 327         """Detect and update the default role if required"""
 328         role = self.get_default_role(self.cloud)
 329         if not role:
 330             return
 331         rconfig = configparser.RawConfigParser()
 332         rconfig.read(rally_conf)
 333         if not rconfig.has_section('openstack'):
 334             rconfig.add_section('openstack')
 335         rconfig.set('openstack', 'swift_operator_role', role.name)
 336         with open(rally_conf, 'wb') as config_file:
 337             rconfig.write(config_file)
 338
 339     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 340         """Print rally logs in res dir"""
 341         if not os.path.exists(self.res_dir):
 342             os.makedirs(self.res_dir)
 343         rconfig = configparser.RawConfigParser()
 344         rconfig.read(rally_conf)
 345         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 346         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 347         with open(rally_conf, 'wb') as config_file:
 348             rconfig.write(config_file)
 349
 350     @staticmethod
 351     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 352         """Clean Rally config"""
 353         rconfig = configparser.RawConfigParser()
 354         rconfig.read(rally_conf)
 355         if rconfig.has_option('openstack', 'img_name_regex'):
 356             rconfig.remove_option('openstack', 'img_name_regex')
 357         if rconfig.has_option('openstack', 'swift_operator_role'):
 358             rconfig.remove_option('openstack', 'swift_operator_role')
 359         if rconfig.has_option('DEFAULT', 'log-file'):
 360             rconfig.remove_option('DEFAULT', 'log-file')
 361         if rconfig.has_option('DEFAULT', 'log_dir'):
 362             rconfig.remove_option('DEFAULT', 'log_dir')
 363         with open(rally_conf, 'wb') as config_file:
 364             rconfig.write(config_file)
 365
 366     def update_scenario_section(self):
 367         """Update scenario section in tempest.conf"""
 368         rconfig = configparser.RawConfigParser()
 369         rconfig.read(self.conf_file)
 370         filename = getattr(
 371             config.CONF, '{}_image'.format(self.case_name), self.filename)
 372         if not rconfig.has_section('scenario'):
 373             rconfig.add_section('scenario')
 374         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 375         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 376         rconfig.set('scenario', 'img_disk_format', getattr(
 377             config.CONF, '{}_image_format'.format(self.case_name),
 378             self.image_format))
 379         extra_properties = self.extra_properties.copy()
 380         extra_properties.update(
 381             getattr(config.CONF, '{}_extra_properties'.format(
 382                 self.case_name), {}))
 383         rconfig.set(
 384             'scenario', 'img_properties',
 385             conf_utils.convert_dict_to_ini(extra_properties))
 386         with open(self.conf_file, 'wb') as config_file:
 387             rconfig.write(config_file)
 388
 389     def configure(self, **kwargs):  # pylint: disable=unused-argument
 390         """
 391         Create all openstack resources for tempest-based testcases and write
 392         tempest.conf.
 393         """
 394         if not os.path.exists(self.res_dir):
 395             os.makedirs(self.res_dir)
 396         compute_cnt = len(self.cloud.list_hypervisors())
 397
 398         self.image_alt = self.publish_image_alt()
 399         self.flavor_alt = self.create_flavor_alt()
 400         LOGGER.debug("flavor: %s", self.flavor_alt)
 401
 402         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 403         if not self.conf_file:
 404             raise Exception("Tempest verifier configuring failed")
 405         conf_utils.configure_tempest_update_params(
 406             self.conf_file,
 407             image_id=self.image.id,
 408             flavor_id=self.flavor.id,
 409             compute_cnt=compute_cnt,
 410             image_alt_id=self.image_alt.id,
 411             flavor_alt_id=self.flavor_alt.id,
 412             domain_name=self.cloud.auth.get("project_domain_name", "Default"))
 413         self.update_scenario_section()
 414         self.backup_tempest_config(self.conf_file, self.res_dir)
 415
 416     def run(self, **kwargs):
 417         self.start_time = time.time()
 418         try:
 419             assert super(TempestCommon, self).run(
 420                 **kwargs) == testcase.TestCase.EX_OK
 421             if not os.path.exists(self.res_dir):
 422                 os.makedirs(self.res_dir)
 423             self.update_rally_regex()
 424             self.update_default_role()
 425             self.update_rally_logs()
 426             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 427             self.configure(**kwargs)
 428             self.generate_test_list(**kwargs)
 429             self.apply_tempest_blacklist()
 430             self.run_verifier_tests(**kwargs)
 431             self.parse_verifier_result()
 432             self.generate_report()
 433             res = testcase.TestCase.EX_OK
 434         except Exception:  # pylint: disable=broad-except
 435             LOGGER.exception('Error with run')
 436             self.result = 0
 437             res = testcase.TestCase.EX_RUN_ERROR
 438         self.stop_time = time.time()
 439         return res
 440
 441     def clean(self):
 442         """
 443         Cleanup all OpenStack objects. Should be called on completion.
 444         """
 445         self.clean_rally_conf()
 446         if self.image_alt:
 447             self.cloud.delete_image(self.image_alt)
 448         if self.flavor_alt:
 449             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 450         super(TempestCommon, self).clean()