functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         environ = dict(
  60             os.environ,
  61             OS_USERNAME=self.project.user.name,
  62             OS_PROJECT_NAME=self.project.project.name,
  63             OS_PROJECT_ID=self.project.project.id,
  64             OS_PASSWORD=self.project.password)
  65         self.deployment_id = conf_utils.create_rally_deployment(
  66             environ=environ)
  67         if not self.deployment_id:
  68             raise Exception("Deployment create failed")
  69         self.verifier_id = conf_utils.create_verifier()
  70         if not self.verifier_id:
  71             raise Exception("Verifier create failed")
  72         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  73             self.verifier_id)
  74         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  75             self.verifier_id, self.deployment_id)
  76         self.verification_id = None
  77         self.res_dir = os.path.join(
  78             getattr(config.CONF, 'dir_results'), self.case_name)
  79         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  80         self.list = os.path.join(self.res_dir, 'test_list.txt')
  81         self.conf_file = None
  82         self.image_alt = None
  83         self.flavor_alt = None
  84         self.services = []
  85         try:
  86             self.services = kwargs['run']['args']['services']
  87         except Exception:  # pylint: disable=broad-except
  88             pass
  89         self.neutron_extensions = []
  90         try:
  91             self.neutron_extensions = kwargs['run']['args'][
  92                 'neutron_extensions']
  93         except Exception:  # pylint: disable=broad-except
  94             pass
  95
  96     def check_services(self):
  97         """Check the mandatory services."""
  98         for service in self.services:
  99             try:
 100                 self.cloud.search_services(service)[0]
 101             except Exception:  # pylint: disable=broad-except
 102                 self.is_skipped = True
 103                 break
 104
 105     def check_extensions(self):
 106         """Check the mandatory network extensions."""
 107         extensions = self.cloud.get_network_extensions()
 108         for network_extension in self.neutron_extensions:
 109             if network_extension not in extensions:
 110                 LOGGER.warning(
 111                     "Cannot find Neutron extension: %s", network_extension)
 112                 self.is_skipped = True
 113                 break
 114
 115     def check_requirements(self):
 116         self.check_services()
 117         self.check_extensions()
 118         if self.is_skipped:
 119             self.project.clean()
 120
 121     @staticmethod
 122     def read_file(filename):
 123         """Read file and return content as a stripped list."""
 124         with open(filename) as src:
 125             return [line.strip() for line in src.readlines()]
 126
 127     @staticmethod
 128     def get_verifier_result(verif_id):
 129         """Retrieve verification results."""
 130         result = {
 131             'num_tests': 0,
 132             'num_success': 0,
 133             'num_failures': 0,
 134             'num_skipped': 0
 135         }
 136         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 137         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 138         proc = subprocess.Popen(cmd,
 139                                 stdout=subprocess.PIPE,
 140                                 stderr=subprocess.STDOUT)
 141         for line in proc.stdout:
 142             LOGGER.info(line.rstrip())
 143             new_line = line.replace(' ', '').split('|')
 144             if 'Tests' in new_line:
 145                 break
 146             if 'Testscount' in new_line:
 147                 result['num_tests'] = int(new_line[2])
 148             elif 'Success' in new_line:
 149                 result['num_success'] = int(new_line[2])
 150             elif 'Skipped' in new_line:
 151                 result['num_skipped'] = int(new_line[2])
 152             elif 'Failures' in new_line:
 153                 result['num_failures'] = int(new_line[2])
 154         return result
 155
 156     @staticmethod
 157     def backup_tempest_config(conf_file, res_dir):
 158         """
 159         Copy config file to tempest results directory
 160         """
 161         if not os.path.exists(res_dir):
 162             os.makedirs(res_dir)
 163         shutil.copyfile(conf_file,
 164                         os.path.join(res_dir, 'tempest.conf'))
 165
 166     def generate_test_list(self, **kwargs):
 167         """Generate test list based on the test mode."""
 168         LOGGER.debug("Generating test case list...")
 169         self.backup_tempest_config(self.conf_file, '/etc')
 170         if kwargs.get('mode') == 'custom':
 171             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 172                 shutil.copyfile(
 173                     conf_utils.TEMPEST_CUSTOM, self.list)
 174             else:
 175                 raise Exception("Tempest test list file %s NOT found."
 176                                 % conf_utils.TEMPEST_CUSTOM)
 177         else:
 178             testr_mode = kwargs.get(
 179                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 180             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 181                 self.verifier_repo_dir, testr_mode, self.list)
 182             output = subprocess.check_output(cmd, shell=True)
 183             LOGGER.info("%s\n%s", cmd, output)
 184         os.remove('/etc/tempest.conf')
 185
 186     def apply_tempest_blacklist(self):
 187         """Exclude blacklisted test cases."""
 188         LOGGER.debug("Applying tempest blacklist...")
 189         if os.path.exists(self.raw_list):
 190             os.remove(self.raw_list)
 191         os.rename(self.list, self.raw_list)
 192         cases_file = self.read_file(self.raw_list)
 193         result_file = open(self.list, 'w')
 194         black_tests = []
 195         try:
 196             deploy_scenario = env.get('DEPLOY_SCENARIO')
 197             if bool(deploy_scenario):
 198                 # if DEPLOY_SCENARIO is set we read the file
 199                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 200                 black_list_yaml = yaml.safe_load(black_list_file)
 201                 black_list_file.close()
 202                 for item in black_list_yaml:
 203                     scenarios = item['scenarios']
 204                     if deploy_scenario in scenarios:
 205                         tests = item['tests']
 206                         for test in tests:
 207                             black_tests.append(test)
 208                         break
 209         except Exception:  # pylint: disable=broad-except
 210             black_tests = []
 211             LOGGER.debug("Tempest blacklist file does not exist.")
 212
 213         for cases_line in cases_file:
 214             for black_tests_line in black_tests:
 215                 if black_tests_line in cases_line:
 216                     break
 217             else:
 218                 result_file.write(str(cases_line) + '\n')
 219         result_file.close()
 220
 221     def run_verifier_tests(self, **kwargs):
 222         """Execute tempest test cases."""
 223         cmd = ["rally", "verify", "start", "--load-list",
 224                self.list]
 225         cmd.extend(kwargs.get('option', []))
 226         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 227
 228         f_stdout = open(
 229             os.path.join(self.res_dir, "tempest.log"), 'w+')
 230
 231         proc = subprocess.Popen(
 232             cmd,
 233             stdout=subprocess.PIPE,
 234             stderr=subprocess.STDOUT,
 235             bufsize=1)
 236
 237         with proc.stdout:
 238             for line in iter(proc.stdout.readline, b''):
 239                 if re.search(r"\} tempest\.", line):
 240                     LOGGER.info(line.rstrip())
 241                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 242                     self.verification_id = re.search(
 243                         r'(?=\(UUID=(.*)\))', line).group(1)
 244                 f_stdout.write(line)
 245         proc.wait()
 246         f_stdout.close()
 247
 248         if self.verification_id is None:
 249             raise Exception('Verification UUID not found')
 250         LOGGER.info('Verification UUID: %s', self.verification_id)
 251
 252         shutil.copy(
 253             "{}/tempest.log".format(self.deployment_dir),
 254             "{}/tempest.debug.log".format(self.res_dir))
 255
 256     def parse_verifier_result(self):
 257         """Parse and save test results."""
 258         stat = self.get_verifier_result(self.verification_id)
 259         try:
 260             num_executed = stat['num_tests'] - stat['num_skipped']
 261             try:
 262                 self.result = 100 * stat['num_success'] / num_executed
 263             except ZeroDivisionError:
 264                 self.result = 0
 265                 if stat['num_tests'] > 0:
 266                     LOGGER.info("All tests have been skipped")
 267                 else:
 268                     LOGGER.error("No test has been executed")
 269                     return
 270
 271             with open(os.path.join(self.res_dir,
 272                                    "tempest.log"), 'r') as logfile:
 273                 output = logfile.read()
 274
 275             success_testcases = []
 276             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 277                                     output):
 278                 success_testcases.append(match)
 279             failed_testcases = []
 280             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 281                                     output):
 282                 failed_testcases.append(match)
 283             skipped_testcases = []
 284             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 285                                     output):
 286                 skipped_testcases.append(match)
 287
 288             self.details = {"tests_number": stat['num_tests'],
 289                             "success_number": stat['num_success'],
 290                             "skipped_number": stat['num_skipped'],
 291                             "failures_number": stat['num_failures'],
 292                             "success": success_testcases,
 293                             "skipped": skipped_testcases,
 294                             "failures": failed_testcases}
 295         except Exception:  # pylint: disable=broad-except
 296             self.result = 0
 297
 298         LOGGER.info("Tempest %s success_rate is %s%%",
 299                     self.case_name, self.result)
 300
 301     def generate_report(self):
 302         """Generate verification report."""
 303         html_file = os.path.join(self.res_dir,
 304                                  "tempest-report.html")
 305         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 306                self.verification_id, "--to", html_file]
 307         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 308                          stderr=subprocess.STDOUT)
 309
 310     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 311         """Set image name as tempest img_name_regex"""
 312         rconfig = configparser.RawConfigParser()
 313         rconfig.read(rally_conf)
 314         if not rconfig.has_section('openstack'):
 315             rconfig.add_section('openstack')
 316         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 317             self.image.name))
 318         with open(rally_conf, 'wb') as config_file:
 319             rconfig.write(config_file)
 320
 321     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 322         """Detect and update the default role if required"""
 323         role = self.get_default_role(self.cloud)
 324         if not role:
 325             return
 326         rconfig = configparser.RawConfigParser()
 327         rconfig.read(rally_conf)
 328         if not rconfig.has_section('openstack'):
 329             rconfig.add_section('openstack')
 330         rconfig.set('openstack', 'swift_operator_role', role.name)
 331         with open(rally_conf, 'wb') as config_file:
 332             rconfig.write(config_file)
 333
 334     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 335         """Print rally logs in res dir"""
 336         if not os.path.exists(self.res_dir):
 337             os.makedirs(self.res_dir)
 338         rconfig = configparser.RawConfigParser()
 339         rconfig.read(rally_conf)
 340         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 341         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 342         with open(rally_conf, 'wb') as config_file:
 343             rconfig.write(config_file)
 344
 345     @staticmethod
 346     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 347         """Clean Rally config"""
 348         rconfig = configparser.RawConfigParser()
 349         rconfig.read(rally_conf)
 350         if rconfig.has_option('openstack', 'img_name_regex'):
 351             rconfig.remove_option('openstack', 'img_name_regex')
 352         if rconfig.has_option('openstack', 'swift_operator_role'):
 353             rconfig.remove_option('openstack', 'swift_operator_role')
 354         if rconfig.has_option('DEFAULT', 'log-file'):
 355             rconfig.remove_option('DEFAULT', 'log-file')
 356         if rconfig.has_option('DEFAULT', 'log_dir'):
 357             rconfig.remove_option('DEFAULT', 'log_dir')
 358         with open(rally_conf, 'wb') as config_file:
 359             rconfig.write(config_file)
 360
 361     def update_scenario_section(self):
 362         """Update scenario section in tempest.conf"""
 363         rconfig = configparser.RawConfigParser()
 364         rconfig.read(self.conf_file)
 365         filename = getattr(
 366             config.CONF, '{}_image'.format(self.case_name), self.filename)
 367         if not rconfig.has_section('scenario'):
 368             rconfig.add_section('scenario')
 369         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 370         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 371         rconfig.set('scenario', 'img_disk_format', getattr(
 372             config.CONF, '{}_image_format'.format(self.case_name),
 373             self.image_format))
 374         extra_properties = self.extra_properties.copy()
 375         extra_properties.update(
 376             getattr(config.CONF, '{}_extra_properties'.format(
 377                 self.case_name), {}))
 378         rconfig.set(
 379             'scenario', 'img_properties',
 380             conf_utils.convert_dict_to_ini(extra_properties))
 381         with open(self.conf_file, 'wb') as config_file:
 382             rconfig.write(config_file)
 383
 384     def configure(self, **kwargs):  # pylint: disable=unused-argument
 385         """
 386         Create all openstack resources for tempest-based testcases and write
 387         tempest.conf.
 388         """
 389         if not os.path.exists(self.res_dir):
 390             os.makedirs(self.res_dir)
 391         compute_cnt = len(self.cloud.list_hypervisors())
 392
 393         self.image_alt = self.publish_image_alt()
 394         self.flavor_alt = self.create_flavor_alt()
 395         LOGGER.debug("flavor: %s", self.flavor_alt)
 396
 397         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 398         if not self.conf_file:
 399             raise Exception("Tempest verifier configuring failed")
 400         conf_utils.configure_tempest_update_params(
 401             self.conf_file, network_name=self.network.name,
 402             image_id=self.image.id,
 403             flavor_id=self.flavor.id,
 404             compute_cnt=compute_cnt,
 405             image_alt_id=self.image_alt.id,
 406             flavor_alt_id=self.flavor_alt.id,
 407             domain_name=self.cloud.auth.get("project_domain_name", "Default"))
 408         self.update_scenario_section()
 409         self.backup_tempest_config(self.conf_file, self.res_dir)
 410
 411     def run(self, **kwargs):
 412         self.start_time = time.time()
 413         try:
 414             assert super(TempestCommon, self).run(
 415                 **kwargs) == testcase.TestCase.EX_OK
 416             if not os.path.exists(self.res_dir):
 417                 os.makedirs(self.res_dir)
 418             self.update_rally_regex()
 419             self.update_default_role()
 420             self.update_rally_logs()
 421             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 422             self.configure(**kwargs)
 423             self.generate_test_list(**kwargs)
 424             self.apply_tempest_blacklist()
 425             self.run_verifier_tests(**kwargs)
 426             self.parse_verifier_result()
 427             self.generate_report()
 428             res = testcase.TestCase.EX_OK
 429         except Exception:  # pylint: disable=broad-except
 430             LOGGER.exception('Error with run')
 431             self.result = 0
 432             res = testcase.TestCase.EX_RUN_ERROR
 433         self.stop_time = time.time()
 434         return res
 435
 436     def clean(self):
 437         """
 438         Cleanup all OpenStack objects. Should be called on completion.
 439         """
 440         self.clean_rally_conf()
 441         if self.image_alt:
 442             self.cloud.delete_image(self.image_alt)
 443         if self.flavor_alt:
 444             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 445         super(TempestCommon, self).clean()