functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         environ = dict(
  60             os.environ,
  61             OS_USERNAME=self.project.user.name,
  62             OS_PROJECT_NAME=self.project.project.name,
  63             OS_PROJECT_ID=self.project.project.id,
  64             OS_PASSWORD=self.project.password)
  65         conf_utils.create_rally_deployment(environ=environ)
  66         conf_utils.create_verifier()
  67         self.verifier_id = conf_utils.get_verifier_id()
  68         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  69             self.verifier_id)
  70         self.deployment_id = conf_utils.get_verifier_deployment_id()
  71         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  72             self.verifier_id, self.deployment_id)
  73         self.verification_id = None
  74         self.res_dir = os.path.join(
  75             getattr(config.CONF, 'dir_results'), self.case_name)
  76         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  77         self.list = os.path.join(self.res_dir, 'test_list.txt')
  78         self.conf_file = None
  79         self.image_alt = None
  80         self.flavor_alt = None
  81         self.services = []
  82         try:
  83             self.services = kwargs['run']['args']['services']
  84         except Exception:  # pylint: disable=broad-except
  85             pass
  86         self.neutron_extensions = []
  87         try:
  88             self.neutron_extensions = kwargs['run']['args'][
  89                 'neutron_extensions']
  90         except Exception:  # pylint: disable=broad-except
  91             pass
  92
  93     def check_services(self):
  94         """Check the mandatory services."""
  95         for service in self.services:
  96             try:
  97                 self.cloud.search_services(service)[0]
  98             except Exception:  # pylint: disable=broad-except
  99                 self.is_skipped = True
 100                 break
 101
 102     def check_extensions(self):
 103         """Check the mandatory network extensions."""
 104         extensions = self.cloud.get_network_extensions()
 105         for network_extension in self.neutron_extensions:
 106             if network_extension not in extensions:
 107                 LOGGER.warning(
 108                     "Cannot find Neutron extension: %s", network_extension)
 109                 self.is_skipped = True
 110                 break
 111
 112     def check_requirements(self):
 113         self.check_services()
 114         self.check_extensions()
 115
 116     @staticmethod
 117     def read_file(filename):
 118         """Read file and return content as a stripped list."""
 119         with open(filename) as src:
 120             return [line.strip() for line in src.readlines()]
 121
 122     @staticmethod
 123     def get_verifier_result(verif_id):
 124         """Retrieve verification results."""
 125         result = {
 126             'num_tests': 0,
 127             'num_success': 0,
 128             'num_failures': 0,
 129             'num_skipped': 0
 130         }
 131         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 132         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 133         proc = subprocess.Popen(cmd,
 134                                 stdout=subprocess.PIPE,
 135                                 stderr=subprocess.STDOUT)
 136         for line in proc.stdout:
 137             LOGGER.info(line.rstrip())
 138             new_line = line.replace(' ', '').split('|')
 139             if 'Tests' in new_line:
 140                 break
 141             if 'Testscount' in new_line:
 142                 result['num_tests'] = int(new_line[2])
 143             elif 'Success' in new_line:
 144                 result['num_success'] = int(new_line[2])
 145             elif 'Skipped' in new_line:
 146                 result['num_skipped'] = int(new_line[2])
 147             elif 'Failures' in new_line:
 148                 result['num_failures'] = int(new_line[2])
 149         return result
 150
 151     @staticmethod
 152     def backup_tempest_config(conf_file, res_dir):
 153         """
 154         Copy config file to tempest results directory
 155         """
 156         if not os.path.exists(res_dir):
 157             os.makedirs(res_dir)
 158         shutil.copyfile(conf_file,
 159                         os.path.join(res_dir, 'tempest.conf'))
 160
 161     def generate_test_list(self, **kwargs):
 162         """Generate test list based on the test mode."""
 163         LOGGER.debug("Generating test case list...")
 164         self.backup_tempest_config(self.conf_file, '/etc')
 165         if kwargs.get('mode') == 'custom':
 166             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 167                 shutil.copyfile(
 168                     conf_utils.TEMPEST_CUSTOM, self.list)
 169             else:
 170                 raise Exception("Tempest test list file %s NOT found."
 171                                 % conf_utils.TEMPEST_CUSTOM)
 172         else:
 173             testr_mode = kwargs.get(
 174                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 175             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 176                 self.verifier_repo_dir, testr_mode, self.list)
 177             output = subprocess.check_output(cmd, shell=True)
 178             LOGGER.info("%s\n%s", cmd, output)
 179         os.remove('/etc/tempest.conf')
 180
 181     def apply_tempest_blacklist(self):
 182         """Exclude blacklisted test cases."""
 183         LOGGER.debug("Applying tempest blacklist...")
 184         if os.path.exists(self.raw_list):
 185             os.remove(self.raw_list)
 186         os.rename(self.list, self.raw_list)
 187         cases_file = self.read_file(self.raw_list)
 188         result_file = open(self.list, 'w')
 189         black_tests = []
 190         try:
 191             deploy_scenario = env.get('DEPLOY_SCENARIO')
 192             if bool(deploy_scenario):
 193                 # if DEPLOY_SCENARIO is set we read the file
 194                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 195                 black_list_yaml = yaml.safe_load(black_list_file)
 196                 black_list_file.close()
 197                 for item in black_list_yaml:
 198                     scenarios = item['scenarios']
 199                     if deploy_scenario in scenarios:
 200                         tests = item['tests']
 201                         for test in tests:
 202                             black_tests.append(test)
 203                         break
 204         except Exception:  # pylint: disable=broad-except
 205             black_tests = []
 206             LOGGER.debug("Tempest blacklist file does not exist.")
 207
 208         for cases_line in cases_file:
 209             for black_tests_line in black_tests:
 210                 if black_tests_line in cases_line:
 211                     break
 212             else:
 213                 result_file.write(str(cases_line) + '\n')
 214         result_file.close()
 215
 216     def run_verifier_tests(self, **kwargs):
 217         """Execute tempest test cases."""
 218         cmd = ["rally", "verify", "start", "--load-list",
 219                self.list]
 220         cmd.extend(kwargs.get('option', []))
 221         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 222
 223         f_stdout = open(
 224             os.path.join(self.res_dir, "tempest.log"), 'w+')
 225
 226         proc = subprocess.Popen(
 227             cmd,
 228             stdout=subprocess.PIPE,
 229             stderr=subprocess.STDOUT,
 230             bufsize=1)
 231
 232         with proc.stdout:
 233             for line in iter(proc.stdout.readline, b''):
 234                 if re.search(r"\} tempest\.", line):
 235                     LOGGER.info(line.rstrip())
 236                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 237                     self.verification_id = re.search(
 238                         r'(?=\(UUID=(.*)\))', line).group(1)
 239                 f_stdout.write(line)
 240         proc.wait()
 241         f_stdout.close()
 242
 243         if self.verification_id is None:
 244             raise Exception('Verification UUID not found')
 245         LOGGER.info('Verification UUID: %s', self.verification_id)
 246
 247         shutil.copy(
 248             "{}/tempest.log".format(self.deployment_dir),
 249             "{}/tempest.debug.log".format(self.res_dir))
 250
 251     def parse_verifier_result(self):
 252         """Parse and save test results."""
 253         stat = self.get_verifier_result(self.verification_id)
 254         try:
 255             num_executed = stat['num_tests'] - stat['num_skipped']
 256             try:
 257                 self.result = 100 * stat['num_success'] / num_executed
 258             except ZeroDivisionError:
 259                 self.result = 0
 260                 if stat['num_tests'] > 0:
 261                     LOGGER.info("All tests have been skipped")
 262                 else:
 263                     LOGGER.error("No test has been executed")
 264                     return
 265
 266             with open(os.path.join(self.res_dir,
 267                                    "tempest.log"), 'r') as logfile:
 268                 output = logfile.read()
 269
 270             success_testcases = []
 271             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 272                                     output):
 273                 success_testcases.append(match)
 274             failed_testcases = []
 275             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 276                                     output):
 277                 failed_testcases.append(match)
 278             skipped_testcases = []
 279             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 280                                     output):
 281                 skipped_testcases.append(match)
 282
 283             self.details = {"tests_number": stat['num_tests'],
 284                             "success_number": stat['num_success'],
 285                             "skipped_number": stat['num_skipped'],
 286                             "failures_number": stat['num_failures'],
 287                             "success": success_testcases,
 288                             "skipped": skipped_testcases,
 289                             "failures": failed_testcases}
 290         except Exception:  # pylint: disable=broad-except
 291             self.result = 0
 292
 293         LOGGER.info("Tempest %s success_rate is %s%%",
 294                     self.case_name, self.result)
 295
 296     def generate_report(self):
 297         """Generate verification report."""
 298         html_file = os.path.join(self.res_dir,
 299                                  "tempest-report.html")
 300         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 301                self.verification_id, "--to", html_file]
 302         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 303                          stderr=subprocess.STDOUT)
 304
 305     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 306         """Set image name as tempest img_name_regex"""
 307         rconfig = configparser.RawConfigParser()
 308         rconfig.read(rally_conf)
 309         if not rconfig.has_section('openstack'):
 310             rconfig.add_section('openstack')
 311         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 312             self.image.name))
 313         with open(rally_conf, 'wb') as config_file:
 314             rconfig.write(config_file)
 315
 316     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 317         """Detect and update the default role if required"""
 318         role = self.get_default_role(self.cloud)
 319         if not role:
 320             return
 321         rconfig = configparser.RawConfigParser()
 322         rconfig.read(rally_conf)
 323         if not rconfig.has_section('openstack'):
 324             rconfig.add_section('openstack')
 325         rconfig.set('openstack', 'swift_operator_role', role.name)
 326         with open(rally_conf, 'wb') as config_file:
 327             rconfig.write(config_file)
 328
 329     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 330         """Print rally logs in res dir"""
 331         if not os.path.exists(self.res_dir):
 332             os.makedirs(self.res_dir)
 333         rconfig = configparser.RawConfigParser()
 334         rconfig.read(rally_conf)
 335         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 336         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 337         with open(rally_conf, 'wb') as config_file:
 338             rconfig.write(config_file)
 339
 340     @staticmethod
 341     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 342         """Clean Rally config"""
 343         rconfig = configparser.RawConfigParser()
 344         rconfig.read(rally_conf)
 345         if rconfig.has_option('openstack', 'img_name_regex'):
 346             rconfig.remove_option('openstack', 'img_name_regex')
 347         if rconfig.has_option('openstack', 'swift_operator_role'):
 348             rconfig.remove_option('openstack', 'swift_operator_role')
 349         if rconfig.has_option('DEFAULT', 'log-file'):
 350             rconfig.remove_option('DEFAULT', 'log-file')
 351         if rconfig.has_option('DEFAULT', 'log_dir'):
 352             rconfig.remove_option('DEFAULT', 'log_dir')
 353         with open(rally_conf, 'wb') as config_file:
 354             rconfig.write(config_file)
 355
 356     def update_scenario_section(self):
 357         """Update scenario section in tempest.conf"""
 358         rconfig = configparser.RawConfigParser()
 359         rconfig.read(self.conf_file)
 360         filename = getattr(
 361             config.CONF, '{}_image'.format(self.case_name), self.filename)
 362         if not rconfig.has_section('scenario'):
 363             rconfig.add_section('scenario')
 364         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 365         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 366         rconfig.set('scenario', 'img_disk_format', getattr(
 367             config.CONF, '{}_image_format'.format(self.case_name),
 368             self.image_format))
 369         extra_properties = self.extra_properties.copy()
 370         if env.get('IMAGE_PROPERTIES'):
 371             extra_properties.update(
 372                 dict((k.strip(), v.strip()) for k, v in (
 373                     item.split(': ') for item in env.get(
 374                         'IMAGE_PROPERTIES').split(','))))
 375         extra_properties.update(
 376             getattr(config.CONF, '{}_extra_properties'.format(
 377                 self.case_name), {}))
 378         rconfig.set(
 379             'scenario', 'img_properties',
 380             conf_utils.convert_dict_to_ini(extra_properties))
 381         with open(self.conf_file, 'wb') as config_file:
 382             rconfig.write(config_file)
 383
 384     def configure(self, **kwargs):  # pylint: disable=unused-argument
 385         """
 386         Create all openstack resources for tempest-based testcases and write
 387         tempest.conf.
 388         """
 389         if not os.path.exists(self.res_dir):
 390             os.makedirs(self.res_dir)
 391         compute_cnt = len(self.cloud.list_hypervisors())
 392
 393         self.image_alt = self.publish_image_alt()
 394         self.flavor_alt = self.create_flavor_alt()
 395         LOGGER.debug("flavor: %s", self.flavor_alt)
 396
 397         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 398         conf_utils.configure_tempest_update_params(
 399             self.conf_file, network_name=self.network.name,
 400             image_id=self.image.id,
 401             flavor_id=self.flavor.id,
 402             compute_cnt=compute_cnt,
 403             image_alt_id=self.image_alt.id,
 404             flavor_alt_id=self.flavor_alt.id,
 405             domain_name=self.cloud.auth.get("project_domain_name", "Default"))
 406         self.update_scenario_section()
 407         self.backup_tempest_config(self.conf_file, self.res_dir)
 408
 409     def run(self, **kwargs):
 410         self.start_time = time.time()
 411         try:
 412             assert super(TempestCommon, self).run(
 413                 **kwargs) == testcase.TestCase.EX_OK
 414             if not os.path.exists(self.res_dir):
 415                 os.makedirs(self.res_dir)
 416             self.update_rally_regex()
 417             self.update_default_role()
 418             self.update_rally_logs()
 419             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 420             self.configure(**kwargs)
 421             self.generate_test_list(**kwargs)
 422             self.apply_tempest_blacklist()
 423             self.run_verifier_tests(**kwargs)
 424             self.parse_verifier_result()
 425             self.generate_report()
 426             res = testcase.TestCase.EX_OK
 427         except Exception:  # pylint: disable=broad-except
 428             LOGGER.exception('Error with run')
 429             self.result = 0
 430             res = testcase.TestCase.EX_RUN_ERROR
 431         self.stop_time = time.time()
 432         return res
 433
 434     def clean(self):
 435         """
 436         Cleanup all OpenStack objects. Should be called on completion.
 437         """
 438         self.clean_rally_conf()
 439         if self.image_alt:
 440             self.cloud.delete_image(self.image_alt)
 441         if self.flavor_alt:
 442             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 443         super(TempestCommon, self).clean()