functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         environ = dict(
  60             os.environ,
  61             OS_USERNAME=self.project.user.name,
  62             OS_PROJECT_NAME=self.project.project.name,
  63             OS_PROJECT_ID=self.project.project.id,
  64             OS_PASSWORD=self.project.password)
  65         conf_utils.create_rally_deployment(environ=environ)
  66         conf_utils.create_verifier()
  67         self.verifier_id = conf_utils.get_verifier_id()
  68         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  69             self.verifier_id)
  70         self.deployment_id = conf_utils.get_verifier_deployment_id()
  71         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  72             self.verifier_id, self.deployment_id)
  73         self.verification_id = None
  74         self.res_dir = os.path.join(
  75             getattr(config.CONF, 'dir_results'), self.case_name)
  76         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  77         self.list = os.path.join(self.res_dir, 'test_list.txt')
  78         self.conf_file = None
  79         self.image_alt = None
  80         self.flavor_alt = None
  81         self.services = []
  82         try:
  83             self.services = kwargs['run']['args']['services']
  84         except Exception:  # pylint: disable=broad-except
  85             pass
  86         self.neutron_extensions = []
  87         try:
  88             self.neutron_extensions = kwargs['run']['args'][
  89                 'neutron_extensions']
  90         except Exception:  # pylint: disable=broad-except
  91             pass
  92
  93     def check_services(self):
  94         """Check the mandatory services."""
  95         for service in self.services:
  96             try:
  97                 self.cloud.search_services(service)[0]
  98             except Exception:  # pylint: disable=broad-except
  99                 self.is_skipped = True
 100                 break
 101
 102     def check_extensions(self):
 103         """Check the mandatory network extensions."""
 104         extensions = self.cloud.get_network_extensions()
 105         for network_extension in self.neutron_extensions:
 106             if network_extension not in extensions:
 107                 LOGGER.warning(
 108                     "Cannot find Neutron extension: %s", network_extension)
 109                 self.is_skipped = True
 110                 break
 111
 112     def check_requirements(self):
 113         self.check_services()
 114         self.check_extensions()
 115
 116     @staticmethod
 117     def read_file(filename):
 118         """Read file and return content as a stripped list."""
 119         with open(filename) as src:
 120             return [line.strip() for line in src.readlines()]
 121
 122     @staticmethod
 123     def get_verifier_result(verif_id):
 124         """Retrieve verification results."""
 125         result = {
 126             'num_tests': 0,
 127             'num_success': 0,
 128             'num_failures': 0,
 129             'num_skipped': 0
 130         }
 131         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 132         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 133         proc = subprocess.Popen(cmd,
 134                                 stdout=subprocess.PIPE,
 135                                 stderr=subprocess.STDOUT)
 136         for line in proc.stdout:
 137             LOGGER.info(line.rstrip())
 138             new_line = line.replace(' ', '').split('|')
 139             if 'Tests' in new_line:
 140                 break
 141             if 'Testscount' in new_line:
 142                 result['num_tests'] = int(new_line[2])
 143             elif 'Success' in new_line:
 144                 result['num_success'] = int(new_line[2])
 145             elif 'Skipped' in new_line:
 146                 result['num_skipped'] = int(new_line[2])
 147             elif 'Failures' in new_line:
 148                 result['num_failures'] = int(new_line[2])
 149         return result
 150
 151     @staticmethod
 152     def backup_tempest_config(conf_file, res_dir):
 153         """
 154         Copy config file to tempest results directory
 155         """
 156         if not os.path.exists(res_dir):
 157             os.makedirs(res_dir)
 158         shutil.copyfile(conf_file,
 159                         os.path.join(res_dir, 'tempest.conf'))
 160
 161     def generate_test_list(self, **kwargs):
 162         """Generate test list based on the test mode."""
 163         LOGGER.debug("Generating test case list...")
 164         self.backup_tempest_config(self.conf_file, '/etc')
 165         if kwargs.get('mode') == 'custom':
 166             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 167                 shutil.copyfile(
 168                     conf_utils.TEMPEST_CUSTOM, self.list)
 169             else:
 170                 raise Exception("Tempest test list file %s NOT found."
 171                                 % conf_utils.TEMPEST_CUSTOM)
 172         else:
 173             testr_mode = kwargs.get(
 174                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 175             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 176                 self.verifier_repo_dir, testr_mode, self.list)
 177             output = subprocess.check_output(cmd, shell=True)
 178             LOGGER.info("%s\n%s", cmd, output)
 179         os.remove('/etc/tempest.conf')
 180
 181     def apply_tempest_blacklist(self):
 182         """Exclude blacklisted test cases."""
 183         LOGGER.debug("Applying tempest blacklist...")
 184         if os.path.exists(self.raw_list):
 185             os.remove(self.raw_list)
 186         os.rename(self.list, self.raw_list)
 187         cases_file = self.read_file(self.raw_list)
 188         result_file = open(self.list, 'w')
 189         black_tests = []
 190         try:
 191             deploy_scenario = env.get('DEPLOY_SCENARIO')
 192             if bool(deploy_scenario):
 193                 # if DEPLOY_SCENARIO is set we read the file
 194                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 195                 black_list_yaml = yaml.safe_load(black_list_file)
 196                 black_list_file.close()
 197                 for item in black_list_yaml:
 198                     scenarios = item['scenarios']
 199                     if deploy_scenario in scenarios:
 200                         tests = item['tests']
 201                         for test in tests:
 202                             black_tests.append(test)
 203                         break
 204         except Exception:  # pylint: disable=broad-except
 205             black_tests = []
 206             LOGGER.debug("Tempest blacklist file does not exist.")
 207
 208         for cases_line in cases_file:
 209             for black_tests_line in black_tests:
 210                 if black_tests_line in cases_line:
 211                     break
 212             else:
 213                 result_file.write(str(cases_line) + '\n')
 214         result_file.close()
 215
 216     def run_verifier_tests(self, **kwargs):
 217         """Execute tempest test cases."""
 218         cmd = ["rally", "verify", "start", "--load-list",
 219                self.list]
 220         cmd.extend(kwargs.get('option', []))
 221         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 222
 223         f_stdout = open(
 224             os.path.join(self.res_dir, "tempest.log"), 'w+')
 225
 226         proc = subprocess.Popen(
 227             cmd,
 228             stdout=subprocess.PIPE,
 229             stderr=subprocess.STDOUT,
 230             bufsize=1)
 231
 232         with proc.stdout:
 233             for line in iter(proc.stdout.readline, b''):
 234                 if re.search(r"\} tempest\.", line):
 235                     LOGGER.info(line.rstrip())
 236                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 237                     self.verification_id = re.search(
 238                         r'(?=\(UUID=(.*)\))', line).group(1)
 239                 f_stdout.write(line)
 240         proc.wait()
 241         f_stdout.close()
 242
 243         if self.verification_id is None:
 244             raise Exception('Verification UUID not found')
 245         LOGGER.info('Verification UUID: %s', self.verification_id)
 246
 247         shutil.copy(
 248             "{}/tempest.log".format(self.deployment_dir),
 249             "{}/tempest.debug.log".format(self.res_dir))
 250
 251     def parse_verifier_result(self):
 252         """Parse and save test results."""
 253         stat = self.get_verifier_result(self.verification_id)
 254         try:
 255             num_executed = stat['num_tests'] - stat['num_skipped']
 256             try:
 257                 self.result = 100 * stat['num_success'] / num_executed
 258             except ZeroDivisionError:
 259                 self.result = 0
 260                 if stat['num_tests'] > 0:
 261                     LOGGER.info("All tests have been skipped")
 262                 else:
 263                     LOGGER.error("No test has been executed")
 264                     return
 265
 266             with open(os.path.join(self.res_dir,
 267                                    "tempest.log"), 'r') as logfile:
 268                 output = logfile.read()
 269
 270             success_testcases = []
 271             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 272                                     output):
 273                 success_testcases.append(match)
 274             failed_testcases = []
 275             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 276                                     output):
 277                 failed_testcases.append(match)
 278             skipped_testcases = []
 279             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 280                                     output):
 281                 skipped_testcases.append(match)
 282
 283             self.details = {"tests_number": stat['num_tests'],
 284                             "success_number": stat['num_success'],
 285                             "skipped_number": stat['num_skipped'],
 286                             "failures_number": stat['num_failures'],
 287                             "success": success_testcases,
 288                             "skipped": skipped_testcases,
 289                             "failures": failed_testcases}
 290         except Exception:  # pylint: disable=broad-except
 291             self.result = 0
 292
 293         LOGGER.info("Tempest %s success_rate is %s%%",
 294                     self.case_name, self.result)
 295
 296     def generate_report(self):
 297         """Generate verification report."""
 298         html_file = os.path.join(self.res_dir,
 299                                  "tempest-report.html")
 300         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 301                self.verification_id, "--to", html_file]
 302         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 303                          stderr=subprocess.STDOUT)
 304
 305     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 306         """Set image name as tempest img_name_regex"""
 307         rconfig = configparser.RawConfigParser()
 308         rconfig.read(rally_conf)
 309         if not rconfig.has_section('openstack'):
 310             rconfig.add_section('openstack')
 311         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 312             self.image.name))
 313         with open(rally_conf, 'wb') as config_file:
 314             rconfig.write(config_file)
 315
 316     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 317         """Detect and update the default role if required"""
 318         role = self.get_default_role(self.cloud)
 319         if not role:
 320             return
 321         rconfig = configparser.RawConfigParser()
 322         rconfig.read(rally_conf)
 323         if not rconfig.has_section('openstack'):
 324             rconfig.add_section('openstack')
 325         rconfig.set('openstack', 'swift_operator_role', role.name)
 326         with open(rally_conf, 'wb') as config_file:
 327             rconfig.write(config_file)
 328
 329     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 330         """Print rally logs in res dir"""
 331         if not os.path.exists(self.res_dir):
 332             os.makedirs(self.res_dir)
 333         rconfig = configparser.RawConfigParser()
 334         rconfig.read(rally_conf)
 335         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 336         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 337         with open(rally_conf, 'wb') as config_file:
 338             rconfig.write(config_file)
 339
 340     @staticmethod
 341     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 342         """Clean Rally config"""
 343         rconfig = configparser.RawConfigParser()
 344         rconfig.read(rally_conf)
 345         if rconfig.has_option('openstack', 'img_name_regex'):
 346             rconfig.remove_option('openstack', 'img_name_regex')
 347         if rconfig.has_option('openstack', 'swift_operator_role'):
 348             rconfig.remove_option('openstack', 'swift_operator_role')
 349         if rconfig.has_option('DEFAULT', 'log-file'):
 350             rconfig.remove_option('DEFAULT', 'log-file')
 351         if rconfig.has_option('DEFAULT', 'log_dir'):
 352             rconfig.remove_option('DEFAULT', 'log_dir')
 353         with open(rally_conf, 'wb') as config_file:
 354             rconfig.write(config_file)
 355
 356     def configure(self, **kwargs):  # pylint: disable=unused-argument
 357         """
 358         Create all openstack resources for tempest-based testcases and write
 359         tempest.conf.
 360         """
 361         if not os.path.exists(self.res_dir):
 362             os.makedirs(self.res_dir)
 363         compute_cnt = len(self.cloud.list_hypervisors())
 364
 365         self.image_alt = self.publish_image_alt()
 366         self.flavor_alt = self.create_flavor_alt()
 367         LOGGER.debug("flavor: %s", self.flavor_alt)
 368
 369         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 370         conf_utils.configure_tempest_update_params(
 371             self.conf_file, network_name=self.network.name,
 372             image_id=self.image.id,
 373             flavor_id=self.flavor.id,
 374             compute_cnt=compute_cnt,
 375             image_alt_id=self.image_alt.id,
 376             flavor_alt_id=self.flavor_alt.id,
 377             domain_name=self.cloud.auth.get("project_domain_name", "Default"))
 378         self.backup_tempest_config(self.conf_file, self.res_dir)
 379
 380     def run(self, **kwargs):
 381         self.start_time = time.time()
 382         try:
 383             assert super(TempestCommon, self).run(
 384                 **kwargs) == testcase.TestCase.EX_OK
 385             if not os.path.exists(self.res_dir):
 386                 os.makedirs(self.res_dir)
 387             self.update_rally_regex()
 388             self.update_default_role()
 389             self.update_rally_logs()
 390             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 391             self.configure(**kwargs)
 392             self.generate_test_list(**kwargs)
 393             self.apply_tempest_blacklist()
 394             self.run_verifier_tests(**kwargs)
 395             self.parse_verifier_result()
 396             self.generate_report()
 397             res = testcase.TestCase.EX_OK
 398         except Exception:  # pylint: disable=broad-except
 399             LOGGER.exception('Error with run')
 400             self.result = 0
 401             res = testcase.TestCase.EX_RUN_ERROR
 402         self.stop_time = time.time()
 403         return res
 404
 405     def clean(self):
 406         """
 407         Cleanup all OpenStack objects. Should be called on completion.
 408         """
 409         self.clean_rally_conf()
 410         if self.image_alt:
 411             self.cloud.delete_image(self.image_alt)
 412         if self.flavor_alt:
 413             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 414         super(TempestCommon, self).clean()