functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         self.verifier_id = conf_utils.get_verifier_id()
  47         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  48             self.verifier_id)
  49         self.deployment_id = conf_utils.get_verifier_deployment_id()
  50         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  51             self.verifier_id, self.deployment_id)
  52         self.verification_id = None
  53         self.res_dir = os.path.join(
  54             getattr(config.CONF, 'dir_results'), self.case_name)
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58         self.image_alt = None
  59         self.flavor_alt = None
  60         self.services = []
  61         try:
  62             self.services = kwargs['run']['args']['services']
  63         except Exception:  # pylint: disable=broad-except
  64             pass
  65         self.neutron_extensions = []
  66         try:
  67             self.neutron_extensions = kwargs['run']['args'][
  68                 'neutron_extensions']
  69         except Exception:  # pylint: disable=broad-except
  70             pass
  71
  72     def check_services(self):
  73         """Check the mandatory services."""
  74         for service in self.services:
  75             try:
  76                 self.cloud.search_services(service)[0]
  77             except Exception:  # pylint: disable=broad-except
  78                 self.is_skipped = True
  79                 break
  80
  81     def check_extensions(self):
  82         """Check the mandatory network extensions."""
  83         extensions = self.cloud.get_network_extensions()
  84         for network_extension in self.neutron_extensions:
  85             if network_extension not in extensions:
  86                 LOGGER.warning(
  87                     "Cannot find Neutron extension: %s", network_extension)
  88                 self.is_skipped = True
  89                 break
  90
  91     def check_requirements(self):
  92         self.check_services()
  93         self.check_extensions()
  94
  95     @staticmethod
  96     def read_file(filename):
  97         """Read file and return content as a stripped list."""
  98         with open(filename) as src:
  99             return [line.strip() for line in src.readlines()]
 100
 101     @staticmethod
 102     def get_verifier_result(verif_id):
 103         """Retrieve verification results."""
 104         result = {
 105             'num_tests': 0,
 106             'num_success': 0,
 107             'num_failures': 0,
 108             'num_skipped': 0
 109         }
 110         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 111         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 112         proc = subprocess.Popen(cmd,
 113                                 stdout=subprocess.PIPE,
 114                                 stderr=subprocess.STDOUT)
 115         for line in proc.stdout:
 116             LOGGER.info(line.rstrip())
 117             new_line = line.replace(' ', '').split('|')
 118             if 'Tests' in new_line:
 119                 break
 120             if 'Testscount' in new_line:
 121                 result['num_tests'] = int(new_line[2])
 122             elif 'Success' in new_line:
 123                 result['num_success'] = int(new_line[2])
 124             elif 'Skipped' in new_line:
 125                 result['num_skipped'] = int(new_line[2])
 126             elif 'Failures' in new_line:
 127                 result['num_failures'] = int(new_line[2])
 128         return result
 129
 130     @staticmethod
 131     def backup_tempest_config(conf_file, res_dir):
 132         """
 133         Copy config file to tempest results directory
 134         """
 135         if not os.path.exists(res_dir):
 136             os.makedirs(res_dir)
 137         shutil.copyfile(conf_file,
 138                         os.path.join(res_dir, 'tempest.conf'))
 139
 140     def generate_test_list(self, **kwargs):
 141         """Generate test list based on the test mode."""
 142         LOGGER.debug("Generating test case list...")
 143         self.backup_tempest_config(self.conf_file, '/etc')
 144         if kwargs.get('mode') == 'custom':
 145             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 146                 shutil.copyfile(
 147                     conf_utils.TEMPEST_CUSTOM, self.list)
 148             else:
 149                 raise Exception("Tempest test list file %s NOT found."
 150                                 % conf_utils.TEMPEST_CUSTOM)
 151         else:
 152             testr_mode = kwargs.get(
 153                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 154             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 155                 self.verifier_repo_dir, testr_mode, self.list)
 156             output = subprocess.check_output(cmd, shell=True)
 157             LOGGER.info("%s\n%s", cmd, output)
 158         os.remove('/etc/tempest.conf')
 159
 160     def apply_tempest_blacklist(self):
 161         """Exclude blacklisted test cases."""
 162         LOGGER.debug("Applying tempest blacklist...")
 163         if os.path.exists(self.raw_list):
 164             os.remove(self.raw_list)
 165         os.rename(self.list, self.raw_list)
 166         cases_file = self.read_file(self.raw_list)
 167         result_file = open(self.list, 'w')
 168         black_tests = []
 169         try:
 170             installer_type = env.get('INSTALLER_TYPE')
 171             deploy_scenario = env.get('DEPLOY_SCENARIO')
 172             if bool(installer_type) * bool(deploy_scenario):
 173                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 174                 # file
 175                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 176                 black_list_yaml = yaml.safe_load(black_list_file)
 177                 black_list_file.close()
 178                 for item in black_list_yaml:
 179                     scenarios = item['scenarios']
 180                     installers = item['installers']
 181                     if (deploy_scenario in scenarios and
 182                             installer_type in installers):
 183                         tests = item['tests']
 184                         for test in tests:
 185                             black_tests.append(test)
 186                         break
 187         except Exception:  # pylint: disable=broad-except
 188             black_tests = []
 189             LOGGER.debug("Tempest blacklist file does not exist.")
 190
 191         for cases_line in cases_file:
 192             for black_tests_line in black_tests:
 193                 if black_tests_line in cases_line:
 194                     break
 195             else:
 196                 result_file.write(str(cases_line) + '\n')
 197         result_file.close()
 198
 199     def run_verifier_tests(self, **kwargs):
 200         """Execute tempest test cases."""
 201         cmd = ["rally", "verify", "start", "--load-list",
 202                self.list]
 203         cmd.extend(kwargs.get('option', []))
 204         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 205
 206         f_stdout = open(
 207             os.path.join(self.res_dir, "tempest.log"), 'w+')
 208
 209         proc = subprocess.Popen(
 210             cmd,
 211             stdout=subprocess.PIPE,
 212             stderr=subprocess.STDOUT,
 213             bufsize=1)
 214
 215         with proc.stdout:
 216             for line in iter(proc.stdout.readline, b''):
 217                 if re.search(r"\} tempest\.", line):
 218                     LOGGER.info(line.rstrip())
 219                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 220                     self.verification_id = re.search(
 221                         r'(?=\(UUID=(.*)\))', line).group(1)
 222                 f_stdout.write(line)
 223         proc.wait()
 224         f_stdout.close()
 225
 226         if self.verification_id is None:
 227             raise Exception('Verification UUID not found')
 228         LOGGER.info('Verification UUID: %s', self.verification_id)
 229
 230         shutil.copy(
 231             "{}/tempest.log".format(self.deployment_dir),
 232             "{}/tempest.debug.log".format(self.res_dir))
 233
 234     def parse_verifier_result(self):
 235         """Parse and save test results."""
 236         stat = self.get_verifier_result(self.verification_id)
 237         try:
 238             num_executed = stat['num_tests'] - stat['num_skipped']
 239             try:
 240                 self.result = 100 * stat['num_success'] / num_executed
 241             except ZeroDivisionError:
 242                 self.result = 0
 243                 if stat['num_tests'] > 0:
 244                     LOGGER.info("All tests have been skipped")
 245                 else:
 246                     LOGGER.error("No test has been executed")
 247                     return
 248
 249             with open(os.path.join(self.res_dir,
 250                                    "tempest.log"), 'r') as logfile:
 251                 output = logfile.read()
 252
 253             success_testcases = []
 254             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 255                                     output):
 256                 success_testcases.append(match)
 257             failed_testcases = []
 258             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 259                                     output):
 260                 failed_testcases.append(match)
 261             skipped_testcases = []
 262             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 263                                     output):
 264                 skipped_testcases.append(match)
 265
 266             self.details = {"tests_number": stat['num_tests'],
 267                             "success_number": stat['num_success'],
 268                             "skipped_number": stat['num_skipped'],
 269                             "failures_number": stat['num_failures'],
 270                             "success": success_testcases,
 271                             "skipped": skipped_testcases,
 272                             "failures": failed_testcases}
 273         except Exception:  # pylint: disable=broad-except
 274             self.result = 0
 275
 276         LOGGER.info("Tempest %s success_rate is %s%%",
 277                     self.case_name, self.result)
 278
 279     def generate_report(self):
 280         """Generate verification report."""
 281         html_file = os.path.join(self.res_dir,
 282                                  "tempest-report.html")
 283         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 284                self.verification_id, "--to", html_file]
 285         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 286                          stderr=subprocess.STDOUT)
 287
 288     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 289         """Set image name as tempest img_name_regex"""
 290         rconfig = configparser.RawConfigParser()
 291         rconfig.read(rally_conf)
 292         if not rconfig.has_section('tempest'):
 293             rconfig.add_section('tempest')
 294         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 295             self.image.name))
 296         with open(rally_conf, 'wb') as config_file:
 297             rconfig.write(config_file)
 298
 299     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 300         """Detect and update the default role if required"""
 301         role = self.get_default_role(self.cloud)
 302         if not role:
 303             return
 304         rconfig = configparser.RawConfigParser()
 305         rconfig.read(rally_conf)
 306         if not rconfig.has_section('tempest'):
 307             rconfig.add_section('tempest')
 308         rconfig.set('tempest', 'swift_operator_role', role.name)
 309         with open(rally_conf, 'wb') as config_file:
 310             rconfig.write(config_file)
 311
 312     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 313         """Print rally logs in res dir"""
 314         if not os.path.exists(self.res_dir):
 315             os.makedirs(self.res_dir)
 316         rconfig = configparser.RawConfigParser()
 317         rconfig.read(rally_conf)
 318         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 319         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 320         with open(rally_conf, 'wb') as config_file:
 321             rconfig.write(config_file)
 322
 323     @staticmethod
 324     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 325         """Clean Rally config"""
 326         rconfig = configparser.RawConfigParser()
 327         rconfig.read(rally_conf)
 328         if rconfig.has_option('tempest', 'img_name_regex'):
 329             rconfig.remove_option('tempest', 'img_name_regex')
 330         if rconfig.has_option('tempest', 'swift_operator_role'):
 331             rconfig.remove_option('tempest', 'swift_operator_role')
 332         if rconfig.has_option('DEFAULT', 'log-file'):
 333             rconfig.remove_option('DEFAULT', 'log-file')
 334         if rconfig.has_option('DEFAULT', 'log_dir'):
 335             rconfig.remove_option('DEFAULT', 'log_dir')
 336         with open(rally_conf, 'wb') as config_file:
 337             rconfig.write(config_file)
 338
 339     def configure(self, **kwargs):  # pylint: disable=unused-argument
 340         """
 341         Create all openstack resources for tempest-based testcases and write
 342         tempest.conf.
 343         """
 344         if not os.path.exists(self.res_dir):
 345             os.makedirs(self.res_dir)
 346         compute_cnt = len(self.cloud.list_hypervisors())
 347
 348         self.image_alt = self.publish_image_alt()
 349         self.flavor_alt = self.create_flavor_alt()
 350         LOGGER.debug("flavor: %s", self.flavor_alt)
 351
 352         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 353         conf_utils.configure_tempest_update_params(
 354             self.conf_file, network_name=self.network.name,
 355             image_id=self.image.id,
 356             flavor_id=self.flavor.id,
 357             compute_cnt=compute_cnt,
 358             image_alt_id=self.image_alt.id,
 359             flavor_alt_id=self.flavor_alt.id)
 360         self.backup_tempest_config(self.conf_file, self.res_dir)
 361
 362     def run(self, **kwargs):
 363         self.start_time = time.time()
 364         try:
 365             assert super(TempestCommon, self).run(
 366                 **kwargs) == testcase.TestCase.EX_OK
 367             self.update_rally_regex()
 368             self.update_default_role()
 369             self.update_rally_logs()
 370             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 371             self.configure(**kwargs)
 372             self.generate_test_list(**kwargs)
 373             self.apply_tempest_blacklist()
 374             self.run_verifier_tests(**kwargs)
 375             self.parse_verifier_result()
 376             self.generate_report()
 377             res = testcase.TestCase.EX_OK
 378         except Exception:  # pylint: disable=broad-except
 379             LOGGER.exception('Error with run')
 380             self.result = 0
 381             res = testcase.TestCase.EX_RUN_ERROR
 382         self.stop_time = time.time()
 383         return res
 384
 385     def clean(self):
 386         """
 387         Cleanup all OpenStack objects. Should be called on completion.
 388         """
 389         self.clean_rally_conf()
 390         if self.image_alt:
 391             self.cloud.delete_image(self.image_alt)
 392         if self.flavor_alt:
 393             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 394         super(TempestCommon, self).clean()