functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  40
  41     def __init__(self, **kwargs):
  42         if "case_name" not in kwargs:
  43             kwargs["case_name"] = 'tempest'
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.verifier_id = conf_utils.get_verifier_id()
  46         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  47             self.verifier_id)
  48         self.deployment_id = conf_utils.get_verifier_deployment_id()
  49         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  50             self.verifier_id, self.deployment_id)
  51         self.verification_id = None
  52         self.res_dir = os.path.join(
  53             getattr(config.CONF, 'dir_results'), self.case_name)
  54         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  55         self.list = os.path.join(self.res_dir, 'test_list.txt')
  56         self.conf_file = None
  57         self.image_alt = None
  58         self.flavor_alt = None
  59         self.services = []
  60         try:
  61             self.services = kwargs['run']['args']['services']
  62         except Exception:  # pylint: disable=broad-except
  63             pass
  64         self.neutron_extensions = []
  65         try:
  66             self.neutron_extensions = kwargs['run']['args'][
  67                 'neutron_extensions']
  68         except Exception:  # pylint: disable=broad-except
  69             pass
  70
  71     def check_services(self):
  72         """Check the mandatory services."""
  73         for service in self.services:
  74             try:
  75                 self.cloud.search_services(service)[0]
  76             except Exception:  # pylint: disable=broad-except
  77                 self.is_skipped = True
  78                 break
  79
  80     def check_extensions(self):
  81         """Check the mandatory network extensions."""
  82         extensions = self.cloud.get_network_extensions()
  83         for network_extension in self.neutron_extensions:
  84             if network_extension not in extensions:
  85                 LOGGER.warning(
  86                     "Cannot find Neutron extension: %s", network_extension)
  87                 self.is_skipped = True
  88                 break
  89
  90     def check_requirements(self):
  91         self.check_services()
  92         self.check_extensions()
  93
  94     @staticmethod
  95     def read_file(filename):
  96         """Read file and return content as a stripped list."""
  97         with open(filename) as src:
  98             return [line.strip() for line in src.readlines()]
  99
 100     @staticmethod
 101     def get_verifier_result(verif_id):
 102         """Retrieve verification results."""
 103         result = {
 104             'num_tests': 0,
 105             'num_success': 0,
 106             'num_failures': 0,
 107             'num_skipped': 0
 108         }
 109         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 110         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 111         proc = subprocess.Popen(cmd,
 112                                 stdout=subprocess.PIPE,
 113                                 stderr=subprocess.STDOUT)
 114         for line in proc.stdout:
 115             LOGGER.info(line.rstrip())
 116             new_line = line.replace(' ', '').split('|')
 117             if 'Tests' in new_line:
 118                 break
 119             if 'Testscount' in new_line:
 120                 result['num_tests'] = int(new_line[2])
 121             elif 'Success' in new_line:
 122                 result['num_success'] = int(new_line[2])
 123             elif 'Skipped' in new_line:
 124                 result['num_skipped'] = int(new_line[2])
 125             elif 'Failures' in new_line:
 126                 result['num_failures'] = int(new_line[2])
 127         return result
 128
 129     @staticmethod
 130     def backup_tempest_config(conf_file, res_dir):
 131         """
 132         Copy config file to tempest results directory
 133         """
 134         if not os.path.exists(res_dir):
 135             os.makedirs(res_dir)
 136         shutil.copyfile(conf_file,
 137                         os.path.join(res_dir, 'tempest.conf'))
 138
 139     def generate_test_list(self, **kwargs):
 140         """Generate test list based on the test mode."""
 141         LOGGER.debug("Generating test case list...")
 142         self.backup_tempest_config(self.conf_file, '/etc')
 143         if kwargs.get('mode') == 'custom':
 144             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 145                 shutil.copyfile(
 146                     conf_utils.TEMPEST_CUSTOM, self.list)
 147             else:
 148                 raise Exception("Tempest test list file %s NOT found."
 149                                 % conf_utils.TEMPEST_CUSTOM)
 150         else:
 151             testr_mode = kwargs.get(
 152                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 153             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 154                 self.verifier_repo_dir, testr_mode, self.list)
 155             output = subprocess.check_output(cmd, shell=True)
 156             LOGGER.info("%s\n%s", cmd, output)
 157         os.remove('/etc/tempest.conf')
 158
 159     def apply_tempest_blacklist(self):
 160         """Exclude blacklisted test cases."""
 161         LOGGER.debug("Applying tempest blacklist...")
 162         if os.path.exists(self.raw_list):
 163             os.remove(self.raw_list)
 164         os.rename(self.list, self.raw_list)
 165         cases_file = self.read_file(self.raw_list)
 166         result_file = open(self.list, 'w')
 167         black_tests = []
 168         try:
 169             installer_type = env.get('INSTALLER_TYPE')
 170             deploy_scenario = env.get('DEPLOY_SCENARIO')
 171             if bool(installer_type) * bool(deploy_scenario):
 172                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 173                 # file
 174                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 175                 black_list_yaml = yaml.safe_load(black_list_file)
 176                 black_list_file.close()
 177                 for item in black_list_yaml:
 178                     scenarios = item['scenarios']
 179                     installers = item['installers']
 180                     if (deploy_scenario in scenarios and
 181                             installer_type in installers):
 182                         tests = item['tests']
 183                         for test in tests:
 184                             black_tests.append(test)
 185                         break
 186         except Exception:  # pylint: disable=broad-except
 187             black_tests = []
 188             LOGGER.debug("Tempest blacklist file does not exist.")
 189
 190         for cases_line in cases_file:
 191             for black_tests_line in black_tests:
 192                 if black_tests_line in cases_line:
 193                     break
 194             else:
 195                 result_file.write(str(cases_line) + '\n')
 196         result_file.close()
 197
 198     def run_verifier_tests(self, **kwargs):
 199         """Execute tempest test cases."""
 200         cmd = ["rally", "verify", "start", "--load-list",
 201                self.list]
 202         cmd.extend(kwargs.get('option', []))
 203         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 204
 205         f_stdout = open(
 206             os.path.join(self.res_dir, "tempest.log"), 'w+')
 207
 208         proc = subprocess.Popen(
 209             cmd,
 210             stdout=subprocess.PIPE,
 211             stderr=subprocess.STDOUT,
 212             bufsize=1)
 213
 214         with proc.stdout:
 215             for line in iter(proc.stdout.readline, b''):
 216                 if re.search(r"\} tempest\.", line):
 217                     LOGGER.info(line.rstrip())
 218                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 219                     self.verification_id = re.search(
 220                         r'(?=\(UUID=(.*)\))', line).group(1)
 221                 f_stdout.write(line)
 222         proc.wait()
 223         f_stdout.close()
 224
 225         if self.verification_id is None:
 226             raise Exception('Verification UUID not found')
 227         LOGGER.info('Verification UUID: %s', self.verification_id)
 228
 229     def parse_verifier_result(self):
 230         """Parse and save test results."""
 231         stat = self.get_verifier_result(self.verification_id)
 232         try:
 233             num_executed = stat['num_tests'] - stat['num_skipped']
 234             try:
 235                 self.result = 100 * stat['num_success'] / num_executed
 236             except ZeroDivisionError:
 237                 self.result = 0
 238                 if stat['num_tests'] > 0:
 239                     LOGGER.info("All tests have been skipped")
 240                 else:
 241                     LOGGER.error("No test has been executed")
 242                     return
 243
 244             with open(os.path.join(self.res_dir,
 245                                    "tempest.log"), 'r') as logfile:
 246                 output = logfile.read()
 247
 248             success_testcases = []
 249             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 250                                     output):
 251                 success_testcases.append(match)
 252             failed_testcases = []
 253             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 254                                     output):
 255                 failed_testcases.append(match)
 256             skipped_testcases = []
 257             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 258                                     output):
 259                 skipped_testcases.append(match)
 260
 261             self.details = {"tests_number": stat['num_tests'],
 262                             "success_number": stat['num_success'],
 263                             "skipped_number": stat['num_skipped'],
 264                             "failures_number": stat['num_failures'],
 265                             "success": success_testcases,
 266                             "skipped": skipped_testcases,
 267                             "failures": failed_testcases}
 268         except Exception:  # pylint: disable=broad-except
 269             self.result = 0
 270
 271         LOGGER.info("Tempest %s success_rate is %s%%",
 272                     self.case_name, self.result)
 273
 274     def generate_report(self):
 275         """Generate verification report."""
 276         html_file = os.path.join(self.res_dir,
 277                                  "tempest-report.html")
 278         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 279                self.verification_id, "--to", html_file]
 280         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 281                          stderr=subprocess.STDOUT)
 282
 283     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 284         """Set image name as tempest img_name_regex"""
 285         rconfig = configparser.RawConfigParser()
 286         rconfig.read(rally_conf)
 287         if not rconfig.has_section('tempest'):
 288             rconfig.add_section('tempest')
 289         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 290             self.image.name))
 291         with open(rally_conf, 'wb') as config_file:
 292             rconfig.write(config_file)
 293
 294     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 295         """Detect and update the default role if required"""
 296         role = self.get_default_role(self.cloud)
 297         if not role:
 298             return
 299         rconfig = configparser.RawConfigParser()
 300         rconfig.read(rally_conf)
 301         if not rconfig.has_section('tempest'):
 302             rconfig.add_section('tempest')
 303         rconfig.set('tempest', 'swift_operator_role', role.name)
 304         with open(rally_conf, 'wb') as config_file:
 305             rconfig.write(config_file)
 306
 307     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 308         """Print rally logs in res dir"""
 309         if not os.path.exists(self.res_dir):
 310             os.makedirs(self.res_dir)
 311         rconfig = configparser.RawConfigParser()
 312         rconfig.read(rally_conf)
 313         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 314         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 315         with open(rally_conf, 'wb') as config_file:
 316             rconfig.write(config_file)
 317
 318     @staticmethod
 319     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 320         """Clean Rally config"""
 321         rconfig = configparser.RawConfigParser()
 322         rconfig.read(rally_conf)
 323         if rconfig.has_option('tempest', 'img_name_regex'):
 324             rconfig.remove_option('tempest', 'img_name_regex')
 325         if rconfig.has_option('tempest', 'swift_operator_role'):
 326             rconfig.remove_option('tempest', 'swift_operator_role')
 327         if rconfig.has_option('DEFAULT', 'log-file'):
 328             rconfig.remove_option('DEFAULT', 'log-file')
 329         if rconfig.has_option('DEFAULT', 'log_dir'):
 330             rconfig.remove_option('DEFAULT', 'log_dir')
 331         with open(rally_conf, 'wb') as config_file:
 332             rconfig.write(config_file)
 333
 334     def configure(self, **kwargs):  # pylint: disable=unused-argument
 335         """
 336         Create all openstack resources for tempest-based testcases and write
 337         tempest.conf.
 338         """
 339         if not os.path.exists(self.res_dir):
 340             os.makedirs(self.res_dir)
 341         compute_cnt = len(self.cloud.list_hypervisors())
 342
 343         self.image_alt = self.publish_image_alt()
 344         self.flavor_alt = self.create_flavor_alt()
 345         LOGGER.debug("flavor: %s", self.flavor_alt)
 346
 347         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 348         conf_utils.configure_tempest_update_params(
 349             self.conf_file, network_name=self.network.id,
 350             image_id=self.image.id,
 351             flavor_id=self.flavor.id,
 352             compute_cnt=compute_cnt,
 353             image_alt_id=self.image_alt.id,
 354             flavor_alt_id=self.flavor_alt.id)
 355         self.backup_tempest_config(self.conf_file, self.res_dir)
 356
 357     def run(self, **kwargs):
 358         self.start_time = time.time()
 359         try:
 360             assert super(TempestCommon, self).run(
 361                 **kwargs) == testcase.TestCase.EX_OK
 362             self.update_rally_regex()
 363             self.update_default_role()
 364             self.update_rally_logs()
 365             self.configure(**kwargs)
 366             self.generate_test_list(**kwargs)
 367             self.apply_tempest_blacklist()
 368             self.run_verifier_tests(**kwargs)
 369             self.parse_verifier_result()
 370             self.generate_report()
 371             res = testcase.TestCase.EX_OK
 372         except Exception:  # pylint: disable=broad-except
 373             LOGGER.exception('Error with run')
 374             self.result = 0
 375             res = testcase.TestCase.EX_RUN_ERROR
 376         self.stop_time = time.time()
 377         return res
 378
 379     def clean(self):
 380         """
 381         Cleanup all OpenStack objects. Should be called on completion.
 382         """
 383         self.clean_rally_conf()
 384         if self.image_alt:
 385             self.cloud.delete_image(self.image_alt)
 386         if self.flavor_alt:
 387             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 388         super(TempestCommon, self).clean()