functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     shared_network = True
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         self.verifier_id = conf_utils.get_verifier_id()
  47         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  48             self.verifier_id)
  49         self.deployment_id = conf_utils.get_verifier_deployment_id()
  50         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  51             self.verifier_id, self.deployment_id)
  52         self.verification_id = None
  53         self.res_dir = os.path.join(
  54             getattr(config.CONF, 'dir_results'), self.case_name)
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58         self.image_alt = None
  59         self.flavor_alt = None
  60         self.services = []
  61         try:
  62             self.services = kwargs['run']['args']['services']
  63         except Exception:  # pylint: disable=broad-except
  64             pass
  65         self.neutron_extensions = []
  66         try:
  67             self.neutron_extensions = kwargs['run']['args'][
  68                 'neutron_extensions']
  69         except Exception:  # pylint: disable=broad-except
  70             pass
  71
  72     def check_services(self):
  73         """Check the mandatory services."""
  74         for service in self.services:
  75             try:
  76                 self.cloud.search_services(service)[0]
  77             except Exception:  # pylint: disable=broad-except
  78                 self.is_skipped = True
  79                 break
  80
  81     def check_extensions(self):
  82         """Check the mandatory network extensions."""
  83         extensions = self.cloud.get_network_extensions()
  84         for network_extension in self.neutron_extensions:
  85             if network_extension not in extensions:
  86                 LOGGER.warning(
  87                     "Cannot find Neutron extension: %s", network_extension)
  88                 self.is_skipped = True
  89                 break
  90
  91     def check_requirements(self):
  92         self.check_services()
  93         self.check_extensions()
  94
  95     @staticmethod
  96     def read_file(filename):
  97         """Read file and return content as a stripped list."""
  98         with open(filename) as src:
  99             return [line.strip() for line in src.readlines()]
 100
 101     @staticmethod
 102     def get_verifier_result(verif_id):
 103         """Retrieve verification results."""
 104         result = {
 105             'num_tests': 0,
 106             'num_success': 0,
 107             'num_failures': 0,
 108             'num_skipped': 0
 109         }
 110         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 111         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 112         proc = subprocess.Popen(cmd,
 113                                 stdout=subprocess.PIPE,
 114                                 stderr=subprocess.STDOUT)
 115         for line in proc.stdout:
 116             LOGGER.info(line.rstrip())
 117             new_line = line.replace(' ', '').split('|')
 118             if 'Tests' in new_line:
 119                 break
 120             if 'Testscount' in new_line:
 121                 result['num_tests'] = int(new_line[2])
 122             elif 'Success' in new_line:
 123                 result['num_success'] = int(new_line[2])
 124             elif 'Skipped' in new_line:
 125                 result['num_skipped'] = int(new_line[2])
 126             elif 'Failures' in new_line:
 127                 result['num_failures'] = int(new_line[2])
 128         return result
 129
 130     @staticmethod
 131     def backup_tempest_config(conf_file, res_dir):
 132         """
 133         Copy config file to tempest results directory
 134         """
 135         if not os.path.exists(res_dir):
 136             os.makedirs(res_dir)
 137         shutil.copyfile(conf_file,
 138                         os.path.join(res_dir, 'tempest.conf'))
 139
 140     def generate_test_list(self, **kwargs):
 141         """Generate test list based on the test mode."""
 142         LOGGER.debug("Generating test case list...")
 143         self.backup_tempest_config(self.conf_file, '/etc')
 144         if kwargs.get('mode') == 'custom':
 145             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 146                 shutil.copyfile(
 147                     conf_utils.TEMPEST_CUSTOM, self.list)
 148             else:
 149                 raise Exception("Tempest test list file %s NOT found."
 150                                 % conf_utils.TEMPEST_CUSTOM)
 151         else:
 152             testr_mode = kwargs.get(
 153                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 154             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 155                 self.verifier_repo_dir, testr_mode, self.list)
 156             output = subprocess.check_output(cmd, shell=True)
 157             LOGGER.info("%s\n%s", cmd, output)
 158         os.remove('/etc/tempest.conf')
 159
 160     def apply_tempest_blacklist(self):
 161         """Exclude blacklisted test cases."""
 162         LOGGER.debug("Applying tempest blacklist...")
 163         if os.path.exists(self.raw_list):
 164             os.remove(self.raw_list)
 165         os.rename(self.list, self.raw_list)
 166         cases_file = self.read_file(self.raw_list)
 167         result_file = open(self.list, 'w')
 168         black_tests = []
 169         try:
 170             deploy_scenario = env.get('DEPLOY_SCENARIO')
 171             if bool(deploy_scenario):
 172                 # if DEPLOY_SCENARIO is set we read the file
 173                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 174                 black_list_yaml = yaml.safe_load(black_list_file)
 175                 black_list_file.close()
 176                 for item in black_list_yaml:
 177                     scenarios = item['scenarios']
 178                     if deploy_scenario in scenarios:
 179                         tests = item['tests']
 180                         for test in tests:
 181                             black_tests.append(test)
 182                         break
 183         except Exception:  # pylint: disable=broad-except
 184             black_tests = []
 185             LOGGER.debug("Tempest blacklist file does not exist.")
 186
 187         for cases_line in cases_file:
 188             for black_tests_line in black_tests:
 189                 if black_tests_line in cases_line:
 190                     break
 191             else:
 192                 result_file.write(str(cases_line) + '\n')
 193         result_file.close()
 194
 195     def run_verifier_tests(self, **kwargs):
 196         """Execute tempest test cases."""
 197         cmd = ["rally", "verify", "start", "--load-list",
 198                self.list]
 199         cmd.extend(kwargs.get('option', []))
 200         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 201
 202         f_stdout = open(
 203             os.path.join(self.res_dir, "tempest.log"), 'w+')
 204
 205         proc = subprocess.Popen(
 206             cmd,
 207             stdout=subprocess.PIPE,
 208             stderr=subprocess.STDOUT,
 209             bufsize=1)
 210
 211         with proc.stdout:
 212             for line in iter(proc.stdout.readline, b''):
 213                 if re.search(r"\} tempest\.", line):
 214                     LOGGER.info(line.rstrip())
 215                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 216                     self.verification_id = re.search(
 217                         r'(?=\(UUID=(.*)\))', line).group(1)
 218                 f_stdout.write(line)
 219         proc.wait()
 220         f_stdout.close()
 221
 222         if self.verification_id is None:
 223             raise Exception('Verification UUID not found')
 224         LOGGER.info('Verification UUID: %s', self.verification_id)
 225
 226         shutil.copy(
 227             "{}/tempest.log".format(self.deployment_dir),
 228             "{}/tempest.debug.log".format(self.res_dir))
 229
 230     def parse_verifier_result(self):
 231         """Parse and save test results."""
 232         stat = self.get_verifier_result(self.verification_id)
 233         try:
 234             num_executed = stat['num_tests'] - stat['num_skipped']
 235             try:
 236                 self.result = 100 * stat['num_success'] / num_executed
 237             except ZeroDivisionError:
 238                 self.result = 0
 239                 if stat['num_tests'] > 0:
 240                     LOGGER.info("All tests have been skipped")
 241                 else:
 242                     LOGGER.error("No test has been executed")
 243                     return
 244
 245             with open(os.path.join(self.res_dir,
 246                                    "tempest.log"), 'r') as logfile:
 247                 output = logfile.read()
 248
 249             success_testcases = []
 250             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 251                                     output):
 252                 success_testcases.append(match)
 253             failed_testcases = []
 254             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 255                                     output):
 256                 failed_testcases.append(match)
 257             skipped_testcases = []
 258             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 259                                     output):
 260                 skipped_testcases.append(match)
 261
 262             self.details = {"tests_number": stat['num_tests'],
 263                             "success_number": stat['num_success'],
 264                             "skipped_number": stat['num_skipped'],
 265                             "failures_number": stat['num_failures'],
 266                             "success": success_testcases,
 267                             "skipped": skipped_testcases,
 268                             "failures": failed_testcases}
 269         except Exception:  # pylint: disable=broad-except
 270             self.result = 0
 271
 272         LOGGER.info("Tempest %s success_rate is %s%%",
 273                     self.case_name, self.result)
 274
 275     def generate_report(self):
 276         """Generate verification report."""
 277         html_file = os.path.join(self.res_dir,
 278                                  "tempest-report.html")
 279         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 280                self.verification_id, "--to", html_file]
 281         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 282                          stderr=subprocess.STDOUT)
 283
 284     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 285         """Set image name as tempest img_name_regex"""
 286         rconfig = configparser.RawConfigParser()
 287         rconfig.read(rally_conf)
 288         if not rconfig.has_section('tempest'):
 289             rconfig.add_section('tempest')
 290         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 291             self.image.name))
 292         with open(rally_conf, 'wb') as config_file:
 293             rconfig.write(config_file)
 294
 295     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 296         """Detect and update the default role if required"""
 297         role = self.get_default_role(self.cloud)
 298         if not role:
 299             return
 300         rconfig = configparser.RawConfigParser()
 301         rconfig.read(rally_conf)
 302         if not rconfig.has_section('tempest'):
 303             rconfig.add_section('tempest')
 304         rconfig.set('tempest', 'swift_operator_role', role.name)
 305         with open(rally_conf, 'wb') as config_file:
 306             rconfig.write(config_file)
 307
 308     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 309         """Print rally logs in res dir"""
 310         if not os.path.exists(self.res_dir):
 311             os.makedirs(self.res_dir)
 312         rconfig = configparser.RawConfigParser()
 313         rconfig.read(rally_conf)
 314         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 315         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 316         with open(rally_conf, 'wb') as config_file:
 317             rconfig.write(config_file)
 318
 319     @staticmethod
 320     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 321         """Clean Rally config"""
 322         rconfig = configparser.RawConfigParser()
 323         rconfig.read(rally_conf)
 324         if rconfig.has_option('tempest', 'img_name_regex'):
 325             rconfig.remove_option('tempest', 'img_name_regex')
 326         if rconfig.has_option('tempest', 'swift_operator_role'):
 327             rconfig.remove_option('tempest', 'swift_operator_role')
 328         if rconfig.has_option('DEFAULT', 'log-file'):
 329             rconfig.remove_option('DEFAULT', 'log-file')
 330         if rconfig.has_option('DEFAULT', 'log_dir'):
 331             rconfig.remove_option('DEFAULT', 'log_dir')
 332         with open(rally_conf, 'wb') as config_file:
 333             rconfig.write(config_file)
 334
 335     def configure(self, **kwargs):  # pylint: disable=unused-argument
 336         """
 337         Create all openstack resources for tempest-based testcases and write
 338         tempest.conf.
 339         """
 340         if not os.path.exists(self.res_dir):
 341             os.makedirs(self.res_dir)
 342         compute_cnt = len(self.cloud.list_hypervisors())
 343
 344         self.image_alt = self.publish_image_alt()
 345         self.flavor_alt = self.create_flavor_alt()
 346         LOGGER.debug("flavor: %s", self.flavor_alt)
 347
 348         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 349         conf_utils.configure_tempest_update_params(
 350             self.conf_file, network_name=self.network.name,
 351             image_id=self.image.id,
 352             flavor_id=self.flavor.id,
 353             compute_cnt=compute_cnt,
 354             image_alt_id=self.image_alt.id,
 355             flavor_alt_id=self.flavor_alt.id)
 356         self.backup_tempest_config(self.conf_file, self.res_dir)
 357
 358     def run(self, **kwargs):
 359         self.start_time = time.time()
 360         try:
 361             assert super(TempestCommon, self).run(
 362                 **kwargs) == testcase.TestCase.EX_OK
 363             if not os.path.exists(self.res_dir):
 364                 os.makedirs(self.res_dir)
 365             self.update_rally_regex()
 366             self.update_default_role()
 367             self.update_rally_logs()
 368             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 369             self.configure(**kwargs)
 370             self.generate_test_list(**kwargs)
 371             self.apply_tempest_blacklist()
 372             self.run_verifier_tests(**kwargs)
 373             self.parse_verifier_result()
 374             self.generate_report()
 375             res = testcase.TestCase.EX_OK
 376         except Exception:  # pylint: disable=broad-except
 377             LOGGER.exception('Error with run')
 378             self.result = 0
 379             res = testcase.TestCase.EX_RUN_ERROR
 380         self.stop_time = time.time()
 381         return res
 382
 383     def clean(self):
 384         """
 385         Cleanup all OpenStack objects. Should be called on completion.
 386         """
 387         self.clean_rally_conf()
 388         if self.image_alt:
 389             self.cloud.delete_image(self.image_alt)
 390         if self.flavor_alt:
 391             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 392         super(TempestCommon, self).clean()