functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  40
  41     def __init__(self, **kwargs):
  42         if "case_name" not in kwargs:
  43             kwargs["case_name"] = 'tempest'
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.verifier_id = conf_utils.get_verifier_id()
  46         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  47             self.verifier_id)
  48         self.deployment_id = conf_utils.get_verifier_deployment_id()
  49         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  50             self.verifier_id, self.deployment_id)
  51         self.verification_id = None
  52         self.res_dir = os.path.join(
  53             getattr(config.CONF, 'dir_results'), self.case_name)
  54         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  55         self.list = os.path.join(self.res_dir, 'test_list.txt')
  56         self.conf_file = None
  57         self.image_alt = None
  58         self.flavor_alt = None
  59         self.services = []
  60         try:
  61             self.services = kwargs['run']['args']['services']
  62         except Exception:  # pylint: disable=broad-except
  63             pass
  64         self.neutron_extensions = []
  65         try:
  66             self.neutron_extensions = kwargs['run']['args'][
  67                 'neutron_extensions']
  68         except Exception:  # pylint: disable=broad-except
  69             pass
  70
  71     def check_services(self):
  72         """Check the mandatory services."""
  73         for service in self.services:
  74             try:
  75                 self.cloud.search_services(service)[0]
  76             except Exception:  # pylint: disable=broad-except
  77                 self.is_skipped = True
  78                 break
  79
  80     def check_extensions(self):
  81         """Check the mandatory network extensions."""
  82         extensions = self.cloud.get_network_extensions()
  83         for network_extension in self.neutron_extensions:
  84             if network_extension not in extensions:
  85                 LOGGER.warning(
  86                     "Cannot find Neutron extension: %s", network_extension)
  87                 self.is_skipped = True
  88                 break
  89
  90     def check_requirements(self):
  91         self.check_services()
  92         self.check_extensions()
  93
  94     @staticmethod
  95     def read_file(filename):
  96         """Read file and return content as a stripped list."""
  97         with open(filename) as src:
  98             return [line.strip() for line in src.readlines()]
  99
 100     @staticmethod
 101     def get_verifier_result(verif_id):
 102         """Retrieve verification results."""
 103         result = {
 104             'num_tests': 0,
 105             'num_success': 0,
 106             'num_failures': 0,
 107             'num_skipped': 0
 108         }
 109         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 110         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 111         proc = subprocess.Popen(cmd,
 112                                 stdout=subprocess.PIPE,
 113                                 stderr=subprocess.STDOUT)
 114         for line in proc.stdout:
 115             new_line = line.replace(' ', '').split('|')
 116             if 'Tests' in new_line:
 117                 break
 118             LOGGER.info(line)
 119             if 'Testscount' in new_line:
 120                 result['num_tests'] = int(new_line[2])
 121             elif 'Success' in new_line:
 122                 result['num_success'] = int(new_line[2])
 123             elif 'Skipped' in new_line:
 124                 result['num_skipped'] = int(new_line[2])
 125             elif 'Failures' in new_line:
 126                 result['num_failures'] = int(new_line[2])
 127         return result
 128
 129     @staticmethod
 130     def backup_tempest_config(conf_file, res_dir):
 131         """
 132         Copy config file to tempest results directory
 133         """
 134         if not os.path.exists(res_dir):
 135             os.makedirs(res_dir)
 136         shutil.copyfile(conf_file,
 137                         os.path.join(res_dir, 'tempest.conf'))
 138
 139     def generate_test_list(self, **kwargs):
 140         """Generate test list based on the test mode."""
 141         LOGGER.debug("Generating test case list...")
 142         self.backup_tempest_config(self.conf_file, '/etc')
 143         if kwargs.get('mode') == 'custom':
 144             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 145                 shutil.copyfile(
 146                     conf_utils.TEMPEST_CUSTOM, self.list)
 147             else:
 148                 raise Exception("Tempest test list file %s NOT found."
 149                                 % conf_utils.TEMPEST_CUSTOM)
 150         else:
 151             testr_mode = kwargs.get(
 152                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 153             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 154                 self.verifier_repo_dir, testr_mode, self.list)
 155             output = subprocess.check_output(cmd, shell=True)
 156             LOGGER.info("%s\n%s", cmd, output)
 157         os.remove('/etc/tempest.conf')
 158
 159     def apply_tempest_blacklist(self):
 160         """Exclude blacklisted test cases."""
 161         LOGGER.debug("Applying tempest blacklist...")
 162         if os.path.exists(self.raw_list):
 163             os.remove(self.raw_list)
 164         os.rename(self.list, self.raw_list)
 165         cases_file = self.read_file(self.raw_list)
 166         result_file = open(self.list, 'w')
 167         black_tests = []
 168         try:
 169             installer_type = env.get('INSTALLER_TYPE')
 170             deploy_scenario = env.get('DEPLOY_SCENARIO')
 171             if bool(installer_type) * bool(deploy_scenario):
 172                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 173                 # file
 174                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 175                 black_list_yaml = yaml.safe_load(black_list_file)
 176                 black_list_file.close()
 177                 for item in black_list_yaml:
 178                     scenarios = item['scenarios']
 179                     installers = item['installers']
 180                     if (deploy_scenario in scenarios and
 181                             installer_type in installers):
 182                         tests = item['tests']
 183                         for test in tests:
 184                             black_tests.append(test)
 185                         break
 186         except Exception:  # pylint: disable=broad-except
 187             black_tests = []
 188             LOGGER.debug("Tempest blacklist file does not exist.")
 189
 190         for cases_line in cases_file:
 191             for black_tests_line in black_tests:
 192                 if black_tests_line in cases_line:
 193                     break
 194             else:
 195                 result_file.write(str(cases_line) + '\n')
 196         result_file.close()
 197
 198     def run_verifier_tests(self, **kwargs):
 199         """Execute tempest test cases."""
 200         cmd = ["rally", "verify", "start", "--load-list",
 201                self.list]
 202         cmd.extend(kwargs.get('option', []))
 203         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 204
 205         f_stdout = open(
 206             os.path.join(self.res_dir, "tempest.log"), 'w+')
 207         f_stderr = open(
 208             os.path.join(self.res_dir,
 209                          "tempest-error.log"), 'w+')
 210
 211         proc = subprocess.Popen(
 212             cmd,
 213             stdout=subprocess.PIPE,
 214             stderr=f_stderr,
 215             bufsize=1)
 216
 217         with proc.stdout:
 218             for line in iter(proc.stdout.readline, b''):
 219                 if re.search(r"\} tempest\.", line):
 220                     LOGGER.info(line.replace('\n', ''))
 221                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 222                     self.verification_id = re.search(
 223                         r'(?=\(UUID=(.*)\))', line).group(1)
 224                     LOGGER.info('Verification UUID: %s', self.verification_id)
 225                 f_stdout.write(line)
 226         proc.wait()
 227
 228         f_stdout.close()
 229         f_stderr.close()
 230
 231         if self.verification_id is None:
 232             raise Exception('Verification UUID not found')
 233
 234     def parse_verifier_result(self):
 235         """Parse and save test results."""
 236         stat = self.get_verifier_result(self.verification_id)
 237         try:
 238             num_executed = stat['num_tests'] - stat['num_skipped']
 239             try:
 240                 self.result = 100 * stat['num_success'] / num_executed
 241             except ZeroDivisionError:
 242                 self.result = 0
 243                 if stat['num_tests'] > 0:
 244                     LOGGER.info("All tests have been skipped")
 245                 else:
 246                     LOGGER.error("No test has been executed")
 247                     return
 248
 249             with open(os.path.join(self.res_dir,
 250                                    "tempest-error.log"), 'r') as logfile:
 251                 output = logfile.read()
 252
 253             success_testcases = []
 254             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 255                                     output):
 256                 success_testcases.append(match)
 257             failed_testcases = []
 258             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 259                                     output):
 260                 failed_testcases.append(match)
 261             skipped_testcases = []
 262             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 263                                     output):
 264                 skipped_testcases.append(match)
 265
 266             self.details = {"tests_number": stat['num_tests'],
 267                             "success_number": stat['num_success'],
 268                             "skipped_number": stat['num_skipped'],
 269                             "failures_number": stat['num_failures'],
 270                             "success": success_testcases,
 271                             "skipped": skipped_testcases,
 272                             "failures": failed_testcases}
 273         except Exception:  # pylint: disable=broad-except
 274             self.result = 0
 275
 276         LOGGER.info("Tempest %s success_rate is %s%%",
 277                     self.case_name, self.result)
 278
 279     def generate_report(self):
 280         """Generate verification report."""
 281         html_file = os.path.join(self.res_dir,
 282                                  "tempest-report.html")
 283         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 284                self.verification_id, "--to", html_file]
 285         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 286                          stderr=subprocess.STDOUT)
 287
 288     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 289         """Set image name as tempest img_name_regex"""
 290         rconfig = configparser.RawConfigParser()
 291         rconfig.read(rally_conf)
 292         if not rconfig.has_section('tempest'):
 293             rconfig.add_section('tempest')
 294         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 295             self.image.name))
 296         with open(rally_conf, 'wb') as config_file:
 297             rconfig.write(config_file)
 298
 299     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 300         """Detect and update the default role if required"""
 301         role = self.get_default_role(self.cloud)
 302         if not role:
 303             return
 304         rconfig = configparser.RawConfigParser()
 305         rconfig.read(rally_conf)
 306         if not rconfig.has_section('tempest'):
 307             rconfig.add_section('tempest')
 308         rconfig.set('tempest', 'swift_operator_role', '^{}$'.format(role.name))
 309         with open(rally_conf, 'wb') as config_file:
 310             rconfig.write(config_file)
 311
 312     def configure(self, **kwargs):  # pylint: disable=unused-argument
 313         """
 314         Create all openstack resources for tempest-based testcases and write
 315         tempest.conf.
 316         """
 317         if not os.path.exists(self.res_dir):
 318             os.makedirs(self.res_dir)
 319         compute_cnt = len(self.cloud.list_hypervisors())
 320
 321         self.image_alt = self.publish_image_alt()
 322         self.flavor_alt = self.create_flavor_alt()
 323         LOGGER.debug("flavor: %s", self.flavor_alt)
 324
 325         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 326         conf_utils.configure_tempest_update_params(
 327             self.conf_file, network_name=self.network.id,
 328             image_id=self.image.id,
 329             flavor_id=self.flavor.id,
 330             compute_cnt=compute_cnt,
 331             image_alt_id=self.image_alt.id,
 332             flavor_alt_id=self.flavor_alt.id)
 333         self.backup_tempest_config(self.conf_file, self.res_dir)
 334
 335     def run(self, **kwargs):
 336         self.start_time = time.time()
 337         try:
 338             assert super(TempestCommon, self).run(
 339                 **kwargs) == testcase.TestCase.EX_OK
 340             self.update_rally_regex()
 341             self.update_default_role()
 342             self.configure(**kwargs)
 343             self.generate_test_list(**kwargs)
 344             self.apply_tempest_blacklist()
 345             self.run_verifier_tests(**kwargs)
 346             self.parse_verifier_result()
 347             self.generate_report()
 348             res = testcase.TestCase.EX_OK
 349         except Exception:  # pylint: disable=broad-except
 350             LOGGER.exception('Error with run')
 351             self.result = 0
 352             res = testcase.TestCase.EX_RUN_ERROR
 353         self.stop_time = time.time()
 354         return res
 355
 356     def clean(self):
 357         """
 358         Cleanup all OpenStack objects. Should be called on completion.
 359         """
 360         super(TempestCommon, self).clean()
 361         if self.image_alt:
 362             self.cloud.delete_image(self.image_alt)
 363         if self.flavor_alt:
 364             self.orig_cloud.delete_flavor(self.flavor_alt.id)