functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39
  40     def __init__(self, **kwargs):
  41         if "case_name" not in kwargs:
  42             kwargs["case_name"] = 'tempest'
  43         super(TempestCommon, self).__init__(**kwargs)
  44         self.verifier_id = conf_utils.get_verifier_id()
  45         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  46             self.verifier_id)
  47         self.deployment_id = conf_utils.get_verifier_deployment_id()
  48         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  49             self.verifier_id, self.deployment_id)
  50         self.verification_id = None
  51         self.res_dir = os.path.join(
  52             getattr(config.CONF, 'dir_results'), self.case_name)
  53         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  54         self.list = os.path.join(self.res_dir, 'test_list.txt')
  55         self.conf_file = None
  56         self.image_alt = None
  57         self.flavor_alt = None
  58         self.services = []
  59         try:
  60             self.services = kwargs['run']['args']['services']
  61         except Exception:  # pylint: disable=broad-except
  62             pass
  63         self.neutron_extensions = []
  64         try:
  65             self.neutron_extensions = kwargs['run']['args'][
  66                 'neutron_extensions']
  67         except Exception:  # pylint: disable=broad-except
  68             pass
  69
  70     def check_services(self):
  71         """Check the mandatory services."""
  72         for service in self.services:
  73             try:
  74                 self.cloud.search_services(service)[0]
  75             except Exception:  # pylint: disable=broad-except
  76                 self.is_skipped = True
  77                 break
  78
  79     def check_extensions(self):
  80         """Check the mandatory network extensions."""
  81         extensions = self.cloud.get_network_extensions()
  82         for network_extension in self.neutron_extensions:
  83             if network_extension not in extensions:
  84                 LOGGER.warning(
  85                     "Cannot find Neutron extension: %s", network_extension)
  86                 self.is_skipped = True
  87                 break
  88
  89     def check_requirements(self):
  90         self.check_services()
  91         self.check_extensions()
  92
  93     @staticmethod
  94     def read_file(filename):
  95         """Read file and return content as a stripped list."""
  96         with open(filename) as src:
  97             return [line.strip() for line in src.readlines()]
  98
  99     @staticmethod
 100     def get_verifier_result(verif_id):
 101         """Retrieve verification results."""
 102         result = {
 103             'num_tests': 0,
 104             'num_success': 0,
 105             'num_failures': 0,
 106             'num_skipped': 0
 107         }
 108         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 109         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 110         proc = subprocess.Popen(cmd,
 111                                 stdout=subprocess.PIPE,
 112                                 stderr=subprocess.STDOUT)
 113         for line in proc.stdout:
 114             new_line = line.replace(' ', '').split('|')
 115             if 'Tests' in new_line:
 116                 break
 117             LOGGER.info(line)
 118             if 'Testscount' in new_line:
 119                 result['num_tests'] = int(new_line[2])
 120             elif 'Success' in new_line:
 121                 result['num_success'] = int(new_line[2])
 122             elif 'Skipped' in new_line:
 123                 result['num_skipped'] = int(new_line[2])
 124             elif 'Failures' in new_line:
 125                 result['num_failures'] = int(new_line[2])
 126         return result
 127
 128     @staticmethod
 129     def backup_tempest_config(conf_file, res_dir):
 130         """
 131         Copy config file to tempest results directory
 132         """
 133         if not os.path.exists(res_dir):
 134             os.makedirs(res_dir)
 135         shutil.copyfile(conf_file,
 136                         os.path.join(res_dir, 'tempest.conf'))
 137
 138     def generate_test_list(self, **kwargs):
 139         """Generate test list based on the test mode."""
 140         LOGGER.debug("Generating test case list...")
 141         self.backup_tempest_config(self.conf_file, '/etc')
 142         if kwargs.get('mode') == 'custom':
 143             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 144                 shutil.copyfile(
 145                     conf_utils.TEMPEST_CUSTOM, self.list)
 146             else:
 147                 raise Exception("Tempest test list file %s NOT found."
 148                                 % conf_utils.TEMPEST_CUSTOM)
 149         else:
 150             testr_mode = kwargs.get(
 151                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 152             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 153                 self.verifier_repo_dir, testr_mode, self.list)
 154             output = subprocess.check_output(cmd, shell=True)
 155             LOGGER.info("%s\n%s", cmd, output)
 156         os.remove('/etc/tempest.conf')
 157
 158     def apply_tempest_blacklist(self):
 159         """Exclude blacklisted test cases."""
 160         LOGGER.debug("Applying tempest blacklist...")
 161         if os.path.exists(self.raw_list):
 162             os.remove(self.raw_list)
 163         os.rename(self.list, self.raw_list)
 164         cases_file = self.read_file(self.raw_list)
 165         result_file = open(self.list, 'w')
 166         black_tests = []
 167         try:
 168             installer_type = env.get('INSTALLER_TYPE')
 169             deploy_scenario = env.get('DEPLOY_SCENARIO')
 170             if bool(installer_type) * bool(deploy_scenario):
 171                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 172                 # file
 173                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 174                 black_list_yaml = yaml.safe_load(black_list_file)
 175                 black_list_file.close()
 176                 for item in black_list_yaml:
 177                     scenarios = item['scenarios']
 178                     installers = item['installers']
 179                     if (deploy_scenario in scenarios and
 180                             installer_type in installers):
 181                         tests = item['tests']
 182                         for test in tests:
 183                             black_tests.append(test)
 184                         break
 185         except Exception:  # pylint: disable=broad-except
 186             black_tests = []
 187             LOGGER.debug("Tempest blacklist file does not exist.")
 188
 189         for cases_line in cases_file:
 190             for black_tests_line in black_tests:
 191                 if black_tests_line in cases_line:
 192                     break
 193             else:
 194                 result_file.write(str(cases_line) + '\n')
 195         result_file.close()
 196
 197     def run_verifier_tests(self, **kwargs):
 198         """Execute tempest test cases."""
 199         cmd = ["rally", "verify", "start", "--load-list",
 200                self.list]
 201         cmd.extend(kwargs.get('option', []))
 202         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 203
 204         f_stdout = open(
 205             os.path.join(self.res_dir, "tempest.log"), 'w+')
 206         f_stderr = open(
 207             os.path.join(self.res_dir,
 208                          "tempest-error.log"), 'w+')
 209
 210         proc = subprocess.Popen(
 211             cmd,
 212             stdout=subprocess.PIPE,
 213             stderr=f_stderr,
 214             bufsize=1)
 215
 216         with proc.stdout:
 217             for line in iter(proc.stdout.readline, b''):
 218                 if re.search(r"\} tempest\.", line):
 219                     LOGGER.info(line.replace('\n', ''))
 220                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 221                     self.verification_id = re.search(
 222                         r'(?=\(UUID=(.*)\))', line).group(1)
 223                     LOGGER.info('Verification UUID: %s', self.verification_id)
 224                 f_stdout.write(line)
 225         proc.wait()
 226
 227         f_stdout.close()
 228         f_stderr.close()
 229
 230         if self.verification_id is None:
 231             raise Exception('Verification UUID not found')
 232
 233     def parse_verifier_result(self):
 234         """Parse and save test results."""
 235         stat = self.get_verifier_result(self.verification_id)
 236         try:
 237             num_executed = stat['num_tests'] - stat['num_skipped']
 238             try:
 239                 self.result = 100 * stat['num_success'] / num_executed
 240             except ZeroDivisionError:
 241                 self.result = 0
 242                 if stat['num_tests'] > 0:
 243                     LOGGER.info("All tests have been skipped")
 244                 else:
 245                     LOGGER.error("No test has been executed")
 246                     return
 247
 248             with open(os.path.join(self.res_dir,
 249                                    "tempest-error.log"), 'r') as logfile:
 250                 output = logfile.read()
 251
 252             success_testcases = []
 253             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 254                                     output):
 255                 success_testcases.append(match)
 256             failed_testcases = []
 257             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 258                                     output):
 259                 failed_testcases.append(match)
 260             skipped_testcases = []
 261             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 262                                     output):
 263                 skipped_testcases.append(match)
 264
 265             self.details = {"tests_number": stat['num_tests'],
 266                             "success_number": stat['num_success'],
 267                             "skipped_number": stat['num_skipped'],
 268                             "failures_number": stat['num_failures'],
 269                             "success": success_testcases,
 270                             "skipped": skipped_testcases,
 271                             "failures": failed_testcases}
 272         except Exception:  # pylint: disable=broad-except
 273             self.result = 0
 274
 275         LOGGER.info("Tempest %s success_rate is %s%%",
 276                     self.case_name, self.result)
 277
 278     def generate_report(self):
 279         """Generate verification report."""
 280         html_file = os.path.join(self.res_dir,
 281                                  "tempest-report.html")
 282         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 283                self.verification_id, "--to", html_file]
 284         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 285                          stderr=subprocess.STDOUT)
 286
 287     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 288         """Set image name as tempest img_name_regex"""
 289         rconfig = configparser.RawConfigParser()
 290         rconfig.read(rally_conf)
 291         if not rconfig.has_section('tempest'):
 292             rconfig.add_section('tempest')
 293         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 294             self.image.name))
 295         with open(rally_conf, 'wb') as config_file:
 296             rconfig.write(config_file)
 297
 298     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 299         """Detect and update the default role if required"""
 300         role = self.get_default_role(self.cloud)
 301         if not role:
 302             return
 303         rconfig = configparser.RawConfigParser()
 304         rconfig.read(rally_conf)
 305         if not rconfig.has_section('tempest'):
 306             rconfig.add_section('tempest')
 307         rconfig.set('tempest', 'swift_operator_role', '^{}$'.format(role.name))
 308         with open(rally_conf, 'wb') as config_file:
 309             rconfig.write(config_file)
 310
 311     def configure(self, **kwargs):  # pylint: disable=unused-argument
 312         """
 313         Create all openstack resources for tempest-based testcases and write
 314         tempest.conf.
 315         """
 316         if not os.path.exists(self.res_dir):
 317             os.makedirs(self.res_dir)
 318         compute_cnt = len(self.cloud.list_hypervisors())
 319
 320         self.image_alt = self.publish_image(
 321             '{}-img_alt_{}'.format(self.case_name, self.guid))
 322         self.flavor_alt = self.create_flavor_alt()
 323         LOGGER.debug("flavor: %s", self.flavor_alt)
 324
 325         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 326         conf_utils.configure_tempest_update_params(
 327             self.conf_file, network_name=self.network.id,
 328             image_id=self.image.id,
 329             flavor_id=self.flavor.id,
 330             compute_cnt=compute_cnt,
 331             image_alt_id=self.image_alt.id,
 332             flavor_alt_id=self.flavor_alt.id)
 333         self.backup_tempest_config(self.conf_file, self.res_dir)
 334
 335     def run(self, **kwargs):
 336         self.start_time = time.time()
 337         try:
 338             assert super(TempestCommon, self).run(
 339                 **kwargs) == testcase.TestCase.EX_OK
 340             self.update_rally_regex()
 341             self.update_default_role()
 342             self.configure(**kwargs)
 343             self.generate_test_list(**kwargs)
 344             self.apply_tempest_blacklist()
 345             self.run_verifier_tests(**kwargs)
 346             self.parse_verifier_result()
 347             self.generate_report()
 348             res = testcase.TestCase.EX_OK
 349         except Exception:  # pylint: disable=broad-except
 350             LOGGER.exception('Error with run')
 351             self.result = 0
 352             res = testcase.TestCase.EX_RUN_ERROR
 353         self.stop_time = time.time()
 354         return res
 355
 356     def clean(self):
 357         """
 358         Cleanup all OpenStack objects. Should be called on completion.
 359         """
 360         super(TempestCommon, self).clean()
 361         if self.image_alt:
 362             self.cloud.delete_image(self.image_alt)
 363         if self.flavor_alt:
 364             self.orig_cloud.delete_flavor(self.flavor_alt.id)