functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39
  40     def __init__(self, **kwargs):
  41         if "case_name" not in kwargs:
  42             kwargs["case_name"] = 'tempest'
  43         super(TempestCommon, self).__init__(**kwargs)
  44         self.verifier_id = conf_utils.get_verifier_id()
  45         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  46             self.verifier_id)
  47         self.deployment_id = conf_utils.get_verifier_deployment_id()
  48         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  49             self.verifier_id, self.deployment_id)
  50         self.verification_id = None
  51         self.res_dir = os.path.join(
  52             getattr(config.CONF, 'dir_results'), self.case_name)
  53         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  54         self.list = os.path.join(self.res_dir, 'test_list.txt')
  55         self.conf_file = None
  56         self.image_alt = None
  57         self.flavor_alt = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         """Read file and return content as a stripped list."""
  62         with open(filename) as src:
  63             return [line.strip() for line in src.readlines()]
  64
  65     @staticmethod
  66     def get_verifier_result(verif_id):
  67         """Retrieve verification results."""
  68         result = {
  69             'num_tests': 0,
  70             'num_success': 0,
  71             'num_failures': 0,
  72             'num_skipped': 0
  73         }
  74         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  75         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  76         proc = subprocess.Popen(cmd,
  77                                 stdout=subprocess.PIPE,
  78                                 stderr=subprocess.STDOUT)
  79         for line in proc.stdout:
  80             new_line = line.replace(' ', '').split('|')
  81             if 'Tests' in new_line:
  82                 break
  83             LOGGER.info(line)
  84             if 'Testscount' in new_line:
  85                 result['num_tests'] = int(new_line[2])
  86             elif 'Success' in new_line:
  87                 result['num_success'] = int(new_line[2])
  88             elif 'Skipped' in new_line:
  89                 result['num_skipped'] = int(new_line[2])
  90             elif 'Failures' in new_line:
  91                 result['num_failures'] = int(new_line[2])
  92         return result
  93
  94     @staticmethod
  95     def backup_tempest_config(conf_file, res_dir):
  96         """
  97         Copy config file to tempest results directory
  98         """
  99         if not os.path.exists(res_dir):
 100             os.makedirs(res_dir)
 101         shutil.copyfile(conf_file,
 102                         os.path.join(res_dir, 'tempest.conf'))
 103
 104     def generate_test_list(self, **kwargs):
 105         """Generate test list based on the test mode."""
 106         LOGGER.debug("Generating test case list...")
 107         self.backup_tempest_config(self.conf_file, '/etc')
 108         if kwargs.get('mode') == 'custom':
 109             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 110                 shutil.copyfile(
 111                     conf_utils.TEMPEST_CUSTOM, self.list)
 112             else:
 113                 raise Exception("Tempest test list file %s NOT found."
 114                                 % conf_utils.TEMPEST_CUSTOM)
 115         else:
 116             testr_mode = kwargs.get(
 117                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 118             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 119                 self.verifier_repo_dir, testr_mode, self.list)
 120             output = subprocess.check_output(cmd, shell=True)
 121             LOGGER.info("%s\n%s", cmd, output)
 122         os.remove('/etc/tempest.conf')
 123
 124     def apply_tempest_blacklist(self):
 125         """Exclude blacklisted test cases."""
 126         LOGGER.debug("Applying tempest blacklist...")
 127         if os.path.exists(self.raw_list):
 128             os.remove(self.raw_list)
 129         os.rename(self.list, self.raw_list)
 130         cases_file = self.read_file(self.raw_list)
 131         result_file = open(self.list, 'w')
 132         black_tests = []
 133         try:
 134             installer_type = env.get('INSTALLER_TYPE')
 135             deploy_scenario = env.get('DEPLOY_SCENARIO')
 136             if bool(installer_type) * bool(deploy_scenario):
 137                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 138                 # file
 139                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 140                 black_list_yaml = yaml.safe_load(black_list_file)
 141                 black_list_file.close()
 142                 for item in black_list_yaml:
 143                     scenarios = item['scenarios']
 144                     installers = item['installers']
 145                     if (deploy_scenario in scenarios and
 146                             installer_type in installers):
 147                         tests = item['tests']
 148                         for test in tests:
 149                             black_tests.append(test)
 150                         break
 151         except Exception:  # pylint: disable=broad-except
 152             black_tests = []
 153             LOGGER.debug("Tempest blacklist file does not exist.")
 154
 155         for cases_line in cases_file:
 156             for black_tests_line in black_tests:
 157                 if black_tests_line in cases_line:
 158                     break
 159             else:
 160                 result_file.write(str(cases_line) + '\n')
 161         result_file.close()
 162
 163     def run_verifier_tests(self, **kwargs):
 164         """Execute tempest test cases."""
 165         cmd = ["rally", "verify", "start", "--load-list",
 166                self.list]
 167         cmd.extend(kwargs.get('option', []))
 168         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 169
 170         f_stdout = open(
 171             os.path.join(self.res_dir, "tempest.log"), 'w+')
 172         f_stderr = open(
 173             os.path.join(self.res_dir,
 174                          "tempest-error.log"), 'w+')
 175
 176         proc = subprocess.Popen(
 177             cmd,
 178             stdout=subprocess.PIPE,
 179             stderr=f_stderr,
 180             bufsize=1)
 181
 182         with proc.stdout:
 183             for line in iter(proc.stdout.readline, b''):
 184                 if re.search(r"\} tempest\.", line):
 185                     LOGGER.info(line.replace('\n', ''))
 186                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 187                     self.verification_id = re.search(
 188                         r'(?=\(UUID=(.*)\))', line).group(1)
 189                     LOGGER.info('Verification UUID: %s', self.verification_id)
 190                 f_stdout.write(line)
 191         proc.wait()
 192
 193         f_stdout.close()
 194         f_stderr.close()
 195
 196         if self.verification_id is None:
 197             raise Exception('Verification UUID not found')
 198
 199     def parse_verifier_result(self):
 200         """Parse and save test results."""
 201         stat = self.get_verifier_result(self.verification_id)
 202         try:
 203             num_executed = stat['num_tests'] - stat['num_skipped']
 204             try:
 205                 self.result = 100 * stat['num_success'] / num_executed
 206             except ZeroDivisionError:
 207                 self.result = 0
 208                 if stat['num_tests'] > 0:
 209                     LOGGER.info("All tests have been skipped")
 210                 else:
 211                     LOGGER.error("No test has been executed")
 212                     return
 213
 214             with open(os.path.join(self.res_dir,
 215                                    "tempest-error.log"), 'r') as logfile:
 216                 output = logfile.read()
 217
 218             success_testcases = []
 219             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 220                                     output):
 221                 success_testcases.append(match)
 222             failed_testcases = []
 223             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 224                                     output):
 225                 failed_testcases.append(match)
 226             skipped_testcases = []
 227             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 228                                     output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests_number": stat['num_tests'],
 232                             "success_number": stat['num_success'],
 233                             "skipped_number": stat['num_skipped'],
 234                             "failures_number": stat['num_failures'],
 235                             "success": success_testcases,
 236                             "skipped": skipped_testcases,
 237                             "failures": failed_testcases}
 238         except Exception:  # pylint: disable=broad-except
 239             self.result = 0
 240
 241         LOGGER.info("Tempest %s success_rate is %s%%",
 242                     self.case_name, self.result)
 243
 244     def generate_report(self):
 245         """Generate verification report."""
 246         html_file = os.path.join(self.res_dir,
 247                                  "tempest-report.html")
 248         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 249                self.verification_id, "--to", html_file]
 250         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 251                          stderr=subprocess.STDOUT)
 252
 253     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 254         """Set image name as tempest img_name_regex"""
 255         rconfig = configparser.RawConfigParser()
 256         rconfig.read(rally_conf)
 257         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 258             self.image.name))
 259         with open(rally_conf, 'wb') as config_file:
 260             rconfig.write(config_file)
 261
 262     def configure(self, **kwargs):  # pylint: disable=unused-argument
 263         """
 264         Create all openstack resources for tempest-based testcases and write
 265         tempest.conf.
 266         """
 267         if not os.path.exists(self.res_dir):
 268             os.makedirs(self.res_dir)
 269         compute_cnt = len(self.cloud.list_hypervisors())
 270
 271         self.image_alt = self.publish_image(
 272             '{}-img_alt_{}'.format(self.case_name, self.guid))
 273         self.flavor_alt = self.create_flavor_alt()
 274         LOGGER.debug("flavor: %s", self.flavor_alt)
 275
 276         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 277         conf_utils.configure_tempest_update_params(
 278             self.conf_file, network_name=self.network.id,
 279             image_id=self.image.id,
 280             flavor_id=self.flavor.id,
 281             compute_cnt=compute_cnt,
 282             image_alt_id=self.image_alt.id,
 283             flavor_alt_id=self.flavor_alt.id)
 284         self.backup_tempest_config(self.conf_file, self.res_dir)
 285
 286     def run(self, **kwargs):
 287         self.start_time = time.time()
 288         try:
 289             super(TempestCommon, self).run(**kwargs)
 290             self.update_rally_regex()
 291             self.configure(**kwargs)
 292             self.generate_test_list(**kwargs)
 293             self.apply_tempest_blacklist()
 294             self.run_verifier_tests(**kwargs)
 295             self.parse_verifier_result()
 296             self.generate_report()
 297             res = testcase.TestCase.EX_OK
 298         except Exception:  # pylint: disable=broad-except
 299             LOGGER.exception('Error with run')
 300             res = testcase.TestCase.EX_RUN_ERROR
 301         self.stop_time = time.time()
 302         return res
 303
 304     def clean(self):
 305         """
 306         Cleanup all OpenStack objects. Should be called on completion.
 307         """
 308         super(TempestCommon, self).clean()
 309         self.cloud.delete_image(self.image_alt)
 310         self.orig_cloud.delete_flavor(self.flavor_alt.id)