functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39
  40     def __init__(self, **kwargs):
  41         if "case_name" not in kwargs:
  42             kwargs["case_name"] = 'tempest'
  43         super(TempestCommon, self).__init__(**kwargs)
  44         self.verifier_id = conf_utils.get_verifier_id()
  45         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  46             self.verifier_id)
  47         self.deployment_id = conf_utils.get_verifier_deployment_id()
  48         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  49             self.verifier_id, self.deployment_id)
  50         self.verification_id = None
  51         self.res_dir = os.path.join(
  52             getattr(config.CONF, 'dir_results'), self.case_name)
  53         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  54         self.list = os.path.join(self.res_dir, 'test_list.txt')
  55         self.conf_file = None
  56         self.image_alt = None
  57         self.flavor_alt = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         """Read file and return content as a stripped list."""
  62         with open(filename) as src:
  63             return [line.strip() for line in src.readlines()]
  64
  65     @staticmethod
  66     def get_verifier_result(verif_id):
  67         """Retrieve verification results."""
  68         result = {
  69             'num_tests': 0,
  70             'num_success': 0,
  71             'num_failures': 0,
  72             'num_skipped': 0
  73         }
  74         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  75         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  76         proc = subprocess.Popen(cmd,
  77                                 stdout=subprocess.PIPE,
  78                                 stderr=subprocess.STDOUT)
  79         for line in proc.stdout:
  80             new_line = line.replace(' ', '').split('|')
  81             if 'Tests' in new_line:
  82                 break
  83             LOGGER.info(line)
  84             if 'Testscount' in new_line:
  85                 result['num_tests'] = int(new_line[2])
  86             elif 'Success' in new_line:
  87                 result['num_success'] = int(new_line[2])
  88             elif 'Skipped' in new_line:
  89                 result['num_skipped'] = int(new_line[2])
  90             elif 'Failures' in new_line:
  91                 result['num_failures'] = int(new_line[2])
  92         return result
  93
  94     @staticmethod
  95     def backup_tempest_config(conf_file, res_dir):
  96         """
  97         Copy config file to tempest results directory
  98         """
  99         if not os.path.exists(res_dir):
 100             os.makedirs(res_dir)
 101         shutil.copyfile(conf_file,
 102                         os.path.join(res_dir, 'tempest.conf'))
 103
 104     def generate_test_list(self, **kwargs):
 105         """Generate test list based on the test mode."""
 106         LOGGER.debug("Generating test case list...")
 107         self.backup_tempest_config(self.conf_file, '/etc')
 108         if kwargs.get('mode') == 'custom':
 109             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 110                 shutil.copyfile(
 111                     conf_utils.TEMPEST_CUSTOM, self.list)
 112             else:
 113                 raise Exception("Tempest test list file %s NOT found."
 114                                 % conf_utils.TEMPEST_CUSTOM)
 115         else:
 116             testr_mode = kwargs.get(
 117                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 118             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 119                 self.verifier_repo_dir, testr_mode, self.list)
 120             output = subprocess.check_output(cmd, shell=True)
 121             LOGGER.info("%s\n%s", cmd, output)
 122         os.remove('/etc/tempest.conf')
 123
 124     def apply_tempest_blacklist(self):
 125         """Exclude blacklisted test cases."""
 126         LOGGER.debug("Applying tempest blacklist...")
 127         if os.path.exists(self.raw_list):
 128             os.remove(self.raw_list)
 129         os.rename(self.list, self.raw_list)
 130         cases_file = self.read_file(self.raw_list)
 131         result_file = open(self.list, 'w')
 132         black_tests = []
 133         try:
 134             installer_type = env.get('INSTALLER_TYPE')
 135             deploy_scenario = env.get('DEPLOY_SCENARIO')
 136             if bool(installer_type) * bool(deploy_scenario):
 137                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 138                 # file
 139                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 140                 black_list_yaml = yaml.safe_load(black_list_file)
 141                 black_list_file.close()
 142                 for item in black_list_yaml:
 143                     scenarios = item['scenarios']
 144                     installers = item['installers']
 145                     if (deploy_scenario in scenarios and
 146                             installer_type in installers):
 147                         tests = item['tests']
 148                         for test in tests:
 149                             black_tests.append(test)
 150                         break
 151         except Exception:  # pylint: disable=broad-except
 152             black_tests = []
 153             LOGGER.debug("Tempest blacklist file does not exist.")
 154
 155         for cases_line in cases_file:
 156             for black_tests_line in black_tests:
 157                 if black_tests_line in cases_line:
 158                     break
 159             else:
 160                 result_file.write(str(cases_line) + '\n')
 161         result_file.close()
 162
 163     def run_verifier_tests(self, **kwargs):
 164         """Execute tempest test cases."""
 165         cmd = ["rally", "verify", "start", "--load-list",
 166                self.list]
 167         cmd.extend(kwargs.get('option', []))
 168         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 169
 170         f_stdout = open(
 171             os.path.join(self.res_dir, "tempest.log"), 'w+')
 172         f_stderr = open(
 173             os.path.join(self.res_dir,
 174                          "tempest-error.log"), 'w+')
 175
 176         proc = subprocess.Popen(
 177             cmd,
 178             stdout=subprocess.PIPE,
 179             stderr=f_stderr,
 180             bufsize=1)
 181
 182         with proc.stdout:
 183             for line in iter(proc.stdout.readline, b''):
 184                 if re.search(r"\} tempest\.", line):
 185                     LOGGER.info(line.replace('\n', ''))
 186                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 187                     self.verification_id = re.search(
 188                         r'(?=\(UUID=(.*)\))', line).group(1)
 189                     LOGGER.info('Verification UUID: %s', self.verification_id)
 190                 f_stdout.write(line)
 191         proc.wait()
 192
 193         f_stdout.close()
 194         f_stderr.close()
 195
 196         if self.verification_id is None:
 197             raise Exception('Verification UUID not found')
 198
 199     def parse_verifier_result(self):
 200         """Parse and save test results."""
 201         stat = self.get_verifier_result(self.verification_id)
 202         try:
 203             num_executed = stat['num_tests'] - stat['num_skipped']
 204             try:
 205                 self.result = 100 * stat['num_success'] / num_executed
 206             except ZeroDivisionError:
 207                 self.result = 0
 208                 if stat['num_tests'] > 0:
 209                     LOGGER.info("All tests have been skipped")
 210                 else:
 211                     LOGGER.error("No test has been executed")
 212                     return
 213
 214             with open(os.path.join(self.res_dir,
 215                                    "tempest-error.log"), 'r') as logfile:
 216                 output = logfile.read()
 217
 218             success_testcases = []
 219             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 220                                     output):
 221                 success_testcases.append(match)
 222             failed_testcases = []
 223             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 224                                     output):
 225                 failed_testcases.append(match)
 226             skipped_testcases = []
 227             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 228                                     output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests_number": stat['num_tests'],
 232                             "success_number": stat['num_success'],
 233                             "skipped_number": stat['num_skipped'],
 234                             "failures_number": stat['num_failures'],
 235                             "success": success_testcases,
 236                             "skipped": skipped_testcases,
 237                             "failures": failed_testcases}
 238         except Exception:  # pylint: disable=broad-except
 239             self.result = 0
 240
 241         LOGGER.info("Tempest %s success_rate is %s%%",
 242                     self.case_name, self.result)
 243
 244     def generate_report(self):
 245         """Generate verification report."""
 246         html_file = os.path.join(self.res_dir,
 247                                  "tempest-report.html")
 248         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 249                self.verification_id, "--to", html_file]
 250         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 251                          stderr=subprocess.STDOUT)
 252
 253     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 254         """Set image name as tempest img_name_regex"""
 255         rconfig = configparser.RawConfigParser()
 256         rconfig.read(rally_conf)
 257         rconfig.add_section('tempest')
 258         rconfig.set('tempest', 'img_name_regex', '^{}$'.format(
 259             self.image.name))
 260         with open(rally_conf, 'wb') as config_file:
 261             rconfig.write(config_file)
 262
 263     def configure(self, **kwargs):  # pylint: disable=unused-argument
 264         """
 265         Create all openstack resources for tempest-based testcases and write
 266         tempest.conf.
 267         """
 268         if not os.path.exists(self.res_dir):
 269             os.makedirs(self.res_dir)
 270         compute_cnt = len(self.cloud.list_hypervisors())
 271
 272         self.image_alt = self.publish_image(
 273             '{}-img_alt_{}'.format(self.case_name, self.guid))
 274         self.flavor_alt = self.create_flavor_alt()
 275         LOGGER.debug("flavor: %s", self.flavor_alt)
 276
 277         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 278         conf_utils.configure_tempest_update_params(
 279             self.conf_file, network_name=self.network.id,
 280             image_id=self.image.id,
 281             flavor_id=self.flavor.id,
 282             compute_cnt=compute_cnt,
 283             image_alt_id=self.image_alt.id,
 284             flavor_alt_id=self.flavor_alt.id)
 285         self.backup_tempest_config(self.conf_file, self.res_dir)
 286
 287     def run(self, **kwargs):
 288         self.start_time = time.time()
 289         try:
 290             assert super(TempestCommon, self).run(
 291                 **kwargs) == testcase.TestCase.EX_OK
 292             self.update_rally_regex()
 293             self.configure(**kwargs)
 294             self.generate_test_list(**kwargs)
 295             self.apply_tempest_blacklist()
 296             self.run_verifier_tests(**kwargs)
 297             self.parse_verifier_result()
 298             self.generate_report()
 299             res = testcase.TestCase.EX_OK
 300         except Exception:  # pylint: disable=broad-except
 301             LOGGER.exception('Error with run')
 302             self.result = 0
 303             res = testcase.TestCase.EX_RUN_ERROR
 304         self.stop_time = time.time()
 305         return res
 306
 307     def clean(self):
 308         """
 309         Cleanup all OpenStack objects. Should be called on completion.
 310         """
 311         super(TempestCommon, self).clean()
 312         if self.image_alt:
 313             self.cloud.delete_image(self.image_alt)
 314         if self.flavor_alt:
 315             self.orig_cloud.delete_flavor(self.flavor_alt.id)