functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from xtesting.core import testcase
  23 import yaml
  24
  25 from functest.core import singlevm
  26 from functest.opnfv_tests.openstack.tempest import conf_utils
  27 from functest.utils import config
  28 from functest.utils import env
  29
  30 LOGGER = logging.getLogger(__name__)
  31
  32
  33 class TempestCommon(singlevm.VmReady1):
  34     # pylint: disable=too-many-instance-attributes
  35     """TempestCommon testcases implementation class."""
  36
  37     visibility = 'public'
  38
  39     def __init__(self, **kwargs):
  40         if "case_name" not in kwargs:
  41             kwargs["case_name"] = 'tempest'
  42         super(TempestCommon, self).__init__(**kwargs)
  43         self.verifier_id = conf_utils.get_verifier_id()
  44         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  45             self.verifier_id)
  46         self.deployment_id = conf_utils.get_verifier_deployment_id()
  47         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  48             self.verifier_id, self.deployment_id)
  49         self.verification_id = None
  50         self.res_dir = os.path.join(
  51             getattr(config.CONF, 'dir_results'), self.case_name)
  52         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  53         self.list = os.path.join(self.res_dir, 'test_list.txt')
  54         self.conf_file = None
  55         self.image_alt = None
  56         self.flavor_alt = None
  57
  58     @staticmethod
  59     def read_file(filename):
  60         """Read file and return content as a stripped list."""
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     @staticmethod
  65     def get_verifier_result(verif_id):
  66         """Retrieve verification results."""
  67         result = {
  68             'num_tests': 0,
  69             'num_success': 0,
  70             'num_failures': 0,
  71             'num_skipped': 0
  72         }
  73         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  74         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  75         proc = subprocess.Popen(cmd,
  76                                 stdout=subprocess.PIPE,
  77                                 stderr=subprocess.STDOUT)
  78         for line in proc.stdout:
  79             new_line = line.replace(' ', '').split('|')
  80             if 'Tests' in new_line:
  81                 break
  82             LOGGER.info(line)
  83             if 'Testscount' in new_line:
  84                 result['num_tests'] = int(new_line[2])
  85             elif 'Success' in new_line:
  86                 result['num_success'] = int(new_line[2])
  87             elif 'Skipped' in new_line:
  88                 result['num_skipped'] = int(new_line[2])
  89             elif 'Failures' in new_line:
  90                 result['num_failures'] = int(new_line[2])
  91         return result
  92
  93     @staticmethod
  94     def backup_tempest_config(conf_file, res_dir):
  95         """
  96         Copy config file to tempest results directory
  97         """
  98         if not os.path.exists(res_dir):
  99             os.makedirs(res_dir)
 100         shutil.copyfile(conf_file,
 101                         os.path.join(res_dir, 'tempest.conf'))
 102
 103     def generate_test_list(self, **kwargs):
 104         """Generate test list based on the test mode."""
 105         LOGGER.debug("Generating test case list...")
 106         self.backup_tempest_config(self.conf_file, '/etc')
 107         if kwargs.get('mode') == 'custom':
 108             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 109                 shutil.copyfile(
 110                     conf_utils.TEMPEST_CUSTOM, self.list)
 111             else:
 112                 raise Exception("Tempest test list file %s NOT found."
 113                                 % conf_utils.TEMPEST_CUSTOM)
 114         else:
 115             testr_mode = kwargs.get(
 116                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 117             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 118                 self.verifier_repo_dir, testr_mode, self.list)
 119             output = subprocess.check_output(cmd, shell=True)
 120             LOGGER.info("%s\n%s", cmd, output)
 121         os.remove('/etc/tempest.conf')
 122
 123     def apply_tempest_blacklist(self):
 124         """Exclude blacklisted test cases."""
 125         LOGGER.debug("Applying tempest blacklist...")
 126         if os.path.exists(self.raw_list):
 127             os.remove(self.raw_list)
 128         os.rename(self.list, self.raw_list)
 129         cases_file = self.read_file(self.raw_list)
 130         result_file = open(self.list, 'w')
 131         black_tests = []
 132         try:
 133             installer_type = env.get('INSTALLER_TYPE')
 134             deploy_scenario = env.get('DEPLOY_SCENARIO')
 135             if bool(installer_type) * bool(deploy_scenario):
 136                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 137                 # file
 138                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 139                 black_list_yaml = yaml.safe_load(black_list_file)
 140                 black_list_file.close()
 141                 for item in black_list_yaml:
 142                     scenarios = item['scenarios']
 143                     installers = item['installers']
 144                     if (deploy_scenario in scenarios and
 145                             installer_type in installers):
 146                         tests = item['tests']
 147                         for test in tests:
 148                             black_tests.append(test)
 149                         break
 150         except Exception:  # pylint: disable=broad-except
 151             black_tests = []
 152             LOGGER.debug("Tempest blacklist file does not exist.")
 153
 154         for cases_line in cases_file:
 155             for black_tests_line in black_tests:
 156                 if black_tests_line in cases_line:
 157                     break
 158             else:
 159                 result_file.write(str(cases_line) + '\n')
 160         result_file.close()
 161
 162     def run_verifier_tests(self, **kwargs):
 163         """Execute tempest test cases."""
 164         cmd = ["rally", "verify", "start", "--load-list",
 165                self.list]
 166         cmd.extend(kwargs.get('option', []))
 167         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 168
 169         f_stdout = open(
 170             os.path.join(self.res_dir, "tempest.log"), 'w+')
 171         f_stderr = open(
 172             os.path.join(self.res_dir,
 173                          "tempest-error.log"), 'w+')
 174
 175         proc = subprocess.Popen(
 176             cmd,
 177             stdout=subprocess.PIPE,
 178             stderr=f_stderr,
 179             bufsize=1)
 180
 181         with proc.stdout:
 182             for line in iter(proc.stdout.readline, b''):
 183                 if re.search(r"\} tempest\.", line):
 184                     LOGGER.info(line.replace('\n', ''))
 185                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 186                     self.verification_id = re.search(
 187                         r'(?=\(UUID=(.*)\))', line).group(1)
 188                     LOGGER.info('Verification UUID: %s', self.verification_id)
 189                 f_stdout.write(line)
 190         proc.wait()
 191
 192         f_stdout.close()
 193         f_stderr.close()
 194
 195         if self.verification_id is None:
 196             raise Exception('Verification UUID not found')
 197
 198     def parse_verifier_result(self):
 199         """Parse and save test results."""
 200         stat = self.get_verifier_result(self.verification_id)
 201         try:
 202             num_executed = stat['num_tests'] - stat['num_skipped']
 203             try:
 204                 self.result = 100 * stat['num_success'] / num_executed
 205             except ZeroDivisionError:
 206                 self.result = 0
 207                 if stat['num_tests'] > 0:
 208                     LOGGER.info("All tests have been skipped")
 209                 else:
 210                     LOGGER.error("No test has been executed")
 211                     return
 212
 213             with open(os.path.join(self.res_dir,
 214                                    "tempest-error.log"), 'r') as logfile:
 215                 output = logfile.read()
 216
 217             success_testcases = []
 218             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 219                                     output):
 220                 success_testcases.append(match)
 221             failed_testcases = []
 222             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 223                                     output):
 224                 failed_testcases.append(match)
 225             skipped_testcases = []
 226             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 227                                     output):
 228                 skipped_testcases.append(match)
 229
 230             self.details = {"tests_number": stat['num_tests'],
 231                             "success_number": stat['num_success'],
 232                             "skipped_number": stat['num_skipped'],
 233                             "failures_number": stat['num_failures'],
 234                             "success": success_testcases,
 235                             "skipped": skipped_testcases,
 236                             "failures": failed_testcases}
 237         except Exception:  # pylint: disable=broad-except
 238             self.result = 0
 239
 240         LOGGER.info("Tempest %s success_rate is %s%%",
 241                     self.case_name, self.result)
 242
 243     def generate_report(self):
 244         """Generate verification report."""
 245         html_file = os.path.join(self.res_dir,
 246                                  "tempest-report.html")
 247         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 248                self.verification_id, "--to", html_file]
 249         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 250                          stderr=subprocess.STDOUT)
 251
 252     def configure(self, **kwargs):  # pylint: disable=unused-argument
 253         """
 254         Create all openstack resources for tempest-based testcases and write
 255         tempest.conf.
 256         """
 257         if not os.path.exists(self.res_dir):
 258             os.makedirs(self.res_dir)
 259         compute_cnt = len(self.cloud.list_hypervisors())
 260
 261         self.image_alt = self.publish_image(
 262             '{}-img_alt_{}'.format(self.case_name, self.guid))
 263         self.flavor_alt = self.create_flavor_alt()
 264         LOGGER.debug("flavor: %s", self.flavor_alt)
 265
 266         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 267         conf_utils.configure_tempest_update_params(
 268             self.conf_file, network_name=self.network.id,
 269             image_id=self.image.id,
 270             flavor_id=self.flavor.id,
 271             compute_cnt=compute_cnt,
 272             image_alt_id=self.image_alt.id,
 273             flavor_alt_id=self.flavor_alt.id)
 274         self.backup_tempest_config(self.conf_file, self.res_dir)
 275
 276     def run(self, **kwargs):
 277         self.start_time = time.time()
 278         try:
 279             super(TempestCommon, self).run(**kwargs)
 280             self.configure(**kwargs)
 281             self.generate_test_list(**kwargs)
 282             self.apply_tempest_blacklist()
 283             self.run_verifier_tests(**kwargs)
 284             self.parse_verifier_result()
 285             self.generate_report()
 286             res = testcase.TestCase.EX_OK
 287         except Exception:  # pylint: disable=broad-except
 288             LOGGER.exception('Error with run')
 289             res = testcase.TestCase.EX_RUN_ERROR
 290         self.stop_time = time.time()
 291         return res
 292
 293     def clean(self):
 294         """
 295         Cleanup all OpenStack objects. Should be called on completion.
 296         """
 297         super(TempestCommon, self).clean()
 298         self.cloud.delete_image(self.image_alt)
 299         self.orig_cloud.delete_flavor(self.flavor_alt.id)