functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     TEMPEST_RESULTS_DIR = os.path.join(
  39         getattr(config.CONF, 'dir_results'), 'tempest')
  40
  41     visibility = 'public'
  42
  43     def __init__(self, **kwargs):
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.mode = ""
  46         self.option = []
  47         self.verifier_id = conf_utils.get_verifier_id()
  48         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  49             self.verifier_id)
  50         self.deployment_id = conf_utils.get_verifier_deployment_id()
  51         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  52             self.verifier_id, self.deployment_id)
  53         self.verification_id = None
  54         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58         self.image_alt = None
  59         self.flavor_alt = None
  60
  61     @staticmethod
  62     def read_file(filename):
  63         """Read file and return content as a stripped list."""
  64         with open(filename) as src:
  65             return [line.strip() for line in src.readlines()]
  66
  67     @staticmethod
  68     def get_verifier_result(verif_id):
  69         """Retrieve verification results."""
  70         result = {
  71             'num_tests': 0,
  72             'num_success': 0,
  73             'num_failures': 0,
  74             'num_skipped': 0
  75         }
  76         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  77         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  78         proc = subprocess.Popen(cmd,
  79                                 stdout=subprocess.PIPE,
  80                                 stderr=subprocess.STDOUT)
  81         for line in proc.stdout:
  82             new_line = line.replace(' ', '').split('|')
  83             if 'Tests' in new_line:
  84                 break
  85             LOGGER.info(line)
  86             if 'Testscount' in new_line:
  87                 result['num_tests'] = int(new_line[2])
  88             elif 'Success' in new_line:
  89                 result['num_success'] = int(new_line[2])
  90             elif 'Skipped' in new_line:
  91                 result['num_skipped'] = int(new_line[2])
  92             elif 'Failures' in new_line:
  93                 result['num_failures'] = int(new_line[2])
  94         return result
  95
  96     @staticmethod
  97     def backup_tempest_config(conf_file, res_dir):
  98         """
  99         Copy config file to tempest results directory
 100         """
 101         if not os.path.exists(res_dir):
 102             os.makedirs(res_dir)
 103         shutil.copyfile(conf_file,
 104                         os.path.join(res_dir, 'tempest.conf'))
 105
 106     def generate_test_list(self):
 107         """Generate test list based on the test mode."""
 108         LOGGER.debug("Generating test case list...")
 109         self.backup_tempest_config(self.conf_file, '/etc')
 110         if self.mode == 'custom':
 111             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 112                 shutil.copyfile(
 113                     conf_utils.TEMPEST_CUSTOM, self.list)
 114             else:
 115                 raise Exception("Tempest test list file %s NOT found."
 116                                 % conf_utils.TEMPEST_CUSTOM)
 117         else:
 118             if self.mode == 'smoke':
 119                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 120             elif self.mode == 'full':
 121                 testr_mode = r"'^tempest\.'"
 122             else:
 123                 testr_mode = self.mode
 124             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 125                 self.verifier_repo_dir, testr_mode, self.list)
 126             output = subprocess.check_output(cmd, shell=True)
 127             LOGGER.info("%s\n%s", cmd, output)
 128         os.remove('/etc/tempest.conf')
 129
 130     def apply_tempest_blacklist(self):
 131         """Exclude blacklisted test cases."""
 132         LOGGER.debug("Applying tempest blacklist...")
 133         if os.path.exists(self.raw_list):
 134             os.remove(self.raw_list)
 135         os.rename(self.list, self.raw_list)
 136         cases_file = self.read_file(self.raw_list)
 137         result_file = open(self.list, 'w')
 138         black_tests = []
 139         try:
 140             installer_type = env.get('INSTALLER_TYPE')
 141             deploy_scenario = env.get('DEPLOY_SCENARIO')
 142             if bool(installer_type) * bool(deploy_scenario):
 143                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 144                 # file
 145                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 146                 black_list_yaml = yaml.safe_load(black_list_file)
 147                 black_list_file.close()
 148                 for item in black_list_yaml:
 149                     scenarios = item['scenarios']
 150                     installers = item['installers']
 151                     if (deploy_scenario in scenarios and
 152                             installer_type in installers):
 153                         tests = item['tests']
 154                         for test in tests:
 155                             black_tests.append(test)
 156                         break
 157         except Exception:  # pylint: disable=broad-except
 158             black_tests = []
 159             LOGGER.debug("Tempest blacklist file does not exist.")
 160
 161         for cases_line in cases_file:
 162             for black_tests_line in black_tests:
 163                 if black_tests_line in cases_line:
 164                     break
 165             else:
 166                 result_file.write(str(cases_line) + '\n')
 167         result_file.close()
 168
 169     def run_verifier_tests(self):
 170         """Execute tempest test cases."""
 171         cmd = ["rally", "verify", "start", "--load-list",
 172                self.list]
 173         cmd.extend(self.option)
 174         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 175
 176         f_stdout = open(
 177             os.path.join(self.res_dir, "tempest.log"), 'w+')
 178         f_stderr = open(
 179             os.path.join(self.res_dir,
 180                          "tempest-error.log"), 'w+')
 181
 182         proc = subprocess.Popen(
 183             cmd,
 184             stdout=subprocess.PIPE,
 185             stderr=f_stderr,
 186             bufsize=1)
 187
 188         with proc.stdout:
 189             for line in iter(proc.stdout.readline, b''):
 190                 if re.search(r"\} tempest\.", line):
 191                     LOGGER.info(line.replace('\n', ''))
 192                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 193                     self.verification_id = re.search(
 194                         r'(?=\(UUID=(.*)\))', line).group(1)
 195                     LOGGER.info('Verification UUID: %s', self.verification_id)
 196                 f_stdout.write(line)
 197         proc.wait()
 198
 199         f_stdout.close()
 200         f_stderr.close()
 201
 202         if self.verification_id is None:
 203             raise Exception('Verification UUID not found')
 204
 205     def parse_verifier_result(self):
 206         """Parse and save test results."""
 207         stat = self.get_verifier_result(self.verification_id)
 208         try:
 209             num_executed = stat['num_tests'] - stat['num_skipped']
 210             try:
 211                 self.result = 100 * stat['num_success'] / num_executed
 212             except ZeroDivisionError:
 213                 self.result = 0
 214                 if stat['num_tests'] > 0:
 215                     LOGGER.info("All tests have been skipped")
 216                 else:
 217                     LOGGER.error("No test has been executed")
 218                     return
 219
 220             with open(os.path.join(self.res_dir,
 221                                    "tempest-error.log"), 'r') as logfile:
 222                 output = logfile.read()
 223
 224             success_testcases = []
 225             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 226                                     output):
 227                 success_testcases.append(match)
 228             failed_testcases = []
 229             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 230                                     output):
 231                 failed_testcases.append(match)
 232             skipped_testcases = []
 233             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 234                                     output):
 235                 skipped_testcases.append(match)
 236
 237             self.details = {"tests_number": stat['num_tests'],
 238                             "success_number": stat['num_success'],
 239                             "skipped_number": stat['num_skipped'],
 240                             "failures_number": stat['num_failures'],
 241                             "success": success_testcases,
 242                             "skipped": skipped_testcases,
 243                             "failures": failed_testcases}
 244         except Exception:  # pylint: disable=broad-except
 245             self.result = 0
 246
 247         LOGGER.info("Tempest %s success_rate is %s%%",
 248                     self.case_name, self.result)
 249
 250     def generate_report(self):
 251         """Generate verification report."""
 252         html_file = os.path.join(self.res_dir,
 253                                  "tempest-report.html")
 254         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 255                self.verification_id, "--to", html_file]
 256         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 257                          stderr=subprocess.STDOUT)
 258
 259     def configure(self, **kwargs):  # pylint: disable=unused-argument
 260         """
 261         Create all openstack resources for tempest-based testcases and write
 262         tempest.conf.
 263         """
 264         if not os.path.exists(self.res_dir):
 265             os.makedirs(self.res_dir)
 266         compute_cnt = len(self.cloud.list_hypervisors())
 267
 268         LOGGER.info("Creating two images for Tempest suite")
 269         self.image_alt = self.publish_image(
 270             '{}-img_alt_{}'.format(self.case_name, self.guid))
 271         LOGGER.debug("image_alt: %s", self.image_alt)
 272         self.flavor_alt = self.create_flavor(
 273             '{}-flavor_alt_{}'.format(self.case_name, self.guid))
 274         LOGGER.debug("flavor: %s", self.flavor_alt)
 275
 276         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 277         conf_utils.configure_tempest_update_params(
 278             self.conf_file, network_name=self.network.id,
 279             image_id=self.image.id,
 280             flavor_id=self.flavor.id,
 281             compute_cnt=compute_cnt,
 282             image_alt_id=self.image_alt.id,
 283             flavor_alt_id=self.flavor_alt.id)
 284         self.backup_tempest_config(self.conf_file, self.res_dir)
 285
 286     def run(self, **kwargs):
 287         self.start_time = time.time()
 288         try:
 289             super(TempestCommon, self).run(**kwargs)
 290             self.configure(**kwargs)
 291             self.generate_test_list()
 292             self.apply_tempest_blacklist()
 293             self.run_verifier_tests()
 294             self.parse_verifier_result()
 295             self.generate_report()
 296             res = testcase.TestCase.EX_OK
 297         except Exception:  # pylint: disable=broad-except
 298             LOGGER.exception('Error with run')
 299             res = testcase.TestCase.EX_RUN_ERROR
 300         self.stop_time = time.time()
 301         return res
 302
 303     def clean(self):
 304         """
 305         Cleanup all OpenStack objects. Should be called on completion.
 306         """
 307         super(TempestCommon, self).clean()
 308         self.cloud.delete_image(self.image_alt)
 309         self.orig_cloud.delete_flavor(self.flavor_alt.id)
 310
 311
 312 class TempestSmokeSerial(TempestCommon):
 313     """Tempest smoke serial testcase implementation."""
 314     def __init__(self, **kwargs):
 315         if "case_name" not in kwargs:
 316             kwargs["case_name"] = 'tempest_smoke_serial'
 317         TempestCommon.__init__(self, **kwargs)
 318         self.mode = "smoke"
 319         self.option = ["--concurrency", "1"]
 320
 321
 322 class TempestNeutronTrunk(TempestCommon):
 323     """Tempest neutron trunk testcase implementation."""
 324     def __init__(self, **kwargs):
 325         if "case_name" not in kwargs:
 326             kwargs["case_name"] = 'neutron_trunk'
 327         TempestCommon.__init__(self, **kwargs)
 328         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 329         self.res_dir = os.path.join(
 330             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 331         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 332         self.list = os.path.join(self.res_dir, 'test_list.txt')
 333
 334     def configure(self, **kwargs):
 335         super(TempestNeutronTrunk, self).configure(**kwargs)
 336         rconfig = configparser.RawConfigParser()
 337         rconfig.read(self.conf_file)
 338         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 339         with open(self.conf_file, 'wb') as config_file:
 340             rconfig.write(config_file)
 341
 342
 343 class TempestBarbican(TempestCommon):
 344     """Tempest Barbican testcase implementation."""
 345     def __init__(self, **kwargs):
 346         if "case_name" not in kwargs:
 347             kwargs["case_name"] = 'barbican'
 348         TempestCommon.__init__(self, **kwargs)
 349         self.mode = "'barbican_tempest_plugin.tests.(api|scenario)'"
 350         self.res_dir = os.path.join(
 351             getattr(config.CONF, 'dir_results'), 'barbican')
 352         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 353         self.list = os.path.join(self.res_dir, 'test_list.txt')
 354
 355
 356 class TempestSmokeParallel(TempestCommon):
 357     """Tempest smoke parallel testcase implementation."""
 358     def __init__(self, **kwargs):
 359         if "case_name" not in kwargs:
 360             kwargs["case_name"] = 'tempest_smoke_parallel'
 361         TempestCommon.__init__(self, **kwargs)
 362         self.mode = "smoke"
 363
 364
 365 class TempestFullParallel(TempestCommon):
 366     """Tempest full parallel testcase implementation."""
 367     def __init__(self, **kwargs):
 368         if "case_name" not in kwargs:
 369             kwargs["case_name"] = 'tempest_full_parallel'
 370         TempestCommon.__init__(self, **kwargs)
 371         self.mode = "full"
 372
 373
 374 class TempestCustom(TempestCommon):
 375     """Tempest custom testcase implementation."""
 376     def __init__(self, **kwargs):
 377         if "case_name" not in kwargs:
 378             kwargs["case_name"] = 'tempest_custom'
 379         TempestCommon.__init__(self, **kwargs)
 380         self.mode = "custom"
 381         self.option = ["--concurrency", "1"]
 382
 383
 384 class TempestDefcore(TempestCommon):
 385     """Tempest Defcore testcase implementation."""
 386     def __init__(self, **kwargs):
 387         if "case_name" not in kwargs:
 388             kwargs["case_name"] = 'tempest_defcore'
 389         TempestCommon.__init__(self, **kwargs)
 390         self.mode = "defcore"
 391         self.option = ["--concurrency", "1"]