functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     TEMPEST_RESULTS_DIR = os.path.join(
  39         getattr(config.CONF, 'dir_results'), 'tempest')
  40
  41     visibility = 'public'
  42
  43     def __init__(self, **kwargs):
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.mode = ""
  46         self.option = []
  47         self.verifier_id = conf_utils.get_verifier_id()
  48         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  49             self.verifier_id)
  50         self.deployment_id = conf_utils.get_verifier_deployment_id()
  51         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  52             self.verifier_id, self.deployment_id)
  53         self.verification_id = None
  54         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58         self.image_alt = None
  59         self.flavor_alt = None
  60
  61     @staticmethod
  62     def read_file(filename):
  63         """Read file and return content as a stripped list."""
  64         with open(filename) as src:
  65             return [line.strip() for line in src.readlines()]
  66
  67     @staticmethod
  68     def get_verifier_result(verif_id):
  69         """Retrieve verification results."""
  70         result = {
  71             'num_tests': 0,
  72             'num_success': 0,
  73             'num_failures': 0,
  74             'num_skipped': 0
  75         }
  76         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  77         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  78         proc = subprocess.Popen(cmd,
  79                                 stdout=subprocess.PIPE,
  80                                 stderr=subprocess.STDOUT)
  81         for line in proc.stdout:
  82             new_line = line.replace(' ', '').split('|')
  83             if 'Tests' in new_line:
  84                 break
  85             LOGGER.info(line)
  86             if 'Testscount' in new_line:
  87                 result['num_tests'] = int(new_line[2])
  88             elif 'Success' in new_line:
  89                 result['num_success'] = int(new_line[2])
  90             elif 'Skipped' in new_line:
  91                 result['num_skipped'] = int(new_line[2])
  92             elif 'Failures' in new_line:
  93                 result['num_failures'] = int(new_line[2])
  94         return result
  95
  96     @staticmethod
  97     def backup_tempest_config(conf_file, res_dir):
  98         """
  99         Copy config file to tempest results directory
 100         """
 101         if not os.path.exists(res_dir):
 102             os.makedirs(res_dir)
 103         shutil.copyfile(conf_file,
 104                         os.path.join(res_dir, 'tempest.conf'))
 105
 106     def generate_test_list(self):
 107         """Generate test list based on the test mode."""
 108         LOGGER.debug("Generating test case list...")
 109         self.backup_tempest_config(self.conf_file, '/etc')
 110         if self.mode == 'custom':
 111             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 112                 shutil.copyfile(
 113                     conf_utils.TEMPEST_CUSTOM, self.list)
 114             else:
 115                 raise Exception("Tempest test list file %s NOT found."
 116                                 % conf_utils.TEMPEST_CUSTOM)
 117         else:
 118             if self.mode == 'smoke':
 119                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 120             elif self.mode == 'full':
 121                 testr_mode = r"'^tempest\.'"
 122             else:
 123                 testr_mode = self.mode
 124             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 125                 self.verifier_repo_dir, testr_mode, self.list)
 126             output = subprocess.check_output(cmd, shell=True)
 127             LOGGER.info("%s\n%s", cmd, output)
 128         os.remove('/etc/tempest.conf')
 129
 130     def apply_tempest_blacklist(self):
 131         """Exclude blacklisted test cases."""
 132         LOGGER.debug("Applying tempest blacklist...")
 133         if os.path.exists(self.raw_list):
 134             os.remove(self.raw_list)
 135         os.rename(self.list, self.raw_list)
 136         cases_file = self.read_file(self.raw_list)
 137         result_file = open(self.list, 'w')
 138         black_tests = []
 139         try:
 140             installer_type = env.get('INSTALLER_TYPE')
 141             deploy_scenario = env.get('DEPLOY_SCENARIO')
 142             if bool(installer_type) * bool(deploy_scenario):
 143                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 144                 # file
 145                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 146                 black_list_yaml = yaml.safe_load(black_list_file)
 147                 black_list_file.close()
 148                 for item in black_list_yaml:
 149                     scenarios = item['scenarios']
 150                     installers = item['installers']
 151                     if (deploy_scenario in scenarios and
 152                             installer_type in installers):
 153                         tests = item['tests']
 154                         for test in tests:
 155                             black_tests.append(test)
 156                         break
 157         except Exception:  # pylint: disable=broad-except
 158             black_tests = []
 159             LOGGER.debug("Tempest blacklist file does not exist.")
 160
 161         for cases_line in cases_file:
 162             for black_tests_line in black_tests:
 163                 if black_tests_line in cases_line:
 164                     break
 165             else:
 166                 result_file.write(str(cases_line) + '\n')
 167         result_file.close()
 168
 169     def run_verifier_tests(self):
 170         """Execute tempest test cases."""
 171         cmd = ["rally", "verify", "start", "--load-list",
 172                self.list]
 173         cmd.extend(self.option)
 174         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 175
 176         f_stdout = open(
 177             os.path.join(self.res_dir, "tempest.log"), 'w+')
 178         f_stderr = open(
 179             os.path.join(self.res_dir,
 180                          "tempest-error.log"), 'w+')
 181
 182         proc = subprocess.Popen(
 183             cmd,
 184             stdout=subprocess.PIPE,
 185             stderr=f_stderr,
 186             bufsize=1)
 187
 188         with proc.stdout:
 189             for line in iter(proc.stdout.readline, b''):
 190                 if re.search(r"\} tempest\.", line):
 191                     LOGGER.info(line.replace('\n', ''))
 192                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 193                     self.verification_id = re.search(
 194                         r'(?=\(UUID=(.*)\))', line).group(1)
 195                     LOGGER.info('Verification UUID: %s', self.verification_id)
 196                 f_stdout.write(line)
 197         proc.wait()
 198
 199         f_stdout.close()
 200         f_stderr.close()
 201
 202         if self.verification_id is None:
 203             raise Exception('Verification UUID not found')
 204
 205     def parse_verifier_result(self):
 206         """Parse and save test results."""
 207         stat = self.get_verifier_result(self.verification_id)
 208         try:
 209             num_executed = stat['num_tests'] - stat['num_skipped']
 210             try:
 211                 self.result = 100 * stat['num_success'] / num_executed
 212             except ZeroDivisionError:
 213                 self.result = 0
 214                 if stat['num_tests'] > 0:
 215                     LOGGER.info("All tests have been skipped")
 216                 else:
 217                     LOGGER.error("No test has been executed")
 218                     return
 219
 220             with open(os.path.join(self.res_dir,
 221                                    "tempest-error.log"), 'r') as logfile:
 222                 output = logfile.read()
 223
 224             success_testcases = []
 225             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 226                                     output):
 227                 success_testcases.append(match)
 228             failed_testcases = []
 229             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 230                                     output):
 231                 failed_testcases.append(match)
 232             skipped_testcases = []
 233             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 234                                     output):
 235                 skipped_testcases.append(match)
 236
 237             self.details = {"tests_number": stat['num_tests'],
 238                             "success_number": stat['num_success'],
 239                             "skipped_number": stat['num_skipped'],
 240                             "failures_number": stat['num_failures'],
 241                             "success": success_testcases,
 242                             "skipped": skipped_testcases,
 243                             "failures": failed_testcases}
 244         except Exception:  # pylint: disable=broad-except
 245             self.result = 0
 246
 247         LOGGER.info("Tempest %s success_rate is %s%%",
 248                     self.case_name, self.result)
 249
 250     def generate_report(self):
 251         """Generate verification report."""
 252         html_file = os.path.join(self.res_dir,
 253                                  "tempest-report.html")
 254         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 255                self.verification_id, "--to", html_file]
 256         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 257                          stderr=subprocess.STDOUT)
 258
 259     def configure(self, **kwargs):  # pylint: disable=unused-argument
 260         """
 261         Create all openstack resources for tempest-based testcases and write
 262         tempest.conf.
 263         """
 264         if not os.path.exists(self.res_dir):
 265             os.makedirs(self.res_dir)
 266         compute_cnt = len(self.cloud.list_hypervisors())
 267
 268         self.image_alt = self.publish_image(
 269             '{}-img_alt_{}'.format(self.case_name, self.guid))
 270         self.flavor_alt = self.create_flavor_alt()
 271         LOGGER.debug("flavor: %s", self.flavor_alt)
 272
 273         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 274         conf_utils.configure_tempest_update_params(
 275             self.conf_file, network_name=self.network.id,
 276             image_id=self.image.id,
 277             flavor_id=self.flavor.id,
 278             compute_cnt=compute_cnt,
 279             image_alt_id=self.image_alt.id,
 280             flavor_alt_id=self.flavor_alt.id)
 281         self.backup_tempest_config(self.conf_file, self.res_dir)
 282
 283     def run(self, **kwargs):
 284         self.start_time = time.time()
 285         try:
 286             super(TempestCommon, self).run(**kwargs)
 287             self.configure(**kwargs)
 288             self.generate_test_list()
 289             self.apply_tempest_blacklist()
 290             self.run_verifier_tests()
 291             self.parse_verifier_result()
 292             self.generate_report()
 293             res = testcase.TestCase.EX_OK
 294         except Exception:  # pylint: disable=broad-except
 295             LOGGER.exception('Error with run')
 296             res = testcase.TestCase.EX_RUN_ERROR
 297         self.stop_time = time.time()
 298         return res
 299
 300     def clean(self):
 301         """
 302         Cleanup all OpenStack objects. Should be called on completion.
 303         """
 304         super(TempestCommon, self).clean()
 305         self.cloud.delete_image(self.image_alt)
 306         self.orig_cloud.delete_flavor(self.flavor_alt.id)
 307
 308
 309 class TempestSmokeSerial(TempestCommon):
 310     """Tempest smoke serial testcase implementation."""
 311     def __init__(self, **kwargs):
 312         if "case_name" not in kwargs:
 313             kwargs["case_name"] = 'tempest_smoke_serial'
 314         TempestCommon.__init__(self, **kwargs)
 315         self.mode = "smoke"
 316         self.option = ["--concurrency", "1"]
 317
 318
 319 class TempestNeutronTrunk(TempestCommon):
 320     """Tempest neutron trunk testcase implementation."""
 321     def __init__(self, **kwargs):
 322         if "case_name" not in kwargs:
 323             kwargs["case_name"] = 'neutron_trunk'
 324         TempestCommon.__init__(self, **kwargs)
 325         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 326         self.res_dir = os.path.join(
 327             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 328         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 329         self.list = os.path.join(self.res_dir, 'test_list.txt')
 330
 331     def configure(self, **kwargs):
 332         super(TempestNeutronTrunk, self).configure(**kwargs)
 333         rconfig = configparser.RawConfigParser()
 334         rconfig.read(self.conf_file)
 335         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 336         with open(self.conf_file, 'wb') as config_file:
 337             rconfig.write(config_file)
 338
 339
 340 class TempestBarbican(TempestCommon):
 341     """Tempest Barbican testcase implementation."""
 342     def __init__(self, **kwargs):
 343         if "case_name" not in kwargs:
 344             kwargs["case_name"] = 'barbican'
 345         TempestCommon.__init__(self, **kwargs)
 346         self.mode = "'barbican_tempest_plugin.tests.(api|scenario)'"
 347         self.res_dir = os.path.join(
 348             getattr(config.CONF, 'dir_results'), 'barbican')
 349         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 350         self.list = os.path.join(self.res_dir, 'test_list.txt')
 351
 352
 353 class TempestSmokeParallel(TempestCommon):
 354     """Tempest smoke parallel testcase implementation."""
 355     def __init__(self, **kwargs):
 356         if "case_name" not in kwargs:
 357             kwargs["case_name"] = 'tempest_smoke_parallel'
 358         TempestCommon.__init__(self, **kwargs)
 359         self.mode = "smoke"
 360
 361
 362 class TempestFullParallel(TempestCommon):
 363     """Tempest full parallel testcase implementation."""
 364     def __init__(self, **kwargs):
 365         if "case_name" not in kwargs:
 366             kwargs["case_name"] = 'tempest_full_parallel'
 367         TempestCommon.__init__(self, **kwargs)
 368         self.mode = "full"
 369
 370
 371 class TempestCustom(TempestCommon):
 372     """Tempest custom testcase implementation."""
 373     def __init__(self, **kwargs):
 374         if "case_name" not in kwargs:
 375             kwargs["case_name"] = 'tempest_custom'
 376         TempestCommon.__init__(self, **kwargs)
 377         self.mode = "custom"
 378         self.option = ["--concurrency", "1"]
 379
 380
 381 class TempestDefcore(TempestCommon):
 382     """Tempest Defcore testcase implementation."""
 383     def __init__(self, **kwargs):
 384         if "case_name" not in kwargs:
 385             kwargs["case_name"] = 'tempest_defcore'
 386         TempestCommon.__init__(self, **kwargs)
 387         self.mode = "defcore"
 388         self.option = ["--concurrency", "1"]