functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady1):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     TEMPEST_RESULTS_DIR = os.path.join(
  39         getattr(config.CONF, 'dir_results'), 'tempest')
  40
  41     visibility = 'public'
  42
  43     def __init__(self, **kwargs):
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.mode = ""
  46         self.option = []
  47         self.verifier_id = conf_utils.get_verifier_id()
  48         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  49             self.verifier_id)
  50         self.deployment_id = conf_utils.get_verifier_deployment_id()
  51         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  52             self.verifier_id, self.deployment_id)
  53         self.verification_id = None
  54         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58         self.image_alt = None
  59         self.flavor_alt = None
  60
  61     @staticmethod
  62     def read_file(filename):
  63         """Read file and return content as a stripped list."""
  64         with open(filename) as src:
  65             return [line.strip() for line in src.readlines()]
  66
  67     @staticmethod
  68     def get_verifier_result(verif_id):
  69         """Retrieve verification results."""
  70         result = {
  71             'num_tests': 0,
  72             'num_success': 0,
  73             'num_failures': 0,
  74             'num_skipped': 0
  75         }
  76         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  77         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  78         proc = subprocess.Popen(cmd,
  79                                 stdout=subprocess.PIPE,
  80                                 stderr=subprocess.STDOUT)
  81         for line in proc.stdout:
  82             new_line = line.replace(' ', '').split('|')
  83             if 'Tests' in new_line:
  84                 break
  85             LOGGER.info(line)
  86             if 'Testscount' in new_line:
  87                 result['num_tests'] = int(new_line[2])
  88             elif 'Success' in new_line:
  89                 result['num_success'] = int(new_line[2])
  90             elif 'Skipped' in new_line:
  91                 result['num_skipped'] = int(new_line[2])
  92             elif 'Failures' in new_line:
  93                 result['num_failures'] = int(new_line[2])
  94         return result
  95
  96     @staticmethod
  97     def backup_tempest_config(conf_file, res_dir):
  98         """
  99         Copy config file to tempest results directory
 100         """
 101         if not os.path.exists(res_dir):
 102             os.makedirs(res_dir)
 103         shutil.copyfile(conf_file,
 104                         os.path.join(res_dir, 'tempest.conf'))
 105
 106     def generate_test_list(self):
 107         """Generate test list based on the test mode."""
 108         LOGGER.debug("Generating test case list...")
 109         self.backup_tempest_config(self.conf_file, '/etc')
 110         if self.mode == 'custom':
 111             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 112                 shutil.copyfile(
 113                     conf_utils.TEMPEST_CUSTOM, self.list)
 114             else:
 115                 raise Exception("Tempest test list file %s NOT found."
 116                                 % conf_utils.TEMPEST_CUSTOM)
 117         else:
 118             if self.mode == 'smoke':
 119                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 120             elif self.mode == 'full':
 121                 testr_mode = r"'^tempest\.'"
 122             else:
 123                 testr_mode = self.mode
 124             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 125                 self.verifier_repo_dir, testr_mode, self.list)
 126             output = subprocess.check_output(cmd, shell=True)
 127             LOGGER.info("%s\n%s", cmd, output)
 128         os.remove('/etc/tempest.conf')
 129
 130     def apply_tempest_blacklist(self):
 131         """Exclude blacklisted test cases."""
 132         LOGGER.debug("Applying tempest blacklist...")
 133         if os.path.exists(self.raw_list):
 134             os.remove(self.raw_list)
 135         os.rename(self.list, self.raw_list)
 136         cases_file = self.read_file(self.raw_list)
 137         result_file = open(self.list, 'w')
 138         black_tests = []
 139         try:
 140             installer_type = env.get('INSTALLER_TYPE')
 141             deploy_scenario = env.get('DEPLOY_SCENARIO')
 142             if bool(installer_type) * bool(deploy_scenario):
 143                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 144                 # file
 145                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 146                 black_list_yaml = yaml.safe_load(black_list_file)
 147                 black_list_file.close()
 148                 for item in black_list_yaml:
 149                     scenarios = item['scenarios']
 150                     installers = item['installers']
 151                     if (deploy_scenario in scenarios and
 152                             installer_type in installers):
 153                         tests = item['tests']
 154                         for test in tests:
 155                             black_tests.append(test)
 156                         break
 157         except Exception:  # pylint: disable=broad-except
 158             black_tests = []
 159             LOGGER.debug("Tempest blacklist file does not exist.")
 160
 161         for cases_line in cases_file:
 162             for black_tests_line in black_tests:
 163                 if black_tests_line in cases_line:
 164                     break
 165             else:
 166                 result_file.write(str(cases_line) + '\n')
 167         result_file.close()
 168
 169     def run_verifier_tests(self):
 170         """Execute tempest test cases."""
 171         cmd = ["rally", "verify", "start", "--load-list",
 172                self.list]
 173         cmd.extend(self.option)
 174         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 175
 176         f_stdout = open(
 177             os.path.join(self.res_dir, "tempest.log"), 'w+')
 178         f_stderr = open(
 179             os.path.join(self.res_dir,
 180                          "tempest-error.log"), 'w+')
 181
 182         proc = subprocess.Popen(
 183             cmd,
 184             stdout=subprocess.PIPE,
 185             stderr=f_stderr,
 186             bufsize=1)
 187
 188         with proc.stdout:
 189             for line in iter(proc.stdout.readline, b''):
 190                 if re.search(r"\} tempest\.", line):
 191                     LOGGER.info(line.replace('\n', ''))
 192                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 193                     self.verification_id = re.search(
 194                         r'(?=\(UUID=(.*)\))', line).group(1)
 195                     LOGGER.info('Verification UUID: %s', self.verification_id)
 196                 f_stdout.write(line)
 197         proc.wait()
 198
 199         f_stdout.close()
 200         f_stderr.close()
 201
 202         if self.verification_id is None:
 203             raise Exception('Verification UUID not found')
 204
 205     def parse_verifier_result(self):
 206         """Parse and save test results."""
 207         stat = self.get_verifier_result(self.verification_id)
 208         try:
 209             num_executed = stat['num_tests'] - stat['num_skipped']
 210             try:
 211                 self.result = 100 * stat['num_success'] / num_executed
 212             except ZeroDivisionError:
 213                 self.result = 0
 214                 if stat['num_tests'] > 0:
 215                     LOGGER.info("All tests have been skipped")
 216                 else:
 217                     LOGGER.error("No test has been executed")
 218                     return
 219
 220             with open(os.path.join(self.res_dir,
 221                                    "tempest-error.log"), 'r') as logfile:
 222                 output = logfile.read()
 223
 224             success_testcases = []
 225             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 226                                     output):
 227                 success_testcases.append(match)
 228             failed_testcases = []
 229             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 230                                     output):
 231                 failed_testcases.append(match)
 232             skipped_testcases = []
 233             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 234                                     output):
 235                 skipped_testcases.append(match)
 236
 237             self.details = {"tests_number": stat['num_tests'],
 238                             "success_number": stat['num_success'],
 239                             "skipped_number": stat['num_skipped'],
 240                             "failures_number": stat['num_failures'],
 241                             "success": success_testcases,
 242                             "skipped": skipped_testcases,
 243                             "failures": failed_testcases}
 244         except Exception:  # pylint: disable=broad-except
 245             self.result = 0
 246
 247         LOGGER.info("Tempest %s success_rate is %s%%",
 248                     self.case_name, self.result)
 249
 250     def generate_report(self):
 251         """Generate verification report."""
 252         html_file = os.path.join(self.res_dir,
 253                                  "tempest-report.html")
 254         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 255                self.verification_id, "--to", html_file]
 256         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 257                          stderr=subprocess.STDOUT)
 258
 259     def configure(self, **kwargs):  # pylint: disable=unused-argument
 260         """
 261         Create all openstack resources for tempest-based testcases and write
 262         tempest.conf.
 263         """
 264         if not os.path.exists(self.res_dir):
 265             os.makedirs(self.res_dir)
 266         compute_cnt = len(self.cloud.list_hypervisors())
 267
 268         LOGGER.info("Creating two images for Tempest suite")
 269
 270         self.image_alt = self.cloud.create_image(
 271             '{}-img_alt_{}'.format(self.case_name, self.guid),
 272             filename=getattr(
 273                 config.CONF, '{}_image'.format(self.case_name),
 274                 self.filename),
 275             meta=getattr(
 276                 config.CONF, '{}_extra_properties'.format(self.case_name),
 277                 self.extra_properties),
 278             visibility=getattr(
 279                 config.CONF, '{}_visibility'.format(self.case_name),
 280                 self.visibility))
 281         LOGGER.debug("image_alt: %s", self.image_alt)
 282
 283         self.flavor_alt = self.orig_cloud.create_flavor(
 284             '{}-flavor_alt_{}'.format(self.case_name, self.guid),
 285             getattr(config.CONF, '{}_flavor_ram'.format(self.case_name),
 286                     self.flavor_ram),
 287             getattr(config.CONF, '{}_flavor_vcpus'.format(self.case_name),
 288                     self.flavor_vcpus),
 289             getattr(config.CONF, '{}_flavor_disk'.format(self.case_name),
 290                     self.flavor_disk))
 291         LOGGER.debug("flavor: %s", self.flavor_alt)
 292         self.orig_cloud.set_flavor_specs(
 293             self.flavor_alt.id, getattr(config.CONF, 'flavor_extra_specs', {}))
 294
 295         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 296         conf_utils.configure_tempest_update_params(
 297             self.conf_file, network_name=self.network.id,
 298             image_id=self.image.id,
 299             flavor_id=self.flavor.id,
 300             compute_cnt=compute_cnt,
 301             image_alt_id=self.image_alt.id,
 302             flavor_alt_id=self.flavor_alt.id)
 303         self.backup_tempest_config(self.conf_file, self.res_dir)
 304
 305     def run(self, **kwargs):
 306         self.start_time = time.time()
 307         try:
 308             super(TempestCommon, self).run(**kwargs)
 309             self.configure(**kwargs)
 310             self.generate_test_list()
 311             self.apply_tempest_blacklist()
 312             self.run_verifier_tests()
 313             self.parse_verifier_result()
 314             self.generate_report()
 315             res = testcase.TestCase.EX_OK
 316         except Exception:  # pylint: disable=broad-except
 317             LOGGER.exception('Error with run')
 318             res = testcase.TestCase.EX_RUN_ERROR
 319         self.stop_time = time.time()
 320         return res
 321
 322     def clean(self):
 323         """
 324         Cleanup all OpenStack objects. Should be called on completion.
 325         """
 326         super(TempestCommon, self).clean()
 327         self.cloud.delete_image(self.image_alt)
 328         self.orig_cloud.delete_flavor(self.flavor_alt.id)
 329
 330
 331 class TempestSmokeSerial(TempestCommon):
 332     """Tempest smoke serial testcase implementation."""
 333     def __init__(self, **kwargs):
 334         if "case_name" not in kwargs:
 335             kwargs["case_name"] = 'tempest_smoke_serial'
 336         TempestCommon.__init__(self, **kwargs)
 337         self.mode = "smoke"
 338         self.option = ["--concurrency", "1"]
 339
 340
 341 class TempestNeutronTrunk(TempestCommon):
 342     """Tempest neutron trunk testcase implementation."""
 343     def __init__(self, **kwargs):
 344         if "case_name" not in kwargs:
 345             kwargs["case_name"] = 'neutron_trunk'
 346         TempestCommon.__init__(self, **kwargs)
 347         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 348         self.res_dir = os.path.join(
 349             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 350         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 351         self.list = os.path.join(self.res_dir, 'test_list.txt')
 352
 353     def configure(self, **kwargs):
 354         super(TempestNeutronTrunk, self).configure(**kwargs)
 355         rconfig = configparser.RawConfigParser()
 356         rconfig.read(self.conf_file)
 357         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 358         with open(self.conf_file, 'wb') as config_file:
 359             rconfig.write(config_file)
 360
 361
 362 class TempestBarbican(TempestCommon):
 363     """Tempest Barbican testcase implementation."""
 364     def __init__(self, **kwargs):
 365         if "case_name" not in kwargs:
 366             kwargs["case_name"] = 'barbican'
 367         TempestCommon.__init__(self, **kwargs)
 368         self.mode = "'barbican_tempest_plugin.tests.(api|scenario)'"
 369         self.res_dir = os.path.join(
 370             getattr(config.CONF, 'dir_results'), 'barbican')
 371         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 372         self.list = os.path.join(self.res_dir, 'test_list.txt')
 373
 374
 375 class TempestSmokeParallel(TempestCommon):
 376     """Tempest smoke parallel testcase implementation."""
 377     def __init__(self, **kwargs):
 378         if "case_name" not in kwargs:
 379             kwargs["case_name"] = 'tempest_smoke_parallel'
 380         TempestCommon.__init__(self, **kwargs)
 381         self.mode = "smoke"
 382
 383
 384 class TempestFullParallel(TempestCommon):
 385     """Tempest full parallel testcase implementation."""
 386     def __init__(self, **kwargs):
 387         if "case_name" not in kwargs:
 388             kwargs["case_name"] = 'tempest_full_parallel'
 389         TempestCommon.__init__(self, **kwargs)
 390         self.mode = "full"
 391
 392
 393 class TempestCustom(TempestCommon):
 394     """Tempest custom testcase implementation."""
 395     def __init__(self, **kwargs):
 396         if "case_name" not in kwargs:
 397             kwargs["case_name"] = 'tempest_custom'
 398         TempestCommon.__init__(self, **kwargs)
 399         self.mode = "custom"
 400         self.option = ["--concurrency", "1"]
 401
 402
 403 class TempestDefcore(TempestCommon):
 404     """Tempest Defcore testcase implementation."""
 405     def __init__(self, **kwargs):
 406         if "case_name" not in kwargs:
 407             kwargs["case_name"] = 'tempest_defcore'
 408         TempestCommon.__init__(self, **kwargs)
 409         self.mode = "defcore"
 410         self.option = ["--concurrency", "1"]