functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import pkg_resources
  16 import re
  17 import shutil
  18 import subprocess
  19 import time
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.tempest import conf_utils
  25 from functest.utils.constants import CONST
  26 import functest.utils.functest_utils as ft_utils
  27
  28 """ logging configuration """
  29 logger = logging.getLogger(__name__)
  30
  31
  32 class TempestCommon(testcase.OSGCTestCase):
  33
  34     def __init__(self, **kwargs):
  35         super(TempestCommon, self).__init__(**kwargs)
  36         self.MODE = ""
  37         self.OPTION = ""
  38         self.VERIFIER_ID = conf_utils.get_verifier_id()
  39         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  40             self.VERIFIER_ID)
  41         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  42         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  43             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  44         self.VERIFICATION_ID = None
  45
  46     @staticmethod
  47     def read_file(filename):
  48         with open(filename) as src:
  49             return [line.strip() for line in src.readlines()]
  50
  51     def generate_test_list(self, verifier_repo_dir):
  52         logger.debug("Generating test case list...")
  53         if self.MODE == 'defcore':
  54             shutil.copyfile(
  55                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  56         elif self.MODE == 'custom':
  57             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  58                 shutil.copyfile(
  59                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  60             else:
  61                 raise Exception("Tempest test list file %s NOT found."
  62                                 % conf_utils.TEMPEST_CUSTOM)
  63         else:
  64             if self.MODE == 'smoke':
  65                 testr_mode = "smoke"
  66             elif self.MODE == 'feature_multisite':
  67                 testr_mode = "'[Kk]ingbird'"
  68             elif self.MODE == 'full':
  69                 testr_mode = ""
  70             else:
  71                 testr_mode = 'tempest.api.' + self.MODE
  72             cmd = ("cd {0};"
  73                    "testr list-tests {1} > {2};"
  74                    "cd -;".format(verifier_repo_dir,
  75                                   testr_mode,
  76                                   conf_utils.TEMPEST_RAW_LIST))
  77             ft_utils.execute_command(cmd)
  78
  79     def apply_tempest_blacklist(self):
  80         logger.debug("Applying tempest blacklist...")
  81         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  82         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  83         black_tests = []
  84         try:
  85             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  86             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  87             if (bool(installer_type) * bool(deploy_scenario)):
  88                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  89                 # file
  90                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  91                 black_list_yaml = yaml.safe_load(black_list_file)
  92                 black_list_file.close()
  93                 for item in black_list_yaml:
  94                     scenarios = item['scenarios']
  95                     installers = item['installers']
  96                     if (deploy_scenario in scenarios and
  97                             installer_type in installers):
  98                         tests = item['tests']
  99                         for test in tests:
 100                             black_tests.append(test)
 101                         break
 102         except Exception:
 103             black_tests = []
 104             logger.debug("Tempest blacklist file does not exist.")
 105
 106         for cases_line in cases_file:
 107             for black_tests_line in black_tests:
 108                 if black_tests_line in cases_line:
 109                     break
 110             else:
 111                 result_file.write(str(cases_line) + '\n')
 112         result_file.close()
 113
 114     def run_verifier_tests(self):
 115         self.OPTION += (" --load-list {} --detailed"
 116                         .format(conf_utils.TEMPEST_LIST))
 117
 118         cmd_line = "rally verify start " + self.OPTION
 119         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 120
 121         header = ("Tempest environment:\n"
 122                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 123                   (CONST.__getattribute__('INSTALLER_TYPE'),
 124                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 125                    CONST.__getattribute__('NODE_NAME'),
 126                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 127
 128         f_stdout = open(
 129             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 130         f_stderr = open(
 131             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 132                          "tempest-error.log"), 'w+')
 133         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 134                                   "environment.log"), 'w+')
 135         f_env.write(header)
 136
 137         p = subprocess.Popen(
 138             cmd_line, shell=True,
 139             stdout=subprocess.PIPE,
 140             stderr=f_stderr,
 141             bufsize=1)
 142
 143         with p.stdout:
 144             for line in iter(p.stdout.readline, b''):
 145                 if re.search("\} tempest\.", line):
 146                     logger.info(line.replace('\n', ''))
 147                 elif re.search('Starting verification', line):
 148                     logger.info(line.replace('\n', ''))
 149                     first_pos = line.index("UUID=") + len("UUID=")
 150                     last_pos = line.index(") for deployment")
 151                     self.VERIFICATION_ID = line[first_pos:last_pos]
 152                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 153                 f_stdout.write(line)
 154         p.wait()
 155
 156         f_stdout.close()
 157         f_stderr.close()
 158         f_env.close()
 159
 160     def parse_verifier_result(self):
 161         if self.VERIFICATION_ID is None:
 162             raise Exception('Verification UUID not found')
 163
 164         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 165         logger.info("Showing result for a verification: '%s'." % cmd_line)
 166         p = subprocess.Popen(cmd_line,
 167                              shell=True,
 168                              stdout=subprocess.PIPE,
 169                              stderr=subprocess.STDOUT)
 170         for line in p.stdout:
 171             new_line = line.replace(' ', '').split('|')
 172             if 'Tests' in new_line:
 173                 break
 174
 175             logger.info(line)
 176             if 'Testscount' in new_line:
 177                 num_tests = new_line[2]
 178             elif 'Success' in new_line:
 179                 num_success = new_line[2]
 180             elif 'Skipped' in new_line:
 181                 num_skipped = new_line[2]
 182             elif 'Failures' in new_line:
 183                 num_failures = new_line[2]
 184
 185         try:
 186             num_executed = int(num_tests) - int(num_skipped)
 187             try:
 188                 self.result = 100 * int(num_success) / int(num_executed)
 189             except ZeroDivisionError:
 190                 logger.error("No test has been executed")
 191                 self.result = 0
 192                 return
 193
 194             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 195                                    "tempest.log"), 'r') as logfile:
 196                 output = logfile.read()
 197
 198             success_testcases = []
 199             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 200                 success_testcases.append(match)
 201             failed_testcases = []
 202             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 203                 failed_testcases.append(match)
 204             skipped_testcases = []
 205             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 206                 skipped_testcases.append(match)
 207
 208             self.details = {"tests": int(num_tests),
 209                             "failures": int(num_failures),
 210                             "success": success_testcases,
 211                             "errors": failed_testcases,
 212                             "skipped": skipped_testcases}
 213         except Exception:
 214             self.result = 0
 215
 216         logger.info("Tempest %s success_rate is %s%%"
 217                     % (self.case_name, self.result))
 218
 219     def run(self):
 220
 221         self.start_time = time.time()
 222         try:
 223             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 224                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 225             image_and_flavor = conf_utils.create_tempest_resources()
 226             conf_utils.configure_tempest(
 227                 self.DEPLOYMENT_DIR,
 228                 IMAGE_ID=image_and_flavor.get("image_id"),
 229                 FLAVOR_ID=image_and_flavor.get("flavor_id"),
 230                 MODE=self.MODE)
 231             self.generate_test_list(self.VERIFIER_REPO_DIR)
 232             self.apply_tempest_blacklist()
 233             self.run_verifier_tests()
 234             self.parse_verifier_result()
 235             res = testcase.TestCase.EX_OK
 236         except Exception as e:
 237             logger.error('Error with run: %s' % e)
 238             res = testcase.TestCase.EX_RUN_ERROR
 239
 240         self.stop_time = time.time()
 241         return res
 242
 243
 244 class TempestSmokeSerial(TempestCommon):
 245
 246     def __init__(self, **kwargs):
 247         if "case_name" not in kwargs:
 248             kwargs["case_name"] = 'tempest_smoke_serial'
 249         TempestCommon.__init__(self, **kwargs)
 250         self.MODE = "smoke"
 251         self.OPTION = "--concurrency 1"
 252
 253
 254 class TempestSmokeParallel(TempestCommon):
 255
 256     def __init__(self, **kwargs):
 257         if "case_name" not in kwargs:
 258             kwargs["case_name"] = 'tempest_smoke_parallel'
 259         TempestCommon.__init__(self, **kwargs)
 260         self.MODE = "smoke"
 261         self.OPTION = ""
 262
 263
 264 class TempestFullParallel(TempestCommon):
 265
 266     def __init__(self, **kwargs):
 267         if "case_name" not in kwargs:
 268             kwargs["case_name"] = 'tempest_full_parallel'
 269         TempestCommon.__init__(self, **kwargs)
 270         self.MODE = "full"
 271
 272
 273 class TempestMultisite(TempestCommon):
 274
 275     def __init__(self, **kwargs):
 276         if "case_name" not in kwargs:
 277             kwargs["case_name"] = 'multisite'
 278         TempestCommon.__init__(self, **kwargs)
 279         self.MODE = "feature_multisite"
 280         self.OPTION = "--concurrency 1"
 281         conf_utils.install_verifier_ext(
 282             pkg_resources.resource_filename('kingbird', '..'))
 283
 284
 285 class TempestCustom(TempestCommon):
 286
 287     def __init__(self, **kwargs):
 288         if "case_name" not in kwargs:
 289             kwargs["case_name"] = 'tempest_custom'
 290         TempestCommon.__init__(self, **kwargs)
 291         self.MODE = "custom"
 292         self.OPTION = "--concurrency 1"
 293
 294
 295 class TempestDefcore(TempestCommon):
 296
 297     def __init__(self, **kwargs):
 298         if "case_name" not in kwargs:
 299             kwargs["case_name"] = 'tempest_defcore'
 300         TempestCommon.__init__(self, **kwargs)
 301         self.MODE = "defcore"
 302         self.OPTION = "--concurrency 1"