functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 import os
  12 import re
  13 import shutil
  14 import subprocess
  15 import time
  16
  17 import yaml
  18
  19 from functest.core import testcase
  20 from functest.opnfv_tests.openstack.tempest import conf_utils
  21 from functest.utils.constants import CONST
  22 import functest.utils.functest_logger as ft_logger
  23 import functest.utils.functest_utils as ft_utils
  24
  25 """ logging configuration """
  26 logger = ft_logger.Logger("Tempest").getLogger()
  27
  28
  29 class TempestCommon(testcase.TestCase):
  30
  31     def __init__(self, case_name=''):
  32         super(TempestCommon, self).__init__(case_name)
  33         self.MODE = ""
  34         self.OPTION = ""
  35         self.VERIFIER_ID = conf_utils.get_verifier_id()
  36         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  37             self.VERIFIER_ID)
  38         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  39         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  40             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  41         self.VERIFICATION_ID = None
  42
  43     @staticmethod
  44     def read_file(filename):
  45         with open(filename) as src:
  46             return [line.strip() for line in src.readlines()]
  47
  48     def generate_test_list(self, verifier_repo_dir):
  49         logger.debug("Generating test case list...")
  50         if self.MODE == 'defcore':
  51             shutil.copyfile(
  52                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  53         elif self.MODE == 'custom':
  54             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  55                 shutil.copyfile(
  56                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  57             else:
  58                 raise Exception("Tempest test list file %s NOT found."
  59                                 % conf_utils.TEMPEST_CUSTOM)
  60         else:
  61             if self.MODE == 'smoke':
  62                 testr_mode = "smoke"
  63             elif self.MODE == 'feature_multisite':
  64                 testr_mode = "'[Kk]ingbird'"
  65             elif self.MODE == 'full':
  66                 testr_mode = ""
  67             else:
  68                 testr_mode = 'tempest.api.' + self.MODE
  69             cmd = ("cd {0};"
  70                    "testr list-tests {1} > {2};"
  71                    "cd -;".format(verifier_repo_dir,
  72                                   testr_mode,
  73                                   conf_utils.TEMPEST_RAW_LIST))
  74             ft_utils.execute_command(cmd)
  75
  76     def apply_tempest_blacklist(self):
  77         logger.debug("Applying tempest blacklist...")
  78         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  79         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  80         black_tests = []
  81         try:
  82             installer_type = CONST.INSTALLER_TYPE
  83             deploy_scenario = CONST.DEPLOY_SCENARIO
  84             if (bool(installer_type) * bool(deploy_scenario)):
  85                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  86                 # file
  87                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  88                 black_list_yaml = yaml.safe_load(black_list_file)
  89                 black_list_file.close()
  90                 for item in black_list_yaml:
  91                     scenarios = item['scenarios']
  92                     installers = item['installers']
  93                     if (deploy_scenario in scenarios and
  94                             installer_type in installers):
  95                         tests = item['tests']
  96                         for test in tests:
  97                             black_tests.append(test)
  98                         break
  99         except Exception:
 100             black_tests = []
 101             logger.debug("Tempest blacklist file does not exist.")
 102
 103         for cases_line in cases_file:
 104             for black_tests_line in black_tests:
 105                 if black_tests_line in cases_line:
 106                     break
 107             else:
 108                 result_file.write(str(cases_line) + '\n')
 109         result_file.close()
 110
 111     def run_verifier_tests(self):
 112         self.OPTION += (" --load-list {} --detailed"
 113                         .format(conf_utils.TEMPEST_LIST))
 114
 115         cmd_line = "rally verify start " + self.OPTION
 116         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 117
 118         header = ("Tempest environment:\n"
 119                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 120                   (CONST.INSTALLER_TYPE,
 121                    CONST.DEPLOY_SCENARIO,
 122                    CONST.NODE_NAME,
 123                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 124
 125         f_stdout = open(
 126             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 127         f_stderr = open(
 128             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 129                          "tempest-error.log"), 'w+')
 130         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 131                                   "environment.log"), 'w+')
 132         f_env.write(header)
 133
 134         p = subprocess.Popen(
 135             cmd_line, shell=True,
 136             stdout=subprocess.PIPE,
 137             stderr=f_stderr,
 138             bufsize=1)
 139
 140         with p.stdout:
 141             for line in iter(p.stdout.readline, b''):
 142                 if re.search("\} tempest\.", line):
 143                     logger.info(line.replace('\n', ''))
 144                 elif re.search('Starting verification', line):
 145                     logger.info(line.replace('\n', ''))
 146                     first_pos = line.index("UUID=") + len("UUID=")
 147                     last_pos = line.index(") for deployment")
 148                     self.VERIFICATION_ID = line[first_pos:last_pos]
 149                     logger.debug('Verication UUID: %s' % self.VERIFICATION_ID)
 150                 f_stdout.write(line)
 151         p.wait()
 152
 153         f_stdout.close()
 154         f_stderr.close()
 155         f_env.close()
 156
 157     def parse_verifier_result(self):
 158         if self.VERIFICATION_ID is None:
 159             raise Exception('Verification UUID not found')
 160
 161         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 162         logger.info("Showing result for a verification: '%s'." % cmd_line)
 163         p = subprocess.Popen(cmd_line,
 164                              shell=True,
 165                              stdout=subprocess.PIPE,
 166                              stderr=subprocess.STDOUT)
 167         for line in p.stdout:
 168             new_line = line.replace(' ', '').split('|')
 169             if 'Tests' in new_line:
 170                 break
 171
 172             logger.info(line)
 173             if 'Testscount' in new_line:
 174                 num_tests = new_line[2]
 175             elif 'Success' in new_line:
 176                 num_success = new_line[2]
 177             elif 'Skipped' in new_line:
 178                 num_skipped = new_line[2]
 179             elif 'Failures' in new_line:
 180                 num_failures = new_line[2]
 181
 182         try:
 183             num_executed = int(num_tests) - int(num_skipped)
 184             success_rate = 100 * int(num_success) / int(num_executed)
 185             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 186                                    "tempest.log"), 'r') as logfile:
 187                 output = logfile.read()
 188
 189             error_logs = ""
 190             for match in re.findall('(.*?)[. ]*fail ', output):
 191                 error_logs += match
 192             skipped_testcase = ""
 193             for match in re.findall('(.*?)[. ]*skip:', output):
 194                 skipped_testcase += match
 195
 196             self.details = {"tests": int(num_tests),
 197                             "failures": int(num_failures),
 198                             "errors": error_logs,
 199                             "skipped": skipped_testcase}
 200         except Exception:
 201             success_rate = 0
 202
 203         self.criteria = ft_utils.check_success_rate(
 204             self.case_name, success_rate)
 205         logger.info("Tempest %s success_rate is %s%%, is marked as %s"
 206                     % (self.case_name, success_rate, self.criteria))
 207
 208     def run(self):
 209
 210         self.start_time = time.time()
 211
 212         if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 213             os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 214
 215         try:
 216             image_and_flavor = conf_utils.create_tempest_resources()
 217             conf_utils.configure_tempest(
 218                 self.DEPLOYMENT_DIR,
 219                 IMAGE_ID=image_and_flavor.get("image_id"),
 220                 FLAVOR_ID=image_and_flavor.get("flavor_id"),
 221                 MODE=self.MODE)
 222             self.generate_test_list(self.VERIFIER_REPO_DIR)
 223             self.apply_tempest_blacklist()
 224             self.run_verifier_tests()
 225             self.parse_verifier_result()
 226             res = testcase.TestCase.EX_OK
 227         except Exception as e:
 228             logger.error('Error with run: %s' % e)
 229             res = testcase.TestCase.EX_RUN_ERROR
 230
 231         self.stop_time = time.time()
 232         return res
 233
 234
 235 class TempestSmokeSerial(TempestCommon):
 236
 237     def __init__(self, case_name='tempest_smoke_serial'):
 238         TempestCommon.__init__(self, case_name)
 239         self.MODE = "smoke"
 240         self.OPTION = "--concurrency 1"
 241
 242
 243 class TempestSmokeParallel(TempestCommon):
 244
 245     def __init__(self, case_name='tempest_smoke_parallel'):
 246         TempestCommon.__init__(self, case_name)
 247         self.MODE = "smoke"
 248         self.OPTION = ""
 249
 250
 251 class TempestFullParallel(TempestCommon):
 252
 253     def __init__(self, case_name="tempest_full_parallel"):
 254         TempestCommon.__init__(self, case_name)
 255         self.MODE = "full"
 256
 257
 258 class TempestMultisite(TempestCommon):
 259
 260     def __init__(self, case_name="multisite"):
 261         TempestCommon.__init__(self, case_name)
 262         self.MODE = "feature_multisite"
 263         self.OPTION = "--concurrency 1"
 264         conf_utils.install_verifier_ext(CONST.dir_repo_kingbird)
 265
 266
 267 class TempestCustom(TempestCommon):
 268
 269     def __init__(self, case_name="tempest_custom"):
 270         TempestCommon.__init__(self, case_name)
 271         self.MODE = "custom"
 272         self.OPTION = "--concurrency 1"
 273
 274
 275 class TempestDefcore(TempestCommon):
 276
 277     def __init__(self, case_name="tempest_defcore"):
 278         TempestCommon.__init__(self, case_name)
 279         self.MODE = "defcore"
 280         self.OPTION = "--concurrency 1"