functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.tempest import conf_utils
  24 from functest.utils.constants import CONST
  25 import functest.utils.functest_utils as ft_utils
  26
  27 """ logging configuration """
  28 logger = logging.getLogger(__name__)
  29
  30
  31 class TempestCommon(testcase.TestCase):
  32
  33     def __init__(self, **kwargs):
  34         super(TempestCommon, self).__init__(**kwargs)
  35         self.MODE = ""
  36         self.OPTION = ""
  37         self.VERIFIER_ID = conf_utils.get_verifier_id()
  38         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  39             self.VERIFIER_ID)
  40         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  41         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  42             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  43         self.VERIFICATION_ID = None
  44
  45     @staticmethod
  46     def read_file(filename):
  47         with open(filename) as src:
  48             return [line.strip() for line in src.readlines()]
  49
  50     def generate_test_list(self, verifier_repo_dir):
  51         logger.debug("Generating test case list...")
  52         if self.MODE == 'defcore':
  53             shutil.copyfile(
  54                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  55         elif self.MODE == 'custom':
  56             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  57                 shutil.copyfile(
  58                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  59             else:
  60                 raise Exception("Tempest test list file %s NOT found."
  61                                 % conf_utils.TEMPEST_CUSTOM)
  62         else:
  63             if self.MODE == 'smoke':
  64                 testr_mode = "smoke"
  65             elif self.MODE == 'feature_multisite':
  66                 testr_mode = "'[Kk]ingbird'"
  67             elif self.MODE == 'full':
  68                 testr_mode = ""
  69             else:
  70                 testr_mode = 'tempest.api.' + self.MODE
  71             cmd = ("cd {0};"
  72                    "testr list-tests {1} > {2};"
  73                    "cd -;".format(verifier_repo_dir,
  74                                   testr_mode,
  75                                   conf_utils.TEMPEST_RAW_LIST))
  76             ft_utils.execute_command(cmd)
  77
  78     def apply_tempest_blacklist(self):
  79         logger.debug("Applying tempest blacklist...")
  80         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  81         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  82         black_tests = []
  83         try:
  84             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  85             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  86             if (bool(installer_type) * bool(deploy_scenario)):
  87                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  88                 # file
  89                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  90                 black_list_yaml = yaml.safe_load(black_list_file)
  91                 black_list_file.close()
  92                 for item in black_list_yaml:
  93                     scenarios = item['scenarios']
  94                     installers = item['installers']
  95                     if (deploy_scenario in scenarios and
  96                             installer_type in installers):
  97                         tests = item['tests']
  98                         for test in tests:
  99                             black_tests.append(test)
 100                         break
 101         except Exception:
 102             black_tests = []
 103             logger.debug("Tempest blacklist file does not exist.")
 104
 105         for cases_line in cases_file:
 106             for black_tests_line in black_tests:
 107                 if black_tests_line in cases_line:
 108                     break
 109             else:
 110                 result_file.write(str(cases_line) + '\n')
 111         result_file.close()
 112
 113     def run_verifier_tests(self):
 114         self.OPTION += (" --load-list {} --detailed"
 115                         .format(conf_utils.TEMPEST_LIST))
 116
 117         cmd_line = "rally verify start " + self.OPTION
 118         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 119
 120         header = ("Tempest environment:\n"
 121                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 122                   (CONST.__getattribute__('INSTALLER_TYPE'),
 123                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 124                    CONST.__getattribute__('NODE_NAME'),
 125                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 126
 127         f_stdout = open(
 128             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 129         f_stderr = open(
 130             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 131                          "tempest-error.log"), 'w+')
 132         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 133                                   "environment.log"), 'w+')
 134         f_env.write(header)
 135
 136         p = subprocess.Popen(
 137             cmd_line, shell=True,
 138             stdout=subprocess.PIPE,
 139             stderr=f_stderr,
 140             bufsize=1)
 141
 142         with p.stdout:
 143             for line in iter(p.stdout.readline, b''):
 144                 if re.search("\} tempest\.", line):
 145                     logger.info(line.replace('\n', ''))
 146                 elif re.search('Starting verification', line):
 147                     logger.info(line.replace('\n', ''))
 148                     first_pos = line.index("UUID=") + len("UUID=")
 149                     last_pos = line.index(") for deployment")
 150                     self.VERIFICATION_ID = line[first_pos:last_pos]
 151                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 152                 f_stdout.write(line)
 153         p.wait()
 154
 155         f_stdout.close()
 156         f_stderr.close()
 157         f_env.close()
 158
 159     def parse_verifier_result(self):
 160         if self.VERIFICATION_ID is None:
 161             raise Exception('Verification UUID not found')
 162
 163         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 164         logger.info("Showing result for a verification: '%s'." % cmd_line)
 165         p = subprocess.Popen(cmd_line,
 166                              shell=True,
 167                              stdout=subprocess.PIPE,
 168                              stderr=subprocess.STDOUT)
 169         for line in p.stdout:
 170             new_line = line.replace(' ', '').split('|')
 171             if 'Tests' in new_line:
 172                 break
 173
 174             logger.info(line)
 175             if 'Testscount' in new_line:
 176                 num_tests = new_line[2]
 177             elif 'Success' in new_line:
 178                 num_success = new_line[2]
 179             elif 'Skipped' in new_line:
 180                 num_skipped = new_line[2]
 181             elif 'Failures' in new_line:
 182                 num_failures = new_line[2]
 183
 184         try:
 185             num_executed = int(num_tests) - int(num_skipped)
 186             try:
 187                 self.result = 100 * int(num_success) / int(num_executed)
 188             except ZeroDivisionError:
 189                 logger.error("No test has been executed")
 190                 self.result = 0
 191                 return
 192
 193             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 194                                    "tempest.log"), 'r') as logfile:
 195                 output = logfile.read()
 196
 197             error_logs = ""
 198             for match in re.findall('(.*?)[. ]*fail ', output):
 199                 error_logs += match
 200             skipped_testcase = ""
 201             for match in re.findall('(.*?)[. ]*skip:', output):
 202                 skipped_testcase += match
 203
 204             self.details = {"tests": int(num_tests),
 205                             "failures": int(num_failures),
 206                             "errors": error_logs,
 207                             "skipped": skipped_testcase}
 208         except Exception:
 209             self.result = 0
 210
 211         logger.info("Tempest %s success_rate is %s%%"
 212                     % (self.case_name, self.result))
 213
 214     def run(self):
 215
 216         self.start_time = time.time()
 217         try:
 218             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 219                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 220             image_and_flavor = conf_utils.create_tempest_resources()
 221             conf_utils.configure_tempest(
 222                 self.DEPLOYMENT_DIR,
 223                 IMAGE_ID=image_and_flavor.get("image_id"),
 224                 FLAVOR_ID=image_and_flavor.get("flavor_id"),
 225                 MODE=self.MODE)
 226             self.generate_test_list(self.VERIFIER_REPO_DIR)
 227             self.apply_tempest_blacklist()
 228             self.run_verifier_tests()
 229             self.parse_verifier_result()
 230             res = testcase.TestCase.EX_OK
 231         except Exception as e:
 232             logger.error('Error with run: %s' % e)
 233             res = testcase.TestCase.EX_RUN_ERROR
 234
 235         self.stop_time = time.time()
 236         return res
 237
 238
 239 class TempestSmokeSerial(TempestCommon):
 240
 241     def __init__(self, **kwargs):
 242         if "case_name" not in kwargs:
 243             kwargs["case_name"] = 'tempest_smoke_serial'
 244         TempestCommon.__init__(self, **kwargs)
 245         self.MODE = "smoke"
 246         self.OPTION = "--concurrency 1"
 247
 248
 249 class TempestSmokeParallel(TempestCommon):
 250
 251     def __init__(self, **kwargs):
 252         if "case_name" not in kwargs:
 253             kwargs["case_name"] = 'tempest_smoke_parallel'
 254         TempestCommon.__init__(self, **kwargs)
 255         self.MODE = "smoke"
 256         self.OPTION = ""
 257
 258
 259 class TempestFullParallel(TempestCommon):
 260
 261     def __init__(self, **kwargs):
 262         if "case_name" not in kwargs:
 263             kwargs["case_name"] = 'tempest_full_parallel'
 264         TempestCommon.__init__(self, **kwargs)
 265         self.MODE = "full"
 266
 267
 268 class TempestMultisite(TempestCommon):
 269
 270     def __init__(self, **kwargs):
 271         if "case_name" not in kwargs:
 272             kwargs["case_name"] = 'multisite'
 273         TempestCommon.__init__(self, **kwargs)
 274         self.MODE = "feature_multisite"
 275         self.OPTION = "--concurrency 1"
 276         conf_utils.install_verifier_ext(
 277             CONST.__getattribute__('dir_repo_kingbird'))
 278
 279
 280 class TempestCustom(TempestCommon):
 281
 282     def __init__(self, **kwargs):
 283         if "case_name" not in kwargs:
 284             kwargs["case_name"] = 'tempest_custom'
 285         TempestCommon.__init__(self, **kwargs)
 286         self.MODE = "custom"
 287         self.OPTION = "--concurrency 1"
 288
 289
 290 class TempestDefcore(TempestCommon):
 291
 292     def __init__(self, **kwargs):
 293         if "case_name" not in kwargs:
 294             kwargs["case_name"] = 'tempest_defcore'
 295         TempestCommon.__init__(self, **kwargs)
 296         self.MODE = "defcore"
 297         self.OPTION = "--concurrency 1"