functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.tempest import conf_utils
  24 from functest.utils.constants import CONST
  25 import functest.utils.functest_utils as ft_utils
  26
  27 """ logging configuration """
  28 logger = logging.getLogger(__name__)
  29
  30
  31 class TempestCommon(testcase.OSGCTestCase):
  32
  33     def __init__(self, **kwargs):
  34         super(TempestCommon, self).__init__(**kwargs)
  35         self.MODE = ""
  36         self.OPTION = ""
  37         self.VERIFIER_ID = conf_utils.get_verifier_id()
  38         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  39             self.VERIFIER_ID)
  40         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  41         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  42             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  43         self.VERIFICATION_ID = None
  44
  45     @staticmethod
  46     def read_file(filename):
  47         with open(filename) as src:
  48             return [line.strip() for line in src.readlines()]
  49
  50     def generate_test_list(self, verifier_repo_dir):
  51         logger.debug("Generating test case list...")
  52         if self.MODE == 'defcore':
  53             shutil.copyfile(
  54                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  55         elif self.MODE == 'custom':
  56             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  57                 shutil.copyfile(
  58                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  59             else:
  60                 raise Exception("Tempest test list file %s NOT found."
  61                                 % conf_utils.TEMPEST_CUSTOM)
  62         else:
  63             if self.MODE == 'smoke':
  64                 testr_mode = "smoke"
  65             elif self.MODE == 'full':
  66                 testr_mode = ""
  67             else:
  68                 testr_mode = 'tempest.api.' + self.MODE
  69             cmd = ("cd {0};"
  70                    "testr list-tests {1} > {2};"
  71                    "cd -;".format(verifier_repo_dir,
  72                                   testr_mode,
  73                                   conf_utils.TEMPEST_RAW_LIST))
  74             ft_utils.execute_command(cmd)
  75
  76     def apply_tempest_blacklist(self):
  77         logger.debug("Applying tempest blacklist...")
  78         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  79         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  80         black_tests = []
  81         try:
  82             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  83             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  84             if (bool(installer_type) * bool(deploy_scenario)):
  85                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  86                 # file
  87                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  88                 black_list_yaml = yaml.safe_load(black_list_file)
  89                 black_list_file.close()
  90                 for item in black_list_yaml:
  91                     scenarios = item['scenarios']
  92                     installers = item['installers']
  93                     if (deploy_scenario in scenarios and
  94                             installer_type in installers):
  95                         tests = item['tests']
  96                         for test in tests:
  97                             black_tests.append(test)
  98                         break
  99         except Exception:
 100             black_tests = []
 101             logger.debug("Tempest blacklist file does not exist.")
 102
 103         for cases_line in cases_file:
 104             for black_tests_line in black_tests:
 105                 if black_tests_line in cases_line:
 106                     break
 107             else:
 108                 result_file.write(str(cases_line) + '\n')
 109         result_file.close()
 110
 111     def run_verifier_tests(self):
 112         self.OPTION += (" --load-list {} --detailed"
 113                         .format(conf_utils.TEMPEST_LIST))
 114
 115         cmd_line = "rally verify start " + self.OPTION
 116         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 117
 118         header = ("Tempest environment:\n"
 119                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 120                   (CONST.__getattribute__('INSTALLER_TYPE'),
 121                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 122                    CONST.__getattribute__('NODE_NAME'),
 123                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 124
 125         f_stdout = open(
 126             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 127         f_stderr = open(
 128             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 129                          "tempest-error.log"), 'w+')
 130         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 131                                   "environment.log"), 'w+')
 132         f_env.write(header)
 133
 134         p = subprocess.Popen(
 135             cmd_line, shell=True,
 136             stdout=subprocess.PIPE,
 137             stderr=f_stderr,
 138             bufsize=1)
 139
 140         with p.stdout:
 141             for line in iter(p.stdout.readline, b''):
 142                 if re.search("\} tempest\.", line):
 143                     logger.info(line.replace('\n', ''))
 144                 elif re.search('Starting verification', line):
 145                     logger.info(line.replace('\n', ''))
 146                     first_pos = line.index("UUID=") + len("UUID=")
 147                     last_pos = line.index(") for deployment")
 148                     self.VERIFICATION_ID = line[first_pos:last_pos]
 149                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 150                 f_stdout.write(line)
 151         p.wait()
 152
 153         f_stdout.close()
 154         f_stderr.close()
 155         f_env.close()
 156
 157     def parse_verifier_result(self):
 158         if self.VERIFICATION_ID is None:
 159             raise Exception('Verification UUID not found')
 160
 161         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 162         logger.info("Showing result for a verification: '%s'." % cmd_line)
 163         p = subprocess.Popen(cmd_line,
 164                              shell=True,
 165                              stdout=subprocess.PIPE,
 166                              stderr=subprocess.STDOUT)
 167         for line in p.stdout:
 168             new_line = line.replace(' ', '').split('|')
 169             if 'Tests' in new_line:
 170                 break
 171
 172             logger.info(line)
 173             if 'Testscount' in new_line:
 174                 num_tests = new_line[2]
 175             elif 'Success' in new_line:
 176                 num_success = new_line[2]
 177             elif 'Skipped' in new_line:
 178                 num_skipped = new_line[2]
 179             elif 'Failures' in new_line:
 180                 num_failures = new_line[2]
 181
 182         try:
 183             num_executed = int(num_tests) - int(num_skipped)
 184             try:
 185                 self.result = 100 * int(num_success) / int(num_executed)
 186             except ZeroDivisionError:
 187                 self.result = 0
 188                 if int(num_tests) > 0:
 189                     logger.info("All tests have been skipped")
 190                 else:
 191                     logger.error("No test has been executed")
 192                     return
 193
 194             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 195                                    "tempest.log"), 'r') as logfile:
 196                 output = logfile.read()
 197
 198             success_testcases = []
 199             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 200                 success_testcases.append(match)
 201             failed_testcases = []
 202             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 203                 failed_testcases.append(match)
 204             skipped_testcases = []
 205             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 206                 skipped_testcases.append(match)
 207
 208             self.details = {"tests": int(num_tests),
 209                             "failures": int(num_failures),
 210                             "success": success_testcases,
 211                             "errors": failed_testcases,
 212                             "skipped": skipped_testcases}
 213         except Exception:
 214             self.result = 0
 215
 216         logger.info("Tempest %s success_rate is %s%%"
 217                     % (self.case_name, self.result))
 218
 219     def run(self):
 220
 221         self.start_time = time.time()
 222         try:
 223             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 224                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 225             image_and_flavor = conf_utils.create_tempest_resources()
 226             conf_utils.configure_tempest(
 227                 self.DEPLOYMENT_DIR,
 228                 IMAGE_ID=image_and_flavor.get("image_id"),
 229                 FLAVOR_ID=image_and_flavor.get("flavor_id"),
 230                 MODE=self.MODE)
 231             self.generate_test_list(self.VERIFIER_REPO_DIR)
 232             self.apply_tempest_blacklist()
 233             self.run_verifier_tests()
 234             self.parse_verifier_result()
 235             res = testcase.TestCase.EX_OK
 236         except Exception as e:
 237             logger.error('Error with run: %s' % e)
 238             res = testcase.TestCase.EX_RUN_ERROR
 239
 240         self.stop_time = time.time()
 241         return res
 242
 243
 244 class TempestSmokeSerial(TempestCommon):
 245
 246     def __init__(self, **kwargs):
 247         if "case_name" not in kwargs:
 248             kwargs["case_name"] = 'tempest_smoke_serial'
 249         TempestCommon.__init__(self, **kwargs)
 250         self.MODE = "smoke"
 251         self.OPTION = "--concurrency 1"
 252
 253
 254 class TempestSmokeParallel(TempestCommon):
 255
 256     def __init__(self, **kwargs):
 257         if "case_name" not in kwargs:
 258             kwargs["case_name"] = 'tempest_smoke_parallel'
 259         TempestCommon.__init__(self, **kwargs)
 260         self.MODE = "smoke"
 261         self.OPTION = ""
 262
 263
 264 class TempestFullParallel(TempestCommon):
 265
 266     def __init__(self, **kwargs):
 267         if "case_name" not in kwargs:
 268             kwargs["case_name"] = 'tempest_full_parallel'
 269         TempestCommon.__init__(self, **kwargs)
 270         self.MODE = "full"
 271
 272
 273 class TempestCustom(TempestCommon):
 274
 275     def __init__(self, **kwargs):
 276         if "case_name" not in kwargs:
 277             kwargs["case_name"] = 'tempest_custom'
 278         TempestCommon.__init__(self, **kwargs)
 279         self.MODE = "custom"
 280         self.OPTION = "--concurrency 1"
 281
 282
 283 class TempestDefcore(TempestCommon):
 284
 285     def __init__(self, **kwargs):
 286         if "case_name" not in kwargs:
 287             kwargs["case_name"] = 'tempest_defcore'
 288         TempestCommon.__init__(self, **kwargs)
 289         self.MODE = "defcore"
 290         self.OPTION = "--concurrency 1"