functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.tempest import conf_utils
  24 from functest.utils.constants import CONST
  25 import functest.utils.functest_utils as ft_utils
  26
  27 """ logging configuration """
  28 logger = logging.getLogger(__name__)
  29
  30
  31 class TempestCommon(testcase.OSGCTestCase):
  32
  33     def __init__(self, **kwargs):
  34         super(TempestCommon, self).__init__(**kwargs)
  35         self.MODE = ""
  36         self.OPTION = ""
  37         self.VERIFIER_ID = conf_utils.get_verifier_id()
  38         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  39             self.VERIFIER_ID)
  40         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  41         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  42             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  43         self.VERIFICATION_ID = None
  44
  45     @staticmethod
  46     def read_file(filename):
  47         with open(filename) as src:
  48             return [line.strip() for line in src.readlines()]
  49
  50     def generate_test_list(self, verifier_repo_dir):
  51         logger.debug("Generating test case list...")
  52         if self.MODE == 'defcore':
  53             shutil.copyfile(
  54                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  55         elif self.MODE == 'custom':
  56             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  57                 shutil.copyfile(
  58                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  59             else:
  60                 raise Exception("Tempest test list file %s NOT found."
  61                                 % conf_utils.TEMPEST_CUSTOM)
  62         else:
  63             if self.MODE == 'smoke':
  64                 testr_mode = "smoke"
  65             elif self.MODE == 'full':
  66                 testr_mode = ""
  67             else:
  68                 testr_mode = 'tempest.api.' + self.MODE
  69             cmd = ("cd {0};"
  70                    "testr list-tests {1} > {2};"
  71                    "cd -;".format(verifier_repo_dir,
  72                                   testr_mode,
  73                                   conf_utils.TEMPEST_RAW_LIST))
  74             ft_utils.execute_command(cmd)
  75
  76     def apply_tempest_blacklist(self):
  77         logger.debug("Applying tempest blacklist...")
  78         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  79         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  80         black_tests = []
  81         try:
  82             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  83             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  84             if (bool(installer_type) * bool(deploy_scenario)):
  85                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  86                 # file
  87                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  88                 black_list_yaml = yaml.safe_load(black_list_file)
  89                 black_list_file.close()
  90                 for item in black_list_yaml:
  91                     scenarios = item['scenarios']
  92                     installers = item['installers']
  93                     if (deploy_scenario in scenarios and
  94                             installer_type in installers):
  95                         tests = item['tests']
  96                         for test in tests:
  97                             black_tests.append(test)
  98                         break
  99         except Exception:
 100             black_tests = []
 101             logger.debug("Tempest blacklist file does not exist.")
 102
 103         for cases_line in cases_file:
 104             for black_tests_line in black_tests:
 105                 if black_tests_line in cases_line:
 106                     break
 107             else:
 108                 result_file.write(str(cases_line) + '\n')
 109         result_file.close()
 110
 111     def run_verifier_tests(self):
 112         self.OPTION += (" --load-list {} --detailed"
 113                         .format(conf_utils.TEMPEST_LIST))
 114
 115         cmd_line = "rally verify start " + self.OPTION
 116         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 117
 118         header = ("Tempest environment:\n"
 119                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 120                   (CONST.__getattribute__('INSTALLER_TYPE'),
 121                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 122                    CONST.__getattribute__('NODE_NAME'),
 123                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 124
 125         f_stdout = open(
 126             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 127         f_stderr = open(
 128             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 129                          "tempest-error.log"), 'w+')
 130         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 131                                   "environment.log"), 'w+')
 132         f_env.write(header)
 133
 134         p = subprocess.Popen(
 135             cmd_line, shell=True,
 136             stdout=subprocess.PIPE,
 137             stderr=f_stderr,
 138             bufsize=1)
 139
 140         with p.stdout:
 141             for line in iter(p.stdout.readline, b''):
 142                 if re.search("\} tempest\.", line):
 143                     logger.info(line.replace('\n', ''))
 144                 elif re.search('Starting verification', line):
 145                     logger.info(line.replace('\n', ''))
 146                     first_pos = line.index("UUID=") + len("UUID=")
 147                     last_pos = line.index(") for deployment")
 148                     self.VERIFICATION_ID = line[first_pos:last_pos]
 149                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 150                 f_stdout.write(line)
 151         p.wait()
 152
 153         f_stdout.close()
 154         f_stderr.close()
 155         f_env.close()
 156
 157     def parse_verifier_result(self):
 158         if self.VERIFICATION_ID is None:
 159             raise Exception('Verification UUID not found')
 160
 161         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 162         logger.info("Showing result for a verification: '%s'." % cmd_line)
 163         p = subprocess.Popen(cmd_line,
 164                              shell=True,
 165                              stdout=subprocess.PIPE,
 166                              stderr=subprocess.STDOUT)
 167         for line in p.stdout:
 168             new_line = line.replace(' ', '').split('|')
 169             if 'Tests' in new_line:
 170                 break
 171
 172             logger.info(line)
 173             if 'Testscount' in new_line:
 174                 num_tests = new_line[2]
 175             elif 'Success' in new_line:
 176                 num_success = new_line[2]
 177             elif 'Skipped' in new_line:
 178                 num_skipped = new_line[2]
 179             elif 'Failures' in new_line:
 180                 num_failures = new_line[2]
 181
 182         try:
 183             num_executed = int(num_tests) - int(num_skipped)
 184             try:
 185                 self.result = 100 * int(num_success) / int(num_executed)
 186             except ZeroDivisionError:
 187                 logger.error("No test has been executed")
 188                 self.result = 0
 189                 return
 190
 191             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 192                                    "tempest.log"), 'r') as logfile:
 193                 output = logfile.read()
 194
 195             success_testcases = []
 196             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 197                 success_testcases.append(match)
 198             failed_testcases = []
 199             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 200                 failed_testcases.append(match)
 201             skipped_testcases = []
 202             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 203                 skipped_testcases.append(match)
 204
 205             self.details = {"tests": int(num_tests),
 206                             "failures": int(num_failures),
 207                             "success": success_testcases,
 208                             "errors": failed_testcases,
 209                             "skipped": skipped_testcases}
 210         except Exception:
 211             self.result = 0
 212
 213         logger.info("Tempest %s success_rate is %s%%"
 214                     % (self.case_name, self.result))
 215
 216     def run(self):
 217
 218         self.start_time = time.time()
 219         try:
 220             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 221                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 222             image_and_flavor = conf_utils.create_tempest_resources()
 223             conf_utils.configure_tempest(
 224                 self.DEPLOYMENT_DIR,
 225                 IMAGE_ID=image_and_flavor.get("image_id"),
 226                 FLAVOR_ID=image_and_flavor.get("flavor_id"),
 227                 MODE=self.MODE)
 228             self.generate_test_list(self.VERIFIER_REPO_DIR)
 229             self.apply_tempest_blacklist()
 230             self.run_verifier_tests()
 231             self.parse_verifier_result()
 232             res = testcase.TestCase.EX_OK
 233         except Exception as e:
 234             logger.error('Error with run: %s' % e)
 235             res = testcase.TestCase.EX_RUN_ERROR
 236
 237         self.stop_time = time.time()
 238         return res
 239
 240
 241 class TempestSmokeSerial(TempestCommon):
 242
 243     def __init__(self, **kwargs):
 244         if "case_name" not in kwargs:
 245             kwargs["case_name"] = 'tempest_smoke_serial'
 246         TempestCommon.__init__(self, **kwargs)
 247         self.MODE = "smoke"
 248         self.OPTION = "--concurrency 1"
 249
 250
 251 class TempestSmokeParallel(TempestCommon):
 252
 253     def __init__(self, **kwargs):
 254         if "case_name" not in kwargs:
 255             kwargs["case_name"] = 'tempest_smoke_parallel'
 256         TempestCommon.__init__(self, **kwargs)
 257         self.MODE = "smoke"
 258         self.OPTION = ""
 259
 260
 261 class TempestFullParallel(TempestCommon):
 262
 263     def __init__(self, **kwargs):
 264         if "case_name" not in kwargs:
 265             kwargs["case_name"] = 'tempest_full_parallel'
 266         TempestCommon.__init__(self, **kwargs)
 267         self.MODE = "full"
 268
 269
 270 class TempestCustom(TempestCommon):
 271
 272     def __init__(self, **kwargs):
 273         if "case_name" not in kwargs:
 274             kwargs["case_name"] = 'tempest_custom'
 275         TempestCommon.__init__(self, **kwargs)
 276         self.MODE = "custom"
 277         self.OPTION = "--concurrency 1"
 278
 279
 280 class TempestDefcore(TempestCommon):
 281
 282     def __init__(self, **kwargs):
 283         if "case_name" not in kwargs:
 284             kwargs["case_name"] = 'tempest_defcore'
 285         TempestCommon.__init__(self, **kwargs)
 286         self.MODE = "defcore"
 287         self.OPTION = "--concurrency 1"