3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 from __future__ import division
23 from functest.core import testcase
24 from functest.opnfv_tests.openstack.tempest import conf_utils
25 from functest.utils.constants import CONST
26 import functest.utils.functest_utils as ft_utils
28 """ logging configuration """
29 logger = logging.getLogger(__name__)
32 class TempestCommon(testcase.OSGCTestCase):
34 def __init__(self, **kwargs):
35 super(TempestCommon, self).__init__(**kwargs)
38 self.VERIFIER_ID = conf_utils.get_verifier_id()
39 self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
41 self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
42 self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
43 self.VERIFIER_ID, self.DEPLOYMENT_ID)
44 self.VERIFICATION_ID = None
47 def read_file(filename):
48 with open(filename) as src:
49 return [line.strip() for line in src.readlines()]
51 def generate_test_list(self, verifier_repo_dir):
52 logger.debug("Generating test case list...")
53 if self.MODE == 'defcore':
55 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
56 elif self.MODE == 'custom':
57 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
59 conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
61 raise Exception("Tempest test list file %s NOT found."
62 % conf_utils.TEMPEST_CUSTOM)
64 if self.MODE == 'smoke':
66 elif self.MODE == 'feature_multisite':
67 testr_mode = "'[Kk]ingbird'"
68 elif self.MODE == 'full':
71 testr_mode = 'tempest.api.' + self.MODE
73 "testr list-tests {1} > {2};"
74 "cd -;".format(verifier_repo_dir,
76 conf_utils.TEMPEST_RAW_LIST))
77 ft_utils.execute_command(cmd)
79 def apply_tempest_blacklist(self):
80 logger.debug("Applying tempest blacklist...")
81 cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
82 result_file = open(conf_utils.TEMPEST_LIST, 'w')
85 installer_type = CONST.__getattribute__('INSTALLER_TYPE')
86 deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
87 if (bool(installer_type) * bool(deploy_scenario)):
88 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
90 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
91 black_list_yaml = yaml.safe_load(black_list_file)
92 black_list_file.close()
93 for item in black_list_yaml:
94 scenarios = item['scenarios']
95 installers = item['installers']
96 if (deploy_scenario in scenarios and
97 installer_type in installers):
100 black_tests.append(test)
104 logger.debug("Tempest blacklist file does not exist.")
106 for cases_line in cases_file:
107 for black_tests_line in black_tests:
108 if black_tests_line in cases_line:
111 result_file.write(str(cases_line) + '\n')
114 def run_verifier_tests(self):
115 self.OPTION += (" --load-list {} --detailed"
116 .format(conf_utils.TEMPEST_LIST))
118 cmd_line = "rally verify start " + self.OPTION
119 logger.info("Starting Tempest test suite: '%s'." % cmd_line)
121 header = ("Tempest environment:\n"
122 " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
123 (CONST.__getattribute__('INSTALLER_TYPE'),
124 CONST.__getattribute__('DEPLOY_SCENARIO'),
125 CONST.__getattribute__('NODE_NAME'),
126 time.strftime("%a %b %d %H:%M:%S %Z %Y")))
129 os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
131 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
132 "tempest-error.log"), 'w+')
133 f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
134 "environment.log"), 'w+')
137 p = subprocess.Popen(
138 cmd_line, shell=True,
139 stdout=subprocess.PIPE,
144 for line in iter(p.stdout.readline, b''):
145 if re.search("\} tempest\.", line):
146 logger.info(line.replace('\n', ''))
147 elif re.search('Starting verification', line):
148 logger.info(line.replace('\n', ''))
149 first_pos = line.index("UUID=") + len("UUID=")
150 last_pos = line.index(") for deployment")
151 self.VERIFICATION_ID = line[first_pos:last_pos]
152 logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
160 def parse_verifier_result(self):
161 if self.VERIFICATION_ID is None:
162 raise Exception('Verification UUID not found')
164 cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
165 logger.info("Showing result for a verification: '%s'." % cmd_line)
166 p = subprocess.Popen(cmd_line,
168 stdout=subprocess.PIPE,
169 stderr=subprocess.STDOUT)
170 for line in p.stdout:
171 new_line = line.replace(' ', '').split('|')
172 if 'Tests' in new_line:
176 if 'Testscount' in new_line:
177 num_tests = new_line[2]
178 elif 'Success' in new_line:
179 num_success = new_line[2]
180 elif 'Skipped' in new_line:
181 num_skipped = new_line[2]
182 elif 'Failures' in new_line:
183 num_failures = new_line[2]
186 num_executed = int(num_tests) - int(num_skipped)
188 self.result = 100 * int(num_success) / int(num_executed)
189 except ZeroDivisionError:
190 logger.error("No test has been executed")
194 with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
195 "tempest.log"), 'r') as logfile:
196 output = logfile.read()
199 for match in re.findall('(.*?)[. ]*fail ', output):
201 skipped_testcase = ""
202 for match in re.findall('(.*?)[. ]*skip:', output):
203 skipped_testcase += match
205 self.details = {"tests": int(num_tests),
206 "failures": int(num_failures),
207 "errors": error_logs,
208 "skipped": skipped_testcase}
212 logger.info("Tempest %s success_rate is %s%%"
213 % (self.case_name, self.result))
217 self.start_time = time.time()
219 if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
220 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
221 image_and_flavor = conf_utils.create_tempest_resources()
222 conf_utils.configure_tempest(
224 IMAGE_ID=image_and_flavor.get("image_id"),
225 FLAVOR_ID=image_and_flavor.get("flavor_id"),
227 self.generate_test_list(self.VERIFIER_REPO_DIR)
228 self.apply_tempest_blacklist()
229 self.run_verifier_tests()
230 self.parse_verifier_result()
231 res = testcase.TestCase.EX_OK
232 except Exception as e:
233 logger.error('Error with run: %s' % e)
234 res = testcase.TestCase.EX_RUN_ERROR
236 self.stop_time = time.time()
240 class TempestSmokeSerial(TempestCommon):
242 def __init__(self, **kwargs):
243 if "case_name" not in kwargs:
244 kwargs["case_name"] = 'tempest_smoke_serial'
245 TempestCommon.__init__(self, **kwargs)
247 self.OPTION = "--concurrency 1"
250 class TempestSmokeParallel(TempestCommon):
252 def __init__(self, **kwargs):
253 if "case_name" not in kwargs:
254 kwargs["case_name"] = 'tempest_smoke_parallel'
255 TempestCommon.__init__(self, **kwargs)
260 class TempestFullParallel(TempestCommon):
262 def __init__(self, **kwargs):
263 if "case_name" not in kwargs:
264 kwargs["case_name"] = 'tempest_full_parallel'
265 TempestCommon.__init__(self, **kwargs)
269 class TempestMultisite(TempestCommon):
271 def __init__(self, **kwargs):
272 if "case_name" not in kwargs:
273 kwargs["case_name"] = 'multisite'
274 TempestCommon.__init__(self, **kwargs)
275 self.MODE = "feature_multisite"
276 self.OPTION = "--concurrency 1"
277 conf_utils.install_verifier_ext(
278 pkg_resources.resource_filename('kingbird', '..'))
281 class TempestCustom(TempestCommon):
283 def __init__(self, **kwargs):
284 if "case_name" not in kwargs:
285 kwargs["case_name"] = 'tempest_custom'
286 TempestCommon.__init__(self, **kwargs)
288 self.OPTION = "--concurrency 1"
291 class TempestDefcore(TempestCommon):
293 def __init__(self, **kwargs):
294 if "case_name" not in kwargs:
295 kwargs["case_name"] = 'tempest_defcore'
296 TempestCommon.__init__(self, **kwargs)
297 self.MODE = "defcore"
298 self.OPTION = "--concurrency 1"