3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 from __future__ import division
22 from functest.core import testcase
23 from functest.opnfv_tests.openstack.tempest import conf_utils
24 from functest.utils.constants import CONST
25 import functest.utils.functest_utils as ft_utils
27 """ logging configuration """
28 logger = logging.getLogger(__name__)
31 class TempestCommon(testcase.TestCase):
33 def __init__(self, **kwargs):
34 super(TempestCommon, self).__init__(**kwargs)
37 self.VERIFIER_ID = conf_utils.get_verifier_id()
38 self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
40 self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
41 self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
42 self.VERIFIER_ID, self.DEPLOYMENT_ID)
43 self.VERIFICATION_ID = None
46 def read_file(filename):
47 with open(filename) as src:
48 return [line.strip() for line in src.readlines()]
50 def generate_test_list(self, verifier_repo_dir):
51 logger.debug("Generating test case list...")
52 if self.MODE == 'defcore':
54 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
55 elif self.MODE == 'custom':
56 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
58 conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
60 raise Exception("Tempest test list file %s NOT found."
61 % conf_utils.TEMPEST_CUSTOM)
63 if self.MODE == 'smoke':
65 elif self.MODE == 'feature_multisite':
66 testr_mode = "'[Kk]ingbird'"
67 elif self.MODE == 'full':
70 testr_mode = 'tempest.api.' + self.MODE
72 "testr list-tests {1} > {2};"
73 "cd -;".format(verifier_repo_dir,
75 conf_utils.TEMPEST_RAW_LIST))
76 ft_utils.execute_command(cmd)
78 def apply_tempest_blacklist(self):
79 logger.debug("Applying tempest blacklist...")
80 cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
81 result_file = open(conf_utils.TEMPEST_LIST, 'w')
84 installer_type = CONST.__getattribute__('INSTALLER_TYPE')
85 deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
86 if (bool(installer_type) * bool(deploy_scenario)):
87 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
89 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
90 black_list_yaml = yaml.safe_load(black_list_file)
91 black_list_file.close()
92 for item in black_list_yaml:
93 scenarios = item['scenarios']
94 installers = item['installers']
95 if (deploy_scenario in scenarios and
96 installer_type in installers):
99 black_tests.append(test)
103 logger.debug("Tempest blacklist file does not exist.")
105 for cases_line in cases_file:
106 for black_tests_line in black_tests:
107 if black_tests_line in cases_line:
110 result_file.write(str(cases_line) + '\n')
113 def run_verifier_tests(self):
114 self.OPTION += (" --load-list {} --detailed"
115 .format(conf_utils.TEMPEST_LIST))
117 cmd_line = "rally verify start " + self.OPTION
118 logger.info("Starting Tempest test suite: '%s'." % cmd_line)
120 header = ("Tempest environment:\n"
121 " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
122 (CONST.__getattribute__('INSTALLER_TYPE'),
123 CONST.__getattribute__('DEPLOY_SCENARIO'),
124 CONST.__getattribute__('NODE_NAME'),
125 time.strftime("%a %b %d %H:%M:%S %Z %Y")))
128 os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
130 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
131 "tempest-error.log"), 'w+')
132 f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
133 "environment.log"), 'w+')
136 p = subprocess.Popen(
137 cmd_line, shell=True,
138 stdout=subprocess.PIPE,
143 for line in iter(p.stdout.readline, b''):
144 if re.search("\} tempest\.", line):
145 logger.info(line.replace('\n', ''))
146 elif re.search('Starting verification', line):
147 logger.info(line.replace('\n', ''))
148 first_pos = line.index("UUID=") + len("UUID=")
149 last_pos = line.index(") for deployment")
150 self.VERIFICATION_ID = line[first_pos:last_pos]
151 logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
159 def parse_verifier_result(self):
160 if self.VERIFICATION_ID is None:
161 raise Exception('Verification UUID not found')
163 cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
164 logger.info("Showing result for a verification: '%s'." % cmd_line)
165 p = subprocess.Popen(cmd_line,
167 stdout=subprocess.PIPE,
168 stderr=subprocess.STDOUT)
169 for line in p.stdout:
170 new_line = line.replace(' ', '').split('|')
171 if 'Tests' in new_line:
175 if 'Testscount' in new_line:
176 num_tests = new_line[2]
177 elif 'Success' in new_line:
178 num_success = new_line[2]
179 elif 'Skipped' in new_line:
180 num_skipped = new_line[2]
181 elif 'Failures' in new_line:
182 num_failures = new_line[2]
185 num_executed = int(num_tests) - int(num_skipped)
187 self.result = 100 * int(num_success) / int(num_executed)
188 except ZeroDivisionError:
189 logger.error("No test has been executed")
193 with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
194 "tempest.log"), 'r') as logfile:
195 output = logfile.read()
198 for match in re.findall('(.*?)[. ]*fail ', output):
200 skipped_testcase = ""
201 for match in re.findall('(.*?)[. ]*skip:', output):
202 skipped_testcase += match
204 self.details = {"tests": int(num_tests),
205 "failures": int(num_failures),
206 "errors": error_logs,
207 "skipped": skipped_testcase}
211 logger.info("Tempest %s success_rate is %s%%"
212 % (self.case_name, self.result))
216 self.start_time = time.time()
218 if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
219 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
220 image_and_flavor = conf_utils.create_tempest_resources()
221 conf_utils.configure_tempest(
223 IMAGE_ID=image_and_flavor.get("image_id"),
224 FLAVOR_ID=image_and_flavor.get("flavor_id"),
226 self.generate_test_list(self.VERIFIER_REPO_DIR)
227 self.apply_tempest_blacklist()
228 self.run_verifier_tests()
229 self.parse_verifier_result()
230 res = testcase.TestCase.EX_OK
231 except Exception as e:
232 logger.error('Error with run: %s' % e)
233 res = testcase.TestCase.EX_RUN_ERROR
235 self.stop_time = time.time()
239 class TempestSmokeSerial(TempestCommon):
241 def __init__(self, **kwargs):
242 if "case_name" not in kwargs:
243 kwargs["case_name"] = 'tempest_smoke_serial'
244 TempestCommon.__init__(self, **kwargs)
246 self.OPTION = "--concurrency 1"
249 class TempestSmokeParallel(TempestCommon):
251 def __init__(self, **kwargs):
252 if "case_name" not in kwargs:
253 kwargs["case_name"] = 'tempest_smoke_parallel'
254 TempestCommon.__init__(self, **kwargs)
259 class TempestFullParallel(TempestCommon):
261 def __init__(self, **kwargs):
262 if "case_name" not in kwargs:
263 kwargs["case_name"] = 'tempest_full_parallel'
264 TempestCommon.__init__(self, **kwargs)
268 class TempestMultisite(TempestCommon):
270 def __init__(self, **kwargs):
271 if "case_name" not in kwargs:
272 kwargs["case_name"] = 'multisite'
273 TempestCommon.__init__(self, **kwargs)
274 self.MODE = "feature_multisite"
275 self.OPTION = "--concurrency 1"
276 conf_utils.install_verifier_ext(
277 CONST.__getattribute__('dir_repo_kingbird'))
280 class TempestCustom(TempestCommon):
282 def __init__(self, **kwargs):
283 if "case_name" not in kwargs:
284 kwargs["case_name"] = 'tempest_custom'
285 TempestCommon.__init__(self, **kwargs)
287 self.OPTION = "--concurrency 1"
290 class TempestDefcore(TempestCommon):
292 def __init__(self, **kwargs):
293 if "case_name" not in kwargs:
294 kwargs["case_name"] = 'tempest_defcore'
295 TempestCommon.__init__(self, **kwargs)
296 self.MODE = "defcore"
297 self.OPTION = "--concurrency 1"