3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 from __future__ import division
22 from functest.core import testcase
23 from functest.opnfv_tests.openstack.tempest import conf_utils
24 from functest.utils.constants import CONST
25 import functest.utils.functest_utils as ft_utils
27 """ logging configuration """
28 logger = logging.getLogger(__name__)
31 class TempestCommon(testcase.OSGCTestCase):
33 def __init__(self, **kwargs):
34 super(TempestCommon, self).__init__(**kwargs)
37 self.VERIFIER_ID = conf_utils.get_verifier_id()
38 self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
40 self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
41 self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
42 self.VERIFIER_ID, self.DEPLOYMENT_ID)
43 self.VERIFICATION_ID = None
46 def read_file(filename):
47 with open(filename) as src:
48 return [line.strip() for line in src.readlines()]
50 def generate_test_list(self, verifier_repo_dir):
51 logger.debug("Generating test case list...")
52 if self.MODE == 'defcore':
54 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
55 elif self.MODE == 'custom':
56 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
58 conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
60 raise Exception("Tempest test list file %s NOT found."
61 % conf_utils.TEMPEST_CUSTOM)
63 if self.MODE == 'smoke':
65 elif self.MODE == 'full':
68 testr_mode = 'tempest.api.' + self.MODE
70 "testr list-tests {1} > {2};"
71 "cd -;".format(verifier_repo_dir,
73 conf_utils.TEMPEST_RAW_LIST))
74 ft_utils.execute_command(cmd)
76 def apply_tempest_blacklist(self):
77 logger.debug("Applying tempest blacklist...")
78 cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
79 result_file = open(conf_utils.TEMPEST_LIST, 'w')
82 installer_type = CONST.__getattribute__('INSTALLER_TYPE')
83 deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
84 if (bool(installer_type) * bool(deploy_scenario)):
85 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
87 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
88 black_list_yaml = yaml.safe_load(black_list_file)
89 black_list_file.close()
90 for item in black_list_yaml:
91 scenarios = item['scenarios']
92 installers = item['installers']
93 if (deploy_scenario in scenarios and
94 installer_type in installers):
97 black_tests.append(test)
101 logger.debug("Tempest blacklist file does not exist.")
103 for cases_line in cases_file:
104 for black_tests_line in black_tests:
105 if black_tests_line in cases_line:
108 result_file.write(str(cases_line) + '\n')
111 def run_verifier_tests(self):
112 self.OPTION += (" --load-list {} --detailed"
113 .format(conf_utils.TEMPEST_LIST))
115 cmd_line = "rally verify start " + self.OPTION
116 logger.info("Starting Tempest test suite: '%s'." % cmd_line)
118 header = ("Tempest environment:\n"
119 " SUT: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
120 (CONST.__getattribute__('INSTALLER_TYPE'),
121 CONST.__getattribute__('DEPLOY_SCENARIO'),
122 CONST.__getattribute__('NODE_NAME'),
123 time.strftime("%a %b %d %H:%M:%S %Z %Y")))
126 os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
128 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
129 "tempest-error.log"), 'w+')
130 f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
131 "environment.log"), 'w+')
134 p = subprocess.Popen(
135 cmd_line, shell=True,
136 stdout=subprocess.PIPE,
141 for line in iter(p.stdout.readline, b''):
142 if re.search("\} tempest\.", line):
143 logger.info(line.replace('\n', ''))
144 elif re.search('Starting verification', line):
145 logger.info(line.replace('\n', ''))
146 first_pos = line.index("UUID=") + len("UUID=")
147 last_pos = line.index(") for deployment")
148 self.VERIFICATION_ID = line[first_pos:last_pos]
149 logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
157 def parse_verifier_result(self):
158 if self.VERIFICATION_ID is None:
159 raise Exception('Verification UUID not found')
161 cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
162 logger.info("Showing result for a verification: '%s'." % cmd_line)
163 p = subprocess.Popen(cmd_line,
165 stdout=subprocess.PIPE,
166 stderr=subprocess.STDOUT)
167 for line in p.stdout:
168 new_line = line.replace(' ', '').split('|')
169 if 'Tests' in new_line:
173 if 'Testscount' in new_line:
174 num_tests = new_line[2]
175 elif 'Success' in new_line:
176 num_success = new_line[2]
177 elif 'Skipped' in new_line:
178 num_skipped = new_line[2]
179 elif 'Failures' in new_line:
180 num_failures = new_line[2]
183 num_executed = int(num_tests) - int(num_skipped)
185 self.result = 100 * int(num_success) / int(num_executed)
186 except ZeroDivisionError:
187 logger.error("No test has been executed")
191 with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
192 "tempest.log"), 'r') as logfile:
193 output = logfile.read()
195 success_testcases = []
196 for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
197 success_testcases.append(match)
198 failed_testcases = []
199 for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
200 failed_testcases.append(match)
201 skipped_testcases = []
202 for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
203 skipped_testcases.append(match)
205 self.details = {"tests": int(num_tests),
206 "failures": int(num_failures),
207 "success": success_testcases,
208 "errors": failed_testcases,
209 "skipped": skipped_testcases}
213 logger.info("Tempest %s success_rate is %s%%"
214 % (self.case_name, self.result))
218 self.start_time = time.time()
220 if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
221 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
222 image_and_flavor = conf_utils.create_tempest_resources()
223 conf_utils.configure_tempest(
225 IMAGE_ID=image_and_flavor.get("image_id"),
226 FLAVOR_ID=image_and_flavor.get("flavor_id"),
228 self.generate_test_list(self.VERIFIER_REPO_DIR)
229 self.apply_tempest_blacklist()
230 self.run_verifier_tests()
231 self.parse_verifier_result()
232 res = testcase.TestCase.EX_OK
233 except Exception as e:
234 logger.error('Error with run: %s' % e)
235 res = testcase.TestCase.EX_RUN_ERROR
237 self.stop_time = time.time()
241 class TempestSmokeSerial(TempestCommon):
243 def __init__(self, **kwargs):
244 if "case_name" not in kwargs:
245 kwargs["case_name"] = 'tempest_smoke_serial'
246 TempestCommon.__init__(self, **kwargs)
248 self.OPTION = "--concurrency 1"
251 class TempestSmokeParallel(TempestCommon):
253 def __init__(self, **kwargs):
254 if "case_name" not in kwargs:
255 kwargs["case_name"] = 'tempest_smoke_parallel'
256 TempestCommon.__init__(self, **kwargs)
261 class TempestFullParallel(TempestCommon):
263 def __init__(self, **kwargs):
264 if "case_name" not in kwargs:
265 kwargs["case_name"] = 'tempest_full_parallel'
266 TempestCommon.__init__(self, **kwargs)
270 class TempestCustom(TempestCommon):
272 def __init__(self, **kwargs):
273 if "case_name" not in kwargs:
274 kwargs["case_name"] = 'tempest_custom'
275 TempestCommon.__init__(self, **kwargs)
277 self.OPTION = "--concurrency 1"
280 class TempestDefcore(TempestCommon):
282 def __init__(self, **kwargs):
283 if "case_name" not in kwargs:
284 kwargs["case_name"] = 'tempest_defcore'
285 TempestCommon.__init__(self, **kwargs)
286 self.MODE = "defcore"
287 self.OPTION = "--concurrency 1"