3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
19 from functest.core import testcase_base
20 from functest.opnfv_tests.openstack.tempest import conf_utils
21 from functest.utils.constants import CONST
22 import functest.utils.functest_logger as ft_logger
23 import functest.utils.functest_utils as ft_utils
24 import functest.utils.openstack_utils as os_utils
26 """ logging configuration """
27 logger = ft_logger.Logger("Tempest").getLogger()
30 class TempestCommon(testcase_base.TestcaseBase):
33 super(TempestCommon, self).__init__()
38 self.VERIFIER_ID = conf_utils.get_verifier_id()
39 self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
41 self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
42 self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
43 self.VERIFIER_ID, self.DEPLOYMENT_ID)
44 self.VERIFICATION_ID = None
47 def read_file(filename):
48 with open(filename) as src:
49 return [line.strip() for line in src.readlines()]
51 def create_tempest_resources(self):
52 keystone_client = os_utils.get_keystone_client()
54 logger.debug("Creating tenant and user for Tempest suite")
55 tenant_id = os_utils.create_tenant(
57 CONST.tempest_identity_tenant_name,
58 CONST.tempest_identity_tenant_description)
60 logger.error("Failed to create %s tenant"
61 % CONST.tempest_identity_tenant_name)
63 user_id = os_utils.create_user(keystone_client,
64 CONST.tempest_identity_user_name,
65 CONST.tempest_identity_user_password,
68 logger.error("Failed to create %s user" %
69 CONST.tempest_identity_user_name)
71 logger.debug("Creating private network for Tempest suite")
72 network_dic = os_utils.create_shared_network_full(
73 CONST.tempest_private_net_name,
74 CONST.tempest_private_subnet_name,
75 CONST.tempest_router_name,
76 CONST.tempest_private_subnet_cidr)
77 if network_dic is None:
78 raise Exception('Failed to create private network')
80 if CONST.tempest_use_custom_images:
81 # adding alternative image should be trivial should we need it
82 logger.debug("Creating image for Tempest suite")
83 _, self.IMAGE_ID = os_utils.get_or_create_image(
84 CONST.openstack_image_name, conf_utils.GLANCE_IMAGE_PATH,
85 CONST.openstack_image_disk_format)
86 if self.IMAGE_ID is None:
87 raise Exception('Failed to create image')
89 if CONST.tempest_use_custom_flavors:
90 # adding alternative flavor should be trivial should we need it
91 logger.debug("Creating flavor for Tempest suite")
92 _, self.FLAVOR_ID = os_utils.get_or_create_flavor(
93 CONST.openstack_flavor_name,
94 CONST.openstack_flavor_ram,
95 CONST.openstack_flavor_disk,
96 CONST.openstack_flavor_vcpus)
97 if self.FLAVOR_ID is None:
98 raise Exception('Failed to create flavor')
100 def generate_test_list(self, verifier_repo_dir):
101 logger.debug("Generating test case list...")
102 if self.MODE == 'defcore':
104 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
105 elif self.MODE == 'custom':
106 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
108 conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
110 raise Exception("Tempest test list file %s NOT found."
111 % conf_utils.TEMPEST_CUSTOM)
113 if self.MODE == 'smoke':
115 elif self.MODE == 'feature_multisite':
116 testr_mode = "'[Kk]ingbird'"
117 elif self.MODE == 'full':
120 testr_mode = 'tempest.api.' + self.MODE
122 "testr list-tests {1} > {2};"
123 "cd -;".format(verifier_repo_dir,
125 conf_utils.TEMPEST_RAW_LIST))
126 ft_utils.execute_command(cmd)
128 def apply_tempest_blacklist(self):
129 logger.debug("Applying tempest blacklist...")
130 cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
131 result_file = open(conf_utils.TEMPEST_LIST, 'w')
134 installer_type = CONST.INSTALLER_TYPE
135 deploy_scenario = CONST.DEPLOY_SCENARIO
136 if (bool(installer_type) * bool(deploy_scenario)):
137 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
139 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
140 black_list_yaml = yaml.safe_load(black_list_file)
141 black_list_file.close()
142 for item in black_list_yaml:
143 scenarios = item['scenarios']
144 installers = item['installers']
145 if (deploy_scenario in scenarios and
146 installer_type in installers):
147 tests = item['tests']
149 black_tests.append(test)
153 logger.debug("Tempest blacklist file does not exist.")
155 for cases_line in cases_file:
156 for black_tests_line in black_tests:
157 if black_tests_line in cases_line:
160 result_file.write(str(cases_line) + '\n')
163 def _parse_verification_id(line):
164 first_pos = line.index("UUID=") + len("UUID=")
165 last_pos = line.index(") for deployment")
166 return line[first_pos:last_pos]
168 def run_verifier_tests(self):
169 self.OPTION += (" --load-list {} --detailed"
170 .format(conf_utils.TEMPEST_LIST))
172 cmd_line = "rally verify start " + self.OPTION
173 logger.info("Starting Tempest test suite: '%s'." % cmd_line)
175 header = ("Tempest environment:\n"
176 " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
177 (CONST.INSTALLER_TYPE,
178 CONST.DEPLOY_SCENARIO,
180 time.strftime("%a %b %d %H:%M:%S %Z %Y")))
183 os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
185 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
186 "tempest-error.log"), 'w+')
187 f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
188 "environment.log"), 'w+')
191 p = subprocess.Popen(
192 cmd_line, shell=True,
193 stdout=subprocess.PIPE,
198 for line in iter(p.stdout.readline, b''):
199 if re.search("\} tempest\.", line):
200 logger.info(line.replace('\n', ''))
201 elif re.search('Starting verification', line):
202 logger.info(line.replace('\n', ''))
203 first_pos = line.index("UUID=") + len("UUID=")
204 last_pos = line.index(") for deployment")
205 self.VERIFICATION_ID = line[first_pos:last_pos]
206 logger.debug('Verication UUID: %s' % self.VERIFICATION_ID)
214 def parse_verifier_result(self):
215 if self.VERIFICATION_ID is None:
216 raise Exception('Verification UUID not found')
218 cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
219 logger.info("Showing result for a verification: '%s'." % cmd_line)
220 p = subprocess.Popen(cmd_line,
222 stdout=subprocess.PIPE,
223 stderr=subprocess.STDOUT)
224 for line in p.stdout:
225 new_line = line.replace(' ', '').split('|')
226 if 'Tests' in new_line:
230 if 'Testscount' in new_line:
231 num_tests = new_line[2]
232 elif 'Success' in new_line:
233 num_success = new_line[2]
234 elif 'Skipped' in new_line:
235 num_skipped = new_line[2]
236 elif 'Failures' in new_line:
237 num_failures = new_line[2]
240 num_executed = int(num_tests) - int(num_skipped)
241 success_rate = 100 * int(num_success) / int(num_executed)
242 with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
243 "tempest.log"), 'r') as logfile:
244 output = logfile.read()
247 for match in re.findall('(.*?)[. ]*fail ', output):
249 skipped_testcase = ""
250 for match in re.findall('(.*?)[. ]*skip:', output):
251 skipped_testcase += match
253 self.details = {"tests": int(num_tests),
254 "failures": int(num_failures),
255 "errors": error_logs,
256 "skipped": skipped_testcase}
260 self.criteria = ft_utils.check_success_rate(
261 self.case_name, success_rate)
262 logger.info("Tempest %s success_rate is %s%%, is marked as %s"
263 % (self.case_name, success_rate, self.criteria))
267 self.start_time = time.time()
269 if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
270 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
273 self.create_tempest_resources()
274 conf_utils.configure_tempest(self.DEPLOYMENT_DIR,
278 self.generate_test_list(self.VERIFIER_REPO_DIR)
279 self.apply_tempest_blacklist()
280 self.run_verifier_tests()
281 self.parse_verifier_result()
282 res = testcase_base.TestcaseBase.EX_OK
283 except Exception as e:
284 logger.error('Error with run: %s' % e)
285 res = testcase_base.TestcaseBase.EX_RUN_ERROR
287 self.stop_time = time.time()
291 class TempestSmokeSerial(TempestCommon):
294 TempestCommon.__init__(self)
295 self.case_name = "tempest_smoke_serial"
297 self.OPTION = "--concurrency 1"
300 class TempestSmokeParallel(TempestCommon):
303 TempestCommon.__init__(self)
304 self.case_name = "tempest_smoke_parallel"
309 class TempestFullParallel(TempestCommon):
312 TempestCommon.__init__(self)
313 self.case_name = "tempest_full_parallel"
317 class TempestMultisite(TempestCommon):
320 TempestCommon.__init__(self)
321 self.case_name = "multisite"
322 self.MODE = "feature_multisite"
323 self.OPTION = "--concurrency 1"
324 conf_utils.install_verifier_ext(CONST.dir_repo_kingbird)
327 class TempestCustom(TempestCommon):
330 TempestCommon.__init__(self)
331 self.case_name = "tempest_custom"
333 self.OPTION = "--concurrency 1"
336 class TempestDefcore(TempestCommon):
339 TempestCommon.__init__(self)
340 self.case_name = "tempest_defcore"
341 self.MODE = "defcore"
342 self.OPTION = "--concurrency 1"