3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
19 from functest.core import testcase_base
20 from functest.opnfv_tests.openstack.tempest import conf_utils
21 from functest.utils.constants import CONST
22 import functest.utils.functest_logger as ft_logger
23 import functest.utils.functest_utils as ft_utils
24 import functest.utils.openstack_utils as os_utils
26 """ logging configuration """
27 logger = ft_logger.Logger("Tempest").getLogger()
30 class TempestCommon(testcase_base.TestcaseBase):
33 super(TempestCommon, self).__init__()
38 self.VERIFIER_ID = self.get_verifier_id()
39 self.VERIFIER_REPO_DIR = self.get_verifier_repo_dir()
40 self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
41 self.DEPLOYMENT_DIR = self.get_verifier_deployment_dir()
42 self.VERIFICATION_ID = None
45 def get_verifier_id():
47 Returns verifer id for current Tempest
49 cmd = ("rally verify list-verifiers | awk '/" +
50 CONST.tempest_deployment_name +
52 p = subprocess.Popen(cmd, shell=True,
53 stdout=subprocess.PIPE,
54 stderr=subprocess.STDOUT)
55 deployment_uuid = p.stdout.readline().rstrip()
56 if deployment_uuid == "":
57 logger.error("Tempest verifier not found.")
58 raise Exception('Error with command:%s' % cmd)
59 return deployment_uuid
62 def get_verifier_deployment_id():
64 Returns deployment id for active Rally deployment
66 cmd = ("rally deployment list | awk '/" +
67 CONST.rally_deployment_name +
69 p = subprocess.Popen(cmd, shell=True,
70 stdout=subprocess.PIPE,
71 stderr=subprocess.STDOUT)
72 deployment_uuid = p.stdout.readline().rstrip()
73 if deployment_uuid == "":
74 logger.error("Rally deployment not found.")
75 raise Exception('Error with command:%s' % cmd)
76 return deployment_uuid
78 def get_verifier_repo_dir(self):
80 Returns installed verfier repo directory for Tempest
82 if not self.VERIFIER_ID:
83 self.VERIFIER_ID = self.get_verifier_id()
85 return os.path.join(CONST.dir_rally_inst,
87 'verifier-{}'.format(self.VERIFIER_ID),
90 def get_verifier_deployment_dir(self):
92 Returns Rally deployment directory for current verifier
94 if not self.VERIFIER_ID:
95 self.VERIFIER_ID = self.get_verifier_id()
97 if not self.DEPLOYMENT_ID:
98 self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
100 return os.path.join(CONST.dir_rally_inst,
102 'verifier-{}'.format(self.VERIFIER_ID),
103 'for-deployment-{}'.format(self.DEPLOYMENT_ID))
106 def read_file(filename):
107 with open(filename) as src:
108 return [line.strip() for line in src.readlines()]
110 def create_tempest_resources(self):
111 keystone_client = os_utils.get_keystone_client()
113 logger.debug("Creating tenant and user for Tempest suite")
114 tenant_id = os_utils.create_tenant(
116 CONST.tempest_identity_tenant_name,
117 CONST.tempest_identity_tenant_description)
119 logger.error("Error : Failed to create %s tenant"
120 % CONST.tempest_identity_tenant_name)
122 user_id = os_utils.create_user(keystone_client,
123 CONST.tempest_identity_user_name,
124 CONST.tempest_identity_user_password,
127 logger.error("Error : Failed to create %s user" %
128 CONST.tempest_identity_user_name)
130 logger.debug("Creating private network for Tempest suite")
131 network_dic = os_utils.create_shared_network_full(
132 CONST.tempest_private_net_name,
133 CONST.tempest_private_subnet_name,
134 CONST.tempest_router_name,
135 CONST.tempest_private_subnet_cidr)
137 return testcase_base.TestcaseBase.EX_RUN_ERROR
139 if CONST.tempest_use_custom_images:
140 # adding alternative image should be trivial should we need it
141 logger.debug("Creating image for Tempest suite")
142 _, self.IMAGE_ID = os_utils.get_or_create_image(
143 CONST.openstack_image_name, conf_utils.GLANCE_IMAGE_PATH,
144 CONST.openstack_image_disk_format)
145 if not self.IMAGE_ID:
146 return testcase_base.TestcaseBase.EX_RUN_ERROR
148 if CONST.tempest_use_custom_flavors:
149 # adding alternative flavor should be trivial should we need it
150 logger.debug("Creating flavor for Tempest suite")
151 _, self.FLAVOR_ID = os_utils.get_or_create_flavor(
152 CONST.openstack_flavor_name,
153 CONST.openstack_flavor_ram,
154 CONST.openstack_flavor_disk,
155 CONST.openstack_flavor_vcpus)
156 if not self.FLAVOR_ID:
157 return testcase_base.TestcaseBase.EX_RUN_ERROR
159 return testcase_base.TestcaseBase.EX_OK
161 def generate_test_list(self, verifier_repo_dir):
162 logger.debug("Generating test case list...")
163 if self.MODE == 'defcore':
165 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
166 elif self.MODE == 'custom':
167 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
169 conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
171 logger.error("Tempest test list file %s NOT found."
172 % conf_utils.TEMPEST_CUSTOM)
173 return testcase_base.TestcaseBase.EX_RUN_ERROR
175 if self.MODE == 'smoke':
177 elif self.MODE == 'feature_multisite':
178 testr_mode = " | grep -i kingbird "
179 elif self.MODE == 'full':
182 testr_mode = 'tempest.api.' + self.MODE
184 "testr list-tests {1} > {2};"
185 "cd -;".format(verifier_repo_dir,
187 conf_utils.TEMPEST_RAW_LIST))
188 ft_utils.execute_command(cmd)
190 return testcase_base.TestcaseBase.EX_OK
192 def apply_tempest_blacklist(self):
193 logger.debug("Applying tempest blacklist...")
194 cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
195 result_file = open(conf_utils.TEMPEST_LIST, 'w')
198 installer_type = CONST.INSTALLER_TYPE
199 deploy_scenario = CONST.DEPLOY_SCENARIO
200 if (bool(installer_type) * bool(deploy_scenario)):
201 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
203 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
204 black_list_yaml = yaml.safe_load(black_list_file)
205 black_list_file.close()
206 for item in black_list_yaml:
207 scenarios = item['scenarios']
208 installers = item['installers']
209 if (deploy_scenario in scenarios and
210 installer_type in installers):
211 tests = item['tests']
213 black_tests.append(test)
217 logger.debug("Tempest blacklist file does not exist.")
219 for cases_line in cases_file:
220 for black_tests_line in black_tests:
221 if black_tests_line in cases_line:
224 result_file.write(str(cases_line) + '\n')
226 return testcase_base.TestcaseBase.EX_OK
228 def _parse_verification_id(line):
229 first_pos = line.index("UUID=") + len("UUID=")
230 last_pos = line.index(") for deployment")
231 return line[first_pos:last_pos]
233 def run_verifier_tests(self):
234 self.OPTION += (" --load-list {}".format(conf_utils.TEMPEST_LIST))
236 cmd_line = "rally verify start " + self.OPTION
237 logger.info("Starting Tempest test suite: '%s'." % cmd_line)
239 header = ("Tempest environment:\n"
240 " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" %
241 (CONST.INSTALLER_TYPE,
242 CONST.DEPLOY_SCENARIO,
244 time.strftime("%a %b %d %H:%M:%S %Z %Y")))
247 os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
249 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
250 "tempest-error.log"), 'w+')
251 f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
252 "environment.log"), 'w+')
255 p = subprocess.Popen(
256 cmd_line, shell=True,
257 stdout=subprocess.PIPE,
262 for line in iter(p.stdout.readline, b''):
263 if re.search("\} tempest\.", line):
264 logger.info(line.replace('\n', ''))
265 elif re.search('Starting verification', line):
266 logger.info(line.replace('\n', ''))
267 first_pos = line.index("UUID=") + len("UUID=")
268 last_pos = line.index(") for deployment")
269 self.VERIFICATION_ID = line[first_pos:last_pos]
270 logger.debug('Verication UUID: %s' % self.VERIFICATION_ID)
278 def parse_verifier_result(self):
279 if not self.VERIFICATION_ID:
280 raise Exception('Verification UUID not found')
282 cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
283 logger.info("Showing result for a verification: '%s'." % cmd_line)
284 p = subprocess.Popen(cmd_line,
286 stdout=subprocess.PIPE,
287 stderr=subprocess.STDOUT)
288 for line in p.stdout:
289 new_line = line.replace(' ', '').split('|')
290 if 'Tests' in new_line:
294 if 'Testscount' in new_line:
295 num_tests = new_line[2]
296 elif 'Success' in new_line:
297 num_success = new_line[2]
298 elif 'Skipped' in new_line:
299 num_skipped = new_line[2]
302 num_executed = int(num_tests) - int(num_skipped)
303 success_rate = 100 * int(num_success) / int(num_executed)
307 self.criteria = ft_utils.check_success_rate(
308 self.case_name, success_rate)
309 logger.info("Tempest %s success_rate is %s%%, is marked as %s"
310 % (self.case_name, success_rate, self.criteria))
314 self.start_time = time.time()
316 if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
317 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
320 res = self.create_tempest_resources()
321 if res != testcase_base.TestcaseBase.EX_OK:
324 res = conf_utils.configure_tempest(logger,
328 if res != testcase_base.TestcaseBase.EX_OK:
331 res = self.generate_test_list(self.VERIFIER_REPO_DIR)
332 if res != testcase_base.TestcaseBase.EX_OK:
335 res = self.apply_tempest_blacklist()
336 if res != testcase_base.TestcaseBase.EX_OK:
339 self.run_verifier_tests()
340 self.parse_verifier_result()
342 self.stop_time = time.time()
345 class TempestSmokeSerial(TempestCommon):
348 TempestCommon.__init__(self)
349 self.case_name = "tempest_smoke_serial"
351 self.OPTION = "--concurrency 1"
354 class TempestSmokeParallel(TempestCommon):
357 TempestCommon.__init__(self)
358 self.case_name = "tempest_smoke_parallel"
363 class TempestFullParallel(TempestCommon):
366 TempestCommon.__init__(self)
367 self.case_name = "tempest_full_parallel"
371 class TempestMultisite(TempestCommon):
374 TempestCommon.__init__(self)
375 self.case_name = "multisite"
376 self.MODE = "feature_multisite"
377 self.OPTION = "--concurrency 1"
378 conf_utils.configure_tempest_multisite(logger, self.DEPLOYMENT_DIR)
381 class TempestCustom(TempestCommon):
383 def __init__(self, mode, option):
384 TempestCommon.__init__(self)
385 self.case_name = "tempest_custom"