3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Tempest testcases implementation."""
13 from __future__ import division
23 from snaps.config.flavor import FlavorConfig
24 from snaps.config.network import NetworkConfig, SubnetConfig
25 from snaps.config.project import ProjectConfig
26 from snaps.config.user import UserConfig
27 from snaps.openstack.create_flavor import OpenStackFlavor
28 from snaps.openstack.tests import openstack_tests
29 from snaps.openstack.utils import deploy_utils
30 from xtesting.core import testcase
33 from functest.opnfv_tests.openstack.snaps import snaps_utils
34 from functest.opnfv_tests.openstack.tempest import conf_utils
35 from functest.utils import config
36 from functest.utils import env
38 LOGGER = logging.getLogger(__name__)
41 class TempestCommon(testcase.TestCase):
42 # pylint: disable=too-many-instance-attributes
43 """TempestCommon testcases implementation class."""
45 TEMPEST_RESULTS_DIR = os.path.join(
46 getattr(config.CONF, 'dir_results'), 'tempest')
48 def __init__(self, **kwargs):
49 super(TempestCommon, self).__init__(**kwargs)
50 self.resources = TempestResourcesManager(**kwargs)
53 self.verifier_id = conf_utils.get_verifier_id()
54 self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
56 self.deployment_id = conf_utils.get_verifier_deployment_id()
57 self.deployment_dir = conf_utils.get_verifier_deployment_dir(
58 self.verifier_id, self.deployment_id)
59 self.verification_id = None
60 self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
61 self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
62 self.list = os.path.join(self.res_dir, 'test_list.txt')
66 def read_file(filename):
67 """Read file and return content as a stripped list."""
68 with open(filename) as src:
69 return [line.strip() for line in src.readlines()]
72 def get_verifier_result(verif_id):
73 """Retrieve verification results."""
80 cmd = ["rally", "verify", "show", "--uuid", verif_id]
81 LOGGER.info("Showing result for a verification: '%s'.", cmd)
82 proc = subprocess.Popen(cmd,
83 stdout=subprocess.PIPE,
84 stderr=subprocess.STDOUT)
85 for line in proc.stdout:
86 new_line = line.replace(' ', '').split('|')
87 if 'Tests' in new_line:
90 if 'Testscount' in new_line:
91 result['num_tests'] = int(new_line[2])
92 elif 'Success' in new_line:
93 result['num_success'] = int(new_line[2])
94 elif 'Skipped' in new_line:
95 result['num_skipped'] = int(new_line[2])
96 elif 'Failures' in new_line:
97 result['num_failures'] = int(new_line[2])
101 def backup_tempest_config(conf_file, res_dir):
103 Copy config file to tempest results directory
105 if not os.path.exists(res_dir):
107 shutil.copyfile(conf_file,
108 os.path.join(res_dir, 'tempest.conf'))
110 def generate_test_list(self):
111 """Generate test list based on the test mode."""
112 LOGGER.debug("Generating test case list...")
113 if self.mode == 'custom':
114 if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
116 conf_utils.TEMPEST_CUSTOM, self.list)
118 raise Exception("Tempest test list file %s NOT found."
119 % conf_utils.TEMPEST_CUSTOM)
121 if self.mode == 'smoke':
122 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
123 elif self.mode == 'full':
124 testr_mode = r"'^tempest\.'"
126 testr_mode = self.mode
127 cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
128 self.verifier_repo_dir, testr_mode, self.list)
129 output = subprocess.check_output(cmd, shell=True)
130 LOGGER.info("%s\n%s", cmd, output)
132 def apply_tempest_blacklist(self):
133 """Exclude blacklisted test cases."""
134 LOGGER.debug("Applying tempest blacklist...")
135 if os.path.exists(self.raw_list):
136 os.remove(self.raw_list)
137 os.rename(self.list, self.raw_list)
138 cases_file = self.read_file(self.raw_list)
139 result_file = open(self.list, 'w')
142 installer_type = env.get('INSTALLER_TYPE')
143 deploy_scenario = env.get('DEPLOY_SCENARIO')
144 if bool(installer_type) * bool(deploy_scenario):
145 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
147 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
148 black_list_yaml = yaml.safe_load(black_list_file)
149 black_list_file.close()
150 for item in black_list_yaml:
151 scenarios = item['scenarios']
152 installers = item['installers']
153 if (deploy_scenario in scenarios and
154 installer_type in installers):
155 tests = item['tests']
157 black_tests.append(test)
159 except Exception: # pylint: disable=broad-except
161 LOGGER.debug("Tempest blacklist file does not exist.")
163 for cases_line in cases_file:
164 for black_tests_line in black_tests:
165 if black_tests_line in cases_line:
168 result_file.write(str(cases_line) + '\n')
171 def run_verifier_tests(self):
172 """Execute tempest test cases."""
173 cmd = ["rally", "verify", "start", "--load-list",
175 cmd.extend(self.option)
176 LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
179 os.path.join(self.res_dir, "tempest.log"), 'w+')
181 os.path.join(self.res_dir,
182 "tempest-error.log"), 'w+')
184 proc = subprocess.Popen(
186 stdout=subprocess.PIPE,
191 for line in iter(proc.stdout.readline, b''):
192 if re.search(r"\} tempest\.", line):
193 LOGGER.info(line.replace('\n', ''))
194 elif re.search(r'(?=\(UUID=(.*)\))', line):
195 self.verification_id = re.search(
196 r'(?=\(UUID=(.*)\))', line).group(1)
197 LOGGER.info('Verification UUID: %s', self.verification_id)
204 if self.verification_id is None:
205 raise Exception('Verification UUID not found')
207 def parse_verifier_result(self):
208 """Parse and save test results."""
209 stat = self.get_verifier_result(self.verification_id)
211 num_executed = stat['num_tests'] - stat['num_skipped']
213 self.result = 100 * stat['num_success'] / num_executed
214 except ZeroDivisionError:
216 if stat['num_tests'] > 0:
217 LOGGER.info("All tests have been skipped")
219 LOGGER.error("No test has been executed")
222 with open(os.path.join(self.res_dir,
223 "tempest.log"), 'r') as logfile:
224 output = logfile.read()
226 success_testcases = []
227 for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
228 success_testcases.append(match)
229 failed_testcases = []
230 for match in re.findall(r'.*\{0\} (.*?)[. ]*fail', output):
231 failed_testcases.append(match)
232 skipped_testcases = []
233 for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
234 skipped_testcases.append(match)
236 self.details = {"tests_number": stat['num_tests'],
237 "success_number": stat['num_success'],
238 "skipped_number": stat['num_skipped'],
239 "failures_number": stat['num_failures'],
240 "success": success_testcases,
241 "skipped": skipped_testcases,
242 "failures": failed_testcases}
243 except Exception: # pylint: disable=broad-except
246 LOGGER.info("Tempest %s success_rate is %s%%",
247 self.case_name, self.result)
249 def generate_report(self):
250 """Generate verification report."""
251 html_file = os.path.join(self.res_dir,
252 "tempest-report.html")
253 cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
254 self.verification_id, "--to", html_file]
255 subprocess.Popen(cmd, stdout=subprocess.PIPE,
256 stderr=subprocess.STDOUT)
258 def configure(self, **kwargs): # pylint: disable=unused-argument
260 Create all openstack resources for tempest-based testcases and write
263 if not os.path.exists(self.res_dir):
264 os.makedirs(self.res_dir)
265 resources = self.resources.create()
266 compute_cnt = snaps_utils.get_active_compute_cnt(
267 self.resources.os_creds)
268 self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
269 conf_utils.configure_tempest_update_params(
270 self.conf_file, network_name=resources.get("network_name"),
271 image_id=resources.get("image_id"),
272 flavor_id=resources.get("flavor_id"),
273 compute_cnt=compute_cnt)
274 self.backup_tempest_config(self.conf_file, self.res_dir)
276 def run(self, **kwargs):
277 self.start_time = time.time()
279 self.configure(**kwargs)
280 self.generate_test_list()
281 self.apply_tempest_blacklist()
282 self.run_verifier_tests()
283 self.parse_verifier_result()
284 self.generate_report()
285 res = testcase.TestCase.EX_OK
286 except Exception: # pylint: disable=broad-except
287 LOGGER.exception('Error with run')
288 res = testcase.TestCase.EX_RUN_ERROR
290 self.resources.cleanup()
291 self.stop_time = time.time()
295 class TempestSmokeSerial(TempestCommon):
296 """Tempest smoke serial testcase implementation."""
297 def __init__(self, **kwargs):
298 if "case_name" not in kwargs:
299 kwargs["case_name"] = 'tempest_smoke_serial'
300 TempestCommon.__init__(self, **kwargs)
302 self.option = ["--concurrency", "1"]
305 class TempestNeutronTrunk(TempestCommon):
306 """Tempest neutron trunk testcase implementation."""
307 def __init__(self, **kwargs):
308 if "case_name" not in kwargs:
309 kwargs["case_name"] = 'neutron_trunk'
310 TempestCommon.__init__(self, **kwargs)
311 self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
312 self.res_dir = os.path.join(
313 getattr(config.CONF, 'dir_results'), 'neutron_trunk')
314 self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
315 self.list = os.path.join(self.res_dir, 'test_list.txt')
317 def configure(self, **kwargs):
318 super(TempestNeutronTrunk, self).configure(**kwargs)
319 rconfig = conf_utils.ConfigParser.RawConfigParser()
320 rconfig.read(self.conf_file)
321 rconfig.set('network-feature-enabled', 'api_extensions', 'all')
322 with open(self.conf_file, 'wb') as config_file:
323 rconfig.write(config_file)
326 class TempestSmokeParallel(TempestCommon):
327 """Tempest smoke parallel testcase implementation."""
328 def __init__(self, **kwargs):
329 if "case_name" not in kwargs:
330 kwargs["case_name"] = 'tempest_smoke_parallel'
331 TempestCommon.__init__(self, **kwargs)
335 class TempestFullParallel(TempestCommon):
336 """Tempest full parallel testcase implementation."""
337 def __init__(self, **kwargs):
338 if "case_name" not in kwargs:
339 kwargs["case_name"] = 'tempest_full_parallel'
340 TempestCommon.__init__(self, **kwargs)
344 class TempestCustom(TempestCommon):
345 """Tempest custom testcase implementation."""
346 def __init__(self, **kwargs):
347 if "case_name" not in kwargs:
348 kwargs["case_name"] = 'tempest_custom'
349 TempestCommon.__init__(self, **kwargs)
351 self.option = ["--concurrency", "1"]
354 class TempestDefcore(TempestCommon):
355 """Tempest Defcore testcase implementation."""
356 def __init__(self, **kwargs):
357 if "case_name" not in kwargs:
358 kwargs["case_name"] = 'tempest_defcore'
359 TempestCommon.__init__(self, **kwargs)
360 self.mode = "defcore"
361 self.option = ["--concurrency", "1"]
364 class TempestResourcesManager(object):
365 """Tempest resource manager."""
366 def __init__(self, **kwargs):
367 self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
368 self.guid = '-' + str(uuid.uuid4())
369 self.creators = list()
370 self.cirros_image_config = getattr(
371 config.CONF, 'snaps_images_cirros', None)
373 def _create_project(self):
374 """Create project for tests."""
375 project_creator = deploy_utils.create_project(
376 self.os_creds, ProjectConfig(
378 config.CONF, 'tempest_identity_tenant_name') + self.guid,
380 config.CONF, 'tempest_identity_tenant_description'),
381 domain=self.os_creds.project_domain_name))
382 if project_creator is None or project_creator.get_project() is None:
383 raise Exception("Failed to create tenant")
384 self.creators.append(project_creator)
385 return project_creator.get_project().id
387 def _create_user(self):
388 """Create user for tests."""
389 user_creator = deploy_utils.create_user(
390 self.os_creds, UserConfig(
392 config.CONF, 'tempest_identity_user_name') + self.guid,
394 config.CONF, 'tempest_identity_user_password'),
395 project_name=getattr(
396 config.CONF, 'tempest_identity_tenant_name') + self.guid,
397 domain_name=self.os_creds.user_domain_name))
398 if user_creator is None or user_creator.get_user() is None:
399 raise Exception("Failed to create user")
400 self.creators.append(user_creator)
401 return user_creator.get_user().id
403 def _create_network(self, project_name):
404 """Create network for tests."""
405 tempest_network_type = None
406 tempest_physical_network = None
407 tempest_segmentation_id = None
409 tempest_network_type = getattr(
410 config.CONF, 'tempest_network_type', None)
411 tempest_physical_network = getattr(
412 config.CONF, 'tempest_physical_network', None)
413 tempest_segmentation_id = getattr(
414 config.CONF, 'tempest_segmentation_id', None)
415 tempest_net_name = getattr(
416 config.CONF, 'tempest_private_net_name') + self.guid
418 network_creator = deploy_utils.create_network(
419 self.os_creds, NetworkConfig(
420 name=tempest_net_name,
421 project_name=project_name,
422 network_type=tempest_network_type,
423 physical_network=tempest_physical_network,
424 segmentation_id=tempest_segmentation_id,
425 subnet_settings=[SubnetConfig(
428 'tempest_private_subnet_name') + self.guid,
429 project_name=project_name,
431 config.CONF, 'tempest_private_subnet_cidr'),
432 dns_nameservers=[env.get('NAMESERVER')])]))
433 if network_creator is None or network_creator.get_network() is None:
434 raise Exception("Failed to create private network")
435 self.creators.append(network_creator)
436 return tempest_net_name
438 def _create_image(self, name):
439 """Create image for tests"""
440 os_image_settings = openstack_tests.cirros_image_settings(
442 image_metadata=self.cirros_image_config)
443 image_creator = deploy_utils.create_image(
444 self.os_creds, os_image_settings)
445 if image_creator is None:
446 raise Exception('Failed to create image')
447 self.creators.append(image_creator)
448 return image_creator.get_image().id
450 def _create_flavor(self, name):
451 """Create flavor for tests."""
452 flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
453 flavor_creator = OpenStackFlavor(
454 self.os_creds, FlavorConfig(
456 ram=getattr(config.CONF, 'openstack_flavor_ram'),
457 disk=getattr(config.CONF, 'openstack_flavor_disk'),
458 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
459 metadata=flavor_metadata))
460 flavor = flavor_creator.create()
462 raise Exception('Failed to create flavor')
463 self.creators.append(flavor_creator)
466 def create(self, create_project=False):
467 """Create resources for Tempest test suite."""
469 'tempest_net_name': None,
471 'image_id_alt': None,
473 'flavor_id_alt': None
478 LOGGER.debug("Creating project and user for Tempest suite")
479 project_name = getattr(
480 config.CONF, 'tempest_identity_tenant_name') + self.guid
481 result['project_id'] = self._create_project()
482 result['user_id'] = self._create_user()
483 result['tenant_id'] = result['project_id'] # for compatibility
485 LOGGER.debug("Creating private network for Tempest suite")
486 result['tempest_net_name'] = self._create_network(project_name)
488 LOGGER.debug("Creating two images for Tempest suite")
489 image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
490 result['image_id'] = self._create_image(image_name)
491 image_name = getattr(
492 config.CONF, 'openstack_image_name_alt') + self.guid
493 result['image_id_alt'] = self._create_image(image_name)
495 LOGGER.info("Creating two flavors for Tempest suite")
496 name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
497 result['flavor_id'] = self._create_flavor(name)
500 config.CONF, 'openstack_flavor_name_alt') + self.guid
501 result['flavor_id_alt'] = self._create_flavor(name)
507 Cleanup all OpenStack objects. Should be called on completion.
509 for creator in reversed(self.creators):
512 except Exception as err: # pylint: disable=broad-except
513 LOGGER.error('Unexpected error cleaning - %s', err)