3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Tempest testcases implementation."""
13 from __future__ import division
24 from six.moves import configparser
25 from xtesting.core import testcase
28 from functest.core import singlevm
29 from functest.opnfv_tests.openstack.rally import rally
30 from functest.utils import config
31 from functest.utils import env
32 from functest.utils import functest_utils
34 LOGGER = logging.getLogger(__name__)
37 class TempestCommon(singlevm.VmReady2):
38 # pylint: disable=too-many-instance-attributes,too-many-public-methods
39 """TempestCommon testcases implementation class."""
42 filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
44 tempest_conf_yaml = pkg_resources.resource_filename(
46 'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
47 tempest_custom = pkg_resources.resource_filename(
49 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
50 tempest_blacklist = pkg_resources.resource_filename(
52 'opnfv_tests/openstack/tempest/custom_tests/blacklist.yaml')
53 tempest_public_blacklist = pkg_resources.resource_filename(
55 'opnfv_tests/openstack/tempest/custom_tests/public_blacklist.yaml')
57 def __init__(self, **kwargs):
58 if "case_name" not in kwargs:
59 kwargs["case_name"] = 'tempest'
60 super(TempestCommon, self).__init__(**kwargs)
61 assert self.orig_cloud
64 if self.orig_cloud.get_role("admin"):
65 self.role_name = "admin"
66 elif self.orig_cloud.get_role("Admin"):
67 self.role_name = "Admin"
69 raise Exception("Cannot detect neither admin nor Admin")
70 self.orig_cloud.grant_role(
71 self.role_name, user=self.project.user.id,
72 project=self.project.project.id,
73 domain=self.project.domain.id)
74 self.orig_cloud.grant_role(
75 self.role_name, user=self.project.user.id,
76 domain=self.project.domain.id)
77 self.deployment_id = None
78 self.verifier_id = None
79 self.verifier_repo_dir = None
80 self.deployment_dir = None
81 self.verification_id = None
82 self.res_dir = os.path.join(
83 getattr(config.CONF, 'dir_results'), self.case_name)
84 self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
85 self.list = os.path.join(self.res_dir, 'test_list.txt')
88 self.flavor_alt = None
91 self.services = kwargs['run']['args']['services']
92 except Exception: # pylint: disable=broad-except
94 self.neutron_extensions = []
96 self.neutron_extensions = kwargs['run']['args'][
98 except Exception: # pylint: disable=broad-except
100 self.deny_skipping = kwargs.get("deny_skipping", False)
101 self.tests_count = kwargs.get("tests_count", 0)
103 def check_services(self):
104 """Check the mandatory services."""
105 for service in self.services:
107 self.cloud.search_services(service)[0]
108 except Exception: # pylint: disable=broad-except
109 self.is_skipped = True
112 def check_extensions(self):
113 """Check the mandatory network extensions."""
114 extensions = self.cloud.get_network_extensions()
115 for network_extension in self.neutron_extensions:
116 if network_extension not in extensions:
118 "Cannot find Neutron extension: %s", network_extension)
119 self.is_skipped = True
122 def check_requirements(self):
123 self.check_services()
124 self.check_extensions()
129 def read_file(filename):
130 """Read file and return content as a stripped list."""
131 with open(filename) as src:
132 return [line.strip() for line in src.readlines()]
135 def get_verifier_result(verif_id):
136 """Retrieve verification results."""
143 cmd = ["rally", "verify", "show", "--uuid", verif_id]
144 LOGGER.info("Showing result for a verification: '%s'.", cmd)
145 proc = subprocess.Popen(cmd,
146 stdout=subprocess.PIPE,
147 stderr=subprocess.STDOUT)
148 for line in proc.stdout:
149 LOGGER.info(line.decode("utf-8").rstrip())
150 new_line = line.decode("utf-8").replace(' ', '').split('|')
151 if 'Tests' in new_line:
153 if 'Testscount' in new_line:
154 result['num_tests'] = int(new_line[2])
155 elif 'Success' in new_line:
156 result['num_success'] = int(new_line[2])
157 elif 'Skipped' in new_line:
158 result['num_skipped'] = int(new_line[2])
159 elif 'Failures' in new_line:
160 result['num_failures'] = int(new_line[2])
164 def backup_tempest_config(conf_file, res_dir):
166 Copy config file to tempest results directory
168 if not os.path.exists(res_dir):
170 shutil.copyfile(conf_file,
171 os.path.join(res_dir, 'tempest.conf'))
174 def create_verifier():
175 """Create new verifier"""
176 LOGGER.info("Create verifier from existing repo...")
177 cmd = ['rally', 'verify', 'delete-verifier',
178 '--id', str(getattr(config.CONF, 'tempest_verifier_name')),
181 output = subprocess.check_output(cmd)
182 LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
183 except subprocess.CalledProcessError:
186 cmd = ['rally', 'verify', 'create-verifier',
187 '--source', str(getattr(config.CONF, 'dir_repo_tempest')),
188 '--name', str(getattr(config.CONF, 'tempest_verifier_name')),
189 '--type', 'tempest', '--system-wide']
190 output = subprocess.check_output(cmd)
191 LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
192 return TempestCommon.get_verifier_id()
195 def get_verifier_id():
197 Returns verifier id for current Tempest
199 cmd = ("rally verify list-verifiers | awk '/" +
200 getattr(config.CONF, 'tempest_verifier_name') +
202 proc = subprocess.Popen(cmd, shell=True,
203 stdout=subprocess.PIPE,
204 stderr=subprocess.STDOUT)
205 verifier_uuid = proc.stdout.readline().rstrip()
206 return verifier_uuid.decode("utf-8")
209 def get_verifier_repo_dir(verifier_id):
211 Returns installed verifier repo directory for Tempest
213 return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
215 'verifier-{}'.format(verifier_id),
219 def get_verifier_deployment_dir(verifier_id, deployment_id):
221 Returns Rally deployment directory for current verifier
223 return os.path.join(getattr(config.CONF, 'dir_rally_inst'),
225 'verifier-{}'.format(verifier_id),
226 'for-deployment-{}'.format(deployment_id))
229 def update_tempest_conf_file(conf_file, rconfig):
230 """Update defined paramters into tempest config file"""
231 with open(TempestCommon.tempest_conf_yaml) as yfile:
232 conf_yaml = yaml.safe_load(yfile)
234 sections = rconfig.sections()
235 for section in conf_yaml:
236 if section not in sections:
237 rconfig.add_section(section)
238 sub_conf = conf_yaml.get(section)
239 for key, value in sub_conf.items():
240 rconfig.set(section, key, value)
242 with open(conf_file, 'w') as config_file:
243 rconfig.write(config_file)
246 def configure_tempest_update_params(
247 tempest_conf_file, image_id=None, flavor_id=None,
248 compute_cnt=1, image_alt_id=None, flavor_alt_id=None,
249 admin_role_name='admin', cidr='192.168.120.0/24',
250 domain_id='default'):
251 # pylint: disable=too-many-branches,too-many-arguments
252 # pylint: disable=too-many-statements,too-many-locals
254 Add/update needed parameters into tempest.conf file
256 LOGGER.debug("Updating selected tempest.conf parameters...")
257 rconfig = configparser.RawConfigParser()
258 rconfig.read(tempest_conf_file)
260 'compute', 'volume_device_name', env.get('VOLUME_DEVICE_NAME'))
261 if image_id is not None:
262 rconfig.set('compute', 'image_ref', image_id)
263 if image_alt_id is not None:
264 rconfig.set('compute', 'image_ref_alt', image_alt_id)
265 if flavor_id is not None:
266 rconfig.set('compute', 'flavor_ref', flavor_id)
267 if flavor_alt_id is not None:
268 rconfig.set('compute', 'flavor_ref_alt', flavor_alt_id)
270 # enable multinode tests
271 rconfig.set('compute', 'min_compute_nodes', compute_cnt)
272 rconfig.set('compute-feature-enabled', 'live_migration', True)
273 filters = ['RetryFilter', 'AvailabilityZoneFilter', 'ComputeFilter',
274 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter',
275 'ServerGroupAntiAffinityFilter',
276 'ServerGroupAffinityFilter']
278 'compute-feature-enabled', 'scheduler_available_filters',
279 functest_utils.convert_list_to_ini(filters))
280 if os.environ.get('OS_REGION_NAME'):
281 rconfig.set('identity', 'region', os.environ.get('OS_REGION_NAME'))
282 if env.get("NEW_USER_ROLE").lower() != "member":
284 'auth', 'tempest_roles',
285 functest_utils.convert_list_to_ini([env.get("NEW_USER_ROLE")]))
286 if not json.loads(env.get("USE_DYNAMIC_CREDENTIALS").lower()):
287 rconfig.set('auth', 'use_dynamic_credentials', False)
288 account_file = os.path.join(
289 getattr(config.CONF, 'dir_functest_data'), 'accounts.yaml')
290 assert os.path.exists(
291 account_file), "{} doesn't exist".format(account_file)
292 rconfig.set('auth', 'test_accounts_file', account_file)
293 rconfig.set('identity', 'auth_version', 'v3')
294 rconfig.set('identity', 'admin_role', admin_role_name)
295 rconfig.set('identity', 'default_domain_id', domain_id)
296 if not rconfig.has_section('network'):
297 rconfig.add_section('network')
298 rconfig.set('network', 'default_network', cidr)
299 rconfig.set('network', 'project_network_cidr', cidr)
300 rconfig.set('network', 'project_networks_reachable', False)
302 'identity', 'v3_endpoint_type',
303 os.environ.get('OS_INTERFACE', 'public'))
305 sections = rconfig.sections()
307 'compute', 'volume', 'image', 'network', 'data-processing',
308 'object-storage', 'orchestration']
309 for service in services_list:
310 if service not in sections:
311 rconfig.add_section(service)
312 rconfig.set(service, 'endpoint_type',
313 os.environ.get('OS_INTERFACE', 'public'))
315 LOGGER.debug('Add/Update required params defined in tempest_conf.yaml '
316 'into tempest.conf file')
317 TempestCommon.update_tempest_conf_file(tempest_conf_file, rconfig)
320 def configure_verifier(deployment_dir):
322 Execute rally verify configure-verifier, which generates tempest.conf
324 cmd = ['rally', 'verify', 'configure-verifier', '--reconfigure',
325 '--id', str(getattr(config.CONF, 'tempest_verifier_name'))]
326 output = subprocess.check_output(cmd)
327 LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
329 LOGGER.debug("Looking for tempest.conf file...")
330 tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
331 if not os.path.isfile(tempest_conf_file):
332 LOGGER.error("Tempest configuration file %s NOT found.",
335 return tempest_conf_file
337 def generate_test_list(self, **kwargs):
338 """Generate test list based on the test mode."""
339 LOGGER.debug("Generating test case list...")
340 self.backup_tempest_config(self.conf_file, '/etc')
341 if kwargs.get('mode') == 'custom':
342 if os.path.isfile(self.tempest_custom):
344 self.tempest_custom, self.list)
346 raise Exception("Tempest test list file %s NOT found."
347 % self.tempest_custom)
349 testr_mode = kwargs.get(
350 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
351 cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
352 self.verifier_repo_dir, testr_mode, self.list)
353 output = subprocess.check_output(cmd, shell=True)
354 LOGGER.info("%s\n%s", cmd, output.decode("utf-8"))
355 os.remove('/etc/tempest.conf')
357 def apply_tempest_blacklist(self, black_list):
358 """Exclude blacklisted test cases."""
359 LOGGER.debug("Applying tempest blacklist...")
360 if os.path.exists(self.raw_list):
361 os.remove(self.raw_list)
362 os.rename(self.list, self.raw_list)
363 cases_file = self.read_file(self.raw_list)
364 result_file = open(self.list, 'w')
367 deploy_scenario = env.get('DEPLOY_SCENARIO')
368 if bool(deploy_scenario):
369 # if DEPLOY_SCENARIO is set we read the file
370 black_list_file = open(black_list)
371 black_list_yaml = yaml.safe_load(black_list_file)
372 black_list_file.close()
373 for item in black_list_yaml:
374 scenarios = item['scenarios']
375 in_it = rally.RallyBase.in_iterable_re
376 if in_it(deploy_scenario, scenarios):
377 tests = item['tests']
378 black_tests.extend(tests)
379 except Exception: # pylint: disable=broad-except
381 LOGGER.debug("Tempest blacklist file does not exist.")
383 for cases_line in cases_file:
384 for black_tests_line in black_tests:
385 if re.search(black_tests_line, cases_line):
388 result_file.write(str(cases_line) + '\n')
391 def run_verifier_tests(self, **kwargs):
392 """Execute tempest test cases."""
393 cmd = ["rally", "verify", "start", "--load-list",
395 cmd.extend(kwargs.get('option', []))
396 LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
399 os.path.join(self.res_dir, "tempest.log"), 'w+')
401 proc = subprocess.Popen(
403 stdout=subprocess.PIPE,
404 stderr=subprocess.STDOUT,
408 for line in iter(proc.stdout.readline, b''):
409 if re.search(r"\} tempest\.", line.decode("utf-8")):
410 LOGGER.info(line.rstrip())
411 elif re.search(r'(?=\(UUID=(.*)\))', line.decode("utf-8")):
412 self.verification_id = re.search(
413 r'(?=\(UUID=(.*)\))', line.decode("utf-8")).group(1)
414 f_stdout.write(line.decode("utf-8"))
418 if self.verification_id is None:
419 raise Exception('Verification UUID not found')
420 LOGGER.info('Verification UUID: %s', self.verification_id)
423 "{}/tempest.log".format(self.deployment_dir),
424 "{}/tempest.debug.log".format(self.res_dir))
426 def parse_verifier_result(self):
427 """Parse and save test results."""
428 stat = self.get_verifier_result(self.verification_id)
430 num_executed = stat['num_tests'] - stat['num_skipped']
432 self.result = 100 * stat['num_success'] / num_executed
433 except ZeroDivisionError:
435 if stat['num_tests'] > 0:
436 LOGGER.info("All tests have been skipped")
438 LOGGER.error("No test has been executed")
441 with open(os.path.join(self.res_dir,
442 "rally.log"), 'r') as logfile:
443 output = logfile.read()
445 success_testcases = []
446 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
448 success_testcases.append(match)
449 failed_testcases = []
450 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
452 failed_testcases.append(match)
453 skipped_testcases = []
454 for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',
456 skipped_testcases.append(match)
458 self.details = {"tests_number": stat['num_tests'],
459 "success_number": stat['num_success'],
460 "skipped_number": stat['num_skipped'],
461 "failures_number": stat['num_failures'],
462 "success": success_testcases,
463 "skipped": skipped_testcases,
464 "failures": failed_testcases}
465 except Exception: # pylint: disable=broad-except
468 LOGGER.info("Tempest %s success_rate is %s%%",
469 self.case_name, self.result)
471 def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
472 """Set image name as tempest img_name_regex"""
473 rconfig = configparser.RawConfigParser()
474 rconfig.read(rally_conf)
475 if not rconfig.has_section('openstack'):
476 rconfig.add_section('openstack')
477 rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
479 with open(rally_conf, 'w') as config_file:
480 rconfig.write(config_file)
482 def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
483 """Detect and update the default role if required"""
484 role = self.get_default_role(self.cloud)
487 rconfig = configparser.RawConfigParser()
488 rconfig.read(rally_conf)
489 if not rconfig.has_section('openstack'):
490 rconfig.add_section('openstack')
491 rconfig.set('openstack', 'swift_operator_role', role.name)
492 with open(rally_conf, 'w') as config_file:
493 rconfig.write(config_file)
496 def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
497 """Clean Rally config"""
498 rconfig = configparser.RawConfigParser()
499 rconfig.read(rally_conf)
500 if rconfig.has_option('openstack', 'img_name_regex'):
501 rconfig.remove_option('openstack', 'img_name_regex')
502 if rconfig.has_option('openstack', 'swift_operator_role'):
503 rconfig.remove_option('openstack', 'swift_operator_role')
504 with open(rally_conf, 'w') as config_file:
505 rconfig.write(config_file)
507 def update_network_section(self):
508 """Update network section in tempest.conf"""
509 rconfig = configparser.RawConfigParser()
510 rconfig.read(self.conf_file)
511 if not rconfig.has_section('network'):
512 rconfig.add_section('network')
513 rconfig.set('network', 'public_network_id', self.ext_net.id)
514 rconfig.set('network', 'floating_network_name', self.ext_net.name)
515 with open(self.conf_file, 'w') as config_file:
516 rconfig.write(config_file)
518 def update_compute_section(self):
519 """Update compute section in tempest.conf"""
520 rconfig = configparser.RawConfigParser()
521 rconfig.read(self.conf_file)
522 if not rconfig.has_section('compute'):
523 rconfig.add_section('compute')
524 rconfig.set('compute', 'fixed_network_name', self.network.name)
525 with open(self.conf_file, 'w') as config_file:
526 rconfig.write(config_file)
528 def update_scenario_section(self):
529 """Update scenario section in tempest.conf"""
530 rconfig = configparser.RawConfigParser()
531 rconfig.read(self.conf_file)
533 config.CONF, '{}_image'.format(self.case_name), self.filename)
534 if not rconfig.has_section('scenario'):
535 rconfig.add_section('scenario')
536 rconfig.set('scenario', 'img_file', os.path.basename(filename))
537 rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
538 rconfig.set('scenario', 'img_disk_format', getattr(
539 config.CONF, '{}_image_format'.format(self.case_name),
541 extra_properties = self.extra_properties.copy()
542 if env.get('IMAGE_PROPERTIES'):
543 extra_properties.update(
544 functest_utils.convert_ini_to_dict(
545 env.get('IMAGE_PROPERTIES')))
546 extra_properties.update(
547 getattr(config.CONF, '{}_extra_properties'.format(
548 self.case_name), {}))
550 'scenario', 'img_properties',
551 functest_utils.convert_dict_to_ini(extra_properties))
552 with open(self.conf_file, 'w') as config_file:
553 rconfig.write(config_file)
555 def configure(self, **kwargs): # pylint: disable=unused-argument
557 Create all openstack resources for tempest-based testcases and write
560 if not os.path.exists(self.res_dir):
561 os.makedirs(self.res_dir)
562 self.deployment_id = rally.RallyBase.create_rally_deployment(
563 environ=self.project.get_environ())
564 if not self.deployment_id:
565 raise Exception("Deployment create failed")
566 self.verifier_id = self.create_verifier()
567 if not self.verifier_id:
568 raise Exception("Verifier create failed")
569 self.verifier_repo_dir = self.get_verifier_repo_dir(
571 self.deployment_dir = self.get_verifier_deployment_dir(
572 self.verifier_id, self.deployment_id)
574 compute_cnt = self.count_hypervisors()
575 self.image_alt = self.publish_image_alt()
576 self.flavor_alt = self.create_flavor_alt()
577 LOGGER.debug("flavor: %s", self.flavor_alt)
579 self.conf_file = self.configure_verifier(self.deployment_dir)
580 if not self.conf_file:
581 raise Exception("Tempest verifier configuring failed")
582 self.configure_tempest_update_params(
584 image_id=self.image.id,
585 flavor_id=self.flavor.id,
586 compute_cnt=compute_cnt,
587 image_alt_id=self.image_alt.id,
588 flavor_alt_id=self.flavor_alt.id,
589 admin_role_name=self.role_name, cidr=self.cidr,
590 domain_id=self.project.domain.id)
591 self.update_network_section()
592 self.update_compute_section()
593 self.update_scenario_section()
594 self.backup_tempest_config(self.conf_file, self.res_dir)
596 def run(self, **kwargs):
597 self.start_time = time.time()
599 assert super(TempestCommon, self).run(
600 **kwargs) == testcase.TestCase.EX_OK
601 if not os.path.exists(self.res_dir):
602 os.makedirs(self.res_dir)
603 self.update_rally_regex()
604 self.update_default_role()
605 rally.RallyBase.update_rally_logs(self.res_dir)
606 shutil.copy("/etc/rally/rally.conf", self.res_dir)
607 self.configure(**kwargs)
608 self.generate_test_list(**kwargs)
609 self.apply_tempest_blacklist(TempestCommon.tempest_blacklist)
610 if env.get('PUBLIC_ENDPOINT_ONLY').lower() == 'true':
611 self.apply_tempest_blacklist(
612 TempestCommon.tempest_public_blacklist)
613 self.run_verifier_tests(**kwargs)
614 self.parse_verifier_result()
615 rally.RallyBase.verify_report(
616 os.path.join(self.res_dir, "tempest-report.html"),
617 self.verification_id)
618 rally.RallyBase.verify_report(
619 os.path.join(self.res_dir, "tempest-report.xml"),
620 self.verification_id, "junit-xml")
621 res = testcase.TestCase.EX_OK
622 except Exception: # pylint: disable=broad-except
623 LOGGER.exception('Error with run')
625 res = testcase.TestCase.EX_RUN_ERROR
626 self.stop_time = time.time()
631 Cleanup all OpenStack objects. Should be called on completion.
633 self.clean_rally_conf()
634 rally.RallyBase.clean_rally_logs()
636 self.cloud.delete_image(self.image_alt)
638 self.orig_cloud.delete_flavor(self.flavor_alt.id)
639 super(TempestCommon, self).clean()
641 def is_successful(self):
642 """The overall result of the test."""
643 skips = self.details.get("skipped_number", 0)
644 if skips > 0 and self.deny_skipping:
645 return testcase.TestCase.EX_TESTCASE_FAILED
646 if self.tests_count and (
647 self.details.get("tests_number", 0) != self.tests_count):
648 return testcase.TestCase.EX_TESTCASE_FAILED
649 return super(TempestCommon, self).is_successful()
652 class TempestScenario(TempestCommon):
653 """Tempest scenario testcase implementation class."""
658 def run(self, **kwargs):
659 self.orig_cloud.set_compute_quotas(
660 self.project.project.name,
661 instances=self.quota_instances,
662 cores=self.quota_cores)
663 return super(TempestScenario, self).run(**kwargs)
666 class TempestHorizon(TempestCommon):
667 """Tempest Horizon testcase implementation class."""
669 def configure(self, **kwargs):
670 super(TempestHorizon, self).configure(**kwargs)
671 rconfig = configparser.RawConfigParser()
672 rconfig.read(self.conf_file)
673 if not rconfig.has_section('dashboard'):
674 rconfig.add_section('dashboard')
675 rconfig.set('dashboard', 'dashboard_url', env.get('DASHBOARD_URL'))
676 with open(self.conf_file, 'w') as config_file:
677 rconfig.write(config_file)
678 self.backup_tempest_config(self.conf_file, self.res_dir)