3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Rally testcases implementation."""
13 from __future__ import division
25 from ruamel.yaml import YAML
26 from six.moves import configparser
27 from xtesting.core import testcase
30 from functest.core import singlevm
31 from functest.opnfv_tests.openstack.tempest import conf_utils
32 from functest.utils import config
33 from functest.utils import env
35 LOGGER = logging.getLogger(__name__)
38 class RallyBase(singlevm.VmReady2):
39 """Base class form Rally testcases implementation."""
41 # pylint: disable=too-many-instance-attributes
42 TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
43 'keystone', 'neutron', 'nova', 'quotas']
45 RALLY_DIR = pkg_resources.resource_filename(
46 'functest', 'opnfv_tests/openstack/rally')
47 RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
48 'functest', 'opnfv_tests/openstack/rally/scenario')
49 TEMPLATE_DIR = pkg_resources.resource_filename(
50 'functest', 'opnfv_tests/openstack/rally/scenario/templates')
51 SUPPORT_DIR = pkg_resources.resource_filename(
52 'functest', 'opnfv_tests/openstack/rally/scenario/support')
55 ITERATIONS_AMOUNT = 10
57 BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
58 TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
59 TEMP_DIR = os.path.join(TASK_DIR, 'var')
64 def __init__(self, **kwargs):
65 """Initialize RallyBase object."""
66 super(RallyBase, self).__init__(**kwargs)
67 assert self.orig_cloud
69 if self.orig_cloud.get_role("admin"):
71 elif self.orig_cloud.get_role("Admin"):
74 raise Exception("Cannot detect neither admin nor Admin")
75 self.orig_cloud.grant_role(
76 role_name, user=self.project.user.id,
77 project=self.project.project.id,
78 domain=self.project.domain.id)
79 self.results_dir = os.path.join(
80 getattr(config.CONF, 'dir_results'), self.case_name)
84 self.scenario_dir = ''
86 self.start_time = None
90 self.flavor_alt = None
93 self.network_extensions = []
96 def build_task_args(self, test_name):
97 """Build arguments for the Rally task."""
98 task_args = {'service_list': [test_name]}
99 task_args['image_name'] = str(self.image.name)
100 task_args['flavor_name'] = str(self.flavor.name)
101 task_args['flavor_alt_name'] = str(self.flavor_alt.name)
102 task_args['glance_image_location'] = str(self.filename)
103 task_args['glance_image_format'] = str(self.image_format)
104 task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
105 task_args['sup_dir'] = str(self.SUPPORT_DIR)
106 task_args['users_amount'] = self.USERS_AMOUNT
107 task_args['tenants_amount'] = self.TENANTS_AMOUNT
108 task_args['use_existing_users'] = False
109 task_args['iterations'] = self.ITERATIONS_AMOUNT
110 task_args['concurrency'] = self.CONCURRENCY
111 task_args['smoke'] = self.smoke
114 task_args['floating_network'] = str(self.ext_net.name)
116 task_args['floating_network'] = ''
119 task_args['netid'] = str(self.network.id)
121 task_args['netid'] = ''
125 def _prepare_test_list(self, test_name):
126 """Build the list of test cases to be executed."""
127 test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
128 scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
131 if not os.path.exists(scenario_file_name):
132 scenario_file_name = os.path.join(self.scenario_dir,
135 if not os.path.exists(scenario_file_name):
136 raise Exception("The scenario '%s' does not exist."
137 % scenario_file_name)
139 LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
140 test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
142 if not os.path.exists(self.TEMP_DIR):
143 os.makedirs(self.TEMP_DIR)
145 self.apply_blacklist(scenario_file_name, test_file_name)
146 return test_file_name
149 def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
150 """Set keystone_default_role in rally.conf"""
151 if env.get("NEW_USER_ROLE").lower() != "member":
152 rconfig = configparser.RawConfigParser()
153 rconfig.read(rally_conf)
154 if not rconfig.has_section('openstack'):
155 rconfig.add_section('openstack')
157 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
158 with open(rally_conf, 'wb') as config_file:
159 rconfig.write(config_file)
162 def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
163 """Clean Rally config"""
164 if env.get("NEW_USER_ROLE").lower() != "member":
165 rconfig = configparser.RawConfigParser()
166 rconfig.read(rally_conf)
167 if rconfig.has_option('openstack', 'keystone_default_role'):
168 rconfig.remove_option('openstack', 'keystone_default_role')
169 with open(rally_conf, 'wb') as config_file:
170 rconfig.write(config_file)
173 def get_task_id(cmd_raw):
175 Get task id from command rally result.
178 :return: task_id as string
180 taskid_re = re.compile('^Task +(.*): started$')
181 for line in cmd_raw.splitlines(True):
183 match = taskid_re.match(line)
185 return match.group(1)
189 def task_succeed(json_raw):
191 Parse JSON from rally JSON results.
196 rally_report = json.loads(json_raw)
197 tasks = rally_report.get('tasks')
200 if task.get('status') != 'finished' or \
201 task.get('pass_sla') is not True:
207 def _migration_supported(self):
208 """Determine if migration is supported."""
209 if self.compute_cnt > 1:
213 def _network_trunk_supported(self):
214 """Determine if network trunk service is available"""
215 if 'trunk' in self.network_extensions:
221 """Exclude scenario."""
224 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
225 black_list_yaml = yaml.safe_load(black_list_file)
227 deploy_scenario = env.get('DEPLOY_SCENARIO')
228 if (bool(deploy_scenario) and
229 'scenario' in black_list_yaml.keys()):
230 for item in black_list_yaml['scenario']:
231 scenarios = item['scenarios']
232 in_it = RallyBase.in_iterable_re
233 if in_it(deploy_scenario, scenarios):
234 tests = item['tests']
235 black_tests.extend(tests)
236 except Exception: # pylint: disable=broad-except
237 LOGGER.debug("Scenario exclusion not applied.")
242 def in_iterable_re(needle, haystack):
244 Check if given needle is in the iterable haystack, using regex.
246 :param needle: string to be matched
247 :param haystack: iterable of strings (optionally regex patterns)
248 :return: True if needle is eqial to any of the elements in haystack,
249 or if a nonempty regex pattern in haystack is found in needle.
251 # match without regex
252 if needle in haystack:
255 for pattern in haystack:
256 # match if regex pattern is set and found in the needle
257 if pattern and re.search(pattern, needle) is not None:
263 """Exclude functionalities."""
268 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
269 black_list_yaml = yaml.safe_load(black_list_file)
271 if not self._migration_supported():
272 func_list.append("no_migration")
273 if not self._network_trunk_supported():
274 func_list.append("no_net_trunk_service")
276 if 'functionality' in black_list_yaml.keys():
277 for item in black_list_yaml['functionality']:
278 functions = item['functions']
279 for func in func_list:
280 if func in functions:
281 tests = item['tests']
282 black_tests.extend(tests)
283 except Exception: # pylint: disable=broad-except
284 LOGGER.debug("Functionality exclusion not applied.")
288 def apply_blacklist(self, case_file_name, result_file_name):
289 """Apply blacklist."""
290 LOGGER.debug("Applying blacklist...")
291 cases_file = open(case_file_name, 'r')
292 result_file = open(result_file_name, 'w')
294 black_tests = list(set(self.excl_func() +
295 self.excl_scenario()))
298 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
301 for cases_line in cases_file:
303 for black_tests_line in black_tests:
304 if re.search(black_tests_line,
305 cases_line.strip().rstrip(':')):
309 result_file.write(str(cases_line))
311 if cases_line.isspace():
318 def file_is_empty(file_name):
319 """Determine is a file is empty."""
321 if os.stat(file_name).st_size > 0:
323 except Exception: # pylint: disable=broad-except
328 def _save_results(self, test_name, task_id):
329 """ Generate and save task execution results"""
330 # check for result directory and create it otherwise
331 if not os.path.exists(self.results_dir):
332 LOGGER.debug('%s does not exist, we create it.',
334 os.makedirs(self.results_dir)
336 # put detailed result to log
337 cmd = (["rally", "task", "detailed", "--uuid", task_id])
338 LOGGER.debug('running command: %s', cmd)
339 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
340 LOGGER.info("%s\n%s", " ".join(cmd), output)
342 # save report as JSON
343 report_json_name = '{}.json'.format(test_name)
344 report_json_dir = os.path.join(self.results_dir, report_json_name)
345 cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
346 "--out", report_json_dir])
347 LOGGER.debug('running command: %s', cmd)
348 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
349 LOGGER.info("%s\n%s", " ".join(cmd), output)
351 json_results = open(report_json_dir).read()
352 self._append_summary(json_results, test_name)
354 # parse JSON operation result
355 if self.task_succeed(json_results):
356 LOGGER.info('Test scenario: "%s" OK.', test_name)
358 LOGGER.info('Test scenario: "%s" Failed.', test_name)
360 def run_task(self, test_name):
362 LOGGER.info('Starting test scenario "%s" ...', test_name)
363 LOGGER.debug('running command: %s', self.run_cmd)
364 proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
365 stderr=subprocess.STDOUT)
366 output = proc.communicate()[0]
368 task_id = self.get_task_id(output)
369 LOGGER.debug('task_id : %s', task_id)
371 LOGGER.error("Failed to retrieve task_id")
372 LOGGER.error("Result:\n%s", output)
373 raise Exception("Failed to retrieve task id")
374 self._save_results(test_name, task_id)
376 def _append_summary(self, json_raw, test_name):
377 """Update statistics summary info."""
380 overall_duration = 0.0
382 rally_report = json.loads(json_raw)
383 for task in rally_report.get('tasks'):
384 for subtask in task.get('subtasks'):
385 for workload in subtask.get('workloads'):
386 if workload.get('full_duration'):
387 overall_duration += workload.get('full_duration')
389 if workload.get('data'):
390 nb_tests += len(workload.get('data'))
392 for result in workload.get('data'):
393 if not result.get('error'):
396 scenario_summary = {'test_name': test_name,
397 'overall_duration': overall_duration,
398 'nb_tests': nb_tests,
399 'nb_success': nb_success,
400 'task_status': self.task_succeed(json_raw)}
401 self.summary.append(scenario_summary)
403 def prepare_run(self, **kwargs):
404 """Prepare resources needed by test scenarios."""
406 LOGGER.debug('Validating run tests...')
407 for test in kwargs.get('tests', self.TESTS):
408 if test in self.TESTS:
409 self.tests.append(test)
411 raise Exception("Test name '%s' is invalid" % test)
413 if not os.path.exists(self.TASK_DIR):
414 os.makedirs(self.TASK_DIR)
416 task = os.path.join(self.RALLY_DIR, 'task.yaml')
417 if not os.path.exists(task):
418 LOGGER.error("Task file '%s' does not exist.", task)
419 raise Exception("Task file '{}' does not exist.".
421 self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
422 shutil.copyfile(task, self.task_file)
424 task_macro = os.path.join(self.RALLY_DIR, 'macro')
425 if not os.path.exists(task_macro):
426 LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
427 raise Exception("Task macro dir '{}' does not exist.".
429 macro_dir = os.path.join(self.TASK_DIR, 'macro')
430 if os.path.exists(macro_dir):
431 shutil.rmtree(macro_dir)
432 shutil.copytree(task_macro, macro_dir)
434 self.update_keystone_default_role()
435 self.compute_cnt = len(self.cloud.list_hypervisors())
436 self.network_extensions = self.cloud.get_network_extensions()
437 self.flavor_alt = self.create_flavor_alt()
438 self.services = [service.name for service in
439 self.cloud.list_services()]
441 LOGGER.debug("flavor: %s", self.flavor_alt)
443 def prepare_task(self, test_name):
444 """Prepare resources for test run."""
445 file_name = self._prepare_test_list(test_name)
446 if self.file_is_empty(file_name):
447 LOGGER.info('No tests for scenario "%s"', test_name)
449 self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
450 "--task", self.task_file, "--task-args",
451 str(self.build_task_args(test_name))])
454 def run_tests(self, **kwargs):
456 optional = kwargs.get('optional', [])
457 for test in self.tests:
458 if test in self.services or test not in optional:
459 if self.prepare_task(test):
462 def _generate_report(self):
463 """Generate test execution summary report."""
470 res_table = prettytable.PrettyTable(
472 field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
473 res_table.align['Module'] = "l"
474 res_table.align['Duration'] = "r"
475 res_table.align['Success'] = "r"
477 # for each scenario we draw a row for the table
478 for item in self.summary:
479 if item['task_status'] is True:
481 total_duration += item['overall_duration']
482 total_nb_tests += item['nb_tests']
483 total_nb_success += item['nb_success']
485 success_avg = 100 * item['nb_success'] / item['nb_tests']
486 except ZeroDivisionError:
488 success_str = str("{:0.2f}".format(success_avg)) + '%'
489 duration_str = time.strftime("%H:%M:%S",
490 time.gmtime(item['overall_duration']))
491 res_table.add_row([item['test_name'], duration_str,
492 item['nb_tests'], success_str])
493 payload.append({'module': item['test_name'],
494 'details': {'duration': item['overall_duration'],
495 'nb tests': item['nb_tests'],
496 'success': success_str}})
498 total_duration_str = time.strftime("%H:%M:%S",
499 time.gmtime(total_duration))
501 self.result = 100 * total_nb_success / total_nb_tests
502 except ZeroDivisionError:
504 success_rate = "{:0.2f}".format(self.result)
505 success_rate_str = str(success_rate) + '%'
506 res_table.add_row(["", "", "", ""])
507 res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
510 LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
511 LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
512 self.case_name, success_rate, nb_modules,
514 payload.append({'summary': {'duration': total_duration,
515 'nb tests': total_nb_tests,
516 'nb success': success_rate}})
517 self.details = payload
519 def generate_html_report(self):
520 """Save all task reports as single HTML
523 subprocess.CalledProcessError: if Rally doesn't return 0
528 cmd = ["rally", "task", "report", "--deployment",
529 str(getattr(config.CONF, 'rally_deployment_name')),
530 "--out", "{}/{}.html".format(self.results_dir, self.case_name)]
531 LOGGER.debug('running command: %s', cmd)
532 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
533 LOGGER.info("%s\n%s", " ".join(cmd), output)
536 """Cleanup of OpenStack resources. Should be called on completion."""
537 self.clean_rally_conf()
539 self.orig_cloud.delete_flavor(self.flavor_alt.id)
540 super(RallyBase, self).clean()
542 def is_successful(self):
543 """The overall result of the test."""
544 for item in self.summary:
545 if item['task_status'] is False:
546 return testcase.TestCase.EX_TESTCASE_FAILED
548 return super(RallyBase, self).is_successful()
550 def run(self, **kwargs):
552 self.start_time = time.time()
554 assert super(RallyBase, self).run(
555 **kwargs) == testcase.TestCase.EX_OK
558 OS_USERNAME=self.project.user.name,
559 OS_PROJECT_NAME=self.project.project.name,
560 OS_PROJECT_ID=self.project.project.id,
561 OS_PASSWORD=self.project.password)
563 del environ['OS_TENANT_NAME']
564 del environ['OS_TENANT_ID']
565 except Exception: # pylint: disable=broad-except
567 conf_utils.create_rally_deployment(environ=environ)
568 self.prepare_run(**kwargs)
569 self.run_tests(**kwargs)
570 self._generate_report()
571 self.generate_html_report()
572 res = testcase.TestCase.EX_OK
573 except Exception as exc: # pylint: disable=broad-except
574 LOGGER.error('Error with run: %s', exc)
576 res = testcase.TestCase.EX_RUN_ERROR
577 self.stop_time = time.time()
581 class RallySanity(RallyBase):
582 """Rally sanity testcase implementation."""
584 def __init__(self, **kwargs):
585 """Initialize RallySanity object."""
586 if "case_name" not in kwargs:
587 kwargs["case_name"] = "rally_sanity"
588 super(RallySanity, self).__init__(**kwargs)
590 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
593 class RallyFull(RallyBase):
594 """Rally full testcase implementation."""
596 def __init__(self, **kwargs):
597 """Initialize RallyFull object."""
598 if "case_name" not in kwargs:
599 kwargs["case_name"] = "rally_full"
600 super(RallyFull, self).__init__(**kwargs)
602 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
605 class RallyJobs(RallyBase):
606 """Rally OpenStack CI testcase implementation."""
610 def __init__(self, **kwargs):
611 """Initialize RallyJobs object."""
612 if "case_name" not in kwargs:
613 kwargs["case_name"] = "rally_jobs"
614 super(RallyJobs, self).__init__(**kwargs)
615 self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
616 self.task_yaml = None
618 def prepare_run(self, **kwargs):
619 """Create resources needed by test scenarios."""
620 super(RallyJobs, self).prepare_run(**kwargs)
621 with open(os.path.join(self.RALLY_DIR,
622 'rally_jobs.yaml'), 'r') as task_file:
623 self.task_yaml = yaml.safe_load(task_file)
625 for task in self.task_yaml:
626 if task not in self.tests:
627 raise Exception("Test '%s' not in '%s'" %
630 def apply_blacklist(self, case_file_name, result_file_name):
631 # pylint: disable=too-many-branches
632 """Apply blacklist."""
633 LOGGER.debug("Applying blacklist...")
634 black_tests = list(set(self.excl_func() +
635 self.excl_scenario()))
637 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
639 template = YAML(typ='jinja2')
640 with open(case_file_name, 'r') as fname:
641 cases = template.load(fname)
642 if cases.get("version", 1) == 1:
643 # scenarios in dictionary
644 for name in cases.keys():
645 if self.in_iterable_re(name, black_tests):
648 # workloads in subtasks
649 for sind, subtask in enumerate(cases.get('subtasks', [])):
651 for wind, workload in enumerate(subtask.get('workloads', [])):
652 scenario = workload.get('scenario', {})
653 for name in scenario.keys():
654 if self.in_iterable_re(name, black_tests):
657 for wind in reversed(idx):
658 cases['subtasks'][sind]['workloads'].pop(wind)
659 # scenarios in subtasks
661 for sind, subtask in enumerate(cases.get('subtasks', [])):
662 scenario = subtask.get('scenario', {})
663 for name in scenario.keys():
664 if self.in_iterable_re(name, black_tests):
667 for sind in reversed(idx):
668 cases['subtasks'].pop(sind)
670 with open(result_file_name, 'w') as fname:
671 template.dump(cases, fname)
673 def build_task_args(self, test_name):
674 """Build arguments for the Rally task."""
677 task_args['floating_network'] = str(self.ext_net.name)
679 task_args['floating_network'] = ''
683 def _remove_plugins_extra():
684 inst_dir = getattr(config.CONF, 'dir_rally_inst')
686 shutil.rmtree(os.path.join(inst_dir, 'plugins'))
687 shutil.rmtree(os.path.join(inst_dir, 'extra'))
688 except Exception: # pylint: disable=broad-except
691 def prepare_task(self, test_name):
692 """Prepare resources for test run."""
693 self._remove_plugins_extra()
694 jobs_dir = os.path.join(
695 getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
696 inst_dir = getattr(config.CONF, 'dir_rally_inst')
697 shutil.copytree(os.path.join(jobs_dir, 'plugins'),
698 os.path.join(inst_dir, 'plugins'))
699 shutil.copytree(os.path.join(jobs_dir, 'extra'),
700 os.path.join(inst_dir, 'extra'))
702 task_name = self.task_yaml.get(test_name).get("task")
703 task = os.path.join(jobs_dir, task_name)
704 if not os.path.exists(task):
705 raise Exception("The scenario '%s' does not exist." % task)
706 LOGGER.debug('Scenario fetched from : %s', task)
708 if not os.path.exists(self.TEMP_DIR):
709 os.makedirs(self.TEMP_DIR)
710 task_file_name = os.path.join(self.TEMP_DIR, task_name)
711 self.apply_blacklist(task, task_file_name)
712 self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
713 "--task-args", str(self.build_task_args(test_name))])
717 self._remove_plugins_extra()
718 super(RallyJobs, self).clean()