3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Rally testcases implementation."""
13 from __future__ import division
25 from ruamel.yaml import YAML
26 from six.moves import configparser
27 from xtesting.core import testcase
28 from xtesting.energy import energy
31 from functest.core import singlevm
32 from functest.opnfv_tests.openstack.tempest import conf_utils
33 from functest.utils import config
34 from functest.utils import env
36 LOGGER = logging.getLogger(__name__)
39 class RallyBase(singlevm.VmReady2):
40 """Base class form Rally testcases implementation."""
42 # pylint: disable=too-many-instance-attributes
43 TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
44 'keystone', 'neutron', 'nova', 'quotas']
46 RALLY_DIR = pkg_resources.resource_filename(
47 'functest', 'opnfv_tests/openstack/rally')
48 RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
49 'functest', 'opnfv_tests/openstack/rally/scenario')
50 TEMPLATE_DIR = pkg_resources.resource_filename(
51 'functest', 'opnfv_tests/openstack/rally/scenario/templates')
52 SUPPORT_DIR = pkg_resources.resource_filename(
53 'functest', 'opnfv_tests/openstack/rally/scenario/support')
56 ITERATIONS_AMOUNT = 10
58 BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
59 TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
60 TEMP_DIR = os.path.join(TASK_DIR, 'var')
65 def __init__(self, **kwargs):
66 """Initialize RallyBase object."""
67 super(RallyBase, self).__init__(**kwargs)
68 assert self.orig_cloud
70 if self.orig_cloud.get_role("admin"):
72 elif self.orig_cloud.get_role("Admin"):
75 raise Exception("Cannot detect neither admin nor Admin")
76 self.orig_cloud.grant_role(
77 role_name, user=self.project.user.id,
78 project=self.project.project.id,
79 domain=self.project.domain.id)
80 self.results_dir = os.path.join(
81 getattr(config.CONF, 'dir_results'), self.case_name)
85 self.scenario_dir = ''
88 self.start_time = None
92 self.flavor_alt = None
95 self.network_extensions = []
97 def build_task_args(self, test_name):
98 """Build arguments for the Rally task."""
99 task_args = {'service_list': [test_name]}
100 task_args['image_name'] = str(self.image.name)
101 task_args['flavor_name'] = str(self.flavor.name)
102 task_args['flavor_alt_name'] = str(self.flavor_alt.name)
103 task_args['glance_image_location'] = str(self.filename)
104 task_args['glance_image_format'] = str(self.image_format)
105 task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
106 task_args['sup_dir'] = str(self.SUPPORT_DIR)
107 task_args['users_amount'] = self.USERS_AMOUNT
108 task_args['tenants_amount'] = self.TENANTS_AMOUNT
109 task_args['use_existing_users'] = False
110 task_args['iterations'] = self.ITERATIONS_AMOUNT
111 task_args['concurrency'] = self.CONCURRENCY
112 task_args['smoke'] = self.smoke
115 task_args['floating_network'] = str(self.ext_net.name)
117 task_args['floating_network'] = ''
120 task_args['netid'] = str(self.network.id)
122 task_args['netid'] = ''
126 def _prepare_test_list(self, test_name):
127 """Build the list of test cases to be executed."""
128 test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
129 scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
132 if not os.path.exists(scenario_file_name):
133 scenario_file_name = os.path.join(self.scenario_dir,
136 if not os.path.exists(scenario_file_name):
137 raise Exception("The scenario '%s' does not exist."
138 % scenario_file_name)
140 LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
141 test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
143 if not os.path.exists(self.TEMP_DIR):
144 os.makedirs(self.TEMP_DIR)
146 self.apply_blacklist(scenario_file_name, test_file_name)
147 return test_file_name
150 def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
151 """Set keystone_default_role in rally.conf"""
152 if env.get("NEW_USER_ROLE").lower() != "member":
153 rconfig = configparser.RawConfigParser()
154 rconfig.read(rally_conf)
155 if not rconfig.has_section('openstack'):
156 rconfig.add_section('openstack')
158 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
159 with open(rally_conf, 'wb') as config_file:
160 rconfig.write(config_file)
163 def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
164 """Clean Rally config"""
165 if env.get("NEW_USER_ROLE").lower() != "member":
166 rconfig = configparser.RawConfigParser()
167 rconfig.read(rally_conf)
168 if rconfig.has_option('openstack', 'keystone_default_role'):
169 rconfig.remove_option('openstack', 'keystone_default_role')
170 with open(rally_conf, 'wb') as config_file:
171 rconfig.write(config_file)
174 def get_task_id(cmd_raw):
176 Get task id from command rally result.
179 :return: task_id as string
181 taskid_re = re.compile('^Task +(.*): started$')
182 for line in cmd_raw.splitlines(True):
184 match = taskid_re.match(line)
186 return match.group(1)
190 def task_succeed(json_raw):
192 Parse JSON from rally JSON results.
197 rally_report = json.loads(json_raw)
198 tasks = rally_report.get('tasks')
201 if task.get('status') != 'finished' or \
202 task.get('pass_sla') is not True:
208 def _migration_supported(self):
209 """Determine if migration is supported."""
210 if self.compute_cnt > 1:
214 def _network_trunk_supported(self):
215 """Determine if network trunk service is available"""
216 if 'trunk' in self.network_extensions:
222 """Exclude scenario."""
225 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
226 black_list_yaml = yaml.safe_load(black_list_file)
228 deploy_scenario = env.get('DEPLOY_SCENARIO')
229 if (bool(deploy_scenario) and
230 'scenario' in black_list_yaml.keys()):
231 for item in black_list_yaml['scenario']:
232 scenarios = item['scenarios']
233 in_it = RallyBase.in_iterable_re
234 if in_it(deploy_scenario, scenarios):
235 tests = item['tests']
236 black_tests.extend(tests)
237 except Exception: # pylint: disable=broad-except
238 LOGGER.debug("Scenario exclusion not applied.")
243 def in_iterable_re(needle, haystack):
245 Check if given needle is in the iterable haystack, using regex.
247 :param needle: string to be matched
248 :param haystack: iterable of strings (optionally regex patterns)
249 :return: True if needle is eqial to any of the elements in haystack,
250 or if a nonempty regex pattern in haystack is found in needle.
252 # match without regex
253 if needle in haystack:
256 for pattern in haystack:
257 # match if regex pattern is set and found in the needle
258 if pattern and re.search(pattern, needle) is not None:
264 """Exclude functionalities."""
269 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
270 black_list_yaml = yaml.safe_load(black_list_file)
272 if not self._migration_supported():
273 func_list.append("no_migration")
274 if not self._network_trunk_supported():
275 func_list.append("no_net_trunk_service")
277 if 'functionality' in black_list_yaml.keys():
278 for item in black_list_yaml['functionality']:
279 functions = item['functions']
280 for func in func_list:
281 if func in functions:
282 tests = item['tests']
283 black_tests.extend(tests)
284 except Exception: # pylint: disable=broad-except
285 LOGGER.debug("Functionality exclusion not applied.")
289 def apply_blacklist(self, case_file_name, result_file_name):
290 """Apply blacklist."""
291 LOGGER.debug("Applying blacklist...")
292 cases_file = open(case_file_name, 'r')
293 result_file = open(result_file_name, 'w')
295 black_tests = list(set(self.excl_func() +
296 self.excl_scenario()))
299 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
302 for cases_line in cases_file:
304 for black_tests_line in black_tests:
305 if re.search(black_tests_line,
306 cases_line.strip().rstrip(':')):
310 result_file.write(str(cases_line))
312 if cases_line.isspace():
319 def file_is_empty(file_name):
320 """Determine is a file is empty."""
322 if os.stat(file_name).st_size > 0:
324 except Exception: # pylint: disable=broad-except
329 def _save_results(self, test_name, task_id):
330 """ Generate and save task execution results"""
331 # check for result directory and create it otherwise
332 if not os.path.exists(self.results_dir):
333 LOGGER.debug('%s does not exist, we create it.',
335 os.makedirs(self.results_dir)
337 # put detailed result to log
338 cmd = (["rally", "task", "detailed", "--uuid", task_id])
339 LOGGER.debug('running command: %s', cmd)
340 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
341 LOGGER.info("%s\n%s", " ".join(cmd), output)
343 # save report as JSON
344 report_json_name = '{}.json'.format(test_name)
345 report_json_dir = os.path.join(self.results_dir, report_json_name)
346 cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
347 "--out", report_json_dir])
348 LOGGER.debug('running command: %s', cmd)
349 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
350 LOGGER.info("%s\n%s", " ".join(cmd), output)
352 # save report as HTML
353 report_html_name = '{}.html'.format(test_name)
354 report_html_dir = os.path.join(self.results_dir, report_html_name)
355 cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
356 "--out", report_html_dir])
357 LOGGER.debug('running command: %s', cmd)
358 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
359 LOGGER.info("%s\n%s", " ".join(cmd), output)
361 json_results = open(report_json_dir).read()
362 self._append_summary(json_results, test_name)
364 # parse JSON operation result
365 if self.task_succeed(json_results):
366 LOGGER.info('Test scenario: "%s" OK.', test_name)
368 LOGGER.info('Test scenario: "%s" Failed.', test_name)
370 def run_task(self, test_name):
372 LOGGER.info('Starting test scenario "%s" ...', test_name)
373 LOGGER.debug('running command: %s', self.run_cmd)
374 proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
375 stderr=subprocess.STDOUT)
376 output = proc.communicate()[0]
378 task_id = self.get_task_id(output)
379 LOGGER.debug('task_id : %s', task_id)
381 LOGGER.error("Failed to retrieve task_id")
382 LOGGER.error("Result:\n%s", output)
383 raise Exception("Failed to retrieve task id")
384 self._save_results(test_name, task_id)
386 def _append_summary(self, json_raw, test_name):
387 """Update statistics summary info."""
390 overall_duration = 0.0
392 rally_report = json.loads(json_raw)
393 for task in rally_report.get('tasks'):
394 for subtask in task.get('subtasks'):
395 for workload in subtask.get('workloads'):
396 if workload.get('full_duration'):
397 overall_duration += workload.get('full_duration')
399 if workload.get('data'):
400 nb_tests += len(workload.get('data'))
402 for result in workload.get('data'):
403 if not result.get('error'):
406 scenario_summary = {'test_name': test_name,
407 'overall_duration': overall_duration,
408 'nb_tests': nb_tests,
409 'nb_success': nb_success,
410 'task_status': self.task_succeed(json_raw)}
411 self.summary.append(scenario_summary)
413 def prepare_run(self):
414 """Prepare resources needed by test scenarios."""
416 LOGGER.debug('Validating the test name...')
417 if self.test_name == 'all':
418 self.tests = self.TESTS
419 elif self.test_name in self.TESTS:
420 self.tests = [self.test_name]
422 raise Exception("Test name '%s' is invalid" % self.test_name)
424 if not os.path.exists(self.TASK_DIR):
425 os.makedirs(self.TASK_DIR)
427 task = os.path.join(self.RALLY_DIR, 'task.yaml')
428 if not os.path.exists(task):
429 LOGGER.error("Task file '%s' does not exist.", task)
430 raise Exception("Task file '{}' does not exist.".
432 self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
433 shutil.copyfile(task, self.task_file)
435 task_macro = os.path.join(self.RALLY_DIR, 'macro')
436 if not os.path.exists(task_macro):
437 LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
438 raise Exception("Task macro dir '{}' does not exist.".
440 macro_dir = os.path.join(self.TASK_DIR, 'macro')
441 if os.path.exists(macro_dir):
442 shutil.rmtree(macro_dir)
443 shutil.copytree(task_macro, macro_dir)
445 self.update_keystone_default_role()
446 self.compute_cnt = len(self.cloud.list_hypervisors())
447 self.network_extensions = self.cloud.get_network_extensions()
448 self.flavor_alt = self.create_flavor_alt()
449 LOGGER.debug("flavor: %s", self.flavor_alt)
451 def prepare_task(self, test_name):
452 """Prepare resources for test run."""
453 file_name = self._prepare_test_list(test_name)
454 if self.file_is_empty(file_name):
455 LOGGER.info('No tests for scenario "%s"', test_name)
457 self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
458 "--task", self.task_file, "--task-args",
459 str(self.build_task_args(test_name))])
464 for test in self.tests:
465 if self.prepare_task(test):
468 def _generate_report(self):
469 """Generate test execution summary report."""
476 res_table = prettytable.PrettyTable(
478 field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
479 res_table.align['Module'] = "l"
480 res_table.align['Duration'] = "r"
481 res_table.align['Success'] = "r"
483 # for each scenario we draw a row for the table
484 for item in self.summary:
485 if item['task_status'] is True:
487 total_duration += item['overall_duration']
488 total_nb_tests += item['nb_tests']
489 total_nb_success += item['nb_success']
491 success_avg = 100 * item['nb_success'] / item['nb_tests']
492 except ZeroDivisionError:
494 success_str = str("{:0.2f}".format(success_avg)) + '%'
495 duration_str = time.strftime("%H:%M:%S",
496 time.gmtime(item['overall_duration']))
497 res_table.add_row([item['test_name'], duration_str,
498 item['nb_tests'], success_str])
499 payload.append({'module': item['test_name'],
500 'details': {'duration': item['overall_duration'],
501 'nb tests': item['nb_tests'],
502 'success': success_str}})
504 total_duration_str = time.strftime("%H:%M:%S",
505 time.gmtime(total_duration))
507 self.result = 100 * total_nb_success / total_nb_tests
508 except ZeroDivisionError:
510 success_rate = "{:0.2f}".format(self.result)
511 success_rate_str = str(success_rate) + '%'
512 res_table.add_row(["", "", "", ""])
513 res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
516 LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
517 LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
518 self.case_name, success_rate, nb_modules,
520 payload.append({'summary': {'duration': total_duration,
521 'nb tests': total_nb_tests,
522 'nb success': success_rate}})
523 self.details = payload
526 """Cleanup of OpenStack resources. Should be called on completion."""
527 self.clean_rally_conf()
529 self.orig_cloud.delete_flavor(self.flavor_alt.id)
530 super(RallyBase, self).clean()
532 def is_successful(self):
533 """The overall result of the test."""
534 for item in self.summary:
535 if item['task_status'] is False:
536 return testcase.TestCase.EX_TESTCASE_FAILED
538 return super(RallyBase, self).is_successful()
540 @energy.enable_recording
541 def run(self, **kwargs):
543 self.start_time = time.time()
545 assert super(RallyBase, self).run(
546 **kwargs) == testcase.TestCase.EX_OK
549 OS_USERNAME=self.project.user.name,
550 OS_PROJECT_NAME=self.project.project.name,
551 OS_PROJECT_ID=self.project.project.id,
552 OS_PASSWORD=self.project.password)
554 del environ['OS_TENANT_NAME']
555 del environ['OS_TENANT_ID']
556 except Exception: # pylint: disable=broad-except
558 conf_utils.create_rally_deployment(environ=environ)
561 self._generate_report()
562 res = testcase.TestCase.EX_OK
563 except Exception as exc: # pylint: disable=broad-except
564 LOGGER.error('Error with run: %s', exc)
566 res = testcase.TestCase.EX_RUN_ERROR
567 self.stop_time = time.time()
571 class RallySanity(RallyBase):
572 """Rally sanity testcase implementation."""
574 def __init__(self, **kwargs):
575 """Initialize RallySanity object."""
576 if "case_name" not in kwargs:
577 kwargs["case_name"] = "rally_sanity"
578 super(RallySanity, self).__init__(**kwargs)
579 self.test_name = 'all'
581 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
584 class RallyFull(RallyBase):
585 """Rally full testcase implementation."""
587 def __init__(self, **kwargs):
588 """Initialize RallyFull object."""
589 if "case_name" not in kwargs:
590 kwargs["case_name"] = "rally_full"
591 super(RallyFull, self).__init__(**kwargs)
592 self.test_name = 'all'
594 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
597 class RallyJobs(RallyBase):
598 """Rally OpenStack CI testcase implementation."""
602 def __init__(self, **kwargs):
603 """Initialize RallyJobs object."""
604 if "case_name" not in kwargs:
605 kwargs["case_name"] = "rally_jobs"
606 super(RallyJobs, self).__init__(**kwargs)
607 self.test_name = 'all'
608 self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
609 self.task_yaml = None
611 def prepare_run(self):
612 """Create resources needed by test scenarios."""
613 super(RallyJobs, self).prepare_run()
614 with open(os.path.join(self.RALLY_DIR,
615 'rally_jobs.yaml'), 'r') as task_file:
616 self.task_yaml = yaml.safe_load(task_file)
618 if not all(task in self.task_yaml for task in self.tests):
619 raise Exception("Test '%s' not in '%s'" %
620 (self.test_name, self.tests))
622 def apply_blacklist(self, case_file_name, result_file_name):
623 # pylint: disable=too-many-branches
624 """Apply blacklist."""
625 LOGGER.debug("Applying blacklist...")
626 black_tests = list(set(self.excl_func() +
627 self.excl_scenario()))
629 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
631 template = YAML(typ='jinja2')
632 with open(case_file_name, 'r') as fname:
633 cases = template.load(fname)
634 if cases.get("version", 1) == 1:
635 # scenarios in dictionary
636 for name in cases.keys():
637 if self.in_iterable_re(name, black_tests):
640 # workloads in subtasks
641 for sind, subtask in enumerate(cases.get('subtasks', [])):
643 for wind, workload in enumerate(subtask.get('workloads', [])):
644 scenario = workload.get('scenario', {})
645 for name in scenario.keys():
646 if self.in_iterable_re(name, black_tests):
649 for wind in reversed(idx):
650 cases['subtasks'][sind]['workloads'].pop(wind)
651 # scenarios in subtasks
653 for sind, subtask in enumerate(cases.get('subtasks', [])):
654 scenario = subtask.get('scenario', {})
655 for name in scenario.keys():
656 if self.in_iterable_re(name, black_tests):
659 for sind in reversed(idx):
660 cases['subtasks'].pop(sind)
662 with open(result_file_name, 'w') as fname:
663 template.dump(cases, fname)
665 def build_task_args(self, test_name):
666 """Build arguments for the Rally task."""
669 task_args['floating_network'] = str(self.ext_net.name)
671 task_args['floating_network'] = ''
675 def _remove_plugins_extra():
676 inst_dir = getattr(config.CONF, 'dir_rally_inst')
678 shutil.rmtree(os.path.join(inst_dir, 'plugins'))
679 shutil.rmtree(os.path.join(inst_dir, 'extra'))
680 except Exception: # pylint: disable=broad-except
683 def prepare_task(self, test_name):
684 """Prepare resources for test run."""
685 self._remove_plugins_extra()
686 jobs_dir = os.path.join(
687 getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
688 inst_dir = getattr(config.CONF, 'dir_rally_inst')
689 shutil.copytree(os.path.join(jobs_dir, 'plugins'),
690 os.path.join(inst_dir, 'plugins'))
691 shutil.copytree(os.path.join(jobs_dir, 'extra'),
692 os.path.join(inst_dir, 'extra'))
694 task_name = self.task_yaml.get(test_name).get("task")
695 task = os.path.join(jobs_dir, task_name)
696 if not os.path.exists(task):
697 raise Exception("The scenario '%s' does not exist." % task)
698 LOGGER.debug('Scenario fetched from : %s', task)
700 if not os.path.exists(self.TEMP_DIR):
701 os.makedirs(self.TEMP_DIR)
702 task_file_name = os.path.join(self.TEMP_DIR, task_name)
703 self.apply_blacklist(task, task_file_name)
704 self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
705 "--task-args", str(self.build_task_args(test_name))])
709 self._remove_plugins_extra()
710 super(RallyJobs, self).clean()