3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Rally testcases implementation."""
13 from __future__ import division
27 from functest.core import testcase
28 from functest.energy import energy
29 from functest.opnfv_tests.openstack.snaps import snaps_utils
30 from functest.opnfv_tests.openstack.tempest import conf_utils
31 from functest.utils import config
32 from functest.utils import env
34 from snaps.config.flavor import FlavorConfig
35 from snaps.config.image import ImageConfig
36 from snaps.config.network import NetworkConfig, SubnetConfig
37 from snaps.config.router import RouterConfig
39 from snaps.openstack.create_flavor import OpenStackFlavor
40 from snaps.openstack.utils import deploy_utils
42 LOGGER = logging.getLogger(__name__)
45 class RallyBase(testcase.TestCase):
46 """Base class form Rally testcases implementation."""
48 # pylint: disable=too-many-instance-attributes
49 TESTS = ['authenticate', 'glance', 'ceilometer', 'cinder', 'heat',
50 'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all']
51 GLANCE_IMAGE_NAME = getattr(config.CONF, 'openstack_image_name')
52 GLANCE_IMAGE_FILENAME = getattr(config.CONF, 'openstack_image_file_name')
53 GLANCE_IMAGE_PATH = os.path.join(getattr(
54 config.CONF, 'dir_functest_images'), GLANCE_IMAGE_FILENAME)
55 GLANCE_IMAGE_FORMAT = getattr(config.CONF, 'openstack_image_disk_format')
56 GLANCE_IMAGE_USERNAME = getattr(config.CONF, 'openstack_image_username')
57 GLANCE_IMAGE_EXTRA_PROPERTIES = getattr(
58 config.CONF, 'openstack_extra_properties', {})
59 FLAVOR_NAME = getattr(config.CONF, 'rally_flavor_name')
60 FLAVOR_ALT_NAME = getattr(config.CONF, 'rally_flavor_alt_name')
63 FLAVOR_EXTRA_SPECS = getattr(config.CONF, 'flavor_extra_specs', None)
64 if FLAVOR_EXTRA_SPECS:
68 RALLY_DIR = pkg_resources.resource_filename(
69 'functest', 'opnfv_tests/openstack/rally')
70 RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
71 'functest', 'opnfv_tests/openstack/rally/scenario')
72 TEMPLATE_DIR = pkg_resources.resource_filename(
73 'functest', 'opnfv_tests/openstack/rally/scenario/templates')
74 SUPPORT_DIR = pkg_resources.resource_filename(
75 'functest', 'opnfv_tests/openstack/rally/scenario/support')
78 ITERATIONS_AMOUNT = 10
80 RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally')
81 BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
82 TEMP_DIR = os.path.join(RALLY_DIR, "var")
84 RALLY_PRIVATE_NET_NAME = getattr(config.CONF, 'rally_network_name')
85 RALLY_PRIVATE_SUBNET_NAME = getattr(config.CONF, 'rally_subnet_name')
86 RALLY_PRIVATE_SUBNET_CIDR = getattr(config.CONF, 'rally_subnet_cidr')
87 RALLY_ROUTER_NAME = getattr(config.CONF, 'rally_router_name')
89 def __init__(self, **kwargs):
90 """Initialize RallyBase object."""
91 super(RallyBase, self).__init__(**kwargs)
92 self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
93 self.guid = '-' + str(uuid.uuid4())
97 self.scenario_dir = ''
98 self.image_name = None
99 self.ext_net_name = None
100 self.priv_net_id = None
101 self.flavor_name = None
102 self.flavor_alt_name = None
104 self.test_name = None
105 self.start_time = None
110 def _build_task_args(self, test_file_name):
111 """Build arguments for the Rally task."""
112 task_args = {'service_list': [test_file_name]}
113 task_args['image_name'] = self.image_name
114 task_args['flavor_name'] = self.flavor_name
115 task_args['flavor_alt_name'] = self.flavor_alt_name
116 task_args['glance_image_location'] = self.GLANCE_IMAGE_PATH
117 task_args['glance_image_format'] = self.GLANCE_IMAGE_FORMAT
118 task_args['tmpl_dir'] = self.TEMPLATE_DIR
119 task_args['sup_dir'] = self.SUPPORT_DIR
120 task_args['users_amount'] = self.USERS_AMOUNT
121 task_args['tenants_amount'] = self.TENANTS_AMOUNT
122 task_args['use_existing_users'] = False
123 task_args['iterations'] = self.ITERATIONS_AMOUNT
124 task_args['concurrency'] = self.CONCURRENCY
125 task_args['smoke'] = self.smoke
127 ext_net = self.ext_net_name
129 task_args['floating_network'] = str(ext_net)
131 task_args['floating_network'] = ''
133 net_id = self.priv_net_id
135 task_args['netid'] = str(net_id)
137 task_args['netid'] = ''
141 def _prepare_test_list(self, test_name):
142 """Build the list of test cases to be executed."""
143 test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
144 scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
147 if not os.path.exists(scenario_file_name):
148 scenario_file_name = os.path.join(self.scenario_dir,
151 if not os.path.exists(scenario_file_name):
152 raise Exception("The scenario '%s' does not exist."
153 % scenario_file_name)
155 LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
156 test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
158 if not os.path.exists(self.TEMP_DIR):
159 os.makedirs(self.TEMP_DIR)
161 self._apply_blacklist(scenario_file_name, test_file_name)
162 return test_file_name
165 def get_task_id(cmd_raw):
167 Get task id from command rally result.
170 :return: task_id as string
172 taskid_re = re.compile('^Task +(.*): started$')
173 for line in cmd_raw.splitlines(True):
175 match = taskid_re.match(line)
177 return match.group(1)
181 def task_succeed(json_raw):
183 Parse JSON from rally JSON results.
188 rally_report = json.loads(json_raw)
189 for report in rally_report:
190 if report is None or report.get('result') is None:
193 for result in report.get('result'):
194 if result is None or len(result.get('error')) > 0:
199 def _migration_supported(self):
200 """Determine if migration is supported."""
201 if self.compute_cnt > 1:
207 def get_cmd_output(proc):
208 """Get command stdout."""
210 for line in proc.stdout:
216 """Exclude scenario."""
219 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
220 black_list_yaml = yaml.safe_load(black_list_file)
222 installer_type = env.get('INSTALLER_TYPE')
223 deploy_scenario = env.get('DEPLOY_SCENARIO')
224 if (bool(installer_type) and bool(deploy_scenario) and
225 'scenario' in black_list_yaml.keys()):
226 for item in black_list_yaml['scenario']:
227 scenarios = item['scenarios']
228 installers = item['installers']
229 in_it = RallyBase.in_iterable_re
230 if (in_it(deploy_scenario, scenarios) and
231 in_it(installer_type, installers)):
232 tests = item['tests']
233 black_tests.extend(tests)
234 except Exception: # pylint: disable=broad-except
235 LOGGER.debug("Scenario exclusion not applied.")
240 def in_iterable_re(needle, haystack):
242 Check if given needle is in the iterable haystack, using regex.
244 :param needle: string to be matched
245 :param haystack: iterable of strings (optionally regex patterns)
246 :return: True if needle is eqial to any of the elements in haystack,
247 or if a nonempty regex pattern in haystack is found in needle.
249 # match without regex
250 if needle in haystack:
253 for pattern in haystack:
254 # match if regex pattern is set and found in the needle
255 if pattern and re.search(pattern, needle) is not None:
261 """Exclude functionalities."""
266 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
267 black_list_yaml = yaml.safe_load(black_list_file)
269 if not self._migration_supported():
270 func_list.append("no_migration")
272 if 'functionality' in black_list_yaml.keys():
273 for item in black_list_yaml['functionality']:
274 functions = item['functions']
275 for func in func_list:
276 if func in functions:
277 tests = item['tests']
278 black_tests.extend(tests)
279 except Exception: # pylint: disable=broad-except
280 LOGGER.debug("Functionality exclusion not applied.")
284 def _apply_blacklist(self, case_file_name, result_file_name):
285 """Apply blacklist."""
286 LOGGER.debug("Applying blacklist...")
287 cases_file = open(case_file_name, 'r')
288 result_file = open(result_file_name, 'w')
290 black_tests = list(set(self.excl_func() +
291 self.excl_scenario()))
294 LOGGER.debug("Blacklisted tests: " + str(black_tests))
297 for cases_line in cases_file:
299 for black_tests_line in black_tests:
300 if re.search(black_tests_line,
301 cases_line.strip().rstrip(':')):
305 result_file.write(str(cases_line))
307 if cases_line.isspace():
314 def file_is_empty(file_name):
315 """Determine is a file is empty."""
317 if os.stat(file_name).st_size > 0:
319 except Exception: # pylint: disable=broad-except
324 def _run_task(self, test_name):
326 LOGGER.info('Starting test scenario "%s" ...', test_name)
328 task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
329 if not os.path.exists(task_file):
330 LOGGER.error("Task file '%s' does not exist.", task_file)
331 raise Exception("Task file '%s' does not exist.", task_file)
333 file_name = self._prepare_test_list(test_name)
334 if self.file_is_empty(file_name):
335 LOGGER.info('No tests for scenario "%s"', test_name)
338 cmd = (["rally", "task", "start", "--abort-on-sla-failure", "--task",
339 task_file, "--task-args",
340 str(self._build_task_args(test_name))])
341 LOGGER.debug('running command: %s', cmd)
343 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
344 stderr=subprocess.STDOUT)
345 output = self.get_cmd_output(proc)
346 task_id = self.get_task_id(output)
348 LOGGER.debug('task_id : %s', task_id)
351 LOGGER.error('Failed to retrieve task_id, validating task...')
352 cmd = (["rally", "task", "validate", "--task", task_file,
353 "--task-args", str(self._build_task_args(test_name))])
354 LOGGER.debug('running command: %s', cmd)
355 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
356 stderr=subprocess.STDOUT)
357 output = self.get_cmd_output(proc)
358 LOGGER.error("Task validation result:" + "\n" + output)
361 # check for result directory and create it otherwise
362 if not os.path.exists(self.RESULTS_DIR):
363 LOGGER.debug('%s does not exist, we create it.',
365 os.makedirs(self.RESULTS_DIR)
367 # get and save rally operation JSON result
368 cmd = (["rally", "task", "detailed", task_id])
369 LOGGER.debug('running command: %s', cmd)
370 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
371 stderr=subprocess.STDOUT)
372 json_detailed = self.get_cmd_output(proc)
373 LOGGER.info('%s', json_detailed)
375 cmd = (["rally", "task", "results", task_id])
376 LOGGER.debug('running command: %s', cmd)
377 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
378 stderr=subprocess.STDOUT)
379 json_results = self.get_cmd_output(proc)
380 self._append_summary(json_results, test_name)
381 report_json_name = 'opnfv-{}.json'.format(test_name)
382 report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
383 with open(report_json_dir, 'w') as r_file:
384 LOGGER.debug('saving json file')
385 r_file.write(json_results)
387 # write html report file
388 report_html_name = 'opnfv-{}.html'.format(test_name)
389 report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
390 cmd = (["rally", "task", "report", task_id, "--out", report_html_dir])
391 LOGGER.debug('running command: %s', cmd)
392 subprocess.Popen(cmd, stdout=subprocess.PIPE,
393 stderr=subprocess.STDOUT)
395 # parse JSON operation result
396 if self.task_succeed(json_results):
397 LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
399 LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
401 def _append_summary(self, json_raw, test_name):
402 """Update statistics summary info."""
405 overall_duration = 0.0
407 rally_report = json.loads(json_raw)
408 for report in rally_report:
409 if report.get('full_duration'):
410 overall_duration += report.get('full_duration')
412 if report.get('result'):
413 for result in report.get('result'):
415 if not result.get('error'):
418 scenario_summary = {'test_name': test_name,
419 'overall_duration': overall_duration,
420 'nb_tests': nb_tests,
421 'nb_success': nb_success}
422 self.summary.append(scenario_summary)
424 def _prepare_env(self):
425 """Create resources needed by test scenarios."""
426 LOGGER.debug('Validating the test name...')
427 if self.test_name not in self.TESTS:
428 raise Exception("Test name '%s' is invalid" % self.test_name)
430 network_name = self.RALLY_PRIVATE_NET_NAME + self.guid
431 subnet_name = self.RALLY_PRIVATE_SUBNET_NAME + self.guid
432 router_name = self.RALLY_ROUTER_NAME + self.guid
433 self.image_name = self.GLANCE_IMAGE_NAME + self.guid
434 self.flavor_name = self.FLAVOR_NAME + self.guid
435 self.flavor_alt_name = self.FLAVOR_ALT_NAME + self.guid
436 self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds)
437 self.compute_cnt = snaps_utils.get_active_compute_cnt(self.os_creds)
439 LOGGER.debug("Creating image '%s'...", self.image_name)
440 image_creator = deploy_utils.create_image(
441 self.os_creds, ImageConfig(
442 name=self.image_name,
443 image_file=self.GLANCE_IMAGE_PATH,
444 img_format=self.GLANCE_IMAGE_FORMAT,
445 image_user=self.GLANCE_IMAGE_USERNAME,
447 extra_properties=self.GLANCE_IMAGE_EXTRA_PROPERTIES))
448 if image_creator is None:
449 raise Exception("Failed to create image")
450 self.creators.append(image_creator)
452 LOGGER.debug("Creating network '%s'...", network_name)
454 rally_network_type = getattr(config.CONF, 'rally_network_type', None)
455 rally_physical_network = getattr(
456 config.CONF, 'rally_physical_network', None)
457 rally_segmentation_id = getattr(
458 config.CONF, 'rally_segmentation_id', None)
460 network_creator = deploy_utils.create_network(
461 self.os_creds, NetworkConfig(
464 network_type=rally_network_type,
465 physical_network=rally_physical_network,
466 segmentation_id=rally_segmentation_id,
467 subnet_settings=[SubnetConfig(
469 cidr=self.RALLY_PRIVATE_SUBNET_CIDR)]))
470 if network_creator is None:
471 raise Exception("Failed to create private network")
472 self.priv_net_id = network_creator.get_network().id
473 self.creators.append(network_creator)
475 LOGGER.debug("Creating router '%s'...", router_name)
476 router_creator = deploy_utils.create_router(
477 self.os_creds, RouterConfig(
479 external_gateway=self.ext_net_name,
480 internal_subnets=[subnet_name]))
481 if router_creator is None:
482 raise Exception("Failed to create router")
483 self.creators.append(router_creator)
485 LOGGER.debug("Creating flavor '%s'...", self.flavor_name)
486 flavor_creator = OpenStackFlavor(
487 self.os_creds, FlavorConfig(
488 name=self.flavor_name, ram=self.FLAVOR_RAM, disk=1, vcpus=1,
489 metadata=self.FLAVOR_EXTRA_SPECS))
490 if flavor_creator is None or flavor_creator.create() is None:
491 raise Exception("Failed to create flavor")
492 self.creators.append(flavor_creator)
494 LOGGER.debug("Creating flavor '%s'...", self.flavor_alt_name)
495 flavor_alt_creator = OpenStackFlavor(
496 self.os_creds, FlavorConfig(
497 name=self.flavor_alt_name, ram=self.FLAVOR_RAM_ALT, disk=1,
498 vcpus=1, metadata=self.FLAVOR_EXTRA_SPECS))
499 if flavor_alt_creator is None or flavor_alt_creator.create() is None:
500 raise Exception("Failed to create flavor")
501 self.creators.append(flavor_alt_creator)
503 def _run_tests(self):
505 if self.test_name == 'all':
506 for test in self.TESTS:
507 if test == 'all' or test == 'vm':
511 self._run_task(self.test_name)
513 def _generate_report(self):
514 """Generate test execution summary report."""
520 res_table = prettytable.PrettyTable(
522 field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
523 res_table.align['Module'] = "l"
524 res_table.align['Duration'] = "r"
525 res_table.align['Success'] = "r"
527 # for each scenario we draw a row for the table
528 for item in self.summary:
529 total_duration += item['overall_duration']
530 total_nb_tests += item['nb_tests']
531 total_nb_success += item['nb_success']
533 success_avg = 100 * item['nb_success'] / item['nb_tests']
534 except ZeroDivisionError:
536 success_str = str("{:0.2f}".format(success_avg)) + '%'
537 duration_str = time.strftime("%M:%S",
538 time.gmtime(item['overall_duration']))
539 res_table.add_row([item['test_name'], duration_str,
540 item['nb_tests'], success_str])
541 payload.append({'module': item['test_name'],
542 'details': {'duration': item['overall_duration'],
543 'nb tests': item['nb_tests'],
544 'success': success_str}})
546 total_duration_str = time.strftime("%H:%M:%S",
547 time.gmtime(total_duration))
549 self.result = 100 * total_nb_success / total_nb_tests
550 except ZeroDivisionError:
552 success_rate = "{:0.2f}".format(self.result)
553 success_rate_str = str(success_rate) + '%'
554 res_table.add_row(["", "", "", ""])
555 res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
558 LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
559 LOGGER.info("Rally '%s' success_rate is %s%%",
560 self.case_name, success_rate)
561 payload.append({'summary': {'duration': total_duration,
562 'nb tests': total_nb_tests,
563 'nb success': success_rate}})
564 self.details = payload
567 """Cleanup all OpenStack objects. Should be called on completion."""
568 for creator in reversed(self.creators):
571 except Exception as exc: # pylint: disable=broad-except
572 LOGGER.error('Unexpected error cleaning - %s', exc)
574 @energy.enable_recording
575 def run(self, **kwargs):
577 self.start_time = time.time()
579 conf_utils.create_rally_deployment()
582 self._generate_report()
583 res = testcase.TestCase.EX_OK
584 except Exception as exc: # pylint: disable=broad-except
585 LOGGER.error('Error with run: %s', exc)
586 res = testcase.TestCase.EX_RUN_ERROR
590 self.stop_time = time.time()
594 class RallySanity(RallyBase):
595 """Rally sanity testcase implementation."""
597 def __init__(self, **kwargs):
598 """Initialize RallySanity object."""
599 if "case_name" not in kwargs:
600 kwargs["case_name"] = "rally_sanity"
601 super(RallySanity, self).__init__(**kwargs)
603 self.test_name = 'all'
605 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
608 class RallyFull(RallyBase):
609 """Rally full testcase implementation."""
611 def __init__(self, **kwargs):
612 """Initialize RallyFull object."""
613 if "case_name" not in kwargs:
614 kwargs["case_name"] = "rally_full"
615 super(RallyFull, self).__init__(**kwargs)
617 self.test_name = 'all'
619 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')