3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Rally testcases implementation."""
13 from __future__ import division
27 from functest.core import testcase
28 from functest.energy import energy
29 from functest.opnfv_tests.openstack.snaps import snaps_utils
30 from functest.opnfv_tests.openstack.tempest import conf_utils
31 from functest.utils.constants import CONST
32 from functest.utils import env
34 from snaps.config.flavor import FlavorConfig
35 from snaps.config.image import ImageConfig
36 from snaps.config.network import NetworkConfig, SubnetConfig
37 from snaps.config.router import RouterConfig
39 from snaps.openstack.create_flavor import OpenStackFlavor
40 from snaps.openstack.utils import deploy_utils
42 LOGGER = logging.getLogger(__name__)
45 class RallyBase(testcase.TestCase):
46 """Base class form Rally testcases implementation."""
48 # pylint: disable=too-many-instance-attributes
49 TESTS = ['authenticate', 'glance', 'ceilometer', 'cinder', 'heat',
50 'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all']
51 GLANCE_IMAGE_NAME = getattr(CONST, 'openstack_image_name')
52 GLANCE_IMAGE_FILENAME = getattr(CONST, 'openstack_image_file_name')
53 GLANCE_IMAGE_PATH = os.path.join(getattr(CONST, 'dir_functest_images'),
54 GLANCE_IMAGE_FILENAME)
55 GLANCE_IMAGE_FORMAT = getattr(CONST, 'openstack_image_disk_format')
56 GLANCE_IMAGE_USERNAME = getattr(CONST, 'openstack_image_username')
57 GLANCE_IMAGE_EXTRA_PROPERTIES = getattr(CONST,
58 'openstack_extra_properties', {})
59 FLAVOR_NAME = getattr(CONST, 'rally_flavor_name')
60 FLAVOR_ALT_NAME = getattr(CONST, 'rally_flavor_alt_name')
63 FLAVOR_EXTRA_SPECS = getattr(CONST, 'flavor_extra_specs', None)
64 if FLAVOR_EXTRA_SPECS:
68 RALLY_DIR = pkg_resources.resource_filename(
69 'functest', 'opnfv_tests/openstack/rally')
70 RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
71 'functest', 'opnfv_tests/openstack/rally/scenario')
72 TEMPLATE_DIR = pkg_resources.resource_filename(
73 'functest', 'opnfv_tests/openstack/rally/scenario/templates')
74 SUPPORT_DIR = pkg_resources.resource_filename(
75 'functest', 'opnfv_tests/openstack/rally/scenario/support')
78 ITERATIONS_AMOUNT = 10
80 RESULTS_DIR = os.path.join(getattr(CONST, 'dir_results'), 'rally')
81 BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
82 TEMP_DIR = os.path.join(RALLY_DIR, "var")
84 RALLY_PRIVATE_NET_NAME = getattr(CONST, 'rally_network_name')
85 RALLY_PRIVATE_SUBNET_NAME = getattr(CONST, 'rally_subnet_name')
86 RALLY_PRIVATE_SUBNET_CIDR = getattr(CONST, 'rally_subnet_cidr')
87 RALLY_ROUTER_NAME = getattr(CONST, 'rally_router_name')
89 def __init__(self, **kwargs):
90 """Initialize RallyBase object."""
91 super(RallyBase, self).__init__(**kwargs)
92 self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
93 self.guid = '-' + str(uuid.uuid4())
97 self.scenario_dir = ''
98 self.image_name = None
99 self.ext_net_name = None
100 self.priv_net_id = None
101 self.flavor_name = None
102 self.flavor_alt_name = None
104 self.test_name = None
105 self.start_time = None
110 def _build_task_args(self, test_file_name):
111 """Build arguments for the Rally task."""
112 task_args = {'service_list': [test_file_name]}
113 task_args['image_name'] = self.image_name
114 task_args['flavor_name'] = self.flavor_name
115 task_args['flavor_alt_name'] = self.flavor_alt_name
116 task_args['glance_image_location'] = self.GLANCE_IMAGE_PATH
117 task_args['glance_image_format'] = self.GLANCE_IMAGE_FORMAT
118 task_args['tmpl_dir'] = self.TEMPLATE_DIR
119 task_args['sup_dir'] = self.SUPPORT_DIR
120 task_args['users_amount'] = self.USERS_AMOUNT
121 task_args['tenants_amount'] = self.TENANTS_AMOUNT
122 task_args['use_existing_users'] = False
123 task_args['iterations'] = self.ITERATIONS_AMOUNT
124 task_args['concurrency'] = self.CONCURRENCY
125 task_args['smoke'] = self.smoke
127 ext_net = self.ext_net_name
129 task_args['floating_network'] = str(ext_net)
131 task_args['floating_network'] = ''
133 net_id = self.priv_net_id
135 task_args['netid'] = str(net_id)
137 task_args['netid'] = ''
141 def _prepare_test_list(self, test_name):
142 """Build the list of test cases to be executed."""
143 test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
144 scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
147 if not os.path.exists(scenario_file_name):
148 scenario_file_name = os.path.join(self.scenario_dir,
151 if not os.path.exists(scenario_file_name):
152 raise Exception("The scenario '%s' does not exist."
153 % scenario_file_name)
155 LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
156 test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
158 if not os.path.exists(self.TEMP_DIR):
159 os.makedirs(self.TEMP_DIR)
161 self._apply_blacklist(scenario_file_name, test_file_name)
162 return test_file_name
165 def get_task_id(cmd_raw):
167 Get task id from command rally result.
170 :return: task_id as string
172 taskid_re = re.compile('^Task +(.*): started$')
173 for line in cmd_raw.splitlines(True):
175 match = taskid_re.match(line)
177 return match.group(1)
181 def task_succeed(json_raw):
183 Parse JSON from rally JSON results.
188 rally_report = json.loads(json_raw)
189 for report in rally_report:
190 if report is None or report.get('result') is None:
193 for result in report.get('result'):
194 if result is None or len(result.get('error')) > 0:
199 def _migration_supported(self):
200 """Determine if migration is supported."""
201 if self.compute_cnt > 1:
207 def get_cmd_output(proc):
208 """Get command stdout."""
210 for line in proc.stdout:
216 """Exclude scenario."""
219 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
220 black_list_yaml = yaml.safe_load(black_list_file)
222 installer_type = env.get('INSTALLER_TYPE')
223 deploy_scenario = env.get('DEPLOY_SCENARIO')
224 if (bool(installer_type) and bool(deploy_scenario) and
225 'scenario' in black_list_yaml.keys()):
226 for item in black_list_yaml['scenario']:
227 scenarios = item['scenarios']
228 installers = item['installers']
229 in_it = RallyBase.in_iterable_re
230 if (in_it(deploy_scenario, scenarios) and
231 in_it(installer_type, installers)):
232 tests = item['tests']
233 black_tests.extend(tests)
234 except Exception: # pylint: disable=broad-except
235 LOGGER.debug("Scenario exclusion not applied.")
240 def in_iterable_re(needle, haystack):
242 Check if given needle is in the iterable haystack, using regex.
244 :param needle: string to be matched
245 :param haystack: iterable of strings (optionally regex patterns)
246 :return: True if needle is eqial to any of the elements in haystack,
247 or if a nonempty regex pattern in haystack is found in needle.
249 # match without regex
250 if needle in haystack:
253 for pattern in haystack:
254 # match if regex pattern is set and found in the needle
255 if pattern and re.search(pattern, needle) is not None:
261 """Exclude functionalities."""
266 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
267 black_list_yaml = yaml.safe_load(black_list_file)
269 if not self._migration_supported():
270 func_list.append("no_migration")
272 if 'functionality' in black_list_yaml.keys():
273 for item in black_list_yaml['functionality']:
274 functions = item['functions']
275 for func in func_list:
276 if func in functions:
277 tests = item['tests']
278 black_tests.extend(tests)
279 except Exception: # pylint: disable=broad-except
280 LOGGER.debug("Functionality exclusion not applied.")
284 def _apply_blacklist(self, case_file_name, result_file_name):
285 """Apply blacklist."""
286 LOGGER.debug("Applying blacklist...")
287 cases_file = open(case_file_name, 'r')
288 result_file = open(result_file_name, 'w')
290 black_tests = list(set(self.excl_func() +
291 self.excl_scenario()))
294 LOGGER.debug("Blacklisted tests: " + str(black_tests))
297 for cases_line in cases_file:
299 for black_tests_line in black_tests:
300 if re.search(black_tests_line,
301 cases_line.strip().rstrip(':')):
305 result_file.write(str(cases_line))
307 if cases_line.isspace():
314 def file_is_empty(file_name):
315 """Determine is a file is empty."""
317 if os.stat(file_name).st_size > 0:
319 except Exception: # pylint: disable=broad-except
324 def _run_task(self, test_name):
326 LOGGER.info('Starting test scenario "%s" ...', test_name)
328 task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
329 if not os.path.exists(task_file):
330 LOGGER.error("Task file '%s' does not exist.", task_file)
331 raise Exception("Task file '%s' does not exist.", task_file)
333 file_name = self._prepare_test_list(test_name)
334 if self.file_is_empty(file_name):
335 LOGGER.info('No tests for scenario "%s"', test_name)
338 cmd = (["rally", "task", "start", "--abort-on-sla-failure", "--task",
339 task_file, "--task-args",
340 str(self._build_task_args(test_name))])
341 LOGGER.debug('running command: %s', cmd)
343 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
344 stderr=subprocess.STDOUT)
345 output = self.get_cmd_output(proc)
346 task_id = self.get_task_id(output)
348 LOGGER.debug('task_id : %s', task_id)
351 LOGGER.error('Failed to retrieve task_id, validating task...')
352 cmd = (["rally", "task", "validate", "--task", task_file,
353 "--task-args", str(self._build_task_args(test_name))])
354 LOGGER.debug('running command: %s', cmd)
355 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
356 stderr=subprocess.STDOUT)
357 output = self.get_cmd_output(proc)
358 LOGGER.error("Task validation result:" + "\n" + output)
361 # check for result directory and create it otherwise
362 if not os.path.exists(self.RESULTS_DIR):
363 LOGGER.debug('%s does not exist, we create it.',
365 os.makedirs(self.RESULTS_DIR)
367 # get and save rally operation JSON result
368 cmd = (["rally", "task", "detailed", task_id])
369 LOGGER.debug('running command: %s', cmd)
370 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
371 stderr=subprocess.STDOUT)
372 json_detailed = self.get_cmd_output(proc)
373 LOGGER.info('%s', json_detailed)
375 cmd = (["rally", "task", "results", task_id])
376 LOGGER.debug('running command: %s', cmd)
377 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
378 stderr=subprocess.STDOUT)
379 json_results = self.get_cmd_output(proc)
380 self._append_summary(json_results, test_name)
381 report_json_name = 'opnfv-{}.json'.format(test_name)
382 report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
383 with open(report_json_dir, 'w') as r_file:
384 LOGGER.debug('saving json file')
385 r_file.write(json_results)
387 # write html report file
388 report_html_name = 'opnfv-{}.html'.format(test_name)
389 report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
390 cmd = (["rally", "task", "report", task_id, "--out", report_html_dir])
391 LOGGER.debug('running command: %s', cmd)
392 subprocess.Popen(cmd, stdout=subprocess.PIPE,
393 stderr=subprocess.STDOUT)
395 # parse JSON operation result
396 if self.task_succeed(json_results):
397 LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
399 LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
401 def _append_summary(self, json_raw, test_name):
402 """Update statistics summary info."""
405 overall_duration = 0.0
407 rally_report = json.loads(json_raw)
408 for report in rally_report:
409 if report.get('full_duration'):
410 overall_duration += report.get('full_duration')
412 if report.get('result'):
413 for result in report.get('result'):
415 if not result.get('error'):
418 scenario_summary = {'test_name': test_name,
419 'overall_duration': overall_duration,
420 'nb_tests': nb_tests,
421 'nb_success': nb_success}
422 self.summary.append(scenario_summary)
424 def _prepare_env(self):
425 """Create resources needed by test scenarios."""
426 LOGGER.debug('Validating the test name...')
427 if self.test_name not in self.TESTS:
428 raise Exception("Test name '%s' is invalid" % self.test_name)
430 network_name = self.RALLY_PRIVATE_NET_NAME + self.guid
431 subnet_name = self.RALLY_PRIVATE_SUBNET_NAME + self.guid
432 router_name = self.RALLY_ROUTER_NAME + self.guid
433 self.image_name = self.GLANCE_IMAGE_NAME + self.guid
434 self.flavor_name = self.FLAVOR_NAME + self.guid
435 self.flavor_alt_name = self.FLAVOR_ALT_NAME + self.guid
436 self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds)
437 self.compute_cnt = snaps_utils.get_active_compute_cnt(self.os_creds)
439 LOGGER.debug("Creating image '%s'...", self.image_name)
440 image_creator = deploy_utils.create_image(
441 self.os_creds, ImageConfig(
442 name=self.image_name,
443 image_file=self.GLANCE_IMAGE_PATH,
444 img_format=self.GLANCE_IMAGE_FORMAT,
445 image_user=self.GLANCE_IMAGE_USERNAME,
447 extra_properties=self.GLANCE_IMAGE_EXTRA_PROPERTIES))
448 if image_creator is None:
449 raise Exception("Failed to create image")
450 self.creators.append(image_creator)
452 LOGGER.debug("Creating network '%s'...", network_name)
454 rally_network_type = getattr(CONST, 'rally_network_type', None)
455 rally_physical_network = getattr(CONST, 'rally_physical_network', None)
456 rally_segmentation_id = getattr(CONST, 'rally_segmentation_id', None)
458 network_creator = deploy_utils.create_network(
459 self.os_creds, NetworkConfig(
462 network_type=rally_network_type,
463 physical_network=rally_physical_network,
464 segmentation_id=rally_segmentation_id,
465 subnet_settings=[SubnetConfig(
467 cidr=self.RALLY_PRIVATE_SUBNET_CIDR)]))
468 if network_creator is None:
469 raise Exception("Failed to create private network")
470 self.priv_net_id = network_creator.get_network().id
471 self.creators.append(network_creator)
473 LOGGER.debug("Creating router '%s'...", router_name)
474 router_creator = deploy_utils.create_router(
475 self.os_creds, RouterConfig(
477 external_gateway=self.ext_net_name,
478 internal_subnets=[subnet_name]))
479 if router_creator is None:
480 raise Exception("Failed to create router")
481 self.creators.append(router_creator)
483 LOGGER.debug("Creating flavor '%s'...", self.flavor_name)
484 flavor_creator = OpenStackFlavor(
485 self.os_creds, FlavorConfig(
486 name=self.flavor_name, ram=self.FLAVOR_RAM, disk=1, vcpus=1,
487 metadata=self.FLAVOR_EXTRA_SPECS))
488 if flavor_creator is None or flavor_creator.create() is None:
489 raise Exception("Failed to create flavor")
490 self.creators.append(flavor_creator)
492 LOGGER.debug("Creating flavor '%s'...", self.flavor_alt_name)
493 flavor_alt_creator = OpenStackFlavor(
494 self.os_creds, FlavorConfig(
495 name=self.flavor_alt_name, ram=self.FLAVOR_RAM_ALT, disk=1,
496 vcpus=1, metadata=self.FLAVOR_EXTRA_SPECS))
497 if flavor_alt_creator is None or flavor_alt_creator.create() is None:
498 raise Exception("Failed to create flavor")
499 self.creators.append(flavor_alt_creator)
501 def _run_tests(self):
503 if self.test_name == 'all':
504 for test in self.TESTS:
505 if test == 'all' or test == 'vm':
509 self._run_task(self.test_name)
511 def _generate_report(self):
512 """Generate test execution summary report."""
518 res_table = prettytable.PrettyTable(
520 field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
521 res_table.align['Module'] = "l"
522 res_table.align['Duration'] = "r"
523 res_table.align['Success'] = "r"
525 # for each scenario we draw a row for the table
526 for item in self.summary:
527 total_duration += item['overall_duration']
528 total_nb_tests += item['nb_tests']
529 total_nb_success += item['nb_success']
531 success_avg = 100 * item['nb_success'] / item['nb_tests']
532 except ZeroDivisionError:
534 success_str = str("{:0.2f}".format(success_avg)) + '%'
535 duration_str = time.strftime("%M:%S",
536 time.gmtime(item['overall_duration']))
537 res_table.add_row([item['test_name'], duration_str,
538 item['nb_tests'], success_str])
539 payload.append({'module': item['test_name'],
540 'details': {'duration': item['overall_duration'],
541 'nb tests': item['nb_tests'],
542 'success': success_str}})
544 total_duration_str = time.strftime("%H:%M:%S",
545 time.gmtime(total_duration))
547 self.result = 100 * total_nb_success / total_nb_tests
548 except ZeroDivisionError:
550 success_rate = "{:0.2f}".format(self.result)
551 success_rate_str = str(success_rate) + '%'
552 res_table.add_row(["", "", "", ""])
553 res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
556 LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
557 LOGGER.info("Rally '%s' success_rate is %s%%",
558 self.case_name, success_rate)
559 payload.append({'summary': {'duration': total_duration,
560 'nb tests': total_nb_tests,
561 'nb success': success_rate}})
562 self.details = payload
565 """Cleanup all OpenStack objects. Should be called on completion."""
566 for creator in reversed(self.creators):
569 except Exception as exc: # pylint: disable=broad-except
570 LOGGER.error('Unexpected error cleaning - %s', exc)
572 @energy.enable_recording
573 def run(self, **kwargs):
575 self.start_time = time.time()
577 conf_utils.create_rally_deployment()
580 self._generate_report()
581 res = testcase.TestCase.EX_OK
582 except Exception as exc: # pylint: disable=broad-except
583 LOGGER.error('Error with run: %s', exc)
584 res = testcase.TestCase.EX_RUN_ERROR
588 self.stop_time = time.time()
592 class RallySanity(RallyBase):
593 """Rally sanity testcase implementation."""
595 def __init__(self, **kwargs):
596 """Initialize RallySanity object."""
597 if "case_name" not in kwargs:
598 kwargs["case_name"] = "rally_sanity"
599 super(RallySanity, self).__init__(**kwargs)
601 self.test_name = 'all'
603 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
606 class RallyFull(RallyBase):
607 """Rally full testcase implementation."""
609 def __init__(self, **kwargs):
610 """Initialize RallyFull object."""
611 if "case_name" not in kwargs:
612 kwargs["case_name"] = "rally_full"
613 super(RallyFull, self).__init__(**kwargs)
615 self.test_name = 'all'
617 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')