X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=vsperf;h=6f248f89a83442f11c4fafa8ce1ebf71cbcd72d6;hb=62cc56705f9ec94c59b2b0597ffdd1fa41008982;hp=ee494dcd3f01999aaaea86afd0ed9f4f46136ffb;hpb=9f3fce18b414e10724549dbee8113c6bdfa2f5db;p=vswitchperf.git diff --git a/vsperf b/vsperf index ee494dcd..6f248f89 100755 --- a/vsperf +++ b/vsperf @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2015-2016 Intel Corporation. +# Copyright 2015-2017 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,16 +26,14 @@ import time import datetime import shutil import unittest -import xmlrunner import locale import copy import glob import subprocess - -sys.dont_write_bytecode = True - +import ast +import xmlrunner from conf import settings -from conf import get_test_param +import core.component_factory as component_factory from core.loader import Loader from testcases import PerformanceTestCase from testcases import IntegrationTestCase @@ -44,8 +42,8 @@ from tools import networkcard from tools import functions from tools.pkt_gen import trafficgen from tools.opnfvdashboard import opnfvdashboard -from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS -import core.component_factory as component_factory + +sys.dont_write_bytecode = True VERBOSITY_LEVELS = { 'debug': logging.DEBUG, @@ -86,7 +84,14 @@ def parse_arguments(): value = value.strip() if len(param): if len(value): - results[param] = value + # values are passed inside string from CLI, so we must retype them accordingly + try: + results[param] = ast.literal_eval(value) + except ValueError: + # for backward compatibility, we have to accept strings without quotes + _LOGGER.warning("Adding missing quotes around string value: %s = %s", + param, str(value)) + results[param] = str(value) else: results[param] = True @@ -142,6 +147,8 @@ def parse_arguments(): help='list all system forwarding applications and exit') parser.add_argument('--list-vnfs', action='store_true', help='list all system vnfs and exit') + parser.add_argument('--list-loadgens', action='store_true', + help='list all background load generators') parser.add_argument('--list-settings', action='store_true', help='list effective settings configuration and exit') parser.add_argument('exact_test_name', nargs='*', help='Exact names of\ @@ -169,6 +176,7 @@ def parse_arguments(): group.add_argument('--vswitch', help='vswitch implementation to use') group.add_argument('--fwdapp', help='packet forwarding application to use') group.add_argument('--vnf', help='vnf to use') + group.add_argument('--loadgen', help='loadgen to use') group.add_argument('--sysmetrics', help='system metrics logger to use') group = parser.add_argument_group('test behavior options') group.add_argument('--xunit', action='store_true', @@ -280,6 +288,39 @@ def check_and_set_locale(): _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s", system_locale, locale.getdefaultlocale()) +def get_vswitch_names(rst_files): + """ Function will return a list of vSwitches detected in given ``rst_files``. + """ + vswitch_names = set() + if len(rst_files): + try: + output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines() + for line in output: + match = re.search(r'^\* vSwitch: ([^,]+)', str(line)) + if match: + vswitch_names.add(match.group(1)) + + if len(vswitch_names): + return list(vswitch_names) + + except subprocess.CalledProcessError: + _LOGGER.warning('Cannot detect vSwitches used during testing.') + + # fallback to the default value + return ['vSwitch'] + +def get_build_tag(): + """ Function will return a Jenkins job ID environment variable. + """ + + try: + build_tag = os.environ['BUILD_TAG'] + + except KeyError: + _LOGGER.warning('Cannot detect Jenkins job ID') + build_tag = "none" + + return build_tag def generate_final_report(): """ Function will check if partial test results are available @@ -289,18 +330,15 @@ def generate_final_report(): path = settings.getValue('RESULTS_PATH') # check if there are any results in rst format rst_results = glob.glob(os.path.join(path, 'result*rst')) + pkt_processors = get_vswitch_names(rst_results) if len(rst_results): try: - test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final'])) + test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final'])) # create report caption directly - it is not worth to execute jinja machinery - if settings.getValue('VSWITCH').lower() != 'none': - pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0] - else: - pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0] report_caption = '{}\n{} {}\n{}\n\n'.format( '============================================================', 'Performance report for', - pkt_processor, + ', '.join(pkt_processors), '============================================================') with open(_TEMPLATE_RST['tmp'], 'w') as file_: @@ -312,7 +350,7 @@ def generate_final_report(): if retval == 0 and os.path.isfile(test_report): _LOGGER.info('Overall test report written to "%s"', test_report) else: - _LOGGER.error('Generatrion of overall test report has failed.') + _LOGGER.error('Generation of overall test report has failed.') # remove temporary file os.remove(_TEMPLATE_RST['tmp']) @@ -345,8 +383,7 @@ def enable_sriov(nic_list): or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]: # if not, enable and set appropriate number of VFs if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1): - _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic) - raise + raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic)) else: _LOGGER.debug("SRIOV enabled for NIC %s", nic) @@ -375,8 +412,7 @@ def disable_sriov(nic_list): for nic in nic_list: if networkcard.is_sriov_nic(nic): if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0): - _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic) - raise + raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic)) else: _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0]) @@ -406,23 +442,45 @@ def handle_list_options(args): print(Loader().get_pktfwds_printable()) sys.exit(0) + if args['list_loadgens']: + print(Loader().get_loadgens_printable()) + sys.exit(0) + if args['list_settings']: print(str(settings)) sys.exit(0) if args['list']: - # configure tests - if args['integration']: - testcases = settings.getValue('INTEGRATION_TESTS') + list_testcases(args) + sys.exit(0) + + +def list_testcases(args): + """ Print list of testcases requested by --list CLI argument + + :param args: A dictionary with all CLI arguments + """ + # configure tests + if args['integration']: + testcases = settings.getValue('INTEGRATION_TESTS') + else: + testcases = settings.getValue('PERFORMANCE_TESTS') + + print("Available Tests:") + print("================") + + for test in testcases: + description = functions.format_description(test['Description'], 70) + if len(test['Name']) < 40: + print('* {:40} {}'.format('{}:'.format(test['Name']), description[0])) else: - testcases = settings.getValue('PERFORMANCE_TESTS') + print('* {}'.format('{}:'.format(test['Name']))) + print(' {:40} {}'.format('', description[0])) + for i in range(1, len(description)): + print(' {:40} {}'.format('', description[i])) + - print("Available Tests:") - print("================") - for test in testcases: - print('* %-30s %s' % ('%s:' % test['Name'], test['Description'])) - sys.exit(0) def vsperf_finalize(): @@ -476,7 +534,7 @@ class MockTestCase(unittest.TestCase): on how self.is_pass was set in the constructor""" self.assertTrue(self.is_pass, self.msg) - +# pylint: disable=too-many-locals, too-many-branches, too-many-statements def main(): """Main function. """ @@ -504,8 +562,11 @@ def main(): # than both a settings file and environment variables settings.load_from_dict(args) - # set dpdk and ovs paths accorfing to VNF and VSWITCH - functions.settings_update_paths() + settings.setValue('mode', args['mode']) + + # update paths to trafficgens if required + if settings.getValue('mode') == 'trafficgen': + functions.settings_update_paths() # if required, handle list-* operations handle_list_options(args) @@ -526,7 +587,7 @@ def main(): # configuration validity checks if args['vswitch']: - vswitch_none = 'none' == args['vswitch'].strip().lower() + vswitch_none = args['vswitch'].strip().lower() == 'none' if vswitch_none: settings.setValue('VSWITCH', 'none') else: @@ -555,6 +616,14 @@ def main(): settings.getValue('VNF_DIR')) sys.exit(1) + if args['loadgen']: + loadgens = Loader().get_loadgens() + if args['loadgen'] not in loadgens: + _LOGGER.error('There are no loadgens matching \'%s\' found in' + ' \'%s\'. Exiting...', args['loadgen'], + settings.getValue('LOADGEN_DIR')) + sys.exit(1) + if args['exact_test_name'] and args['tests']: _LOGGER.error("Cannot specify tests with both positional args and --test.") sys.exit(1) @@ -576,16 +645,13 @@ def main(): 'driver' : networkcard.get_driver(tmp_nic), 'device' : networkcard.get_device_name(tmp_nic)}) else: - _LOGGER.error("Invalid network card PCI ID: '%s'", nic) vsperf_finalize() - raise + raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic)) settings.setValue('NICS', nic_list) # for backward compatibility settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list)) - settings.setValue('mode', args['mode']) - # generate results directory name date = datetime.datetime.fromtimestamp(time.time()) results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S') @@ -602,12 +668,9 @@ def main(): _LOGGER.debug("Executing traffic generator:") loader = Loader() # set traffic details, so they can be passed to traffic ctl - traffic = copy.deepcopy(TRAFFIC_DEFAULTS) - traffic.update({'traffic_type': get_test_param('traffic_type', TRAFFIC_DEFAULTS['traffic_type']), - 'bidir': get_test_param('bidirectional', TRAFFIC_DEFAULTS['bidir']), - 'multistream': int(get_test_param('multistream', TRAFFIC_DEFAULTS['multistream'])), - 'stream_type': get_test_param('stream_type', TRAFFIC_DEFAULTS['stream_type']), - 'frame_rate': int(get_test_param('iload', TRAFFIC_DEFAULTS['frame_rate']))}) + traffic = copy.deepcopy(settings.getValue('TRAFFIC')) + + traffic = functions.check_traffic(traffic) traffic_ctl = component_factory.create_traffic( traffic['traffic_type'], @@ -644,7 +707,11 @@ def main(): sys.exit(1) # run tests + # Add pylint exception: Redefinition of test type from + # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase + # pylint: disable=redefined-variable-type suite = unittest.TestSuite() + settings_snapshot = copy.deepcopy(settings.__dict__) for cfg in selected_tests: test_name = cfg.get('Name', '') try: @@ -654,11 +721,13 @@ def main(): test = PerformanceTestCase(cfg) test.run() suite.addTest(MockTestCase('', True, test.name)) - #pylint: disable=broad-except + # pylint: disable=broad-except except (Exception) as ex: _LOGGER.exception("Failed to run test: %s", test_name) suite.addTest(MockTestCase(str(ex), False, test_name)) _LOGGER.info("Continuing with next test...") + finally: + settings.restore_from_dict(settings_snapshot) # generate final rst report with results of all executed TCs generate_final_report() @@ -670,22 +739,24 @@ def main(): if args['opnfvpod']: pod_name = args['opnfvpod'] - installer_name = settings.getValue('OPNFV_INSTALLER') + installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower() opnfv_url = settings.getValue('OPNFV_URL') pkg_list = settings.getValue('PACKAGE_LIST') - int_data = {'vanilla': False, - 'pod': pod_name, + int_data = {'pod': pod_name, + 'build_tag': get_build_tag(), 'installer': installer_name, 'pkg_list': pkg_list, - 'db_url': opnfv_url} - if settings.getValue('VSWITCH').endswith('Vanilla'): - int_data['vanilla'] = True - opnfvdashboard.results2opnfv_dashboard(results_path, int_data) + 'db_url': opnfv_url, + # pass vswitch name from configuration to be used for failed + # TCs; In case of successful TCs it is safer to use vswitch + # name from CSV as TC can override global configuration + 'vswitch': str(settings.getValue('VSWITCH')).lower()} + tc_names = [tc['Name'] for tc in selected_tests] + opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data) # cleanup before exit vsperf_finalize() if __name__ == "__main__": main() -