#!/usr/bin/env python3
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
import os
import sys
import argparse
+import re
import time
import datetime
import shutil
import unittest
-import xmlrunner
import locale
import copy
import glob
import subprocess
-
-sys.dont_write_bytecode = True
-
+import ast
+import xmlrunner
from conf import settings
-from conf import get_test_param
+import core.component_factory as component_factory
from core.loader import Loader
from testcases import PerformanceTestCase
from testcases import IntegrationTestCase
from tools import functions
from tools.pkt_gen import trafficgen
from tools.opnfvdashboard import opnfvdashboard
-from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
-import core.component_factory as component_factory
+
+sys.dont_write_bytecode = True
VERBOSITY_LEVELS = {
'debug': logging.DEBUG,
'critical': logging.CRITICAL
}
-_TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
- 'foot' : 'tools/report/report_foot.rst',
+_CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+
+_TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
+ 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
'final' : 'test_report.rst',
- 'tmp' : 'tools/report/report_tmp_caption.rst'
+ 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
}
+
_LOGGER = logging.getLogger()
def parse_arguments():
This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
values. For multiple overrides use a ; separated list for
- e.g. --test-params 'x=z; y=a,b'
+ e.g. --test-params 'x=z; y=(a,b)'
"""
def __call__(self, parser, namespace, values, option_string=None):
results = {}
- for value in values.split(';'):
- result = [key.strip() for key in value.split('=')]
- if len(result) == 1:
- results[result[0]] = True
- elif len(result) == 2:
- results[result[0]] = result[1]
- else:
- raise argparse.ArgumentTypeError(
- 'expected \'%s\' to be of format \'key=val\' or'
- ' \'key\'' % result)
+ for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
+ param = param.strip()
+ value = value.strip()
+ if len(param):
+ if len(value):
+ # values are passed inside string from CLI, so we must retype them accordingly
+ try:
+ results[param] = ast.literal_eval(value)
+ except ValueError:
+ # for backward compatibility, we have to accept strings without quotes
+ _LOGGER.warning("Adding missing quotes around string value: %s = %s",
+ param, str(value))
+ results[param] = str(value)
+ else:
+ results[param] = True
setattr(namespace, self.dest, results)
help='list all system forwarding applications and exit')
parser.add_argument('--list-vnfs', action='store_true',
help='list all system vnfs and exit')
+ parser.add_argument('--list-loadgens', action='store_true',
+ help='list all background load generators')
parser.add_argument('--list-settings', action='store_true',
help='list effective settings configuration and exit')
parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
group.add_argument('--vswitch', help='vswitch implementation to use')
group.add_argument('--fwdapp', help='packet forwarding application to use')
group.add_argument('--vnf', help='vnf to use')
+ group.add_argument('--loadgen', help='loadgen to use')
group.add_argument('--sysmetrics', help='system metrics logger to use')
group = parser.add_argument_group('test behavior options')
group.add_argument('--xunit', action='store_true',
group.add_argument('--conf-file', action=_ValidateFileAction,
help='settings file')
group.add_argument('--test-params', action=_SplitTestParamsAction,
- help='csv list of test parameters: key=val; e.g.'
- 'including pkt_sizes=x,y; duration=x; '
- 'rfc2544_trials=x ...')
+ help='csv list of test parameters: key=val; e.g. '
+ 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
+ 'GUEST_LOOPBACK=["l2fwd"] ...')
group.add_argument('--opnfvpod', help='name of POD in opnfv')
args = vars(parser.parse_args())
_LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
system_locale, locale.getdefaultlocale())
+def get_vswitch_names(rst_files):
+ """ Function will return a list of vSwitches detected in given ``rst_files``.
+ """
+ vswitch_names = set()
+ if len(rst_files):
+ try:
+ output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
+ for line in output:
+ match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
+ if match:
+ vswitch_names.add(match.group(1))
+
+ if len(vswitch_names):
+ return list(vswitch_names)
+
+ except subprocess.CalledProcessError:
+ _LOGGER.warning('Cannot detect vSwitches used during testing.')
+
+ # fallback to the default value
+ return ['vSwitch']
+
+def get_build_tag():
+ """ Function will return a Jenkins job ID environment variable.
+ """
+
+ try:
+ build_tag = os.environ['BUILD_TAG']
+
+ except KeyError:
+ _LOGGER.warning('Cannot detect Jenkins job ID')
+ build_tag = "none"
+
+ return build_tag
def generate_final_report():
""" Function will check if partial test results are available
path = settings.getValue('RESULTS_PATH')
# check if there are any results in rst format
rst_results = glob.glob(os.path.join(path, 'result*rst'))
+ pkt_processors = get_vswitch_names(rst_results)
if len(rst_results):
try:
- test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
+ test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
# create report caption directly - it is not worth to execute jinja machinery
- if settings.getValue('VSWITCH').lower() != 'none':
- pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
- else:
- pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
report_caption = '{}\n{} {}\n{}\n\n'.format(
'============================================================',
'Performance report for',
- pkt_processor,
+ ', '.join(pkt_processors),
'============================================================')
with open(_TEMPLATE_RST['tmp'], 'w') as file_:
if retval == 0 and os.path.isfile(test_report):
_LOGGER.info('Overall test report written to "%s"', test_report)
else:
- _LOGGER.error('Generatrion of overall test report has failed.')
+ _LOGGER.error('Generation of overall test report has failed.')
# remove temporary file
os.remove(_TEMPLATE_RST['tmp'])
or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
# if not, enable and set appropriate number of VFs
if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
- _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
- raise
+ raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
else:
_LOGGER.debug("SRIOV enabled for NIC %s", nic)
+ # ensure that path to the bind tool is valid
+ functions.settings_update_paths()
+
# WORKAROUND: it has been observed with IXGBE(VF) driver,
# that NIC doesn't correclty dispatch traffic to VFs based
# on their MAC address. Unbind and bind to the same driver
for nic in nic_list:
if networkcard.is_sriov_nic(nic):
if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
- _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
- raise
+ raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
else:
_LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
print(Loader().get_pktfwds_printable())
sys.exit(0)
+ if args['list_loadgens']:
+ print(Loader().get_loadgens_printable())
+ sys.exit(0)
+
if args['list_settings']:
print(str(settings))
sys.exit(0)
if args['list']:
- # configure tests
- if args['integration']:
- testcases = settings.getValue('INTEGRATION_TESTS')
+ list_testcases(args)
+ sys.exit(0)
+
+
+def list_testcases(args):
+ """ Print list of testcases requested by --list CLI argument
+
+ :param args: A dictionary with all CLI arguments
+ """
+ # configure tests
+ if args['integration']:
+ testcases = settings.getValue('INTEGRATION_TESTS')
+ else:
+ testcases = settings.getValue('PERFORMANCE_TESTS')
+
+ print("Available Tests:")
+ print("================")
+
+ for test in testcases:
+ description = functions.format_description(test['Description'], 70)
+ if len(test['Name']) < 40:
+ print('* {:40} {}'.format('{}:'.format(test['Name']), description[0]))
else:
- testcases = settings.getValue('PERFORMANCE_TESTS')
+ print('* {}'.format('{}:'.format(test['Name'])))
+ print(' {:40} {}'.format('', description[0]))
+ for i in range(1, len(description)):
+ print(' {:40} {}'.format('', description[i]))
+
- print("Available Tests:")
- print("================")
- for test in testcases:
- print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
- sys.exit(0)
def vsperf_finalize():
on how self.is_pass was set in the constructor"""
self.assertTrue(self.is_pass, self.msg)
-
+# pylint: disable=too-many-locals, too-many-branches, too-many-statements
def main():
"""Main function.
"""
# configure settings
- settings.load_from_dir('conf')
+ settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
# Load non performance/integration tests
if args['integration']:
- settings.load_from_dir('conf/integration')
+ settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
# load command line parameters first in case there are settings files
# to be used
# than both a settings file and environment variables
settings.load_from_dict(args)
- # set dpdk and ovs paths accorfing to VNF and VSWITCH
- functions.settings_update_paths()
+ settings.setValue('mode', args['mode'])
+
+ # update paths to trafficgens if required
+ if settings.getValue('mode') == 'trafficgen':
+ functions.settings_update_paths()
# if required, handle list-* operations
handle_list_options(args)
# configuration validity checks
if args['vswitch']:
- vswitch_none = 'none' == args['vswitch'].strip().lower()
+ vswitch_none = args['vswitch'].strip().lower() == 'none'
if vswitch_none:
settings.setValue('VSWITCH', 'none')
else:
if args['vnf'] not in vnfs:
_LOGGER.error('there are no vnfs matching \'%s\' found in'
' \'%s\'. exiting...', args['vnf'],
- settings.getValue('vnf_dir'))
+ settings.getValue('VNF_DIR'))
+ sys.exit(1)
+
+ if args['loadgen']:
+ loadgens = Loader().get_loadgens()
+ if args['loadgen'] not in loadgens:
+ _LOGGER.error('There are no loadgens matching \'%s\' found in'
+ ' \'%s\'. Exiting...', args['loadgen'],
+ settings.getValue('LOADGEN_DIR'))
sys.exit(1)
if args['exact_test_name'] and args['tests']:
_LOGGER.error("Cannot specify tests with both positional args and --test.")
sys.exit(1)
- # sriov handling
- settings.setValue('SRIOV_ENABLED', enable_sriov(settings.getValue('WHITELIST_NICS')))
-
# modify NIC configuration to decode enhanced PCI IDs
wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
+ # sriov handling is performed on checked/expanded PCI IDs
+ settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
+
nic_list = []
for nic in wl_nics_orig:
tmp_nic = networkcard.get_nic_info(nic)
'driver' : networkcard.get_driver(tmp_nic),
'device' : networkcard.get_device_name(tmp_nic)})
else:
- _LOGGER.error("Invalid network card PCI ID: '%s'", nic)
vsperf_finalize()
- raise
+ raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
settings.setValue('NICS', nic_list)
# for backward compatibility
settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
- # update global settings
- guest_loopback = get_test_param('guest_loopback', None)
- if guest_loopback:
- tmp_gl = []
- for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))):
- tmp_gl.append(guest_loopback)
- settings.setValue('GUEST_LOOPBACK', tmp_gl)
-
- settings.setValue('mode', args['mode'])
-
# generate results directory name
date = datetime.datetime.fromtimestamp(time.time())
results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
_LOGGER.debug("Executing traffic generator:")
loader = Loader()
# set traffic details, so they can be passed to traffic ctl
- traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
- traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
- 'bidir': get_test_param('bidirectional', 'False'),
- 'multistream': int(get_test_param('multistream', 0)),
- 'stream_type': get_test_param('stream_type', 'L4'),
- 'frame_rate': int(get_test_param('iload', 100))})
+ traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
+
+ traffic = functions.check_traffic(traffic)
traffic_ctl = component_factory.create_traffic(
traffic['traffic_type'],
traffic_ctl.send_traffic(traffic)
_LOGGER.debug("Traffic Results:")
traffic_ctl.print_results()
+
+ # write results into CSV file
+ result_file = os.path.join(results_path, "result.csv")
+ PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
else:
# configure tests
if args['integration']:
sys.exit(1)
# run tests
+ # Add pylint exception: Redefinition of test type from
+ # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase
+ # pylint: disable=redefined-variable-type
suite = unittest.TestSuite()
+ settings_snapshot = copy.deepcopy(settings.__dict__)
for cfg in selected_tests:
test_name = cfg.get('Name', '<Name not set>')
try:
test = PerformanceTestCase(cfg)
test.run()
suite.addTest(MockTestCase('', True, test.name))
- #pylint: disable=broad-except
+ # pylint: disable=broad-except
except (Exception) as ex:
_LOGGER.exception("Failed to run test: %s", test_name)
suite.addTest(MockTestCase(str(ex), False, test_name))
_LOGGER.info("Continuing with next test...")
+ finally:
+ settings.restore_from_dict(settings_snapshot)
# generate final rst report with results of all executed TCs
generate_final_report()
if args['opnfvpod']:
pod_name = args['opnfvpod']
- installer_name = settings.getValue('OPNFV_INSTALLER')
+ installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()
opnfv_url = settings.getValue('OPNFV_URL')
pkg_list = settings.getValue('PACKAGE_LIST')
- int_data = {'cuse': False,
- 'vanilla': False,
- 'pod': pod_name,
+ int_data = {'pod': pod_name,
+ 'build_tag': get_build_tag(),
'installer': installer_name,
'pkg_list': pkg_list,
- 'db_url': opnfv_url}
- if settings.getValue('VSWITCH').endswith('Vanilla'):
- int_data['vanilla'] = True
- if settings.getValue('VNF').endswith('Cuse'):
- int_data['cuse'] = True
- opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
+ 'db_url': opnfv_url,
+ # pass vswitch name from configuration to be used for failed
+ # TCs; In case of successful TCs it is safer to use vswitch
+ # name from CSV as TC can override global configuration
+ 'vswitch': str(settings.getValue('VSWITCH')).lower()}
+ tc_names = [tc['Name'] for tc in selected_tests]
+ opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
# cleanup before exit
vsperf_finalize()
if __name__ == "__main__":
main()
-
Code Review / vswitchperf.git / blobdiff