3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
34 sys.dont_write_bytecode = True
36 from conf import settings
37 from conf import get_test_param
38 from core.loader import Loader
39 from testcases import PerformanceTestCase
40 from testcases import IntegrationTestCase
41 from tools import tasks
42 from tools import networkcard
43 from tools import functions
44 from tools.pkt_gen import trafficgen
45 from tools.opnfvdashboard import opnfvdashboard
46 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
47 import core.component_factory as component_factory
50 'debug': logging.DEBUG,
52 'warning': logging.WARNING,
53 'error': logging.ERROR,
54 'critical': logging.CRITICAL
57 _TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
58 'foot' : 'tools/report/report_foot.rst',
59 'final' : 'test_report.rst',
60 'tmp' : 'tools/report/report_tmp_caption.rst'
63 _LOGGER = logging.getLogger()
65 def parse_arguments():
67 Parse command line arguments.
69 class _SplitTestParamsAction(argparse.Action):
71 Parse and split the '--test-params' argument.
73 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
74 values. For multiple overrides use a ; separated list for
75 e.g. --test-params 'x=z; y=a,b'
77 def __call__(self, parser, namespace, values, option_string=None):
80 for value in values.split(';'):
81 result = [key.strip() for key in value.split('=')]
83 results[result[0]] = True
84 elif len(result) == 2:
85 results[result[0]] = result[1]
87 raise argparse.ArgumentTypeError(
88 'expected \'%s\' to be of format \'key=val\' or'
91 setattr(namespace, self.dest, results)
93 class _ValidateFileAction(argparse.Action):
94 """Validate a file can be read from before using it.
96 def __call__(self, parser, namespace, values, option_string=None):
97 if not os.path.isfile(values):
98 raise argparse.ArgumentTypeError(
99 'the path \'%s\' is not a valid path' % values)
100 elif not os.access(values, os.R_OK):
101 raise argparse.ArgumentTypeError(
102 'the path \'%s\' is not accessible' % values)
104 setattr(namespace, self.dest, values)
106 class _ValidateDirAction(argparse.Action):
107 """Validate a directory can be written to before using it.
109 def __call__(self, parser, namespace, values, option_string=None):
110 if not os.path.isdir(values):
111 raise argparse.ArgumentTypeError(
112 'the path \'%s\' is not a valid path' % values)
113 elif not os.access(values, os.W_OK):
114 raise argparse.ArgumentTypeError(
115 'the path \'%s\' is not accessible' % values)
117 setattr(namespace, self.dest, values)
119 def list_logging_levels():
120 """Give a summary of all available logging levels.
122 :return: List of verbosity level names in decreasing order of
125 return sorted(VERBOSITY_LEVELS.keys(),
126 key=lambda x: VERBOSITY_LEVELS[x])
128 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
129 argparse.ArgumentDefaultsHelpFormatter)
130 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
131 parser.add_argument('--list', '--list-tests', action='store_true',
132 help='list all tests and exit')
133 parser.add_argument('--list-trafficgens', action='store_true',
134 help='list all traffic generators and exit')
135 parser.add_argument('--list-collectors', action='store_true',
136 help='list all system metrics loggers and exit')
137 parser.add_argument('--list-vswitches', action='store_true',
138 help='list all system vswitches and exit')
139 parser.add_argument('--list-fwdapps', action='store_true',
140 help='list all system forwarding applications and exit')
141 parser.add_argument('--list-vnfs', action='store_true',
142 help='list all system vnfs and exit')
143 parser.add_argument('--list-settings', action='store_true',
144 help='list effective settings configuration and exit')
145 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
146 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
147 runs only the two tests with those exact names.\
148 To run all tests omit both positional args and --tests arg.')
150 group = parser.add_argument_group('test selection options')
151 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
152 Values: "normal" - execute vSwitch, VNF and traffic generator;\
153 "trafficgen" - execute only traffic generator; "trafficgen-off" \
154 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
155 and VNF but pause before traffic transmission ', default='normal')
157 group.add_argument('-f', '--test-spec', help='test specification file')
158 group.add_argument('-d', '--test-dir', help='directory containing tests')
159 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
160 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
161 name contains RFC2544 less those containing "p2p"; "!back2back" - \
162 run all tests except those containing back2back')
163 group.add_argument('--verbosity', choices=list_logging_levels(),
165 group.add_argument('--integration', action='store_true', help='execute integration tests')
166 group.add_argument('--trafficgen', help='traffic generator to use')
167 group.add_argument('--vswitch', help='vswitch implementation to use')
168 group.add_argument('--fwdapp', help='packet forwarding application to use')
169 group.add_argument('--vnf', help='vnf to use')
170 group.add_argument('--sysmetrics', help='system metrics logger to use')
171 group = parser.add_argument_group('test behavior options')
172 group.add_argument('--xunit', action='store_true',
173 help='enable xUnit-formatted output')
174 group.add_argument('--xunit-dir', action=_ValidateDirAction,
175 help='output directory of xUnit-formatted output')
176 group.add_argument('--load-env', action='store_true',
177 help='enable loading of settings from the environment')
178 group.add_argument('--conf-file', action=_ValidateFileAction,
179 help='settings file')
180 group.add_argument('--test-params', action=_SplitTestParamsAction,
181 help='csv list of test parameters: key=val; e.g.'
182 'including pkt_sizes=x,y; duration=x; '
183 'rfc2544_tests=x ...')
184 group.add_argument('--opnfvpod', help='name of POD in opnfv')
186 args = vars(parser.parse_args())
191 def configure_logging(level):
192 """Configure logging.
194 log_file_default = os.path.join(
195 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
196 log_file_host_cmds = os.path.join(
197 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
198 log_file_traffic_gen = os.path.join(
199 settings.getValue('LOG_DIR'),
200 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
202 _LOGGER.setLevel(logging.DEBUG)
204 stream_logger = logging.StreamHandler(sys.stdout)
205 stream_logger.setLevel(VERBOSITY_LEVELS[level])
206 stream_logger.setFormatter(logging.Formatter(
207 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
208 _LOGGER.addHandler(stream_logger)
210 file_logger = logging.FileHandler(filename=log_file_default)
211 file_logger.setLevel(logging.DEBUG)
212 _LOGGER.addHandler(file_logger)
214 class CommandFilter(logging.Filter):
215 """Filter out strings beginning with 'cmd :'"""
216 def filter(self, record):
217 return record.getMessage().startswith(tasks.CMD_PREFIX)
219 class TrafficGenCommandFilter(logging.Filter):
220 """Filter out strings beginning with 'gencmd :'"""
221 def filter(self, record):
222 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
224 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
225 cmd_logger.setLevel(logging.DEBUG)
226 cmd_logger.addFilter(CommandFilter())
227 _LOGGER.addHandler(cmd_logger)
229 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
230 gen_logger.setLevel(logging.DEBUG)
231 gen_logger.addFilter(TrafficGenCommandFilter())
232 _LOGGER.addHandler(gen_logger)
235 def apply_filter(tests, tc_filter):
236 """Allow a subset of tests to be conveniently selected
238 :param tests: The list of Tests from which to select.
239 :param tc_filter: A case-insensitive string of comma-separated terms
240 indicating the Tests to select.
241 e.g. 'RFC' - select all tests whose name contains 'RFC'
242 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
244 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
245 or 'burst' and from these remove any containing 'p2p'.
246 e.g. '' - empty string selects all tests.
247 :return: A list of the selected Tests.
249 # if negative filter is first we have to start with full list of tests
250 if tc_filter.strip()[0] == '!':
254 if tc_filter is None:
257 for term in [x.strip() for x in tc_filter.lower().split(",")]:
258 if not term or term[0] != '!':
259 # Add matching tests from 'tests' into results
260 result.extend([test for test in tests \
261 if test['Name'].lower().find(term) >= 0])
263 # Term begins with '!' so we remove matching tests
264 result = [test for test in result \
265 if test['Name'].lower().find(term[1:]) < 0]
270 def check_and_set_locale():
271 """ Function will check locale settings. In case, that it isn't configured
272 properly, then default values specified by DEFAULT_LOCALE will be used.
275 system_locale = locale.getdefaultlocale()
276 if None in system_locale:
277 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
278 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
279 system_locale, locale.getdefaultlocale())
282 def generate_final_report():
283 """ Function will check if partial test results are available
284 and generates final report in rst format.
287 path = settings.getValue('RESULTS_PATH')
288 # check if there are any results in rst format
289 rst_results = glob.glob(os.path.join(path, 'result*rst'))
292 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
293 # create report caption directly - it is not worth to execute jinja machinery
294 if settings.getValue('VSWITCH').lower() != 'none':
295 pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
297 pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
298 report_caption = '{}\n{} {}\n{}\n\n'.format(
299 '============================================================',
300 'Performance report for',
302 '============================================================')
304 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
305 file_.write(report_caption)
307 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
308 ' '.join(rst_results), _TEMPLATE_RST['foot'],
309 test_report), shell=True)
310 if retval == 0 and os.path.isfile(test_report):
311 _LOGGER.info('Overall test report written to "%s"', test_report)
313 _LOGGER.error('Generatrion of overall test report has failed.')
315 # remove temporary file
316 os.remove(_TEMPLATE_RST['tmp'])
318 except subprocess.CalledProcessError:
319 _LOGGER.error('Generatrion of overall test report has failed.')
322 def enable_sriov(nic_list):
323 """ Enable SRIOV for given enhanced PCI IDs
325 :param nic_list: A list of enhanced PCI IDs
327 # detect if sriov is required
330 if networkcard.is_sriov_nic(nic):
331 tmp_nic = nic.split('|')
332 if tmp_nic[0] in sriov_nic:
333 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
334 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
336 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
338 # sriov is required for some NICs
340 for nic in sriov_nic:
341 # check if SRIOV is supported and enough virt interfaces are available
342 if not networkcard.is_sriov_supported(nic) \
343 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
344 # if not, enable and set appropriate number of VFs
345 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
346 _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
349 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
351 # WORKAROUND: it has been observed with IXGBE(VF) driver,
352 # that NIC doesn't correclty dispatch traffic to VFs based
353 # on their MAC address. Unbind and bind to the same driver
355 networkcard.reinit_vfs(nic)
357 # After SRIOV is enabled it takes some time until network drivers
358 # properly initialize all cards.
359 # Wait also in case, that SRIOV was already configured as it can be
360 # configured automatically just before vsperf execution.
368 def disable_sriov(nic_list):
369 """ Disable SRIOV for given PCI IDs
371 :param nic_list: A list of enhanced PCI IDs
374 if networkcard.is_sriov_nic(nic):
375 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
376 _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
379 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
382 def handle_list_options(args):
383 """ Process --list cli arguments if needed
385 :param args: A dictionary with all CLI arguments
387 if args['list_trafficgens']:
388 print(Loader().get_trafficgens_printable())
391 if args['list_collectors']:
392 print(Loader().get_collectors_printable())
395 if args['list_vswitches']:
396 print(Loader().get_vswitches_printable())
399 if args['list_vnfs']:
400 print(Loader().get_vnfs_printable())
403 if args['list_fwdapps']:
404 print(Loader().get_pktfwds_printable())
407 if args['list_settings']:
413 if args['integration']:
414 testcases = settings.getValue('INTEGRATION_TESTS')
416 testcases = settings.getValue('PERFORMANCE_TESTS')
418 print("Available Tests:")
419 print("================")
421 for test in testcases:
422 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
426 def vsperf_finalize():
427 """ Clean up before exit
429 # remove directory if no result files were created
431 results_path = settings.getValue('RESULTS_PATH')
432 if os.path.exists(results_path):
433 files_list = os.listdir(results_path)
435 _LOGGER.info("Removing empty result directory: " + results_path)
436 shutil.rmtree(results_path)
437 except AttributeError:
438 # skip it if parameter doesn't exist
441 # disable SRIOV if needed
443 if settings.getValue('SRIOV_ENABLED'):
444 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
445 except AttributeError:
446 # skip it if parameter doesn't exist
450 class MockTestCase(unittest.TestCase):
451 """Allow use of xmlrunner to generate Jenkins compatible output without
452 using xmlrunner to actually run tests.
455 suite = unittest.TestSuite()
456 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
457 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
458 xmlrunner.XMLTestRunner(...).run(suite)
461 def __init__(self, msg, is_pass, test_name):
464 self.is_pass = is_pass
466 #dynamically create a test method with the right name
467 #but point the method at our generic test method
468 setattr(MockTestCase, test_name, self.generic_test)
470 super(MockTestCase, self).__init__(test_name)
472 def generic_test(self):
473 """Provide a generic function that raises or not based
474 on how self.is_pass was set in the constructor"""
475 self.assertTrue(self.is_pass, self.msg)
481 args = parse_arguments()
485 settings.load_from_dir('conf')
487 # Load non performance/integration tests
488 if args['integration']:
489 settings.load_from_dir('conf/integration')
491 # load command line parameters first in case there are settings files
493 settings.load_from_dict(args)
495 if args['conf_file']:
496 settings.load_from_file(args['conf_file'])
499 settings.load_from_env()
501 # reload command line parameters since these should take higher priority
502 # than both a settings file and environment variables
503 settings.load_from_dict(args)
505 # set dpdk and ovs paths accorfing to VNF and VSWITCH
506 functions.settings_update_paths()
508 # if required, handle list-* operations
509 handle_list_options(args)
511 configure_logging(settings.getValue('VERBOSITY'))
513 # check and fix locale
514 check_and_set_locale()
516 # configure trafficgens
517 if args['trafficgen']:
518 trafficgens = Loader().get_trafficgens()
519 if args['trafficgen'] not in trafficgens:
520 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
521 ' \'%s\'. Exiting...', args['trafficgen'],
522 settings.getValue('TRAFFICGEN_DIR'))
525 # configuration validity checks
527 vswitch_none = 'none' == args['vswitch'].strip().lower()
529 settings.setValue('VSWITCH', 'none')
531 vswitches = Loader().get_vswitches()
532 if args['vswitch'] not in vswitches:
533 _LOGGER.error('There are no vswitches matching \'%s\' found in'
534 ' \'%s\'. Exiting...', args['vswitch'],
535 settings.getValue('VSWITCH_DIR'))
539 settings.setValue('PKTFWD', args['fwdapp'])
540 fwdapps = Loader().get_pktfwds()
541 if args['fwdapp'] not in fwdapps:
542 _LOGGER.error('There are no forwarding application'
543 ' matching \'%s\' found in'
544 ' \'%s\'. Exiting...', args['fwdapp'],
545 settings.getValue('PKTFWD_DIR'))
549 vnfs = Loader().get_vnfs()
550 if args['vnf'] not in vnfs:
551 _LOGGER.error('there are no vnfs matching \'%s\' found in'
552 ' \'%s\'. exiting...', args['vnf'],
553 settings.getValue('vnf_dir'))
556 if args['exact_test_name'] and args['tests']:
557 _LOGGER.error("Cannot specify tests with both positional args and --test.")
560 # modify NIC configuration to decode enhanced PCI IDs
561 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
562 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
564 # sriov handling is performed on checked/expanded PCI IDs
565 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
568 for nic in wl_nics_orig:
569 tmp_nic = networkcard.get_nic_info(nic)
571 nic_list.append({'pci' : tmp_nic,
572 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
573 'mac' : networkcard.get_mac(tmp_nic),
574 'driver' : networkcard.get_driver(tmp_nic),
575 'device' : networkcard.get_device_name(tmp_nic)})
577 _LOGGER.error("Invalid network card PCI ID: '%s'", nic)
581 settings.setValue('NICS', nic_list)
582 # for backward compatibility
583 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
585 # update global settings
586 guest_loopback = get_test_param('guest_loopback', None)
589 for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))):
590 tmp_gl.append(guest_loopback)
591 settings.setValue('GUEST_LOOPBACK', tmp_gl)
593 settings.setValue('mode', args['mode'])
595 # generate results directory name
596 date = datetime.datetime.fromtimestamp(time.time())
597 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
598 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
599 settings.setValue('RESULTS_PATH', results_path)
601 # create results directory
602 if not os.path.exists(results_path):
603 _LOGGER.info("Creating result directory: " + results_path)
604 os.makedirs(results_path)
606 if settings.getValue('mode') == 'trafficgen':
607 # execute only traffic generator
608 _LOGGER.debug("Executing traffic generator:")
610 # set traffic details, so they can be passed to traffic ctl
611 traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
612 traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
613 'bidir': get_test_param('bidirectional', 'False'),
614 'multistream': int(get_test_param('multistream', 0)),
615 'stream_type': get_test_param('stream_type', 'L4'),
616 'frame_rate': int(get_test_param('iload', 100))})
618 traffic_ctl = component_factory.create_traffic(
619 traffic['traffic_type'],
620 loader.get_trafficgen_class())
622 traffic_ctl.send_traffic(traffic)
623 _LOGGER.debug("Traffic Results:")
624 traffic_ctl.print_results()
626 # write results into CSV file
627 result_file = os.path.join(results_path, "result.csv")
628 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
631 if args['integration']:
632 testcases = settings.getValue('INTEGRATION_TESTS')
634 testcases = settings.getValue('PERFORMANCE_TESTS')
636 if args['exact_test_name']:
637 exact_names = args['exact_test_name']
638 # positional args => exact matches only
639 selected_tests = [test for test in testcases if test['Name'] in exact_names]
641 # --tests => apply filter to select requested tests
642 selected_tests = apply_filter(testcases, args['tests'])
644 # Default - run all tests
645 selected_tests = testcases
647 if not len(selected_tests):
648 _LOGGER.error("No tests matched --tests option or positional args. Done.")
653 suite = unittest.TestSuite()
654 for cfg in selected_tests:
655 test_name = cfg.get('Name', '<Name not set>')
657 if args['integration']:
658 test = IntegrationTestCase(cfg)
660 test = PerformanceTestCase(cfg)
662 suite.addTest(MockTestCase('', True, test.name))
663 #pylint: disable=broad-except
664 except (Exception) as ex:
665 _LOGGER.exception("Failed to run test: %s", test_name)
666 suite.addTest(MockTestCase(str(ex), False, test_name))
667 _LOGGER.info("Continuing with next test...")
669 # generate final rst report with results of all executed TCs
670 generate_final_report()
672 if settings.getValue('XUNIT'):
673 xmlrunner.XMLTestRunner(
674 output=settings.getValue('XUNIT_DIR'), outsuffix="",
675 verbosity=0).run(suite)
678 pod_name = args['opnfvpod']
679 installer_name = settings.getValue('OPNFV_INSTALLER')
680 opnfv_url = settings.getValue('OPNFV_URL')
681 pkg_list = settings.getValue('PACKAGE_LIST')
683 int_data = {'vanilla': False,
685 'installer': installer_name,
686 'pkg_list': pkg_list,
688 if settings.getValue('VSWITCH').endswith('Vanilla'):
689 int_data['vanilla'] = True
690 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
692 # cleanup before exit
695 if __name__ == "__main__":