3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
34 sys.dont_write_bytecode = True
36 from conf import settings
37 from conf import get_test_param
38 from core.loader import Loader
39 from testcases import PerformanceTestCase
40 from testcases import IntegrationTestCase
41 from tools import tasks
42 from tools import networkcard
43 from tools.pkt_gen import trafficgen
44 from tools.opnfvdashboard import opnfvdashboard
45 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
46 import core.component_factory as component_factory
49 'debug': logging.DEBUG,
51 'warning': logging.WARNING,
52 'error': logging.ERROR,
53 'critical': logging.CRITICAL
56 _TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
57 'foot' : 'tools/report/report_foot.rst',
58 'final' : 'test_report.rst',
59 'tmp' : 'tools/report/report_tmp_caption.rst'
62 _LOGGER = logging.getLogger()
64 def parse_arguments():
66 Parse command line arguments.
68 class _SplitTestParamsAction(argparse.Action):
70 Parse and split the '--test-params' argument.
72 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
73 values. For multiple overrides use a ; separated list for
74 e.g. --test-params 'x=z; y=a,b'
76 def __call__(self, parser, namespace, values, option_string=None):
79 for value in values.split(';'):
80 result = [key.strip() for key in value.split('=')]
82 results[result[0]] = True
83 elif len(result) == 2:
84 results[result[0]] = result[1]
86 raise argparse.ArgumentTypeError(
87 'expected \'%s\' to be of format \'key=val\' or'
90 setattr(namespace, self.dest, results)
92 class _ValidateFileAction(argparse.Action):
93 """Validate a file can be read from before using it.
95 def __call__(self, parser, namespace, values, option_string=None):
96 if not os.path.isfile(values):
97 raise argparse.ArgumentTypeError(
98 'the path \'%s\' is not a valid path' % values)
99 elif not os.access(values, os.R_OK):
100 raise argparse.ArgumentTypeError(
101 'the path \'%s\' is not accessible' % values)
103 setattr(namespace, self.dest, values)
105 class _ValidateDirAction(argparse.Action):
106 """Validate a directory can be written to before using it.
108 def __call__(self, parser, namespace, values, option_string=None):
109 if not os.path.isdir(values):
110 raise argparse.ArgumentTypeError(
111 'the path \'%s\' is not a valid path' % values)
112 elif not os.access(values, os.W_OK):
113 raise argparse.ArgumentTypeError(
114 'the path \'%s\' is not accessible' % values)
116 setattr(namespace, self.dest, values)
118 def list_logging_levels():
119 """Give a summary of all available logging levels.
121 :return: List of verbosity level names in decreasing order of
124 return sorted(VERBOSITY_LEVELS.keys(),
125 key=lambda x: VERBOSITY_LEVELS[x])
127 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
128 argparse.ArgumentDefaultsHelpFormatter)
129 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
130 parser.add_argument('--list', '--list-tests', action='store_true',
131 help='list all tests and exit')
132 parser.add_argument('--list-trafficgens', action='store_true',
133 help='list all traffic generators and exit')
134 parser.add_argument('--list-collectors', action='store_true',
135 help='list all system metrics loggers and exit')
136 parser.add_argument('--list-vswitches', action='store_true',
137 help='list all system vswitches and exit')
138 parser.add_argument('--list-fwdapps', action='store_true',
139 help='list all system forwarding applications and exit')
140 parser.add_argument('--list-vnfs', action='store_true',
141 help='list all system vnfs and exit')
142 parser.add_argument('--list-settings', action='store_true',
143 help='list effective settings configuration and exit')
144 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
145 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
146 runs only the two tests with those exact names.\
147 To run all tests omit both positional args and --tests arg.')
149 group = parser.add_argument_group('test selection options')
150 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
151 Values: "normal" - execute vSwitch, VNF and traffic generator;\
152 "trafficgen" - execute only traffic generator; "trafficgen-off" \
153 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
154 and VNF but pause before traffic transmission ', default='normal')
156 group.add_argument('-f', '--test-spec', help='test specification file')
157 group.add_argument('-d', '--test-dir', help='directory containing tests')
158 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
159 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
160 name contains RFC2544 less those containing "p2p"')
161 group.add_argument('--verbosity', choices=list_logging_levels(),
163 group.add_argument('--integration', action='store_true', help='execute integration tests')
164 group.add_argument('--trafficgen', help='traffic generator to use')
165 group.add_argument('--vswitch', help='vswitch implementation to use')
166 group.add_argument('--fwdapp', help='packet forwarding application to use')
167 group.add_argument('--vnf', help='vnf to use')
168 group.add_argument('--sysmetrics', help='system metrics logger to use')
169 group = parser.add_argument_group('test behavior options')
170 group.add_argument('--xunit', action='store_true',
171 help='enable xUnit-formatted output')
172 group.add_argument('--xunit-dir', action=_ValidateDirAction,
173 help='output directory of xUnit-formatted output')
174 group.add_argument('--load-env', action='store_true',
175 help='enable loading of settings from the environment')
176 group.add_argument('--conf-file', action=_ValidateFileAction,
177 help='settings file')
178 group.add_argument('--test-params', action=_SplitTestParamsAction,
179 help='csv list of test parameters: key=val; e.g.'
180 'including pkt_sizes=x,y; duration=x; '
181 'rfc2544_trials=x ...')
182 group.add_argument('--opnfvpod', help='name of POD in opnfv')
184 args = vars(parser.parse_args())
189 def configure_logging(level):
190 """Configure logging.
192 log_file_default = os.path.join(
193 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
194 log_file_host_cmds = os.path.join(
195 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
196 log_file_traffic_gen = os.path.join(
197 settings.getValue('LOG_DIR'),
198 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
200 _LOGGER.setLevel(logging.DEBUG)
202 stream_logger = logging.StreamHandler(sys.stdout)
203 stream_logger.setLevel(VERBOSITY_LEVELS[level])
204 stream_logger.setFormatter(logging.Formatter(
205 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
206 _LOGGER.addHandler(stream_logger)
208 file_logger = logging.FileHandler(filename=log_file_default)
209 file_logger.setLevel(logging.DEBUG)
210 _LOGGER.addHandler(file_logger)
212 class CommandFilter(logging.Filter):
213 """Filter out strings beginning with 'cmd :'"""
214 def filter(self, record):
215 return record.getMessage().startswith(tasks.CMD_PREFIX)
217 class TrafficGenCommandFilter(logging.Filter):
218 """Filter out strings beginning with 'gencmd :'"""
219 def filter(self, record):
220 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
222 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
223 cmd_logger.setLevel(logging.DEBUG)
224 cmd_logger.addFilter(CommandFilter())
225 _LOGGER.addHandler(cmd_logger)
227 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
228 gen_logger.setLevel(logging.DEBUG)
229 gen_logger.addFilter(TrafficGenCommandFilter())
230 _LOGGER.addHandler(gen_logger)
233 def apply_filter(tests, tc_filter):
234 """Allow a subset of tests to be conveniently selected
236 :param tests: The list of Tests from which to select.
237 :param tc_filter: A case-insensitive string of comma-separated terms
238 indicating the Tests to select.
239 e.g. 'RFC' - select all tests whose name contains 'RFC'
240 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
242 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
243 or 'burst' and from these remove any containing 'p2p'.
244 e.g. '' - empty string selects all tests.
245 :return: A list of the selected Tests.
248 if tc_filter is None:
251 for term in [x.strip() for x in tc_filter.lower().split(",")]:
252 if not term or term[0] != '!':
253 # Add matching tests from 'tests' into results
254 result.extend([test for test in tests \
255 if test.name.lower().find(term) >= 0])
257 # Term begins with '!' so we remove matching tests
258 result = [test for test in result \
259 if test.name.lower().find(term[1:]) < 0]
264 def check_and_set_locale():
265 """ Function will check locale settings. In case, that it isn't configured
266 properly, then default values specified by DEFAULT_LOCALE will be used.
269 system_locale = locale.getdefaultlocale()
270 if None in system_locale:
271 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
272 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
273 system_locale, locale.getdefaultlocale())
276 def generate_final_report():
277 """ Function will check if partial test results are available
278 and generates final report in rst format.
281 path = settings.getValue('RESULTS_PATH')
282 # check if there are any results in rst format
283 rst_results = glob.glob(os.path.join(path, 'result*rst'))
286 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
287 # create report caption directly - it is not worth to execute jinja machinery
288 if settings.getValue('VSWITCH').lower() != 'none':
289 pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
291 pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
292 report_caption = '{}\n{} {}\n{}\n\n'.format(
293 '============================================================',
294 'Performance report for',
296 '============================================================')
298 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
299 file_.write(report_caption)
301 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
302 ' '.join(rst_results), _TEMPLATE_RST['foot'],
303 test_report), shell=True)
304 if retval == 0 and os.path.isfile(test_report):
305 _LOGGER.info('Overall test report written to "%s"', test_report)
307 _LOGGER.error('Generatrion of overall test report has failed.')
309 # remove temporary file
310 os.remove(_TEMPLATE_RST['tmp'])
312 except subprocess.CalledProcessError:
313 _LOGGER.error('Generatrion of overall test report has failed.')
316 def enable_sriov(nic_list):
317 """ Enable SRIOV for given enhanced PCI IDs
319 :param nic_list: A list of enhanced PCI IDs
321 # detect if sriov is required
324 if networkcard.is_sriov_nic(nic):
325 tmp_nic = nic.split('|')
326 if tmp_nic[0] in sriov_nic:
327 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
328 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
330 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
332 # sriov is required for some NICs
334 for nic in sriov_nic:
335 # check if SRIOV is supported and enough virt interfaces are available
336 if not networkcard.is_sriov_supported(nic) \
337 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
338 # if not, enable and set appropriate number of VFs
339 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
340 _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
343 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
345 # WORKAROUND: it has been observed with IXGBE(VF) driver,
346 # that NIC doesn't correclty dispatch traffic to VFs based
347 # on their MAC address. Unbind and bind to the same driver
349 networkcard.reinit_vfs(nic)
351 # After SRIOV is enabled it takes some time until network drivers
352 # properly initialize all cards.
353 # Wait also in case, that SRIOV was already configured as it can be
354 # configured automatically just before vsperf execution.
362 def disable_sriov(nic_list):
363 """ Disable SRIOV for given PCI IDs
365 :param nic_list: A list of enhanced PCI IDs
368 if networkcard.is_sriov_nic(nic):
369 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
370 _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
373 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
376 def handle_list_options(args):
377 """ Process --list cli arguments if needed
379 :param args: A dictionary with all CLI arguments
381 if args['list_trafficgens']:
382 print(Loader().get_trafficgens_printable())
385 if args['list_collectors']:
386 print(Loader().get_collectors_printable())
389 if args['list_vswitches']:
390 print(Loader().get_vswitches_printable())
393 if args['list_vnfs']:
394 print(Loader().get_vnfs_printable())
397 if args['list_fwdapps']:
398 print(Loader().get_pktfwds_printable())
401 if args['list_settings']:
407 if args['integration']:
408 testcases = settings.getValue('INTEGRATION_TESTS')
410 testcases = settings.getValue('PERFORMANCE_TESTS')
412 print("Available Tests:")
413 print("================")
415 for test in testcases:
416 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
420 def vsperf_finalize():
421 """ Clean up before exit
423 # remove directory if no result files were created
425 results_path = settings.getValue('RESULTS_PATH')
426 if os.path.exists(results_path):
427 files_list = os.listdir(results_path)
429 _LOGGER.info("Removing empty result directory: " + results_path)
430 shutil.rmtree(results_path)
431 except AttributeError:
432 # skip it if parameter doesn't exist
435 # disable SRIOV if needed
437 if settings.getValue('SRIOV_ENABLED'):
438 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
439 except AttributeError:
440 # skip it if parameter doesn't exist
444 class MockTestCase(unittest.TestCase):
445 """Allow use of xmlrunner to generate Jenkins compatible output without
446 using xmlrunner to actually run tests.
449 suite = unittest.TestSuite()
450 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
451 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
452 xmlrunner.XMLTestRunner(...).run(suite)
455 def __init__(self, msg, is_pass, test_name):
458 self.is_pass = is_pass
460 #dynamically create a test method with the right name
461 #but point the method at our generic test method
462 setattr(MockTestCase, test_name, self.generic_test)
464 super(MockTestCase, self).__init__(test_name)
466 def generic_test(self):
467 """Provide a generic function that raises or not based
468 on how self.is_pass was set in the constructor"""
469 self.assertTrue(self.is_pass, self.msg)
475 args = parse_arguments()
479 settings.load_from_dir('conf')
481 # Load non performance/integration tests
482 if args['integration']:
483 settings.load_from_dir('conf/integration')
485 # load command line parameters first in case there are settings files
487 settings.load_from_dict(args)
489 if args['conf_file']:
490 settings.load_from_file(args['conf_file'])
493 settings.load_from_env()
495 # reload command line parameters since these should take higher priority
496 # than both a settings file and environment variables
497 settings.load_from_dict(args)
500 # set dpdk and ovs paths accorfing to VNF and VSWITCH
501 if settings.getValue('VSWITCH').endswith('Vanilla'):
502 # settings paths for Vanilla
503 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
504 elif settings.getValue('VSWITCH').endswith('Vhost'):
505 if settings.getValue('VNF').endswith('Cuse'):
506 # settings paths for Cuse
507 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
508 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
510 # settings paths for VhostUser
511 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
512 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
514 # default - set to VHOST USER but can be changed during enhancement
515 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
516 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
517 if 'none' == settings.getValue('VSWITCH').strip().lower():
520 # if required, handle list-* operations
521 handle_list_options(args)
523 configure_logging(settings.getValue('VERBOSITY'))
525 # check and fix locale
526 check_and_set_locale()
528 # configure trafficgens
529 if args['trafficgen']:
530 trafficgens = Loader().get_trafficgens()
531 if args['trafficgen'] not in trafficgens:
532 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
533 ' \'%s\'. Exiting...', args['trafficgen'],
534 settings.getValue('TRAFFICGEN_DIR'))
537 # configuration validity checks
539 vswitch_none = 'none' == args['vswitch'].strip().lower()
541 settings.setValue('VSWITCH', 'none')
543 vswitches = Loader().get_vswitches()
544 if args['vswitch'] not in vswitches:
545 _LOGGER.error('There are no vswitches matching \'%s\' found in'
546 ' \'%s\'. Exiting...', args['vswitch'],
547 settings.getValue('VSWITCH_DIR'))
551 settings.setValue('PKTFWD', args['fwdapp'])
552 fwdapps = Loader().get_pktfwds()
553 if args['fwdapp'] not in fwdapps:
554 _LOGGER.error('There are no forwarding application'
555 ' matching \'%s\' found in'
556 ' \'%s\'. Exiting...', args['fwdapp'],
557 settings.getValue('PKTFWD_DIR'))
561 vnfs = Loader().get_vnfs()
562 if args['vnf'] not in vnfs:
563 _LOGGER.error('there are no vnfs matching \'%s\' found in'
564 ' \'%s\'. exiting...', args['vnf'],
565 settings.getValue('vnf_dir'))
568 if args['exact_test_name'] and args['tests']:
569 _LOGGER.error("Cannot specify tests with both positional args and --test.")
573 settings.setValue('SRIOV_ENABLED', enable_sriov(settings.getValue('WHITELIST_NICS')))
575 # modify NIC configuration to decode enhanced PCI IDs
576 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
577 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
580 for nic in wl_nics_orig:
581 tmp_nic = networkcard.get_nic_info(nic)
583 nic_list.append({'pci' : tmp_nic,
584 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
585 'mac' : networkcard.get_mac(tmp_nic),
586 'driver' : networkcard.get_driver(tmp_nic),
587 'device' : networkcard.get_device_name(tmp_nic)})
589 _LOGGER.error("Invalid network card PCI ID: '%s'", nic)
593 settings.setValue('NICS', nic_list)
594 # for backward compatibility
595 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
597 # update global settings
598 guest_loopback = get_test_param('guest_loopback', None)
601 for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))):
602 tmp_gl.append(guest_loopback)
603 settings.setValue('GUEST_LOOPBACK', tmp_gl)
605 settings.setValue('mode', args['mode'])
607 # generate results directory name
608 date = datetime.datetime.fromtimestamp(time.time())
609 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
610 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
611 settings.setValue('RESULTS_PATH', results_path)
613 # create results directory
614 if not os.path.exists(results_path):
615 _LOGGER.info("Creating result directory: " + results_path)
616 os.makedirs(results_path)
618 if settings.getValue('mode') == 'trafficgen':
619 # execute only traffic generator
620 _LOGGER.debug("Executing traffic generator:")
622 # set traffic details, so they can be passed to traffic ctl
623 traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
624 traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
625 'bidir': get_test_param('bidirectional', False),
626 'multistream': int(get_test_param('multistream', 0)),
627 'stream_type': get_test_param('stream_type', 'L4'),
628 'frame_rate': int(get_test_param('iload', 100))})
630 traffic_ctl = component_factory.create_traffic(
631 traffic['traffic_type'],
632 loader.get_trafficgen_class())
634 traffic_ctl.send_traffic(traffic)
635 _LOGGER.debug("Traffic Results:")
636 traffic_ctl.print_results()
639 if args['integration']:
640 testcases = settings.getValue('INTEGRATION_TESTS')
642 testcases = settings.getValue('PERFORMANCE_TESTS')
645 for cfg in testcases:
647 if args['integration']:
648 all_tests.append(IntegrationTestCase(cfg))
650 all_tests.append(PerformanceTestCase(cfg))
651 except (Exception) as _:
652 _LOGGER.exception("Failed to create test: %s",
653 cfg.get('Name', '<Name not set>'))
657 # select requested tests
658 if args['exact_test_name']:
659 exact_names = args['exact_test_name']
660 # positional args => exact matches only
661 selected_tests = [test for test in all_tests if test.name in exact_names]
663 # --tests => apply filter to select requested tests
664 selected_tests = apply_filter(all_tests, args['tests'])
666 # Default - run all tests
667 selected_tests = all_tests
669 if not selected_tests:
670 _LOGGER.error("No tests matched --test option or positional args. Done.")
675 suite = unittest.TestSuite()
676 for test in selected_tests:
679 suite.addTest(MockTestCase('', True, test.name))
680 #pylint: disable=broad-except
681 except (Exception) as ex:
682 _LOGGER.exception("Failed to run test: %s", test.name)
683 suite.addTest(MockTestCase(str(ex), False, test.name))
684 _LOGGER.info("Continuing with next test...")
686 # generate final rst report with results of all executed TCs
687 generate_final_report()
689 if settings.getValue('XUNIT'):
690 xmlrunner.XMLTestRunner(
691 output=settings.getValue('XUNIT_DIR'), outsuffix="",
692 verbosity=0).run(suite)
695 pod_name = args['opnfvpod']
696 installer_name = settings.getValue('OPNFV_INSTALLER')
697 opnfv_url = settings.getValue('OPNFV_URL')
698 pkg_list = settings.getValue('PACKAGE_LIST')
700 int_data = {'cuse': False,
703 'installer': installer_name,
704 'pkg_list': pkg_list,
706 if settings.getValue('VSWITCH').endswith('Vanilla'):
707 int_data['vanilla'] = True
708 if settings.getValue('VNF').endswith('Cuse'):
709 int_data['cuse'] = True
710 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
712 # cleanup before exit
715 if __name__ == "__main__":