3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
35 sys.dont_write_bytecode = True
37 from conf import settings
38 from conf import get_test_param
39 from core.loader import Loader
40 from testcases import PerformanceTestCase
41 from testcases import IntegrationTestCase
42 from tools import tasks
43 from tools import networkcard
44 from tools import functions
45 from tools.pkt_gen import trafficgen
46 from tools.opnfvdashboard import opnfvdashboard
47 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
48 import core.component_factory as component_factory
51 'debug': logging.DEBUG,
53 'warning': logging.WARNING,
54 'error': logging.ERROR,
55 'critical': logging.CRITICAL
58 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
60 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
61 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
62 'final' : 'test_report.rst',
63 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
67 _LOGGER = logging.getLogger()
69 def parse_arguments():
71 Parse command line arguments.
73 class _SplitTestParamsAction(argparse.Action):
75 Parse and split the '--test-params' argument.
77 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
78 values. For multiple overrides use a ; separated list for
79 e.g. --test-params 'x=z; y=(a,b)'
81 def __call__(self, parser, namespace, values, option_string=None):
84 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
89 results[param] = value
93 setattr(namespace, self.dest, results)
95 class _ValidateFileAction(argparse.Action):
96 """Validate a file can be read from before using it.
98 def __call__(self, parser, namespace, values, option_string=None):
99 if not os.path.isfile(values):
100 raise argparse.ArgumentTypeError(
101 'the path \'%s\' is not a valid path' % values)
102 elif not os.access(values, os.R_OK):
103 raise argparse.ArgumentTypeError(
104 'the path \'%s\' is not accessible' % values)
106 setattr(namespace, self.dest, values)
108 class _ValidateDirAction(argparse.Action):
109 """Validate a directory can be written to before using it.
111 def __call__(self, parser, namespace, values, option_string=None):
112 if not os.path.isdir(values):
113 raise argparse.ArgumentTypeError(
114 'the path \'%s\' is not a valid path' % values)
115 elif not os.access(values, os.W_OK):
116 raise argparse.ArgumentTypeError(
117 'the path \'%s\' is not accessible' % values)
119 setattr(namespace, self.dest, values)
121 def list_logging_levels():
122 """Give a summary of all available logging levels.
124 :return: List of verbosity level names in decreasing order of
127 return sorted(VERBOSITY_LEVELS.keys(),
128 key=lambda x: VERBOSITY_LEVELS[x])
130 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
131 argparse.ArgumentDefaultsHelpFormatter)
132 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
133 parser.add_argument('--list', '--list-tests', action='store_true',
134 help='list all tests and exit')
135 parser.add_argument('--list-trafficgens', action='store_true',
136 help='list all traffic generators and exit')
137 parser.add_argument('--list-collectors', action='store_true',
138 help='list all system metrics loggers and exit')
139 parser.add_argument('--list-vswitches', action='store_true',
140 help='list all system vswitches and exit')
141 parser.add_argument('--list-fwdapps', action='store_true',
142 help='list all system forwarding applications and exit')
143 parser.add_argument('--list-vnfs', action='store_true',
144 help='list all system vnfs and exit')
145 parser.add_argument('--list-settings', action='store_true',
146 help='list effective settings configuration and exit')
147 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
148 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
149 runs only the two tests with those exact names.\
150 To run all tests omit both positional args and --tests arg.')
152 group = parser.add_argument_group('test selection options')
153 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
154 Values: "normal" - execute vSwitch, VNF and traffic generator;\
155 "trafficgen" - execute only traffic generator; "trafficgen-off" \
156 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
157 and VNF but pause before traffic transmission ', default='normal')
159 group.add_argument('-f', '--test-spec', help='test specification file')
160 group.add_argument('-d', '--test-dir', help='directory containing tests')
161 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
162 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
163 name contains RFC2544 less those containing "p2p"; "!back2back" - \
164 run all tests except those containing back2back')
165 group.add_argument('--verbosity', choices=list_logging_levels(),
167 group.add_argument('--integration', action='store_true', help='execute integration tests')
168 group.add_argument('--trafficgen', help='traffic generator to use')
169 group.add_argument('--vswitch', help='vswitch implementation to use')
170 group.add_argument('--fwdapp', help='packet forwarding application to use')
171 group.add_argument('--vnf', help='vnf to use')
172 group.add_argument('--sysmetrics', help='system metrics logger to use')
173 group = parser.add_argument_group('test behavior options')
174 group.add_argument('--xunit', action='store_true',
175 help='enable xUnit-formatted output')
176 group.add_argument('--xunit-dir', action=_ValidateDirAction,
177 help='output directory of xUnit-formatted output')
178 group.add_argument('--load-env', action='store_true',
179 help='enable loading of settings from the environment')
180 group.add_argument('--conf-file', action=_ValidateFileAction,
181 help='settings file')
182 group.add_argument('--test-params', action=_SplitTestParamsAction,
183 help='csv list of test parameters: key=val; e.g. '
184 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
185 'GUEST_LOOPBACK=["l2fwd"] ...')
186 group.add_argument('--opnfvpod', help='name of POD in opnfv')
188 args = vars(parser.parse_args())
193 def configure_logging(level):
194 """Configure logging.
196 log_file_default = os.path.join(
197 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
198 log_file_host_cmds = os.path.join(
199 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
200 log_file_traffic_gen = os.path.join(
201 settings.getValue('LOG_DIR'),
202 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
204 _LOGGER.setLevel(logging.DEBUG)
206 stream_logger = logging.StreamHandler(sys.stdout)
207 stream_logger.setLevel(VERBOSITY_LEVELS[level])
208 stream_logger.setFormatter(logging.Formatter(
209 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
210 _LOGGER.addHandler(stream_logger)
212 file_logger = logging.FileHandler(filename=log_file_default)
213 file_logger.setLevel(logging.DEBUG)
214 _LOGGER.addHandler(file_logger)
216 class CommandFilter(logging.Filter):
217 """Filter out strings beginning with 'cmd :'"""
218 def filter(self, record):
219 return record.getMessage().startswith(tasks.CMD_PREFIX)
221 class TrafficGenCommandFilter(logging.Filter):
222 """Filter out strings beginning with 'gencmd :'"""
223 def filter(self, record):
224 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
226 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
227 cmd_logger.setLevel(logging.DEBUG)
228 cmd_logger.addFilter(CommandFilter())
229 _LOGGER.addHandler(cmd_logger)
231 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
232 gen_logger.setLevel(logging.DEBUG)
233 gen_logger.addFilter(TrafficGenCommandFilter())
234 _LOGGER.addHandler(gen_logger)
237 def apply_filter(tests, tc_filter):
238 """Allow a subset of tests to be conveniently selected
240 :param tests: The list of Tests from which to select.
241 :param tc_filter: A case-insensitive string of comma-separated terms
242 indicating the Tests to select.
243 e.g. 'RFC' - select all tests whose name contains 'RFC'
244 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
246 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
247 or 'burst' and from these remove any containing 'p2p'.
248 e.g. '' - empty string selects all tests.
249 :return: A list of the selected Tests.
251 # if negative filter is first we have to start with full list of tests
252 if tc_filter.strip()[0] == '!':
256 if tc_filter is None:
259 for term in [x.strip() for x in tc_filter.lower().split(",")]:
260 if not term or term[0] != '!':
261 # Add matching tests from 'tests' into results
262 result.extend([test for test in tests \
263 if test['Name'].lower().find(term) >= 0])
265 # Term begins with '!' so we remove matching tests
266 result = [test for test in result \
267 if test['Name'].lower().find(term[1:]) < 0]
272 def check_and_set_locale():
273 """ Function will check locale settings. In case, that it isn't configured
274 properly, then default values specified by DEFAULT_LOCALE will be used.
277 system_locale = locale.getdefaultlocale()
278 if None in system_locale:
279 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
280 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
281 system_locale, locale.getdefaultlocale())
284 def generate_final_report():
285 """ Function will check if partial test results are available
286 and generates final report in rst format.
289 path = settings.getValue('RESULTS_PATH')
290 # check if there are any results in rst format
291 rst_results = glob.glob(os.path.join(path, 'result*rst'))
294 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
295 # create report caption directly - it is not worth to execute jinja machinery
296 if settings.getValue('VSWITCH').lower() != 'none':
297 pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
299 pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
300 report_caption = '{}\n{} {}\n{}\n\n'.format(
301 '============================================================',
302 'Performance report for',
304 '============================================================')
306 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
307 file_.write(report_caption)
309 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
310 ' '.join(rst_results), _TEMPLATE_RST['foot'],
311 test_report), shell=True)
312 if retval == 0 and os.path.isfile(test_report):
313 _LOGGER.info('Overall test report written to "%s"', test_report)
315 _LOGGER.error('Generatrion of overall test report has failed.')
317 # remove temporary file
318 os.remove(_TEMPLATE_RST['tmp'])
320 except subprocess.CalledProcessError:
321 _LOGGER.error('Generatrion of overall test report has failed.')
324 def enable_sriov(nic_list):
325 """ Enable SRIOV for given enhanced PCI IDs
327 :param nic_list: A list of enhanced PCI IDs
329 # detect if sriov is required
332 if networkcard.is_sriov_nic(nic):
333 tmp_nic = nic.split('|')
334 if tmp_nic[0] in sriov_nic:
335 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
336 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
338 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
340 # sriov is required for some NICs
342 for nic in sriov_nic:
343 # check if SRIOV is supported and enough virt interfaces are available
344 if not networkcard.is_sriov_supported(nic) \
345 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
346 # if not, enable and set appropriate number of VFs
347 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
348 _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
351 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
353 # WORKAROUND: it has been observed with IXGBE(VF) driver,
354 # that NIC doesn't correclty dispatch traffic to VFs based
355 # on their MAC address. Unbind and bind to the same driver
357 networkcard.reinit_vfs(nic)
359 # After SRIOV is enabled it takes some time until network drivers
360 # properly initialize all cards.
361 # Wait also in case, that SRIOV was already configured as it can be
362 # configured automatically just before vsperf execution.
370 def disable_sriov(nic_list):
371 """ Disable SRIOV for given PCI IDs
373 :param nic_list: A list of enhanced PCI IDs
376 if networkcard.is_sriov_nic(nic):
377 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
378 _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
381 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
384 def handle_list_options(args):
385 """ Process --list cli arguments if needed
387 :param args: A dictionary with all CLI arguments
389 if args['list_trafficgens']:
390 print(Loader().get_trafficgens_printable())
393 if args['list_collectors']:
394 print(Loader().get_collectors_printable())
397 if args['list_vswitches']:
398 print(Loader().get_vswitches_printable())
401 if args['list_vnfs']:
402 print(Loader().get_vnfs_printable())
405 if args['list_fwdapps']:
406 print(Loader().get_pktfwds_printable())
409 if args['list_settings']:
415 if args['integration']:
416 testcases = settings.getValue('INTEGRATION_TESTS')
418 testcases = settings.getValue('PERFORMANCE_TESTS')
420 print("Available Tests:")
421 print("================")
423 for test in testcases:
424 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
428 def vsperf_finalize():
429 """ Clean up before exit
431 # remove directory if no result files were created
433 results_path = settings.getValue('RESULTS_PATH')
434 if os.path.exists(results_path):
435 files_list = os.listdir(results_path)
437 _LOGGER.info("Removing empty result directory: " + results_path)
438 shutil.rmtree(results_path)
439 except AttributeError:
440 # skip it if parameter doesn't exist
443 # disable SRIOV if needed
445 if settings.getValue('SRIOV_ENABLED'):
446 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
447 except AttributeError:
448 # skip it if parameter doesn't exist
452 class MockTestCase(unittest.TestCase):
453 """Allow use of xmlrunner to generate Jenkins compatible output without
454 using xmlrunner to actually run tests.
457 suite = unittest.TestSuite()
458 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
459 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
460 xmlrunner.XMLTestRunner(...).run(suite)
463 def __init__(self, msg, is_pass, test_name):
466 self.is_pass = is_pass
468 #dynamically create a test method with the right name
469 #but point the method at our generic test method
470 setattr(MockTestCase, test_name, self.generic_test)
472 super(MockTestCase, self).__init__(test_name)
474 def generic_test(self):
475 """Provide a generic function that raises or not based
476 on how self.is_pass was set in the constructor"""
477 self.assertTrue(self.is_pass, self.msg)
483 args = parse_arguments()
487 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
489 # Load non performance/integration tests
490 if args['integration']:
491 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
493 # load command line parameters first in case there are settings files
495 settings.load_from_dict(args)
497 if args['conf_file']:
498 settings.load_from_file(args['conf_file'])
501 settings.load_from_env()
503 # reload command line parameters since these should take higher priority
504 # than both a settings file and environment variables
505 settings.load_from_dict(args)
507 settings.setValue('mode', args['mode'])
509 # set dpdk and ovs paths accorfing to VNF and VSWITCH
510 if settings.getValue('mode') != 'trafficgen':
511 functions.settings_update_paths()
513 # if required, handle list-* operations
514 handle_list_options(args)
516 configure_logging(settings.getValue('VERBOSITY'))
518 # check and fix locale
519 check_and_set_locale()
521 # configure trafficgens
522 if args['trafficgen']:
523 trafficgens = Loader().get_trafficgens()
524 if args['trafficgen'] not in trafficgens:
525 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
526 ' \'%s\'. Exiting...', args['trafficgen'],
527 settings.getValue('TRAFFICGEN_DIR'))
530 # configuration validity checks
532 vswitch_none = 'none' == args['vswitch'].strip().lower()
534 settings.setValue('VSWITCH', 'none')
536 vswitches = Loader().get_vswitches()
537 if args['vswitch'] not in vswitches:
538 _LOGGER.error('There are no vswitches matching \'%s\' found in'
539 ' \'%s\'. Exiting...', args['vswitch'],
540 settings.getValue('VSWITCH_DIR'))
544 settings.setValue('PKTFWD', args['fwdapp'])
545 fwdapps = Loader().get_pktfwds()
546 if args['fwdapp'] not in fwdapps:
547 _LOGGER.error('There are no forwarding application'
548 ' matching \'%s\' found in'
549 ' \'%s\'. Exiting...', args['fwdapp'],
550 settings.getValue('PKTFWD_DIR'))
554 vnfs = Loader().get_vnfs()
555 if args['vnf'] not in vnfs:
556 _LOGGER.error('there are no vnfs matching \'%s\' found in'
557 ' \'%s\'. exiting...', args['vnf'],
558 settings.getValue('VNF_DIR'))
561 if args['exact_test_name'] and args['tests']:
562 _LOGGER.error("Cannot specify tests with both positional args and --test.")
565 # modify NIC configuration to decode enhanced PCI IDs
566 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
567 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
569 # sriov handling is performed on checked/expanded PCI IDs
570 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
573 for nic in wl_nics_orig:
574 tmp_nic = networkcard.get_nic_info(nic)
576 nic_list.append({'pci' : tmp_nic,
577 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
578 'mac' : networkcard.get_mac(tmp_nic),
579 'driver' : networkcard.get_driver(tmp_nic),
580 'device' : networkcard.get_device_name(tmp_nic)})
582 _LOGGER.error("Invalid network card PCI ID: '%s'", nic)
586 settings.setValue('NICS', nic_list)
587 # for backward compatibility
588 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
590 # generate results directory name
591 date = datetime.datetime.fromtimestamp(time.time())
592 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
593 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
594 settings.setValue('RESULTS_PATH', results_path)
596 # create results directory
597 if not os.path.exists(results_path):
598 _LOGGER.info("Creating result directory: " + results_path)
599 os.makedirs(results_path)
601 if settings.getValue('mode') == 'trafficgen':
602 # execute only traffic generator
603 _LOGGER.debug("Executing traffic generator:")
605 # set traffic details, so they can be passed to traffic ctl
606 traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
607 traffic.update({'traffic_type': get_test_param('traffic_type', TRAFFIC_DEFAULTS['traffic_type']),
608 'bidir': get_test_param('bidirectional', TRAFFIC_DEFAULTS['bidir']),
609 'multistream': int(get_test_param('multistream', TRAFFIC_DEFAULTS['multistream'])),
610 'stream_type': get_test_param('stream_type', TRAFFIC_DEFAULTS['stream_type']),
611 'frame_rate': int(get_test_param('iload', TRAFFIC_DEFAULTS['frame_rate']))})
613 traffic_ctl = component_factory.create_traffic(
614 traffic['traffic_type'],
615 loader.get_trafficgen_class())
617 traffic_ctl.send_traffic(traffic)
618 _LOGGER.debug("Traffic Results:")
619 traffic_ctl.print_results()
621 # write results into CSV file
622 result_file = os.path.join(results_path, "result.csv")
623 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
626 if args['integration']:
627 testcases = settings.getValue('INTEGRATION_TESTS')
629 testcases = settings.getValue('PERFORMANCE_TESTS')
631 if args['exact_test_name']:
632 exact_names = args['exact_test_name']
633 # positional args => exact matches only
634 selected_tests = [test for test in testcases if test['Name'] in exact_names]
636 # --tests => apply filter to select requested tests
637 selected_tests = apply_filter(testcases, args['tests'])
639 # Default - run all tests
640 selected_tests = testcases
642 if not len(selected_tests):
643 _LOGGER.error("No tests matched --tests option or positional args. Done.")
648 suite = unittest.TestSuite()
649 for cfg in selected_tests:
650 test_name = cfg.get('Name', '<Name not set>')
652 if args['integration']:
653 test = IntegrationTestCase(cfg)
655 test = PerformanceTestCase(cfg)
657 suite.addTest(MockTestCase('', True, test.name))
658 #pylint: disable=broad-except
659 except (Exception) as ex:
660 _LOGGER.exception("Failed to run test: %s", test_name)
661 suite.addTest(MockTestCase(str(ex), False, test_name))
662 _LOGGER.info("Continuing with next test...")
664 # generate final rst report with results of all executed TCs
665 generate_final_report()
667 if settings.getValue('XUNIT'):
668 xmlrunner.XMLTestRunner(
669 output=settings.getValue('XUNIT_DIR'), outsuffix="",
670 verbosity=0).run(suite)
673 pod_name = args['opnfvpod']
674 installer_name = settings.getValue('OPNFV_INSTALLER')
675 opnfv_url = settings.getValue('OPNFV_URL')
676 pkg_list = settings.getValue('PACKAGE_LIST')
678 int_data = {'vanilla': False,
680 'installer': installer_name,
681 'pkg_list': pkg_list,
683 if settings.getValue('VSWITCH').endswith('Vanilla'):
684 int_data['vanilla'] = True
685 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
687 # cleanup before exit
690 if __name__ == "__main__":