3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
35 from conf import settings
36 import core.component_factory as component_factory
37 from core.loader import Loader
38 from testcases import PerformanceTestCase
39 from testcases import IntegrationTestCase
40 from tools import tasks
41 from tools import networkcard
42 from tools import functions
43 from tools.pkt_gen import trafficgen
44 from tools.opnfvdashboard import opnfvdashboard
46 sys.dont_write_bytecode = True
49 'debug': logging.DEBUG,
51 'warning': logging.WARNING,
52 'error': logging.ERROR,
53 'critical': logging.CRITICAL
56 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
58 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
59 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
60 'final' : 'test_report.rst',
61 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
65 _LOGGER = logging.getLogger()
67 def parse_arguments():
69 Parse command line arguments.
71 class _SplitTestParamsAction(argparse.Action):
73 Parse and split the '--test-params' argument.
75 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
76 values. For multiple overrides use a ; separated list for
77 e.g. --test-params 'x=z; y=(a,b)'
79 def __call__(self, parser, namespace, values, option_string=None):
82 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
87 # values are passed inside string from CLI, so we must retype them accordingly
89 results[param] = ast.literal_eval(value)
91 # for backward compatibility, we have to accept strings without quotes
92 _LOGGER.warning("Adding missing quotes around string value: %s = %s",
94 results[param] = str(value)
98 setattr(namespace, self.dest, results)
100 class _ValidateFileAction(argparse.Action):
101 """Validate a file can be read from before using it.
103 def __call__(self, parser, namespace, values, option_string=None):
104 if not os.path.isfile(values):
105 raise argparse.ArgumentTypeError(
106 'the path \'%s\' is not a valid path' % values)
107 elif not os.access(values, os.R_OK):
108 raise argparse.ArgumentTypeError(
109 'the path \'%s\' is not accessible' % values)
111 setattr(namespace, self.dest, values)
113 class _ValidateDirAction(argparse.Action):
114 """Validate a directory can be written to before using it.
116 def __call__(self, parser, namespace, values, option_string=None):
117 if not os.path.isdir(values):
118 raise argparse.ArgumentTypeError(
119 'the path \'%s\' is not a valid path' % values)
120 elif not os.access(values, os.W_OK):
121 raise argparse.ArgumentTypeError(
122 'the path \'%s\' is not accessible' % values)
124 setattr(namespace, self.dest, values)
126 def list_logging_levels():
127 """Give a summary of all available logging levels.
129 :return: List of verbosity level names in decreasing order of
132 return sorted(VERBOSITY_LEVELS.keys(),
133 key=lambda x: VERBOSITY_LEVELS[x])
135 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
136 argparse.ArgumentDefaultsHelpFormatter)
137 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
138 parser.add_argument('--list', '--list-tests', action='store_true',
139 help='list all tests and exit')
140 parser.add_argument('--list-trafficgens', action='store_true',
141 help='list all traffic generators and exit')
142 parser.add_argument('--list-collectors', action='store_true',
143 help='list all system metrics loggers and exit')
144 parser.add_argument('--list-vswitches', action='store_true',
145 help='list all system vswitches and exit')
146 parser.add_argument('--list-fwdapps', action='store_true',
147 help='list all system forwarding applications and exit')
148 parser.add_argument('--list-vnfs', action='store_true',
149 help='list all system vnfs and exit')
150 parser.add_argument('--list-settings', action='store_true',
151 help='list effective settings configuration and exit')
152 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
153 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
154 runs only the two tests with those exact names.\
155 To run all tests omit both positional args and --tests arg.')
157 group = parser.add_argument_group('test selection options')
158 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
159 Values: "normal" - execute vSwitch, VNF and traffic generator;\
160 "trafficgen" - execute only traffic generator; "trafficgen-off" \
161 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
162 and VNF but pause before traffic transmission ', default='normal')
164 group.add_argument('-f', '--test-spec', help='test specification file')
165 group.add_argument('-d', '--test-dir', help='directory containing tests')
166 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
167 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
168 name contains RFC2544 less those containing "p2p"; "!back2back" - \
169 run all tests except those containing back2back')
170 group.add_argument('--verbosity', choices=list_logging_levels(),
172 group.add_argument('--integration', action='store_true', help='execute integration tests')
173 group.add_argument('--trafficgen', help='traffic generator to use')
174 group.add_argument('--vswitch', help='vswitch implementation to use')
175 group.add_argument('--fwdapp', help='packet forwarding application to use')
176 group.add_argument('--vnf', help='vnf to use')
177 group.add_argument('--sysmetrics', help='system metrics logger to use')
178 group = parser.add_argument_group('test behavior options')
179 group.add_argument('--xunit', action='store_true',
180 help='enable xUnit-formatted output')
181 group.add_argument('--xunit-dir', action=_ValidateDirAction,
182 help='output directory of xUnit-formatted output')
183 group.add_argument('--load-env', action='store_true',
184 help='enable loading of settings from the environment')
185 group.add_argument('--conf-file', action=_ValidateFileAction,
186 help='settings file')
187 group.add_argument('--test-params', action=_SplitTestParamsAction,
188 help='csv list of test parameters: key=val; e.g. '
189 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
190 'GUEST_LOOPBACK=["l2fwd"] ...')
191 group.add_argument('--opnfvpod', help='name of POD in opnfv')
193 args = vars(parser.parse_args())
198 def configure_logging(level):
199 """Configure logging.
201 log_file_default = os.path.join(
202 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
203 log_file_host_cmds = os.path.join(
204 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
205 log_file_traffic_gen = os.path.join(
206 settings.getValue('LOG_DIR'),
207 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
209 _LOGGER.setLevel(logging.DEBUG)
211 stream_logger = logging.StreamHandler(sys.stdout)
212 stream_logger.setLevel(VERBOSITY_LEVELS[level])
213 stream_logger.setFormatter(logging.Formatter(
214 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
215 _LOGGER.addHandler(stream_logger)
217 file_logger = logging.FileHandler(filename=log_file_default)
218 file_logger.setLevel(logging.DEBUG)
219 _LOGGER.addHandler(file_logger)
221 class CommandFilter(logging.Filter):
222 """Filter out strings beginning with 'cmd :'"""
223 def filter(self, record):
224 return record.getMessage().startswith(tasks.CMD_PREFIX)
226 class TrafficGenCommandFilter(logging.Filter):
227 """Filter out strings beginning with 'gencmd :'"""
228 def filter(self, record):
229 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
231 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
232 cmd_logger.setLevel(logging.DEBUG)
233 cmd_logger.addFilter(CommandFilter())
234 _LOGGER.addHandler(cmd_logger)
236 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
237 gen_logger.setLevel(logging.DEBUG)
238 gen_logger.addFilter(TrafficGenCommandFilter())
239 _LOGGER.addHandler(gen_logger)
242 def apply_filter(tests, tc_filter):
243 """Allow a subset of tests to be conveniently selected
245 :param tests: The list of Tests from which to select.
246 :param tc_filter: A case-insensitive string of comma-separated terms
247 indicating the Tests to select.
248 e.g. 'RFC' - select all tests whose name contains 'RFC'
249 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
251 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
252 or 'burst' and from these remove any containing 'p2p'.
253 e.g. '' - empty string selects all tests.
254 :return: A list of the selected Tests.
256 # if negative filter is first we have to start with full list of tests
257 if tc_filter.strip()[0] == '!':
261 if tc_filter is None:
264 for term in [x.strip() for x in tc_filter.lower().split(",")]:
265 if not term or term[0] != '!':
266 # Add matching tests from 'tests' into results
267 result.extend([test for test in tests \
268 if test['Name'].lower().find(term) >= 0])
270 # Term begins with '!' so we remove matching tests
271 result = [test for test in result \
272 if test['Name'].lower().find(term[1:]) < 0]
277 def check_and_set_locale():
278 """ Function will check locale settings. In case, that it isn't configured
279 properly, then default values specified by DEFAULT_LOCALE will be used.
282 system_locale = locale.getdefaultlocale()
283 if None in system_locale:
284 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
285 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
286 system_locale, locale.getdefaultlocale())
289 def generate_final_report():
290 """ Function will check if partial test results are available
291 and generates final report in rst format.
294 path = settings.getValue('RESULTS_PATH')
295 # check if there are any results in rst format
296 rst_results = glob.glob(os.path.join(path, 'result*rst'))
299 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
300 # create report caption directly - it is not worth to execute jinja machinery
301 if settings.getValue('VSWITCH').lower() != 'none':
302 pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
304 pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
305 report_caption = '{}\n{} {}\n{}\n\n'.format(
306 '============================================================',
307 'Performance report for',
309 '============================================================')
311 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
312 file_.write(report_caption)
314 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
315 ' '.join(rst_results), _TEMPLATE_RST['foot'],
316 test_report), shell=True)
317 if retval == 0 and os.path.isfile(test_report):
318 _LOGGER.info('Overall test report written to "%s"', test_report)
320 _LOGGER.error('Generatrion of overall test report has failed.')
322 # remove temporary file
323 os.remove(_TEMPLATE_RST['tmp'])
325 except subprocess.CalledProcessError:
326 _LOGGER.error('Generatrion of overall test report has failed.')
329 def enable_sriov(nic_list):
330 """ Enable SRIOV for given enhanced PCI IDs
332 :param nic_list: A list of enhanced PCI IDs
334 # detect if sriov is required
337 if networkcard.is_sriov_nic(nic):
338 tmp_nic = nic.split('|')
339 if tmp_nic[0] in sriov_nic:
340 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
341 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
343 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
345 # sriov is required for some NICs
347 for nic in sriov_nic:
348 # check if SRIOV is supported and enough virt interfaces are available
349 if not networkcard.is_sriov_supported(nic) \
350 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
351 # if not, enable and set appropriate number of VFs
352 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
353 raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
355 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
357 # WORKAROUND: it has been observed with IXGBE(VF) driver,
358 # that NIC doesn't correclty dispatch traffic to VFs based
359 # on their MAC address. Unbind and bind to the same driver
361 networkcard.reinit_vfs(nic)
363 # After SRIOV is enabled it takes some time until network drivers
364 # properly initialize all cards.
365 # Wait also in case, that SRIOV was already configured as it can be
366 # configured automatically just before vsperf execution.
374 def disable_sriov(nic_list):
375 """ Disable SRIOV for given PCI IDs
377 :param nic_list: A list of enhanced PCI IDs
380 if networkcard.is_sriov_nic(nic):
381 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
382 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
384 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
387 def handle_list_options(args):
388 """ Process --list cli arguments if needed
390 :param args: A dictionary with all CLI arguments
392 if args['list_trafficgens']:
393 print(Loader().get_trafficgens_printable())
396 if args['list_collectors']:
397 print(Loader().get_collectors_printable())
400 if args['list_vswitches']:
401 print(Loader().get_vswitches_printable())
404 if args['list_vnfs']:
405 print(Loader().get_vnfs_printable())
408 if args['list_fwdapps']:
409 print(Loader().get_pktfwds_printable())
412 if args['list_settings']:
418 if args['integration']:
419 testcases = settings.getValue('INTEGRATION_TESTS')
421 testcases = settings.getValue('PERFORMANCE_TESTS')
423 print("Available Tests:")
424 print("================")
426 for test in testcases:
427 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
431 def vsperf_finalize():
432 """ Clean up before exit
434 # remove directory if no result files were created
436 results_path = settings.getValue('RESULTS_PATH')
437 if os.path.exists(results_path):
438 files_list = os.listdir(results_path)
440 _LOGGER.info("Removing empty result directory: " + results_path)
441 shutil.rmtree(results_path)
442 except AttributeError:
443 # skip it if parameter doesn't exist
446 # disable SRIOV if needed
448 if settings.getValue('SRIOV_ENABLED'):
449 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
450 except AttributeError:
451 # skip it if parameter doesn't exist
455 class MockTestCase(unittest.TestCase):
456 """Allow use of xmlrunner to generate Jenkins compatible output without
457 using xmlrunner to actually run tests.
460 suite = unittest.TestSuite()
461 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
462 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
463 xmlrunner.XMLTestRunner(...).run(suite)
466 def __init__(self, msg, is_pass, test_name):
469 self.is_pass = is_pass
471 #dynamically create a test method with the right name
472 #but point the method at our generic test method
473 setattr(MockTestCase, test_name, self.generic_test)
475 super(MockTestCase, self).__init__(test_name)
477 def generic_test(self):
478 """Provide a generic function that raises or not based
479 on how self.is_pass was set in the constructor"""
480 self.assertTrue(self.is_pass, self.msg)
486 args = parse_arguments()
490 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
492 # Load non performance/integration tests
493 if args['integration']:
494 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
496 # load command line parameters first in case there are settings files
498 settings.load_from_dict(args)
500 if args['conf_file']:
501 settings.load_from_file(args['conf_file'])
504 settings.load_from_env()
506 # reload command line parameters since these should take higher priority
507 # than both a settings file and environment variables
508 settings.load_from_dict(args)
510 settings.setValue('mode', args['mode'])
512 # set dpdk and ovs paths according to VNF and VSWITCH
513 if settings.getValue('mode') != 'trafficgen':
514 functions.settings_update_paths()
516 # if required, handle list-* operations
517 handle_list_options(args)
519 configure_logging(settings.getValue('VERBOSITY'))
521 # check and fix locale
522 check_and_set_locale()
524 # configure trafficgens
525 if args['trafficgen']:
526 trafficgens = Loader().get_trafficgens()
527 if args['trafficgen'] not in trafficgens:
528 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
529 ' \'%s\'. Exiting...', args['trafficgen'],
530 settings.getValue('TRAFFICGEN_DIR'))
533 # configuration validity checks
535 vswitch_none = 'none' == args['vswitch'].strip().lower()
537 settings.setValue('VSWITCH', 'none')
539 vswitches = Loader().get_vswitches()
540 if args['vswitch'] not in vswitches:
541 _LOGGER.error('There are no vswitches matching \'%s\' found in'
542 ' \'%s\'. Exiting...', args['vswitch'],
543 settings.getValue('VSWITCH_DIR'))
547 settings.setValue('PKTFWD', args['fwdapp'])
548 fwdapps = Loader().get_pktfwds()
549 if args['fwdapp'] not in fwdapps:
550 _LOGGER.error('There are no forwarding application'
551 ' matching \'%s\' found in'
552 ' \'%s\'. Exiting...', args['fwdapp'],
553 settings.getValue('PKTFWD_DIR'))
557 vnfs = Loader().get_vnfs()
558 if args['vnf'] not in vnfs:
559 _LOGGER.error('there are no vnfs matching \'%s\' found in'
560 ' \'%s\'. exiting...', args['vnf'],
561 settings.getValue('VNF_DIR'))
564 if args['exact_test_name'] and args['tests']:
565 _LOGGER.error("Cannot specify tests with both positional args and --test.")
568 # modify NIC configuration to decode enhanced PCI IDs
569 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
570 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
572 # sriov handling is performed on checked/expanded PCI IDs
573 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
576 for nic in wl_nics_orig:
577 tmp_nic = networkcard.get_nic_info(nic)
579 nic_list.append({'pci' : tmp_nic,
580 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
581 'mac' : networkcard.get_mac(tmp_nic),
582 'driver' : networkcard.get_driver(tmp_nic),
583 'device' : networkcard.get_device_name(tmp_nic)})
586 raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
588 settings.setValue('NICS', nic_list)
589 # for backward compatibility
590 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
592 # generate results directory name
593 date = datetime.datetime.fromtimestamp(time.time())
594 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
595 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
596 settings.setValue('RESULTS_PATH', results_path)
598 # create results directory
599 if not os.path.exists(results_path):
600 _LOGGER.info("Creating result directory: " + results_path)
601 os.makedirs(results_path)
603 if settings.getValue('mode') == 'trafficgen':
604 # execute only traffic generator
605 _LOGGER.debug("Executing traffic generator:")
607 # set traffic details, so they can be passed to traffic ctl
608 traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
610 traffic_ctl = component_factory.create_traffic(
611 traffic['traffic_type'],
612 loader.get_trafficgen_class())
614 traffic_ctl.send_traffic(traffic)
615 _LOGGER.debug("Traffic Results:")
616 traffic_ctl.print_results()
618 # write results into CSV file
619 result_file = os.path.join(results_path, "result.csv")
620 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
623 if args['integration']:
624 testcases = settings.getValue('INTEGRATION_TESTS')
626 testcases = settings.getValue('PERFORMANCE_TESTS')
628 if args['exact_test_name']:
629 exact_names = args['exact_test_name']
630 # positional args => exact matches only
631 selected_tests = [test for test in testcases if test['Name'] in exact_names]
633 # --tests => apply filter to select requested tests
634 selected_tests = apply_filter(testcases, args['tests'])
636 # Default - run all tests
637 selected_tests = testcases
639 if not len(selected_tests):
640 _LOGGER.error("No tests matched --tests option or positional args. Done.")
645 suite = unittest.TestSuite()
646 for cfg in selected_tests:
647 test_name = cfg.get('Name', '<Name not set>')
649 if args['integration']:
650 test = IntegrationTestCase(cfg)
652 test = PerformanceTestCase(cfg)
654 suite.addTest(MockTestCase('', True, test.name))
655 #pylint: disable=broad-except
656 except (Exception) as ex:
657 _LOGGER.exception("Failed to run test: %s", test_name)
658 suite.addTest(MockTestCase(str(ex), False, test_name))
659 _LOGGER.info("Continuing with next test...")
661 # generate final rst report with results of all executed TCs
662 generate_final_report()
664 if settings.getValue('XUNIT'):
665 xmlrunner.XMLTestRunner(
666 output=settings.getValue('XUNIT_DIR'), outsuffix="",
667 verbosity=0).run(suite)
670 pod_name = args['opnfvpod']
671 installer_name = settings.getValue('OPNFV_INSTALLER')
672 opnfv_url = settings.getValue('OPNFV_URL')
673 pkg_list = settings.getValue('PACKAGE_LIST')
675 int_data = {'vanilla': False,
677 'installer': installer_name,
678 'pkg_list': pkg_list,
680 if settings.getValue('VSWITCH').endswith('Vanilla'):
681 int_data['vanilla'] = True
682 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
684 # cleanup before exit
687 if __name__ == "__main__":