3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
34 sys.dont_write_bytecode = True
36 from conf import settings
37 from conf import get_test_param
38 from core.loader import Loader
39 from testcases import PerformanceTestCase
40 from testcases import IntegrationTestCase
41 from tools import tasks
42 from tools import networkcard
43 from tools import functions
44 from tools.pkt_gen import trafficgen
45 from tools.opnfvdashboard import opnfvdashboard
46 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
47 import core.component_factory as component_factory
50 'debug': logging.DEBUG,
52 'warning': logging.WARNING,
53 'error': logging.ERROR,
54 'critical': logging.CRITICAL
57 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
59 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
60 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
61 'final' : 'test_report.rst',
62 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
66 _LOGGER = logging.getLogger()
68 def parse_arguments():
70 Parse command line arguments.
72 class _SplitTestParamsAction(argparse.Action):
74 Parse and split the '--test-params' argument.
76 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
77 values. For multiple overrides use a ; separated list for
78 e.g. --test-params 'x=z; y=a,b'
80 def __call__(self, parser, namespace, values, option_string=None):
83 for value in values.split(';'):
84 result = [key.strip() for key in value.split('=')]
86 results[result[0]] = True
87 elif len(result) == 2:
88 results[result[0]] = result[1]
90 raise argparse.ArgumentTypeError(
91 'expected \'%s\' to be of format \'key=val\' or'
94 setattr(namespace, self.dest, results)
96 class _ValidateFileAction(argparse.Action):
97 """Validate a file can be read from before using it.
99 def __call__(self, parser, namespace, values, option_string=None):
100 if not os.path.isfile(values):
101 raise argparse.ArgumentTypeError(
102 'the path \'%s\' is not a valid path' % values)
103 elif not os.access(values, os.R_OK):
104 raise argparse.ArgumentTypeError(
105 'the path \'%s\' is not accessible' % values)
107 setattr(namespace, self.dest, values)
109 class _ValidateDirAction(argparse.Action):
110 """Validate a directory can be written to before using it.
112 def __call__(self, parser, namespace, values, option_string=None):
113 if not os.path.isdir(values):
114 raise argparse.ArgumentTypeError(
115 'the path \'%s\' is not a valid path' % values)
116 elif not os.access(values, os.W_OK):
117 raise argparse.ArgumentTypeError(
118 'the path \'%s\' is not accessible' % values)
120 setattr(namespace, self.dest, values)
122 def list_logging_levels():
123 """Give a summary of all available logging levels.
125 :return: List of verbosity level names in decreasing order of
128 return sorted(VERBOSITY_LEVELS.keys(),
129 key=lambda x: VERBOSITY_LEVELS[x])
131 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
132 argparse.ArgumentDefaultsHelpFormatter)
133 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
134 parser.add_argument('--list', '--list-tests', action='store_true',
135 help='list all tests and exit')
136 parser.add_argument('--list-trafficgens', action='store_true',
137 help='list all traffic generators and exit')
138 parser.add_argument('--list-collectors', action='store_true',
139 help='list all system metrics loggers and exit')
140 parser.add_argument('--list-vswitches', action='store_true',
141 help='list all system vswitches and exit')
142 parser.add_argument('--list-fwdapps', action='store_true',
143 help='list all system forwarding applications and exit')
144 parser.add_argument('--list-vnfs', action='store_true',
145 help='list all system vnfs and exit')
146 parser.add_argument('--list-settings', action='store_true',
147 help='list effective settings configuration and exit')
148 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
149 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
150 runs only the two tests with those exact names.\
151 To run all tests omit both positional args and --tests arg.')
153 group = parser.add_argument_group('test selection options')
154 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
155 Values: "normal" - execute vSwitch, VNF and traffic generator;\
156 "trafficgen" - execute only traffic generator; "trafficgen-off" \
157 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
158 and VNF but pause before traffic transmission ', default='normal')
160 group.add_argument('-f', '--test-spec', help='test specification file')
161 group.add_argument('-d', '--test-dir', help='directory containing tests')
162 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
163 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
164 name contains RFC2544 less those containing "p2p"; "!back2back" - \
165 run all tests except those containing back2back')
166 group.add_argument('--verbosity', choices=list_logging_levels(),
168 group.add_argument('--integration', action='store_true', help='execute integration tests')
169 group.add_argument('--trafficgen', help='traffic generator to use')
170 group.add_argument('--vswitch', help='vswitch implementation to use')
171 group.add_argument('--fwdapp', help='packet forwarding application to use')
172 group.add_argument('--vnf', help='vnf to use')
173 group.add_argument('--sysmetrics', help='system metrics logger to use')
174 group = parser.add_argument_group('test behavior options')
175 group.add_argument('--xunit', action='store_true',
176 help='enable xUnit-formatted output')
177 group.add_argument('--xunit-dir', action=_ValidateDirAction,
178 help='output directory of xUnit-formatted output')
179 group.add_argument('--load-env', action='store_true',
180 help='enable loading of settings from the environment')
181 group.add_argument('--conf-file', action=_ValidateFileAction,
182 help='settings file')
183 group.add_argument('--test-params', action=_SplitTestParamsAction,
184 help='csv list of test parameters: key=val; e.g.'
185 'including pkt_sizes=x,y; duration=x; '
186 'rfc2544_tests=x ...')
187 group.add_argument('--opnfvpod', help='name of POD in opnfv')
189 args = vars(parser.parse_args())
194 def configure_logging(level):
195 """Configure logging.
197 log_file_default = os.path.join(
198 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
199 log_file_host_cmds = os.path.join(
200 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
201 log_file_traffic_gen = os.path.join(
202 settings.getValue('LOG_DIR'),
203 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
205 _LOGGER.setLevel(logging.DEBUG)
207 stream_logger = logging.StreamHandler(sys.stdout)
208 stream_logger.setLevel(VERBOSITY_LEVELS[level])
209 stream_logger.setFormatter(logging.Formatter(
210 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
211 _LOGGER.addHandler(stream_logger)
213 file_logger = logging.FileHandler(filename=log_file_default)
214 file_logger.setLevel(logging.DEBUG)
215 _LOGGER.addHandler(file_logger)
217 class CommandFilter(logging.Filter):
218 """Filter out strings beginning with 'cmd :'"""
219 def filter(self, record):
220 return record.getMessage().startswith(tasks.CMD_PREFIX)
222 class TrafficGenCommandFilter(logging.Filter):
223 """Filter out strings beginning with 'gencmd :'"""
224 def filter(self, record):
225 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
227 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
228 cmd_logger.setLevel(logging.DEBUG)
229 cmd_logger.addFilter(CommandFilter())
230 _LOGGER.addHandler(cmd_logger)
232 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
233 gen_logger.setLevel(logging.DEBUG)
234 gen_logger.addFilter(TrafficGenCommandFilter())
235 _LOGGER.addHandler(gen_logger)
238 def apply_filter(tests, tc_filter):
239 """Allow a subset of tests to be conveniently selected
241 :param tests: The list of Tests from which to select.
242 :param tc_filter: A case-insensitive string of comma-separated terms
243 indicating the Tests to select.
244 e.g. 'RFC' - select all tests whose name contains 'RFC'
245 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
247 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
248 or 'burst' and from these remove any containing 'p2p'.
249 e.g. '' - empty string selects all tests.
250 :return: A list of the selected Tests.
252 # if negative filter is first we have to start with full list of tests
253 if tc_filter.strip()[0] == '!':
257 if tc_filter is None:
260 for term in [x.strip() for x in tc_filter.lower().split(",")]:
261 if not term or term[0] != '!':
262 # Add matching tests from 'tests' into results
263 result.extend([test for test in tests \
264 if test['Name'].lower().find(term) >= 0])
266 # Term begins with '!' so we remove matching tests
267 result = [test for test in result \
268 if test['Name'].lower().find(term[1:]) < 0]
273 def check_and_set_locale():
274 """ Function will check locale settings. In case, that it isn't configured
275 properly, then default values specified by DEFAULT_LOCALE will be used.
278 system_locale = locale.getdefaultlocale()
279 if None in system_locale:
280 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
281 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
282 system_locale, locale.getdefaultlocale())
285 def generate_final_report():
286 """ Function will check if partial test results are available
287 and generates final report in rst format.
290 path = settings.getValue('RESULTS_PATH')
291 # check if there are any results in rst format
292 rst_results = glob.glob(os.path.join(path, 'result*rst'))
295 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
296 # create report caption directly - it is not worth to execute jinja machinery
297 if settings.getValue('VSWITCH').lower() != 'none':
298 pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
300 pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]
301 report_caption = '{}\n{} {}\n{}\n\n'.format(
302 '============================================================',
303 'Performance report for',
305 '============================================================')
307 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
308 file_.write(report_caption)
310 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
311 ' '.join(rst_results), _TEMPLATE_RST['foot'],
312 test_report), shell=True)
313 if retval == 0 and os.path.isfile(test_report):
314 _LOGGER.info('Overall test report written to "%s"', test_report)
316 _LOGGER.error('Generatrion of overall test report has failed.')
318 # remove temporary file
319 os.remove(_TEMPLATE_RST['tmp'])
321 except subprocess.CalledProcessError:
322 _LOGGER.error('Generatrion of overall test report has failed.')
325 def enable_sriov(nic_list):
326 """ Enable SRIOV for given enhanced PCI IDs
328 :param nic_list: A list of enhanced PCI IDs
330 # detect if sriov is required
333 if networkcard.is_sriov_nic(nic):
334 tmp_nic = nic.split('|')
335 if tmp_nic[0] in sriov_nic:
336 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
337 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
339 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
341 # sriov is required for some NICs
343 for nic in sriov_nic:
344 # check if SRIOV is supported and enough virt interfaces are available
345 if not networkcard.is_sriov_supported(nic) \
346 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
347 # if not, enable and set appropriate number of VFs
348 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
349 _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
352 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
354 # WORKAROUND: it has been observed with IXGBE(VF) driver,
355 # that NIC doesn't correclty dispatch traffic to VFs based
356 # on their MAC address. Unbind and bind to the same driver
358 networkcard.reinit_vfs(nic)
360 # After SRIOV is enabled it takes some time until network drivers
361 # properly initialize all cards.
362 # Wait also in case, that SRIOV was already configured as it can be
363 # configured automatically just before vsperf execution.
371 def disable_sriov(nic_list):
372 """ Disable SRIOV for given PCI IDs
374 :param nic_list: A list of enhanced PCI IDs
377 if networkcard.is_sriov_nic(nic):
378 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
379 _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
382 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
385 def handle_list_options(args):
386 """ Process --list cli arguments if needed
388 :param args: A dictionary with all CLI arguments
390 if args['list_trafficgens']:
391 print(Loader().get_trafficgens_printable())
394 if args['list_collectors']:
395 print(Loader().get_collectors_printable())
398 if args['list_vswitches']:
399 print(Loader().get_vswitches_printable())
402 if args['list_vnfs']:
403 print(Loader().get_vnfs_printable())
406 if args['list_fwdapps']:
407 print(Loader().get_pktfwds_printable())
410 if args['list_settings']:
416 if args['integration']:
417 testcases = settings.getValue('INTEGRATION_TESTS')
419 testcases = settings.getValue('PERFORMANCE_TESTS')
421 print("Available Tests:")
422 print("================")
424 for test in testcases:
425 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
429 def vsperf_finalize():
430 """ Clean up before exit
432 # remove directory if no result files were created
434 results_path = settings.getValue('RESULTS_PATH')
435 if os.path.exists(results_path):
436 files_list = os.listdir(results_path)
438 _LOGGER.info("Removing empty result directory: " + results_path)
439 shutil.rmtree(results_path)
440 except AttributeError:
441 # skip it if parameter doesn't exist
444 # disable SRIOV if needed
446 if settings.getValue('SRIOV_ENABLED'):
447 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
448 except AttributeError:
449 # skip it if parameter doesn't exist
453 class MockTestCase(unittest.TestCase):
454 """Allow use of xmlrunner to generate Jenkins compatible output without
455 using xmlrunner to actually run tests.
458 suite = unittest.TestSuite()
459 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
460 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
461 xmlrunner.XMLTestRunner(...).run(suite)
464 def __init__(self, msg, is_pass, test_name):
467 self.is_pass = is_pass
469 #dynamically create a test method with the right name
470 #but point the method at our generic test method
471 setattr(MockTestCase, test_name, self.generic_test)
473 super(MockTestCase, self).__init__(test_name)
475 def generic_test(self):
476 """Provide a generic function that raises or not based
477 on how self.is_pass was set in the constructor"""
478 self.assertTrue(self.is_pass, self.msg)
484 args = parse_arguments()
488 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
490 # Load non performance/integration tests
491 if args['integration']:
492 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
494 # load command line parameters first in case there are settings files
496 settings.load_from_dict(args)
498 if args['conf_file']:
499 settings.load_from_file(args['conf_file'])
502 settings.load_from_env()
504 # reload command line parameters since these should take higher priority
505 # than both a settings file and environment variables
506 settings.load_from_dict(args)
508 # set dpdk and ovs paths accorfing to VNF and VSWITCH
509 functions.settings_update_paths()
511 # if required, handle list-* operations
512 handle_list_options(args)
514 configure_logging(settings.getValue('VERBOSITY'))
516 # check and fix locale
517 check_and_set_locale()
519 # configure trafficgens
520 if args['trafficgen']:
521 trafficgens = Loader().get_trafficgens()
522 if args['trafficgen'] not in trafficgens:
523 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
524 ' \'%s\'. Exiting...', args['trafficgen'],
525 settings.getValue('TRAFFICGEN_DIR'))
528 # configuration validity checks
530 vswitch_none = 'none' == args['vswitch'].strip().lower()
532 settings.setValue('VSWITCH', 'none')
534 vswitches = Loader().get_vswitches()
535 if args['vswitch'] not in vswitches:
536 _LOGGER.error('There are no vswitches matching \'%s\' found in'
537 ' \'%s\'. Exiting...', args['vswitch'],
538 settings.getValue('VSWITCH_DIR'))
542 settings.setValue('PKTFWD', args['fwdapp'])
543 fwdapps = Loader().get_pktfwds()
544 if args['fwdapp'] not in fwdapps:
545 _LOGGER.error('There are no forwarding application'
546 ' matching \'%s\' found in'
547 ' \'%s\'. Exiting...', args['fwdapp'],
548 settings.getValue('PKTFWD_DIR'))
552 vnfs = Loader().get_vnfs()
553 if args['vnf'] not in vnfs:
554 _LOGGER.error('there are no vnfs matching \'%s\' found in'
555 ' \'%s\'. exiting...', args['vnf'],
556 settings.getValue('VNF_DIR'))
559 if args['exact_test_name'] and args['tests']:
560 _LOGGER.error("Cannot specify tests with both positional args and --test.")
563 # modify NIC configuration to decode enhanced PCI IDs
564 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
565 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
567 # sriov handling is performed on checked/expanded PCI IDs
568 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
571 for nic in wl_nics_orig:
572 tmp_nic = networkcard.get_nic_info(nic)
574 nic_list.append({'pci' : tmp_nic,
575 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
576 'mac' : networkcard.get_mac(tmp_nic),
577 'driver' : networkcard.get_driver(tmp_nic),
578 'device' : networkcard.get_device_name(tmp_nic)})
580 _LOGGER.error("Invalid network card PCI ID: '%s'", nic)
584 settings.setValue('NICS', nic_list)
585 # for backward compatibility
586 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
588 # update global settings
589 guest_loopback = get_test_param('guest_loopback', None)
592 for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))):
593 tmp_gl.append(guest_loopback)
594 settings.setValue('GUEST_LOOPBACK', tmp_gl)
596 settings.setValue('mode', args['mode'])
598 # generate results directory name
599 date = datetime.datetime.fromtimestamp(time.time())
600 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
601 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
602 settings.setValue('RESULTS_PATH', results_path)
604 # create results directory
605 if not os.path.exists(results_path):
606 _LOGGER.info("Creating result directory: " + results_path)
607 os.makedirs(results_path)
609 if settings.getValue('mode') == 'trafficgen':
610 # execute only traffic generator
611 _LOGGER.debug("Executing traffic generator:")
613 # set traffic details, so they can be passed to traffic ctl
614 traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
615 traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
616 'bidir': get_test_param('bidirectional', 'False'),
617 'multistream': int(get_test_param('multistream', 0)),
618 'stream_type': get_test_param('stream_type', 'L4'),
619 'frame_rate': int(get_test_param('iload', 100))})
621 traffic_ctl = component_factory.create_traffic(
622 traffic['traffic_type'],
623 loader.get_trafficgen_class())
625 traffic_ctl.send_traffic(traffic)
626 _LOGGER.debug("Traffic Results:")
627 traffic_ctl.print_results()
629 # write results into CSV file
630 result_file = os.path.join(results_path, "result.csv")
631 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
634 if args['integration']:
635 testcases = settings.getValue('INTEGRATION_TESTS')
637 testcases = settings.getValue('PERFORMANCE_TESTS')
639 if args['exact_test_name']:
640 exact_names = args['exact_test_name']
641 # positional args => exact matches only
642 selected_tests = [test for test in testcases if test['Name'] in exact_names]
644 # --tests => apply filter to select requested tests
645 selected_tests = apply_filter(testcases, args['tests'])
647 # Default - run all tests
648 selected_tests = testcases
650 if not len(selected_tests):
651 _LOGGER.error("No tests matched --tests option or positional args. Done.")
656 suite = unittest.TestSuite()
657 for cfg in selected_tests:
658 test_name = cfg.get('Name', '<Name not set>')
660 if args['integration']:
661 test = IntegrationTestCase(cfg)
663 test = PerformanceTestCase(cfg)
665 suite.addTest(MockTestCase('', True, test.name))
666 #pylint: disable=broad-except
667 except (Exception) as ex:
668 _LOGGER.exception("Failed to run test: %s", test_name)
669 suite.addTest(MockTestCase(str(ex), False, test_name))
670 _LOGGER.info("Continuing with next test...")
672 # generate final rst report with results of all executed TCs
673 generate_final_report()
675 if settings.getValue('XUNIT'):
676 xmlrunner.XMLTestRunner(
677 output=settings.getValue('XUNIT_DIR'), outsuffix="",
678 verbosity=0).run(suite)
681 pod_name = args['opnfvpod']
682 installer_name = settings.getValue('OPNFV_INSTALLER')
683 opnfv_url = settings.getValue('OPNFV_URL')
684 pkg_list = settings.getValue('PACKAGE_LIST')
686 int_data = {'vanilla': False,
688 'installer': installer_name,
689 'pkg_list': pkg_list,
691 if settings.getValue('VSWITCH').endswith('Vanilla'):
692 int_data['vanilla'] = True
693 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
695 # cleanup before exit
698 if __name__ == "__main__":