3 # Copyright 2015-2017 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
35 from conf import settings
36 import core.component_factory as component_factory
37 from core.loader import Loader
38 from testcases import PerformanceTestCase
39 from testcases import IntegrationTestCase
40 from tools import tasks
41 from tools import networkcard
42 from tools import functions
43 from tools.pkt_gen import trafficgen
44 from tools.opnfvdashboard import opnfvdashboard
46 sys.dont_write_bytecode = True
49 'debug': logging.DEBUG,
51 'warning': logging.WARNING,
52 'error': logging.ERROR,
53 'critical': logging.CRITICAL
56 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
58 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
59 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
60 'final' : 'test_report.rst',
61 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
65 _LOGGER = logging.getLogger()
67 def parse_arguments():
69 Parse command line arguments.
71 class _SplitTestParamsAction(argparse.Action):
73 Parse and split the '--test-params' argument.
75 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
76 values. For multiple overrides use a ; separated list for
77 e.g. --test-params 'x=z; y=(a,b)'
79 def __call__(self, parser, namespace, values, option_string=None):
82 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
87 # values are passed inside string from CLI, so we must retype them accordingly
89 results[param] = ast.literal_eval(value)
91 # for backward compatibility, we have to accept strings without quotes
92 _LOGGER.warning("Adding missing quotes around string value: %s = %s",
94 results[param] = str(value)
98 setattr(namespace, self.dest, results)
100 class _ValidateFileAction(argparse.Action):
101 """Validate a file can be read from before using it.
103 def __call__(self, parser, namespace, values, option_string=None):
104 if not os.path.isfile(values):
105 raise argparse.ArgumentTypeError(
106 'the path \'%s\' is not a valid path' % values)
107 elif not os.access(values, os.R_OK):
108 raise argparse.ArgumentTypeError(
109 'the path \'%s\' is not accessible' % values)
111 setattr(namespace, self.dest, values)
113 class _ValidateDirAction(argparse.Action):
114 """Validate a directory can be written to before using it.
116 def __call__(self, parser, namespace, values, option_string=None):
117 if not os.path.isdir(values):
118 raise argparse.ArgumentTypeError(
119 'the path \'%s\' is not a valid path' % values)
120 elif not os.access(values, os.W_OK):
121 raise argparse.ArgumentTypeError(
122 'the path \'%s\' is not accessible' % values)
124 setattr(namespace, self.dest, values)
126 def list_logging_levels():
127 """Give a summary of all available logging levels.
129 :return: List of verbosity level names in decreasing order of
132 return sorted(VERBOSITY_LEVELS.keys(),
133 key=lambda x: VERBOSITY_LEVELS[x])
135 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
136 argparse.ArgumentDefaultsHelpFormatter)
137 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
138 parser.add_argument('--list', '--list-tests', action='store_true',
139 help='list all tests and exit')
140 parser.add_argument('--list-trafficgens', action='store_true',
141 help='list all traffic generators and exit')
142 parser.add_argument('--list-collectors', action='store_true',
143 help='list all system metrics loggers and exit')
144 parser.add_argument('--list-vswitches', action='store_true',
145 help='list all system vswitches and exit')
146 parser.add_argument('--list-fwdapps', action='store_true',
147 help='list all system forwarding applications and exit')
148 parser.add_argument('--list-vnfs', action='store_true',
149 help='list all system vnfs and exit')
150 parser.add_argument('--list-settings', action='store_true',
151 help='list effective settings configuration and exit')
152 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
153 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
154 runs only the two tests with those exact names.\
155 To run all tests omit both positional args and --tests arg.')
157 group = parser.add_argument_group('test selection options')
158 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
159 Values: "normal" - execute vSwitch, VNF and traffic generator;\
160 "trafficgen" - execute only traffic generator; "trafficgen-off" \
161 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
162 and VNF but pause before traffic transmission ', default='normal')
164 group.add_argument('-f', '--test-spec', help='test specification file')
165 group.add_argument('-d', '--test-dir', help='directory containing tests')
166 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
167 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
168 name contains RFC2544 less those containing "p2p"; "!back2back" - \
169 run all tests except those containing back2back')
170 group.add_argument('--verbosity', choices=list_logging_levels(),
172 group.add_argument('--integration', action='store_true', help='execute integration tests')
173 group.add_argument('--trafficgen', help='traffic generator to use')
174 group.add_argument('--vswitch', help='vswitch implementation to use')
175 group.add_argument('--fwdapp', help='packet forwarding application to use')
176 group.add_argument('--vnf', help='vnf to use')
177 group.add_argument('--sysmetrics', help='system metrics logger to use')
178 group = parser.add_argument_group('test behavior options')
179 group.add_argument('--xunit', action='store_true',
180 help='enable xUnit-formatted output')
181 group.add_argument('--xunit-dir', action=_ValidateDirAction,
182 help='output directory of xUnit-formatted output')
183 group.add_argument('--load-env', action='store_true',
184 help='enable loading of settings from the environment')
185 group.add_argument('--conf-file', action=_ValidateFileAction,
186 help='settings file')
187 group.add_argument('--test-params', action=_SplitTestParamsAction,
188 help='csv list of test parameters: key=val; e.g. '
189 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
190 'GUEST_LOOPBACK=["l2fwd"] ...')
191 group.add_argument('--opnfvpod', help='name of POD in opnfv')
193 args = vars(parser.parse_args())
198 def configure_logging(level):
199 """Configure logging.
201 log_file_default = os.path.join(
202 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
203 log_file_host_cmds = os.path.join(
204 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
205 log_file_traffic_gen = os.path.join(
206 settings.getValue('LOG_DIR'),
207 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
209 _LOGGER.setLevel(logging.DEBUG)
211 stream_logger = logging.StreamHandler(sys.stdout)
212 stream_logger.setLevel(VERBOSITY_LEVELS[level])
213 stream_logger.setFormatter(logging.Formatter(
214 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
215 _LOGGER.addHandler(stream_logger)
217 file_logger = logging.FileHandler(filename=log_file_default)
218 file_logger.setLevel(logging.DEBUG)
219 _LOGGER.addHandler(file_logger)
221 class CommandFilter(logging.Filter):
222 """Filter out strings beginning with 'cmd :'"""
223 def filter(self, record):
224 return record.getMessage().startswith(tasks.CMD_PREFIX)
226 class TrafficGenCommandFilter(logging.Filter):
227 """Filter out strings beginning with 'gencmd :'"""
228 def filter(self, record):
229 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
231 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
232 cmd_logger.setLevel(logging.DEBUG)
233 cmd_logger.addFilter(CommandFilter())
234 _LOGGER.addHandler(cmd_logger)
236 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
237 gen_logger.setLevel(logging.DEBUG)
238 gen_logger.addFilter(TrafficGenCommandFilter())
239 _LOGGER.addHandler(gen_logger)
242 def apply_filter(tests, tc_filter):
243 """Allow a subset of tests to be conveniently selected
245 :param tests: The list of Tests from which to select.
246 :param tc_filter: A case-insensitive string of comma-separated terms
247 indicating the Tests to select.
248 e.g. 'RFC' - select all tests whose name contains 'RFC'
249 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
251 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
252 or 'burst' and from these remove any containing 'p2p'.
253 e.g. '' - empty string selects all tests.
254 :return: A list of the selected Tests.
256 # if negative filter is first we have to start with full list of tests
257 if tc_filter.strip()[0] == '!':
261 if tc_filter is None:
264 for term in [x.strip() for x in tc_filter.lower().split(",")]:
265 if not term or term[0] != '!':
266 # Add matching tests from 'tests' into results
267 result.extend([test for test in tests \
268 if test['Name'].lower().find(term) >= 0])
270 # Term begins with '!' so we remove matching tests
271 result = [test for test in result \
272 if test['Name'].lower().find(term[1:]) < 0]
277 def check_and_set_locale():
278 """ Function will check locale settings. In case, that it isn't configured
279 properly, then default values specified by DEFAULT_LOCALE will be used.
282 system_locale = locale.getdefaultlocale()
283 if None in system_locale:
284 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
285 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
286 system_locale, locale.getdefaultlocale())
288 def get_vswitch_names(rst_files):
289 """ Function will return a list of vSwitches detected in given ``rst_files``.
291 vswitch_names = set()
294 output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
296 match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
298 vswitch_names.add(match.group(1))
300 if len(vswitch_names):
301 return list(vswitch_names)
303 except subprocess.CalledProcessError:
304 _LOGGER.warning('Cannot detect vSwitches used during testing.')
306 # fallback to the default value
311 def generate_final_report():
312 """ Function will check if partial test results are available
313 and generates final report in rst format.
316 path = settings.getValue('RESULTS_PATH')
317 # check if there are any results in rst format
318 rst_results = glob.glob(os.path.join(path, 'result*rst'))
319 pkt_processors = get_vswitch_names(rst_results)
322 test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
323 # create report caption directly - it is not worth to execute jinja machinery
324 report_caption = '{}\n{} {}\n{}\n\n'.format(
325 '============================================================',
326 'Performance report for',
327 ', '.join(pkt_processors),
328 '============================================================')
330 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
331 file_.write(report_caption)
333 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
334 ' '.join(rst_results), _TEMPLATE_RST['foot'],
335 test_report), shell=True)
336 if retval == 0 and os.path.isfile(test_report):
337 _LOGGER.info('Overall test report written to "%s"', test_report)
339 _LOGGER.error('Generatrion of overall test report has failed.')
341 # remove temporary file
342 os.remove(_TEMPLATE_RST['tmp'])
344 except subprocess.CalledProcessError:
345 _LOGGER.error('Generatrion of overall test report has failed.')
348 def enable_sriov(nic_list):
349 """ Enable SRIOV for given enhanced PCI IDs
351 :param nic_list: A list of enhanced PCI IDs
353 # detect if sriov is required
356 if networkcard.is_sriov_nic(nic):
357 tmp_nic = nic.split('|')
358 if tmp_nic[0] in sriov_nic:
359 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
360 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
362 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
364 # sriov is required for some NICs
366 for nic in sriov_nic:
367 # check if SRIOV is supported and enough virt interfaces are available
368 if not networkcard.is_sriov_supported(nic) \
369 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
370 # if not, enable and set appropriate number of VFs
371 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
372 raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
374 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
376 # WORKAROUND: it has been observed with IXGBE(VF) driver,
377 # that NIC doesn't correclty dispatch traffic to VFs based
378 # on their MAC address. Unbind and bind to the same driver
380 networkcard.reinit_vfs(nic)
382 # After SRIOV is enabled it takes some time until network drivers
383 # properly initialize all cards.
384 # Wait also in case, that SRIOV was already configured as it can be
385 # configured automatically just before vsperf execution.
393 def disable_sriov(nic_list):
394 """ Disable SRIOV for given PCI IDs
396 :param nic_list: A list of enhanced PCI IDs
399 if networkcard.is_sriov_nic(nic):
400 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
401 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
403 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
406 def handle_list_options(args):
407 """ Process --list cli arguments if needed
409 :param args: A dictionary with all CLI arguments
411 if args['list_trafficgens']:
412 print(Loader().get_trafficgens_printable())
415 if args['list_collectors']:
416 print(Loader().get_collectors_printable())
419 if args['list_vswitches']:
420 print(Loader().get_vswitches_printable())
423 if args['list_vnfs']:
424 print(Loader().get_vnfs_printable())
427 if args['list_fwdapps']:
428 print(Loader().get_pktfwds_printable())
431 if args['list_settings']:
437 if args['integration']:
438 testcases = settings.getValue('INTEGRATION_TESTS')
440 testcases = settings.getValue('PERFORMANCE_TESTS')
442 print("Available Tests:")
443 print("================")
445 for test in testcases:
446 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
450 def vsperf_finalize():
451 """ Clean up before exit
453 # remove directory if no result files were created
455 results_path = settings.getValue('RESULTS_PATH')
456 if os.path.exists(results_path):
457 files_list = os.listdir(results_path)
459 _LOGGER.info("Removing empty result directory: " + results_path)
460 shutil.rmtree(results_path)
461 except AttributeError:
462 # skip it if parameter doesn't exist
465 # disable SRIOV if needed
467 if settings.getValue('SRIOV_ENABLED'):
468 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
469 except AttributeError:
470 # skip it if parameter doesn't exist
474 class MockTestCase(unittest.TestCase):
475 """Allow use of xmlrunner to generate Jenkins compatible output without
476 using xmlrunner to actually run tests.
479 suite = unittest.TestSuite()
480 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
481 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
482 xmlrunner.XMLTestRunner(...).run(suite)
485 def __init__(self, msg, is_pass, test_name):
488 self.is_pass = is_pass
490 #dynamically create a test method with the right name
491 #but point the method at our generic test method
492 setattr(MockTestCase, test_name, self.generic_test)
494 super(MockTestCase, self).__init__(test_name)
496 def generic_test(self):
497 """Provide a generic function that raises or not based
498 on how self.is_pass was set in the constructor"""
499 self.assertTrue(self.is_pass, self.msg)
501 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
505 args = parse_arguments()
509 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
511 # Load non performance/integration tests
512 if args['integration']:
513 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
515 # load command line parameters first in case there are settings files
517 settings.load_from_dict(args)
519 if args['conf_file']:
520 settings.load_from_file(args['conf_file'])
523 settings.load_from_env()
525 # reload command line parameters since these should take higher priority
526 # than both a settings file and environment variables
527 settings.load_from_dict(args)
529 settings.setValue('mode', args['mode'])
531 # set dpdk and ovs paths according to VNF and VSWITCH
532 if settings.getValue('mode') != 'trafficgen':
533 functions.settings_update_paths()
535 # if required, handle list-* operations
536 handle_list_options(args)
538 configure_logging(settings.getValue('VERBOSITY'))
540 # check and fix locale
541 check_and_set_locale()
543 # configure trafficgens
544 if args['trafficgen']:
545 trafficgens = Loader().get_trafficgens()
546 if args['trafficgen'] not in trafficgens:
547 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
548 ' \'%s\'. Exiting...', args['trafficgen'],
549 settings.getValue('TRAFFICGEN_DIR'))
552 # configuration validity checks
554 vswitch_none = args['vswitch'].strip().lower() == 'none'
556 settings.setValue('VSWITCH', 'none')
558 vswitches = Loader().get_vswitches()
559 if args['vswitch'] not in vswitches:
560 _LOGGER.error('There are no vswitches matching \'%s\' found in'
561 ' \'%s\'. Exiting...', args['vswitch'],
562 settings.getValue('VSWITCH_DIR'))
566 settings.setValue('PKTFWD', args['fwdapp'])
567 fwdapps = Loader().get_pktfwds()
568 if args['fwdapp'] not in fwdapps:
569 _LOGGER.error('There are no forwarding application'
570 ' matching \'%s\' found in'
571 ' \'%s\'. Exiting...', args['fwdapp'],
572 settings.getValue('PKTFWD_DIR'))
576 vnfs = Loader().get_vnfs()
577 if args['vnf'] not in vnfs:
578 _LOGGER.error('there are no vnfs matching \'%s\' found in'
579 ' \'%s\'. exiting...', args['vnf'],
580 settings.getValue('VNF_DIR'))
583 if args['exact_test_name'] and args['tests']:
584 _LOGGER.error("Cannot specify tests with both positional args and --test.")
587 # modify NIC configuration to decode enhanced PCI IDs
588 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
589 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
591 # sriov handling is performed on checked/expanded PCI IDs
592 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
595 for nic in wl_nics_orig:
596 tmp_nic = networkcard.get_nic_info(nic)
598 nic_list.append({'pci' : tmp_nic,
599 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
600 'mac' : networkcard.get_mac(tmp_nic),
601 'driver' : networkcard.get_driver(tmp_nic),
602 'device' : networkcard.get_device_name(tmp_nic)})
605 raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
607 settings.setValue('NICS', nic_list)
608 # for backward compatibility
609 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
611 # generate results directory name
612 date = datetime.datetime.fromtimestamp(time.time())
613 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
614 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
615 settings.setValue('RESULTS_PATH', results_path)
617 # create results directory
618 if not os.path.exists(results_path):
619 _LOGGER.info("Creating result directory: " + results_path)
620 os.makedirs(results_path)
622 if settings.getValue('mode') == 'trafficgen':
623 # execute only traffic generator
624 _LOGGER.debug("Executing traffic generator:")
626 # set traffic details, so they can be passed to traffic ctl
627 traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
629 traffic = functions.check_traffic(traffic)
631 traffic_ctl = component_factory.create_traffic(
632 traffic['traffic_type'],
633 loader.get_trafficgen_class())
635 traffic_ctl.send_traffic(traffic)
636 _LOGGER.debug("Traffic Results:")
637 traffic_ctl.print_results()
639 # write results into CSV file
640 result_file = os.path.join(results_path, "result.csv")
641 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
644 if args['integration']:
645 testcases = settings.getValue('INTEGRATION_TESTS')
647 testcases = settings.getValue('PERFORMANCE_TESTS')
649 if args['exact_test_name']:
650 exact_names = args['exact_test_name']
651 # positional args => exact matches only
652 selected_tests = [test for test in testcases if test['Name'] in exact_names]
654 # --tests => apply filter to select requested tests
655 selected_tests = apply_filter(testcases, args['tests'])
657 # Default - run all tests
658 selected_tests = testcases
660 if not len(selected_tests):
661 _LOGGER.error("No tests matched --tests option or positional args. Done.")
666 # Add pylint exception: Redefinition of test type from
667 # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase
668 # pylint: disable=redefined-variable-type
669 suite = unittest.TestSuite()
670 for cfg in selected_tests:
671 test_name = cfg.get('Name', '<Name not set>')
673 if args['integration']:
674 test = IntegrationTestCase(cfg)
676 test = PerformanceTestCase(cfg)
678 suite.addTest(MockTestCase('', True, test.name))
679 # pylint: disable=broad-except
680 except (Exception) as ex:
681 _LOGGER.exception("Failed to run test: %s", test_name)
682 suite.addTest(MockTestCase(str(ex), False, test_name))
683 _LOGGER.info("Continuing with next test...")
685 # generate final rst report with results of all executed TCs
686 generate_final_report()
688 if settings.getValue('XUNIT'):
689 xmlrunner.XMLTestRunner(
690 output=settings.getValue('XUNIT_DIR'), outsuffix="",
691 verbosity=0).run(suite)
694 pod_name = args['opnfvpod']
695 installer_name = settings.getValue('OPNFV_INSTALLER')
696 opnfv_url = settings.getValue('OPNFV_URL')
697 pkg_list = settings.getValue('PACKAGE_LIST')
699 int_data = {'vanilla': False,
701 'installer': installer_name,
702 'pkg_list': pkg_list,
704 if settings.getValue('VSWITCH').endswith('Vanilla'):
705 int_data['vanilla'] = True
706 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
708 # cleanup before exit
711 if __name__ == "__main__":