3 # Copyright 2015-2017 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
35 from conf import settings
36 import core.component_factory as component_factory
37 from core.loader import Loader
38 from testcases import PerformanceTestCase
39 from testcases import IntegrationTestCase
40 from tools import tasks
41 from tools import networkcard
42 from tools import functions
43 from tools.pkt_gen import trafficgen
44 from tools.opnfvdashboard import opnfvdashboard
46 sys.dont_write_bytecode = True
49 'debug': logging.DEBUG,
51 'warning': logging.WARNING,
52 'error': logging.ERROR,
53 'critical': logging.CRITICAL
56 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
58 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
59 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
60 'final' : 'test_report.rst',
61 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
65 _LOGGER = logging.getLogger()
67 def parse_arguments():
69 Parse command line arguments.
71 class _SplitTestParamsAction(argparse.Action):
73 Parse and split the '--test-params' argument.
75 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
76 values. For multiple overrides use a ; separated list for
77 e.g. --test-params 'x=z; y=(a,b)'
79 def __call__(self, parser, namespace, values, option_string=None):
82 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
87 # values are passed inside string from CLI, so we must retype them accordingly
89 results[param] = ast.literal_eval(value)
91 # for backward compatibility, we have to accept strings without quotes
92 _LOGGER.warning("Adding missing quotes around string value: %s = %s",
94 results[param] = str(value)
98 setattr(namespace, self.dest, results)
100 class _ValidateFileAction(argparse.Action):
101 """Validate a file can be read from before using it.
103 def __call__(self, parser, namespace, values, option_string=None):
104 if not os.path.isfile(values):
105 raise argparse.ArgumentTypeError(
106 'the path \'%s\' is not a valid path' % values)
107 elif not os.access(values, os.R_OK):
108 raise argparse.ArgumentTypeError(
109 'the path \'%s\' is not accessible' % values)
111 setattr(namespace, self.dest, values)
113 class _ValidateDirAction(argparse.Action):
114 """Validate a directory can be written to before using it.
116 def __call__(self, parser, namespace, values, option_string=None):
117 if not os.path.isdir(values):
118 raise argparse.ArgumentTypeError(
119 'the path \'%s\' is not a valid path' % values)
120 elif not os.access(values, os.W_OK):
121 raise argparse.ArgumentTypeError(
122 'the path \'%s\' is not accessible' % values)
124 setattr(namespace, self.dest, values)
126 def list_logging_levels():
127 """Give a summary of all available logging levels.
129 :return: List of verbosity level names in decreasing order of
132 return sorted(VERBOSITY_LEVELS.keys(),
133 key=lambda x: VERBOSITY_LEVELS[x])
135 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
136 argparse.ArgumentDefaultsHelpFormatter)
137 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
138 parser.add_argument('--list', '--list-tests', action='store_true',
139 help='list all tests and exit')
140 parser.add_argument('--list-trafficgens', action='store_true',
141 help='list all traffic generators and exit')
142 parser.add_argument('--list-collectors', action='store_true',
143 help='list all system metrics loggers and exit')
144 parser.add_argument('--list-vswitches', action='store_true',
145 help='list all system vswitches and exit')
146 parser.add_argument('--list-fwdapps', action='store_true',
147 help='list all system forwarding applications and exit')
148 parser.add_argument('--list-vnfs', action='store_true',
149 help='list all system vnfs and exit')
150 parser.add_argument('--list-settings', action='store_true',
151 help='list effective settings configuration and exit')
152 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
153 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
154 runs only the two tests with those exact names.\
155 To run all tests omit both positional args and --tests arg.')
157 group = parser.add_argument_group('test selection options')
158 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
159 Values: "normal" - execute vSwitch, VNF and traffic generator;\
160 "trafficgen" - execute only traffic generator; "trafficgen-off" \
161 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
162 and VNF but pause before traffic transmission ', default='normal')
164 group.add_argument('-f', '--test-spec', help='test specification file')
165 group.add_argument('-d', '--test-dir', help='directory containing tests')
166 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
167 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
168 name contains RFC2544 less those containing "p2p"; "!back2back" - \
169 run all tests except those containing back2back')
170 group.add_argument('--verbosity', choices=list_logging_levels(),
172 group.add_argument('--integration', action='store_true', help='execute integration tests')
173 group.add_argument('--trafficgen', help='traffic generator to use')
174 group.add_argument('--vswitch', help='vswitch implementation to use')
175 group.add_argument('--fwdapp', help='packet forwarding application to use')
176 group.add_argument('--vnf', help='vnf to use')
177 group.add_argument('--sysmetrics', help='system metrics logger to use')
178 group = parser.add_argument_group('test behavior options')
179 group.add_argument('--xunit', action='store_true',
180 help='enable xUnit-formatted output')
181 group.add_argument('--xunit-dir', action=_ValidateDirAction,
182 help='output directory of xUnit-formatted output')
183 group.add_argument('--load-env', action='store_true',
184 help='enable loading of settings from the environment')
185 group.add_argument('--conf-file', action=_ValidateFileAction,
186 help='settings file')
187 group.add_argument('--test-params', action=_SplitTestParamsAction,
188 help='csv list of test parameters: key=val; e.g. '
189 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
190 'GUEST_LOOPBACK=["l2fwd"] ...')
191 group.add_argument('--opnfvpod', help='name of POD in opnfv')
193 args = vars(parser.parse_args())
198 def configure_logging(level):
199 """Configure logging.
201 log_file_default = os.path.join(
202 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
203 log_file_host_cmds = os.path.join(
204 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
205 log_file_traffic_gen = os.path.join(
206 settings.getValue('LOG_DIR'),
207 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
209 _LOGGER.setLevel(logging.DEBUG)
211 stream_logger = logging.StreamHandler(sys.stdout)
212 stream_logger.setLevel(VERBOSITY_LEVELS[level])
213 stream_logger.setFormatter(logging.Formatter(
214 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
215 _LOGGER.addHandler(stream_logger)
217 file_logger = logging.FileHandler(filename=log_file_default)
218 file_logger.setLevel(logging.DEBUG)
219 _LOGGER.addHandler(file_logger)
221 class CommandFilter(logging.Filter):
222 """Filter out strings beginning with 'cmd :'"""
223 def filter(self, record):
224 return record.getMessage().startswith(tasks.CMD_PREFIX)
226 class TrafficGenCommandFilter(logging.Filter):
227 """Filter out strings beginning with 'gencmd :'"""
228 def filter(self, record):
229 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
231 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
232 cmd_logger.setLevel(logging.DEBUG)
233 cmd_logger.addFilter(CommandFilter())
234 _LOGGER.addHandler(cmd_logger)
236 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
237 gen_logger.setLevel(logging.DEBUG)
238 gen_logger.addFilter(TrafficGenCommandFilter())
239 _LOGGER.addHandler(gen_logger)
242 def apply_filter(tests, tc_filter):
243 """Allow a subset of tests to be conveniently selected
245 :param tests: The list of Tests from which to select.
246 :param tc_filter: A case-insensitive string of comma-separated terms
247 indicating the Tests to select.
248 e.g. 'RFC' - select all tests whose name contains 'RFC'
249 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
251 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
252 or 'burst' and from these remove any containing 'p2p'.
253 e.g. '' - empty string selects all tests.
254 :return: A list of the selected Tests.
256 # if negative filter is first we have to start with full list of tests
257 if tc_filter.strip()[0] == '!':
261 if tc_filter is None:
264 for term in [x.strip() for x in tc_filter.lower().split(",")]:
265 if not term or term[0] != '!':
266 # Add matching tests from 'tests' into results
267 result.extend([test for test in tests \
268 if test['Name'].lower().find(term) >= 0])
270 # Term begins with '!' so we remove matching tests
271 result = [test for test in result \
272 if test['Name'].lower().find(term[1:]) < 0]
277 def check_and_set_locale():
278 """ Function will check locale settings. In case, that it isn't configured
279 properly, then default values specified by DEFAULT_LOCALE will be used.
282 system_locale = locale.getdefaultlocale()
283 if None in system_locale:
284 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
285 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
286 system_locale, locale.getdefaultlocale())
288 def get_vswitch_names(rst_files):
289 """ Function will return a list of vSwitches detected in given ``rst_files``.
291 vswitch_names = set()
294 output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
296 match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
298 vswitch_names.add(match.group(1))
300 if len(vswitch_names):
301 return list(vswitch_names)
303 except subprocess.CalledProcessError:
304 _LOGGER.warning('Cannot detect vSwitches used during testing.')
306 # fallback to the default value
310 """ Function will return a Jenkins job ID environment variable.
314 build_tag = os.environ['BUILD_TAG']
317 _LOGGER.warning('Cannot detect Jenkins job ID')
322 def generate_final_report():
323 """ Function will check if partial test results are available
324 and generates final report in rst format.
327 path = settings.getValue('RESULTS_PATH')
328 # check if there are any results in rst format
329 rst_results = glob.glob(os.path.join(path, 'result*rst'))
330 pkt_processors = get_vswitch_names(rst_results)
333 test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
334 # create report caption directly - it is not worth to execute jinja machinery
335 report_caption = '{}\n{} {}\n{}\n\n'.format(
336 '============================================================',
337 'Performance report for',
338 ', '.join(pkt_processors),
339 '============================================================')
341 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
342 file_.write(report_caption)
344 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
345 ' '.join(rst_results), _TEMPLATE_RST['foot'],
346 test_report), shell=True)
347 if retval == 0 and os.path.isfile(test_report):
348 _LOGGER.info('Overall test report written to "%s"', test_report)
350 _LOGGER.error('Generation of overall test report has failed.')
352 # remove temporary file
353 os.remove(_TEMPLATE_RST['tmp'])
355 except subprocess.CalledProcessError:
356 _LOGGER.error('Generatrion of overall test report has failed.')
359 def enable_sriov(nic_list):
360 """ Enable SRIOV for given enhanced PCI IDs
362 :param nic_list: A list of enhanced PCI IDs
364 # detect if sriov is required
367 if networkcard.is_sriov_nic(nic):
368 tmp_nic = nic.split('|')
369 if tmp_nic[0] in sriov_nic:
370 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
371 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
373 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
375 # sriov is required for some NICs
377 for nic in sriov_nic:
378 # check if SRIOV is supported and enough virt interfaces are available
379 if not networkcard.is_sriov_supported(nic) \
380 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
381 # if not, enable and set appropriate number of VFs
382 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
383 raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
385 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
387 # WORKAROUND: it has been observed with IXGBE(VF) driver,
388 # that NIC doesn't correclty dispatch traffic to VFs based
389 # on their MAC address. Unbind and bind to the same driver
391 networkcard.reinit_vfs(nic)
393 # After SRIOV is enabled it takes some time until network drivers
394 # properly initialize all cards.
395 # Wait also in case, that SRIOV was already configured as it can be
396 # configured automatically just before vsperf execution.
404 def disable_sriov(nic_list):
405 """ Disable SRIOV for given PCI IDs
407 :param nic_list: A list of enhanced PCI IDs
410 if networkcard.is_sriov_nic(nic):
411 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
412 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
414 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
417 def handle_list_options(args):
418 """ Process --list cli arguments if needed
420 :param args: A dictionary with all CLI arguments
422 if args['list_trafficgens']:
423 print(Loader().get_trafficgens_printable())
426 if args['list_collectors']:
427 print(Loader().get_collectors_printable())
430 if args['list_vswitches']:
431 print(Loader().get_vswitches_printable())
434 if args['list_vnfs']:
435 print(Loader().get_vnfs_printable())
438 if args['list_fwdapps']:
439 print(Loader().get_pktfwds_printable())
442 if args['list_settings']:
448 if args['integration']:
449 testcases = settings.getValue('INTEGRATION_TESTS')
451 testcases = settings.getValue('PERFORMANCE_TESTS')
453 print("Available Tests:")
454 print("================")
456 for test in testcases:
457 print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
461 def vsperf_finalize():
462 """ Clean up before exit
464 # remove directory if no result files were created
466 results_path = settings.getValue('RESULTS_PATH')
467 if os.path.exists(results_path):
468 files_list = os.listdir(results_path)
470 _LOGGER.info("Removing empty result directory: " + results_path)
471 shutil.rmtree(results_path)
472 except AttributeError:
473 # skip it if parameter doesn't exist
476 # disable SRIOV if needed
478 if settings.getValue('SRIOV_ENABLED'):
479 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
480 except AttributeError:
481 # skip it if parameter doesn't exist
485 class MockTestCase(unittest.TestCase):
486 """Allow use of xmlrunner to generate Jenkins compatible output without
487 using xmlrunner to actually run tests.
490 suite = unittest.TestSuite()
491 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
492 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
493 xmlrunner.XMLTestRunner(...).run(suite)
496 def __init__(self, msg, is_pass, test_name):
499 self.is_pass = is_pass
501 #dynamically create a test method with the right name
502 #but point the method at our generic test method
503 setattr(MockTestCase, test_name, self.generic_test)
505 super(MockTestCase, self).__init__(test_name)
507 def generic_test(self):
508 """Provide a generic function that raises or not based
509 on how self.is_pass was set in the constructor"""
510 self.assertTrue(self.is_pass, self.msg)
512 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
516 args = parse_arguments()
520 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
522 # Load non performance/integration tests
523 if args['integration']:
524 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
526 # load command line parameters first in case there are settings files
528 settings.load_from_dict(args)
530 if args['conf_file']:
531 settings.load_from_file(args['conf_file'])
534 settings.load_from_env()
536 # reload command line parameters since these should take higher priority
537 # than both a settings file and environment variables
538 settings.load_from_dict(args)
540 settings.setValue('mode', args['mode'])
542 # set dpdk and ovs paths according to VNF and VSWITCH
543 if settings.getValue('mode') != 'trafficgen':
544 functions.settings_update_paths()
546 # if required, handle list-* operations
547 handle_list_options(args)
549 configure_logging(settings.getValue('VERBOSITY'))
551 # check and fix locale
552 check_and_set_locale()
554 # configure trafficgens
555 if args['trafficgen']:
556 trafficgens = Loader().get_trafficgens()
557 if args['trafficgen'] not in trafficgens:
558 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
559 ' \'%s\'. Exiting...', args['trafficgen'],
560 settings.getValue('TRAFFICGEN_DIR'))
563 # configuration validity checks
565 vswitch_none = args['vswitch'].strip().lower() == 'none'
567 settings.setValue('VSWITCH', 'none')
569 vswitches = Loader().get_vswitches()
570 if args['vswitch'] not in vswitches:
571 _LOGGER.error('There are no vswitches matching \'%s\' found in'
572 ' \'%s\'. Exiting...', args['vswitch'],
573 settings.getValue('VSWITCH_DIR'))
577 settings.setValue('PKTFWD', args['fwdapp'])
578 fwdapps = Loader().get_pktfwds()
579 if args['fwdapp'] not in fwdapps:
580 _LOGGER.error('There are no forwarding application'
581 ' matching \'%s\' found in'
582 ' \'%s\'. Exiting...', args['fwdapp'],
583 settings.getValue('PKTFWD_DIR'))
587 vnfs = Loader().get_vnfs()
588 if args['vnf'] not in vnfs:
589 _LOGGER.error('there are no vnfs matching \'%s\' found in'
590 ' \'%s\'. exiting...', args['vnf'],
591 settings.getValue('VNF_DIR'))
594 if args['exact_test_name'] and args['tests']:
595 _LOGGER.error("Cannot specify tests with both positional args and --test.")
598 # modify NIC configuration to decode enhanced PCI IDs
599 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
600 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
602 # sriov handling is performed on checked/expanded PCI IDs
603 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
606 for nic in wl_nics_orig:
607 tmp_nic = networkcard.get_nic_info(nic)
609 nic_list.append({'pci' : tmp_nic,
610 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
611 'mac' : networkcard.get_mac(tmp_nic),
612 'driver' : networkcard.get_driver(tmp_nic),
613 'device' : networkcard.get_device_name(tmp_nic)})
616 raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
618 settings.setValue('NICS', nic_list)
619 # for backward compatibility
620 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
622 # generate results directory name
623 date = datetime.datetime.fromtimestamp(time.time())
624 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
625 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
626 settings.setValue('RESULTS_PATH', results_path)
628 # create results directory
629 if not os.path.exists(results_path):
630 _LOGGER.info("Creating result directory: " + results_path)
631 os.makedirs(results_path)
633 if settings.getValue('mode') == 'trafficgen':
634 # execute only traffic generator
635 _LOGGER.debug("Executing traffic generator:")
637 # set traffic details, so they can be passed to traffic ctl
638 traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
640 traffic = functions.check_traffic(traffic)
642 traffic_ctl = component_factory.create_traffic(
643 traffic['traffic_type'],
644 loader.get_trafficgen_class())
646 traffic_ctl.send_traffic(traffic)
647 _LOGGER.debug("Traffic Results:")
648 traffic_ctl.print_results()
650 # write results into CSV file
651 result_file = os.path.join(results_path, "result.csv")
652 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
655 if args['integration']:
656 testcases = settings.getValue('INTEGRATION_TESTS')
658 testcases = settings.getValue('PERFORMANCE_TESTS')
660 if args['exact_test_name']:
661 exact_names = args['exact_test_name']
662 # positional args => exact matches only
663 selected_tests = [test for test in testcases if test['Name'] in exact_names]
665 # --tests => apply filter to select requested tests
666 selected_tests = apply_filter(testcases, args['tests'])
668 # Default - run all tests
669 selected_tests = testcases
671 if not len(selected_tests):
672 _LOGGER.error("No tests matched --tests option or positional args. Done.")
677 # Add pylint exception: Redefinition of test type from
678 # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase
679 # pylint: disable=redefined-variable-type
680 suite = unittest.TestSuite()
681 for cfg in selected_tests:
682 test_name = cfg.get('Name', '<Name not set>')
684 if args['integration']:
685 test = IntegrationTestCase(cfg)
687 test = PerformanceTestCase(cfg)
689 suite.addTest(MockTestCase('', True, test.name))
690 # pylint: disable=broad-except
691 except (Exception) as ex:
692 _LOGGER.exception("Failed to run test: %s", test_name)
693 suite.addTest(MockTestCase(str(ex), False, test_name))
694 _LOGGER.info("Continuing with next test...")
696 # generate final rst report with results of all executed TCs
697 generate_final_report()
699 if settings.getValue('XUNIT'):
700 xmlrunner.XMLTestRunner(
701 output=settings.getValue('XUNIT_DIR'), outsuffix="",
702 verbosity=0).run(suite)
705 pod_name = args['opnfvpod']
706 installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()
707 opnfv_url = settings.getValue('OPNFV_URL')
708 pkg_list = settings.getValue('PACKAGE_LIST')
710 int_data = {'pod': pod_name,
711 'build_tag': get_build_tag(),
712 'installer': installer_name,
713 'pkg_list': pkg_list,
715 # pass vswitch name from configuration to be used for failed
716 # TCs; In case of successful TCs it is safer to use vswitch
717 # name from CSV as TC can override global configuration
718 'vswitch': str(settings.getValue('VSWITCH')).lower()}
719 tc_names = [tc['Name'] for tc in selected_tests]
720 opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
722 # cleanup before exit
725 if __name__ == "__main__":