3 # Copyright 2015-2016 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
34 sys.dont_write_bytecode = True
36 from conf import settings
37 from conf import get_test_param
38 from core.loader import Loader
39 from testcases import PerformanceTestCase
40 from testcases import IntegrationTestCase
41 from tools import tasks
42 from tools.pkt_gen import trafficgen
43 from tools.opnfvdashboard import opnfvdashboard
44 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
45 import core.component_factory as component_factory
48 'debug': logging.DEBUG,
50 'warning': logging.WARNING,
51 'error': logging.ERROR,
52 'critical': logging.CRITICAL
55 _TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
56 'foot' : 'tools/report/report_foot.rst',
57 'final' : 'test_report.rst',
58 'tmp' : 'tools/report/report_tmp_caption.rst'
61 def parse_arguments():
63 Parse command line arguments.
65 class _SplitTestParamsAction(argparse.Action):
67 Parse and split the '--test-params' argument.
69 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
70 values. For multiple overrides use a ; separated list for
71 e.g. --test-params 'x=z; y=a,b'
73 def __call__(self, parser, namespace, values, option_string=None):
76 for value in values.split(';'):
77 result = [key.strip() for key in value.split('=')]
79 results[result[0]] = True
80 elif len(result) == 2:
81 results[result[0]] = result[1]
83 raise argparse.ArgumentTypeError(
84 'expected \'%s\' to be of format \'key=val\' or'
87 setattr(namespace, self.dest, results)
89 class _ValidateFileAction(argparse.Action):
90 """Validate a file can be read from before using it.
92 def __call__(self, parser, namespace, values, option_string=None):
93 if not os.path.isfile(values):
94 raise argparse.ArgumentTypeError(
95 'the path \'%s\' is not a valid path' % values)
96 elif not os.access(values, os.R_OK):
97 raise argparse.ArgumentTypeError(
98 'the path \'%s\' is not accessible' % values)
100 setattr(namespace, self.dest, values)
102 class _ValidateDirAction(argparse.Action):
103 """Validate a directory can be written to before using it.
105 def __call__(self, parser, namespace, values, option_string=None):
106 if not os.path.isdir(values):
107 raise argparse.ArgumentTypeError(
108 'the path \'%s\' is not a valid path' % values)
109 elif not os.access(values, os.W_OK):
110 raise argparse.ArgumentTypeError(
111 'the path \'%s\' is not accessible' % values)
113 setattr(namespace, self.dest, values)
115 def list_logging_levels():
116 """Give a summary of all available logging levels.
118 :return: List of verbosity level names in decreasing order of
121 return sorted(VERBOSITY_LEVELS.keys(),
122 key=lambda x: VERBOSITY_LEVELS[x])
124 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
125 argparse.ArgumentDefaultsHelpFormatter)
126 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
127 parser.add_argument('--list', '--list-tests', action='store_true',
128 help='list all tests and exit')
129 parser.add_argument('--list-trafficgens', action='store_true',
130 help='list all traffic generators and exit')
131 parser.add_argument('--list-collectors', action='store_true',
132 help='list all system metrics loggers and exit')
133 parser.add_argument('--list-vswitches', action='store_true',
134 help='list all system vswitches and exit')
135 parser.add_argument('--list-fwdapps', action='store_true',
136 help='list all system forwarding applications and exit')
137 parser.add_argument('--list-vnfs', action='store_true',
138 help='list all system vnfs and exit')
139 parser.add_argument('--list-settings', action='store_true',
140 help='list effective settings configuration and exit')
141 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
142 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
143 runs only the two tests with those exact names.\
144 To run all tests omit both positional args and --tests arg.')
146 group = parser.add_argument_group('test selection options')
147 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
148 Values: "normal" - execute vSwitch, VNF and traffic generator;\
149 "trafficgen" - execute only traffic generator; "trafficgen-off" \
150 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
151 and VNF but pause before traffic transmission ', default='normal')
153 group.add_argument('-f', '--test-spec', help='test specification file')
154 group.add_argument('-d', '--test-dir', help='directory containing tests')
155 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
156 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
157 name contains RFC2544 less those containing "p2p"')
158 group.add_argument('--verbosity', choices=list_logging_levels(),
160 group.add_argument('--integration', action='store_true', help='execute integration tests')
161 group.add_argument('--trafficgen', help='traffic generator to use')
162 group.add_argument('--vswitch', help='vswitch implementation to use')
163 group.add_argument('--fwdapp', help='packet forwarding application to use')
164 group.add_argument('--vnf', help='vnf to use')
165 group.add_argument('--sysmetrics', help='system metrics logger to use')
166 group = parser.add_argument_group('test behavior options')
167 group.add_argument('--xunit', action='store_true',
168 help='enable xUnit-formatted output')
169 group.add_argument('--xunit-dir', action=_ValidateDirAction,
170 help='output directory of xUnit-formatted output')
171 group.add_argument('--load-env', action='store_true',
172 help='enable loading of settings from the environment')
173 group.add_argument('--conf-file', action=_ValidateFileAction,
174 help='settings file')
175 group.add_argument('--test-params', action=_SplitTestParamsAction,
176 help='csv list of test parameters: key=val; e.g.'
177 'including pkt_sizes=x,y; duration=x; '
178 'rfc2544_trials=x ...')
179 group.add_argument('--opnfvpod', help='name of POD in opnfv')
181 args = vars(parser.parse_args())
186 def configure_logging(level):
187 """Configure logging.
189 log_file_default = os.path.join(
190 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
191 log_file_host_cmds = os.path.join(
192 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
193 log_file_traffic_gen = os.path.join(
194 settings.getValue('LOG_DIR'),
195 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
197 logger = logging.getLogger()
198 logger.setLevel(logging.DEBUG)
200 stream_logger = logging.StreamHandler(sys.stdout)
201 stream_logger.setLevel(VERBOSITY_LEVELS[level])
202 stream_logger.setFormatter(logging.Formatter(
203 '[%(levelname)s] %(asctime)s : (%(name)s) - %(message)s'))
204 logger.addHandler(stream_logger)
206 file_logger = logging.FileHandler(filename=log_file_default)
207 file_logger.setLevel(logging.DEBUG)
208 logger.addHandler(file_logger)
210 class CommandFilter(logging.Filter):
211 """Filter out strings beginning with 'cmd :'"""
212 def filter(self, record):
213 return record.getMessage().startswith(tasks.CMD_PREFIX)
215 class TrafficGenCommandFilter(logging.Filter):
216 """Filter out strings beginning with 'gencmd :'"""
217 def filter(self, record):
218 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
220 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
221 cmd_logger.setLevel(logging.DEBUG)
222 cmd_logger.addFilter(CommandFilter())
223 logger.addHandler(cmd_logger)
225 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
226 gen_logger.setLevel(logging.DEBUG)
227 gen_logger.addFilter(TrafficGenCommandFilter())
228 logger.addHandler(gen_logger)
231 def apply_filter(tests, tc_filter):
232 """Allow a subset of tests to be conveniently selected
234 :param tests: The list of Tests from which to select.
235 :param tc_filter: A case-insensitive string of comma-separated terms
236 indicating the Tests to select.
237 e.g. 'RFC' - select all tests whose name contains 'RFC'
238 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
240 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
241 or 'burst' and from these remove any containing 'p2p'.
242 e.g. '' - empty string selects all tests.
243 :return: A list of the selected Tests.
246 if tc_filter is None:
249 for term in [x.strip() for x in tc_filter.lower().split(",")]:
250 if not term or term[0] != '!':
251 # Add matching tests from 'tests' into results
252 result.extend([test for test in tests \
253 if test.name.lower().find(term) >= 0])
255 # Term begins with '!' so we remove matching tests
256 result = [test for test in result \
257 if test.name.lower().find(term[1:]) < 0]
262 def check_and_set_locale():
263 """ Function will check locale settings. In case, that it isn't configured
264 properly, then default values specified by DEFAULT_LOCALE will be used.
267 system_locale = locale.getdefaultlocale()
268 if None in system_locale:
269 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
270 logging.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
271 system_locale, locale.getdefaultlocale())
274 def generate_final_report(path):
275 """ Function will check if partial test results are available
276 and generates final report in rst format.
279 # check if there are any results in rst format
280 rst_results = glob.glob(os.path.join(path, 'result*rst'))
283 test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
284 # create report caption directly - it is not worth to execute jinja machinery
285 report_caption = '{}\n{} {}\n{}\n\n'.format(
286 '============================================================',
287 'Performance report for',
288 Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0],
290 '============================================================')
292 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
293 file_.write(report_caption)
295 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
296 ' '.join(rst_results), _TEMPLATE_RST['foot'],
297 test_report), shell=True)
298 if retval == 0 and os.path.isfile(test_report):
299 logging.info('Overall test report written to "%s"', test_report)
301 logging.error('Generatrion of overall test report has failed.')
303 # remove temporary file
304 os.remove(_TEMPLATE_RST['tmp'])
306 except subprocess.CalledProcessError:
307 logging.error('Generatrion of overall test report has failed.')
310 class MockTestCase(unittest.TestCase):
311 """Allow use of xmlrunner to generate Jenkins compatible output without
312 using xmlrunner to actually run tests.
315 suite = unittest.TestSuite()
316 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
317 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
318 xmlrunner.XMLTestRunner(...).run(suite)
321 def __init__(self, msg, is_pass, test_name):
324 self.is_pass = is_pass
326 #dynamically create a test method with the right name
327 #but point the method at our generic test method
328 setattr(MockTestCase, test_name, self.generic_test)
330 super(MockTestCase, self).__init__(test_name)
332 def generic_test(self):
333 """Provide a generic function that raises or not based
334 on how self.is_pass was set in the constructor"""
335 self.assertTrue(self.is_pass, self.msg)
341 args = parse_arguments()
345 settings.load_from_dir('conf')
347 # Load non performance/integration tests
348 if args['integration']:
349 settings.load_from_dir('conf/integration')
351 # load command line parameters first in case there are settings files
353 settings.load_from_dict(args)
355 if args['conf_file']:
356 settings.load_from_file(args['conf_file'])
359 settings.load_from_env()
361 # reload command line parameters since these should take higher priority
362 # than both a settings file and environment variables
363 settings.load_from_dict(args)
366 # set dpdk and ovs paths accorfing to VNF and VSWITCH
367 if settings.getValue('VSWITCH').endswith('Vanilla'):
368 # settings paths for Vanilla
369 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
370 elif settings.getValue('VSWITCH').endswith('Vhost'):
371 if settings.getValue('VNF').endswith('Cuse'):
372 # settings paths for Cuse
373 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
374 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
376 # settings paths for VhostUser
377 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
378 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
380 # default - set to VHOST USER but can be changed during enhancement
381 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
382 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
383 if 'none' == settings.getValue('VSWITCH').strip().lower():
386 configure_logging(settings.getValue('VERBOSITY'))
387 logger = logging.getLogger()
389 # check and fix locale
390 check_and_set_locale()
392 # configure trafficgens
393 if args['trafficgen']:
394 trafficgens = Loader().get_trafficgens()
395 if args['trafficgen'] not in trafficgens:
396 logging.error('There are no trafficgens matching \'%s\' found in'
397 ' \'%s\'. Exiting...', args['trafficgen'],
398 settings.getValue('TRAFFICGEN_DIR'))
403 vswitch_none = 'none' == args['vswitch'].strip().lower()
405 settings.setValue('VSWITCH', 'none')
407 vswitches = Loader().get_vswitches()
408 if args['vswitch'] not in vswitches:
409 logging.error('There are no vswitches matching \'%s\' found in'
410 ' \'%s\'. Exiting...', args['vswitch'],
411 settings.getValue('VSWITCH_DIR'))
415 settings.setValue('PKTFWD', args['fwdapp'])
416 fwdapps = Loader().get_pktfwds()
417 if args['fwdapp'] not in fwdapps:
418 logging.error('There are no forwarding application'
419 ' matching \'%s\' found in'
420 ' \'%s\'. Exiting...', args['fwdapp'],
421 settings.getValue('PKTFWD_DIR'))
425 vnfs = Loader().get_vnfs()
426 if args['vnf'] not in vnfs:
427 logging.error('there are no vnfs matching \'%s\' found in'
428 ' \'%s\'. exiting...', args['vnf'],
429 settings.getValue('vnf_dir'))
432 # update global settings
433 guest_loopback = get_test_param('guest_loopback', None)
436 for i in range(len(settings.getValue('GUEST_LOOPBACK'))):
437 tmp_gl.append(guest_loopback)
438 settings.setValue('GUEST_LOOPBACK', tmp_gl)
440 settings.setValue('mode', args['mode'])
442 # generate results directory name
443 date = datetime.datetime.fromtimestamp(time.time())
444 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
445 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
447 # create results directory
448 if not os.path.exists(results_path):
449 logger.info("Creating result directory: " + results_path)
450 os.makedirs(results_path)
452 if settings.getValue('mode') == 'trafficgen':
453 # execute only traffic generator
454 logging.debug("Executing traffic generator:")
456 # set traffic details, so they can be passed to traffic ctl
457 traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
458 traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
459 'bidir': get_test_param('bidirectional', False),
460 'multistream': int(get_test_param('multistream', 0)),
461 'stream_type': get_test_param('stream_type', 'L4'),
462 'frame_rate': int(get_test_param('iload', 100))})
464 traffic_ctl = component_factory.create_traffic(
465 traffic['traffic_type'],
466 loader.get_trafficgen_class())
468 traffic_ctl.send_traffic(traffic)
469 logging.debug("Traffic Results:")
470 traffic_ctl.print_results()
473 if args['integration']:
474 testcases = settings.getValue('INTEGRATION_TESTS')
476 testcases = settings.getValue('PERFORMANCE_TESTS')
479 for cfg in testcases:
481 if args['integration']:
482 all_tests.append(IntegrationTestCase(cfg, results_path))
484 all_tests.append(PerformanceTestCase(cfg, results_path))
485 except (Exception) as _:
486 logger.exception("Failed to create test: %s",
487 cfg.get('Name', '<Name not set>'))
490 # if required, handle list-* operations
493 print("Available Tests:")
494 print("================")
495 for test in all_tests:
496 print('* %-30s %s' % ('%s:' % test.name, test.desc))
499 if args['list_trafficgens']:
500 print(Loader().get_trafficgens_printable())
503 if args['list_collectors']:
504 print(Loader().get_collectors_printable())
507 if args['list_vswitches']:
508 print(Loader().get_vswitches_printable())
511 if args['list_vnfs']:
512 print(Loader().get_vnfs_printable())
515 if args['list_settings']:
519 # select requested tests
520 if args['exact_test_name'] and args['tests']:
521 logger.error("Cannot specify tests with both positional args and --test.")
524 if args['exact_test_name']:
525 exact_names = args['exact_test_name']
526 # positional args => exact matches only
527 selected_tests = [test for test in all_tests if test.name in exact_names]
529 # --tests => apply filter to select requested tests
530 selected_tests = apply_filter(all_tests, args['tests'])
532 # Default - run all tests
533 selected_tests = all_tests
535 if not selected_tests:
536 logger.error("No tests matched --test option or positional args. Done.")
540 suite = unittest.TestSuite()
541 for test in selected_tests:
544 suite.addTest(MockTestCase('', True, test.name))
545 #pylint: disable=broad-except
546 except (Exception) as ex:
547 logger.exception("Failed to run test: %s", test.name)
548 suite.addTest(MockTestCase(str(ex), False, test.name))
549 logger.info("Continuing with next test...")
551 # generate final rst report with results of all executed TCs
552 generate_final_report(results_path)
554 if settings.getValue('XUNIT'):
555 xmlrunner.XMLTestRunner(
556 output=settings.getValue('XUNIT_DIR'), outsuffix="",
557 verbosity=0).run(suite)
560 pod_name = args['opnfvpod']
561 installer_name = settings.getValue('OPNFV_INSTALLER')
562 opnfv_url = settings.getValue('OPNFV_URL')
563 pkg_list = settings.getValue('PACKAGE_LIST')
565 int_data = {'cuse': False,
568 'installer': installer_name,
569 'pkg_list': pkg_list,
571 if settings.getValue('VSWITCH').endswith('Vanilla'):
572 int_data['vanilla'] = True
573 if settings.getValue('VNF').endswith('Cuse'):
574 int_data['cuse'] = True
575 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
577 #remove directory if no result files were created.
578 if os.path.exists(results_path):
579 files_list = os.listdir(results_path)
581 shutil.rmtree(results_path)
583 if __name__ == "__main__":