3 # Copyright 2015 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
31 sys.dont_write_bytecode = True
33 from conf import settings
34 from core.loader import Loader
35 from testcases import TestCase
36 from tools import tasks
37 from tools.pkt_gen import trafficgen
38 from tools.opnfvdashboard import opnfvdashboard
41 'debug': logging.DEBUG,
43 'warning': logging.WARNING,
44 'error': logging.ERROR,
45 'critical': logging.CRITICAL
49 def parse_arguments():
51 Parse command line arguments.
53 class _SplitTestParamsAction(argparse.Action):
55 Parse and split the '--test-params' argument.
57 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
58 values. For multiple overrides use a ; separated list for
59 e.g. --test-params 'x=z; y=a,b'
61 def __call__(self, parser, namespace, values, option_string=None):
64 for value in values.split(';'):
65 result = [key.strip() for key in value.split('=')]
67 results[result[0]] = True
68 elif len(result) == 2:
69 results[result[0]] = result[1]
71 raise argparse.ArgumentTypeError(
72 'expected \'%s\' to be of format \'key=val\' or'
75 setattr(namespace, self.dest, results)
77 class _ValidateFileAction(argparse.Action):
78 """Validate a file can be read from before using it.
80 def __call__(self, parser, namespace, values, option_string=None):
81 if not os.path.isfile(values):
82 raise argparse.ArgumentTypeError(
83 'the path \'%s\' is not a valid path' % values)
84 elif not os.access(values, os.R_OK):
85 raise argparse.ArgumentTypeError(
86 'the path \'%s\' is not accessible' % values)
88 setattr(namespace, self.dest, values)
90 class _ValidateDirAction(argparse.Action):
91 """Validate a directory can be written to before using it.
93 def __call__(self, parser, namespace, values, option_string=None):
94 if not os.path.isdir(values):
95 raise argparse.ArgumentTypeError(
96 'the path \'%s\' is not a valid path' % values)
97 elif not os.access(values, os.W_OK):
98 raise argparse.ArgumentTypeError(
99 'the path \'%s\' is not accessible' % values)
101 setattr(namespace, self.dest, values)
103 def list_logging_levels():
104 """Give a summary of all available logging levels.
106 :return: List of verbosity level names in decreasing order of
109 return sorted(VERBOSITY_LEVELS.keys(),
110 key=lambda x: VERBOSITY_LEVELS[x])
112 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
113 argparse.ArgumentDefaultsHelpFormatter)
114 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
115 parser.add_argument('--list', '--list-tests', action='store_true',
116 help='list all tests and exit')
117 parser.add_argument('--list-trafficgens', action='store_true',
118 help='list all traffic generators and exit')
119 parser.add_argument('--list-collectors', action='store_true',
120 help='list all system metrics loggers and exit')
121 parser.add_argument('--list-vswitches', action='store_true',
122 help='list all system vswitches and exit')
123 parser.add_argument('--list-vnfs', action='store_true',
124 help='list all system vnfs and exit')
125 parser.add_argument('--list-settings', action='store_true',
126 help='list effective settings configuration and exit')
127 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
128 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
129 runs only the two tests with those exact names.\
130 To run all tests omit both positional args and --tests arg.')
132 group = parser.add_argument_group('test selection options')
133 group.add_argument('-f', '--test-spec', help='test specification file')
134 group.add_argument('-d', '--test-dir', help='directory containing tests')
135 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
136 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
137 name contains RFC2544 less those containing "p2p"')
138 group.add_argument('--verbosity', choices=list_logging_levels(),
140 group.add_argument('--trafficgen', help='traffic generator to use')
141 group.add_argument('--vswitch', help='vswitch implementation to use')
142 group.add_argument('--vnf', help='vnf to use')
143 group.add_argument('--duration', help='traffic transmit duration')
144 group.add_argument('--sysmetrics', help='system metrics logger to use')
145 group = parser.add_argument_group('test behavior options')
146 group.add_argument('--xunit', action='store_true',
147 help='enable xUnit-formatted output')
148 group.add_argument('--xunit-dir', action=_ValidateDirAction,
149 help='output directory of xUnit-formatted output')
150 group.add_argument('--load-env', action='store_true',
151 help='enable loading of settings from the environment')
152 group.add_argument('--conf-file', action=_ValidateFileAction,
153 help='settings file')
154 group.add_argument('--test-params', action=_SplitTestParamsAction,
155 help='csv list of test parameters: key=val; e.g.'
156 'including pkt_sizes=x,y; duration=x; '
157 'rfc2544_trials=x ...')
158 group.add_argument('--opnfvpod', help='name of POD in opnfv')
160 args = vars(parser.parse_args())
165 def configure_logging(level):
166 """Configure logging.
168 log_file_default = os.path.join(
169 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
170 log_file_host_cmds = os.path.join(
171 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
172 log_file_traffic_gen = os.path.join(
173 settings.getValue('LOG_DIR'),
174 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
176 logger = logging.getLogger()
177 logger.setLevel(logging.DEBUG)
179 stream_logger = logging.StreamHandler(sys.stdout)
180 stream_logger.setLevel(VERBOSITY_LEVELS[level])
181 stream_logger.setFormatter(logging.Formatter(
182 '[%(levelname)s] %(asctime)s : (%(name)s) - %(message)s'))
183 logger.addHandler(stream_logger)
185 file_logger = logging.FileHandler(filename=log_file_default)
186 file_logger.setLevel(logging.DEBUG)
187 logger.addHandler(file_logger)
189 class CommandFilter(logging.Filter):
190 """Filter out strings beginning with 'cmd :'"""
191 def filter(self, record):
192 return record.getMessage().startswith(tasks.CMD_PREFIX)
194 class TrafficGenCommandFilter(logging.Filter):
195 """Filter out strings beginning with 'gencmd :'"""
196 def filter(self, record):
197 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
199 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
200 cmd_logger.setLevel(logging.DEBUG)
201 cmd_logger.addFilter(CommandFilter())
202 logger.addHandler(cmd_logger)
204 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
205 gen_logger.setLevel(logging.DEBUG)
206 gen_logger.addFilter(TrafficGenCommandFilter())
207 logger.addHandler(gen_logger)
210 def apply_filter(tests, tc_filter):
211 """Allow a subset of tests to be conveniently selected
213 :param tests: The list of Tests from which to select.
214 :param tc_filter: A case-insensitive string of comma-separated terms
215 indicating the Tests to select.
216 e.g. 'RFC' - select all tests whose name contains 'RFC'
217 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
219 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
220 or 'burst' and from these remove any containing 'p2p'.
221 e.g. '' - empty string selects all tests.
222 :return: A list of the selected Tests.
225 if tc_filter is None:
228 for term in [x.strip() for x in tc_filter.lower().split(",")]:
229 if not term or term[0] != '!':
230 # Add matching tests from 'tests' into results
231 result.extend([test for test in tests \
232 if test.name.lower().find(term) >= 0])
234 # Term begins with '!' so we remove matching tests
235 result = [test for test in result \
236 if test.name.lower().find(term[1:]) < 0]
241 def check_and_set_locale():
242 """ Function will check locale settings. In case, that it isn't configured
243 properly, then default values specified by DEFAULT_LOCALE will be used.
246 system_locale = locale.getdefaultlocale()
247 if None in system_locale:
248 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
249 logging.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
250 system_locale, locale.getdefaultlocale())
252 class MockTestCase(unittest.TestCase):
253 """Allow use of xmlrunner to generate Jenkins compatible output without
254 using xmlrunner to actually run tests.
257 suite = unittest.TestSuite()
258 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
259 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
260 xmlrunner.XMLTestRunner(...).run(suite)
263 def __init__(self, msg, is_pass, test_name):
266 self.is_pass = is_pass
268 #dynamically create a test method with the right name
269 #but point the method at our generic test method
270 setattr(MockTestCase, test_name, self.generic_test)
272 super(MockTestCase, self).__init__(test_name)
274 def generic_test(self):
275 """Provide a generic function that raises or not based
276 on how self.is_pass was set in the constructor"""
277 self.assertTrue(self.is_pass, self.msg)
283 args = parse_arguments()
287 settings.load_from_dir('conf')
289 # load command line parameters first in case there are settings files
291 settings.load_from_dict(args)
293 if args['conf_file']:
294 settings.load_from_file(args['conf_file'])
297 settings.load_from_env()
299 # reload command line parameters since these should take higher priority
300 # than both a settings file and environment variables
301 settings.load_from_dict(args)
303 # set dpdk and ovs paths accorfing to VNF and VSWITCH
304 if settings.getValue('VSWITCH').endswith('Vanilla'):
305 # settings paths for Vanilla
306 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
307 elif settings.getValue('VSWITCH').endswith('Vhost'):
308 if settings.getValue('VNF').endswith('Cuse'):
309 # settings paths for Cuse
310 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
311 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
313 # settings paths for VhostUser
314 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
315 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
317 # default - set to VHOST USER but can be changed during enhancement
318 settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
319 settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
321 configure_logging(settings.getValue('VERBOSITY'))
322 logger = logging.getLogger()
324 # check and fix locale
325 check_and_set_locale()
327 # configure trafficgens
329 if args['trafficgen']:
330 trafficgens = Loader().get_trafficgens()
331 if args['trafficgen'] not in trafficgens:
332 logging.error('There are no trafficgens matching \'%s\' found in'
333 ' \'%s\'. Exiting...', args['trafficgen'],
334 settings.getValue('TRAFFICGEN_DIR'))
339 vswitches = Loader().get_vswitches()
340 if args['vswitch'] not in vswitches:
341 logging.error('There are no vswitches matching \'%s\' found in'
342 ' \'%s\'. Exiting...', args['vswitch'],
343 settings.getValue('VSWITCH_DIR'))
347 vnfs = Loader().get_vnfs()
348 if args['vnf'] not in vnfs:
349 logging.error('there are no vnfs matching \'%s\' found in'
350 ' \'%s\'. exiting...', args['vnf'],
351 settings.getValue('vnf_dir'))
355 if args['duration'].isdigit() and int(args['duration']) > 0:
356 settings.setValue('duration', args['duration'])
358 logging.error('The selected Duration is not a number')
361 # generate results directory name
362 date = datetime.datetime.fromtimestamp(time.time())
363 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
364 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
367 testcases = settings.getValue('PERFORMANCE_TESTS')
369 for cfg in testcases:
371 all_tests.append(TestCase(cfg, results_path))
372 except (Exception) as _:
373 logger.exception("Failed to create test: %s",
374 cfg.get('Name', '<Name not set>'))
377 # if required, handle list-* operations
380 print("Available Tests:")
382 for test in all_tests:
383 print('* %-18s%s' % ('%s:' % test.name, test.desc))
386 if args['list_trafficgens']:
387 print(Loader().get_trafficgens_printable())
390 if args['list_collectors']:
391 print(Loader().get_collectors_printable())
394 if args['list_vswitches']:
395 print(Loader().get_vswitches_printable())
398 if args['list_vnfs']:
399 print(Loader().get_vnfs_printable())
402 if args['list_settings']:
406 # select requested tests
407 if args['exact_test_name'] and args['tests']:
408 logger.error("Cannot specify tests with both positional args and --test.")
411 if args['exact_test_name']:
412 exact_names = args['exact_test_name']
413 # positional args => exact matches only
414 selected_tests = [test for test in all_tests if test.name in exact_names]
416 # --tests => apply filter to select requested tests
417 selected_tests = apply_filter(all_tests, args['tests'])
419 # Default - run all tests
420 selected_tests = all_tests
422 if not selected_tests:
423 logger.error("No tests matched --test option or positional args. Done.")
426 # create results directory
427 if not os.path.exists(results_path):
428 logger.info("Creating result directory: " + results_path)
429 os.makedirs(results_path)
432 suite = unittest.TestSuite()
433 for test in selected_tests:
436 suite.addTest(MockTestCase('', True, test.name))
437 #pylint: disable=broad-except
438 except (Exception) as ex:
439 logger.exception("Failed to run test: %s", test.name)
440 suite.addTest(MockTestCase(str(ex), False, test.name))
441 logger.info("Continuing with next test...")
443 if settings.getValue('XUNIT'):
444 xmlrunner.XMLTestRunner(
445 output=settings.getValue('XUNIT_DIR'), outsuffix="",
446 verbosity=0).run(suite)
449 pod_name = args['opnfvpod']
450 installer_name = settings.getValue('OPNFV_INSTALLER')
452 int_data = {'cuse': False,
455 'installer': installer_name}
456 if settings.getValue('VSWITCH').endswith('Vanilla'):
457 int_data['vanilla'] = True
458 if settings.getValue('VNF').endswith('Cuse'):
459 int_data['cuse'] = True
460 opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
462 #remove directory if no result files were created.
463 if os.path.exists(results_path):
464 files_list = os.listdir(results_path)
466 shutil.rmtree(results_path)
468 if __name__ == "__main__":