3 # Copyright 2015 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
30 sys.dont_write_bytecode = True
32 from conf import settings
33 from core.loader import Loader
34 from testcases import TestCase
35 from tools.report import report
36 from tools import tasks
37 from tools.collectors import collector
38 from tools.pkt_gen import trafficgen
39 from vswitches import vswitch
43 'debug': logging.DEBUG,
45 'warning': logging.WARNING,
46 'error': logging.ERROR,
47 'critical': logging.CRITICAL
51 def parse_arguments():
53 Parse command line arguments.
55 class _SplitTestParamsAction(argparse.Action):
57 Parse and split the '--test-params' argument.
59 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
60 values. For multiple overrides use a ; separated list for
61 e.g. --test-params 'x=z; y=a,b'
63 def __call__(self, parser, namespace, values, option_string=None):
66 for value in values.split(';'):
67 result = [key.strip() for key in value.split('=')]
69 results[result[0]] = True
70 elif len(result) == 2:
71 results[result[0]] = result[1]
73 raise argparse.ArgumentTypeError(
74 'expected \'%s\' to be of format \'key=val\' or'
77 setattr(namespace, self.dest, results)
79 class _ValidateFileAction(argparse.Action):
80 """Validate a file can be read from before using it.
82 def __call__(self, parser, namespace, values, option_string=None):
83 if not os.path.isfile(values):
84 raise argparse.ArgumentTypeError(
85 'the path \'%s\' is not a valid path' % values)
86 elif not os.access(values, os.R_OK):
87 raise argparse.ArgumentTypeError(
88 'the path \'%s\' is not accessible' % values)
90 setattr(namespace, self.dest, values)
92 class _ValidateDirAction(argparse.Action):
93 """Validate a directory can be written to before using it.
95 def __call__(self, parser, namespace, values, option_string=None):
96 if not os.path.isdir(values):
97 raise argparse.ArgumentTypeError(
98 'the path \'%s\' is not a valid path' % values)
99 elif not os.access(values, os.W_OK):
100 raise argparse.ArgumentTypeError(
101 'the path \'%s\' is not accessible' % values)
103 setattr(namespace, self.dest, values)
105 def list_logging_levels():
106 """Give a summary of all available logging levels.
108 :return: List of verbosity level names in decreasing order of
111 return sorted(VERBOSITY_LEVELS.keys(),
112 key=lambda x: VERBOSITY_LEVELS[x])
114 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
115 argparse.ArgumentDefaultsHelpFormatter)
116 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
117 parser.add_argument('--list', '--list-tests', action='store_true',
118 help='list all tests and exit')
119 parser.add_argument('--list-trafficgens', action='store_true',
120 help='list all traffic generators and exit')
121 parser.add_argument('--list-collectors', action='store_true',
122 help='list all system metrics loggers and exit')
123 parser.add_argument('--list-vswitches', action='store_true',
124 help='list all system vswitches and exit')
125 parser.add_argument('--list-vnfs', action='store_true',
126 help='list all system vnfs and exit')
127 parser.add_argument('--list-settings', action='store_true',
128 help='list effective settings configuration and exit')
129 parser.add_argument('test', nargs='*', help='test specification(s)')
131 group = parser.add_argument_group('test selection options')
132 group.add_argument('-f', '--test-spec', help='test specification file')
133 group.add_argument('-d', '--test-dir', help='directory containing tests')
134 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
135 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
136 name contains RFC2544 less those containing "p2p"')
137 group.add_argument('--verbosity', choices=list_logging_levels(),
139 group.add_argument('--trafficgen', help='traffic generator to use')
140 group.add_argument('--vswitch', help='vswitch implementation to use')
141 group.add_argument('--vnf', help='vnf to use')
142 group.add_argument('--duration', help='traffic transmit duration')
143 group.add_argument('--sysmetrics', help='system metrics logger to use')
144 group = parser.add_argument_group('test behavior options')
145 group.add_argument('--xunit', action='store_true',
146 help='enable xUnit-formatted output')
147 group.add_argument('--xunit-dir', action=_ValidateDirAction,
148 help='output directory of xUnit-formatted output')
149 group.add_argument('--load-env', action='store_true',
150 help='enable loading of settings from the environment')
151 group.add_argument('--conf-file', action=_ValidateFileAction,
152 help='settings file')
153 group.add_argument('--test-params', action=_SplitTestParamsAction,
154 help='csv list of test parameters: key=val; e.g.'
155 'including pkt_sizes=x,y; duration=x; '
156 'rfc2544_trials=x ...')
158 args = vars(parser.parse_args())
163 def configure_logging(level):
164 """Configure logging.
166 log_file_default = os.path.join(
167 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
168 log_file_host_cmds = os.path.join(
169 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
170 log_file_traffic_gen = os.path.join(
171 settings.getValue('LOG_DIR'),
172 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
173 log_file_sys_metrics = os.path.join(
174 settings.getValue('LOG_DIR'),
175 settings.getValue('LOG_FILE_SYS_METRICS'))
177 logger = logging.getLogger()
178 logger.setLevel(logging.DEBUG)
180 stream_logger = logging.StreamHandler(sys.stdout)
181 stream_logger.setLevel(VERBOSITY_LEVELS[level])
182 stream_logger.setFormatter(logging.Formatter(
183 '[%(levelname)s] %(asctime)s : (%(name)s) - %(message)s'))
184 logger.addHandler(stream_logger)
186 file_logger = logging.FileHandler(filename=log_file_default)
187 file_logger.setLevel(logging.DEBUG)
188 logger.addHandler(file_logger)
190 class CommandFilter(logging.Filter):
191 """Filter out strings beginning with 'cmd :'"""
192 def filter(self, record):
193 return record.getMessage().startswith(tasks.CMD_PREFIX)
195 class TrafficGenCommandFilter(logging.Filter):
196 """Filter out strings beginning with 'gencmd :'"""
197 def filter(self, record):
198 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
200 class SystemMetricsCommandFilter(logging.Filter):
201 """Filter out strings beginning with 'gencmd :'"""
202 def filter(self, record):
203 return record.getMessage().startswith(collector.CMD_PREFIX)
205 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
206 cmd_logger.setLevel(logging.DEBUG)
207 cmd_logger.addFilter(CommandFilter())
208 logger.addHandler(cmd_logger)
210 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
211 gen_logger.setLevel(logging.DEBUG)
212 gen_logger.addFilter(TrafficGenCommandFilter())
213 logger.addHandler(gen_logger)
215 metrics_logger = logging.FileHandler(filename=log_file_sys_metrics)
216 metrics_logger.setLevel(logging.DEBUG)
217 metrics_logger.addFilter(SystemMetricsCommandFilter())
218 logger.addHandler(metrics_logger)
221 def apply_filter(tests, tc_filter):
222 """Allow a subset of tests to be conveniently selected
224 :param tests: The list of Tests from which to select.
225 :param tc_filter: A case-insensitive string of comma-separated terms
226 indicating the Tests to select.
227 e.g. 'RFC' - select all tests whose name contains 'RFC'
228 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
230 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
231 or 'burst' and from these remove any containing 'p2p'.
232 e.g. '' - empty string selects all tests.
233 :return: A list of the selected Tests.
236 if tc_filter is None:
239 for term in [x.strip() for x in tc_filter.lower().split(",")]:
240 if not term or term[0] != '!':
241 # Add matching tests from 'tests' into results
242 result.extend([test for test in tests \
243 if test.name.lower().find(term) >= 0])
245 # Term begins with '!' so we remove matching tests
246 result = [test for test in result \
247 if test.name.lower().find(term[1:]) < 0]
252 class MockTestCase(unittest.TestCase):
253 """Allow use of xmlrunner to generate Jenkins compatible output without
254 using xmlrunner to actually run tests.
257 suite = unittest.TestSuite()
258 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
259 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
260 xmlrunner.XMLTestRunner(...).run(suite)
263 def __init__(self, msg, is_pass, test_name):
266 self.is_pass = is_pass
268 #dynamically create a test method with the right name
269 #but point the method at our generic test method
270 setattr(MockTestCase, test_name, self.generic_test)
272 super(MockTestCase, self).__init__(test_name)
274 def generic_test(self):
275 """Provide a generic function that raises or not based
276 on how self.is_pass was set in the constructor"""
277 self.assertTrue(self.is_pass, self.msg)
283 args = parse_arguments()
287 settings.load_from_dir('conf')
289 # load command line parameters first in case there are settings files
291 settings.load_from_dict(args)
293 if args['conf_file']:
294 settings.load_from_file(args['conf_file'])
297 settings.load_from_env()
299 # reload command line parameters since these should take higher priority
300 # than both a settings file and environment variables
301 settings.load_from_dict(args)
303 configure_logging(settings.getValue('VERBOSITY'))
304 logger = logging.getLogger()
306 # configure trafficgens
308 if args['trafficgen']:
309 trafficgens = Loader().get_trafficgens()
310 if args['trafficgen'] not in trafficgens:
311 logging.error('There are no trafficgens matching \'%s\' found in'
312 ' \'%s\'. Exiting...', args['trafficgen'],
313 settings.getValue('TRAFFICGEN_DIR'))
318 vswitches = Loader().get_vswitches()
319 if args['vswitch'] not in vswitches:
320 logging.error('There are no vswitches matching \'%s\' found in'
321 ' \'%s\'. Exiting...', args['vswitch'],
322 settings.getValue('VSWITCH_DIR'))
326 vnfs = Loader().get_vnfs()
327 if args['vnf'] not in vnfs:
328 logging.error('there are no vnfs matching \'%s\' found in'
329 ' \'%s\'. exiting...', args['vnf'],
330 settings.getValue('vnf_dir'))
334 if args['duration'].isdigit() and int(args['duration']) > 0:
335 settings.setValue('duration', args['duration'])
337 logging.error('The selected Duration is not a number')
340 # generate results directory name
341 date = datetime.datetime.fromtimestamp(time.time())
342 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
343 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
346 testcases = settings.getValue('PERFORMANCE_TESTS')
348 for cfg in testcases:
350 all_tests.append(TestCase(cfg, results_path))
351 except (Exception) as _:
352 logger.exception("Failed to create test: %s",
353 cfg.get('Name', '<Name not set>'))
356 # TODO(BOM) Apply filter to select requested tests
357 all_tests = apply_filter(all_tests, args['tests'])
359 # if required, handle list-* operations
362 print("Available Tests:")
364 for test in all_tests:
365 print('* %-18s%s' % ('%s:' % test.name, test.desc))
368 if args['list_trafficgens']:
369 print(Loader().get_trafficgens_printable())
372 if args['list_collectors']:
373 print(Loader().get_collectors_printable())
376 if args['list_vswitches']:
377 print(Loader().get_vswitches_printable())
380 if args['list_vnfs']:
381 print(Loader().get_vnfs_printable())
384 if args['list_settings']:
388 # create results directory
389 if not os.path.exists(results_path):
390 logger.info("Creating result directory: " + results_path)
391 os.makedirs(results_path)
393 suite = unittest.TestSuite()
396 for test in all_tests:
399 suite.addTest(MockTestCase('', True, test.name))
400 #pylint: disable=broad-except
401 except (Exception) as ex:
402 logger.exception("Failed to run test: %s", test.name)
403 suite.addTest(MockTestCase(str(ex), False, test.name))
404 logger.info("Continuing with next test...")
406 if settings.getValue('XUNIT'):
407 xmlrunner.XMLTestRunner(
408 output=settings.getValue('XUNIT_DIR'), outsuffix="",
409 verbosity=0).run(suite)
411 #remove directory if no result files were created.
412 if os.path.exists(results_path):
413 files_list = os.listdir(results_path)
415 shutil.rmtree(results_path)
417 for file in files_list:
418 # generate report from all csv files
419 if file[-3:] == 'csv':
420 results_csv = os.path.join(results_path, file)
421 if os.path.isfile(results_csv) and os.access(results_csv, os.R_OK):
422 report.generate(testcases, results_csv)
424 if __name__ == "__main__":