3 # Copyright 2015 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
30 sys.dont_write_bytecode = True
32 from conf import settings
33 from core.loader import Loader
34 from testcases import TestCase
35 from tools import tasks
36 from tools.collectors import collector
37 from tools.pkt_gen import trafficgen
38 from vswitches import vswitch
42 'debug': logging.DEBUG,
44 'warning': logging.WARNING,
45 'error': logging.ERROR,
46 'critical': logging.CRITICAL
50 def parse_arguments():
52 Parse command line arguments.
54 class _SplitTestParamsAction(argparse.Action):
56 Parse and split the '--test-params' argument.
58 This expects either 'x=y' or 'x' (implicit true) values.
60 def __call__(self, parser, namespace, values, option_string=None):
63 for value in values.split(';'):
64 result = [key.strip() for key in value.split('=')]
66 results[result[0]] = True
67 elif len(result) == 2:
68 results[result[0]] = result[1]
70 raise argparse.ArgumentTypeError(
71 'expected \'%s\' to be of format \'key=val\' or'
74 setattr(namespace, self.dest, results)
76 class _ValidateFileAction(argparse.Action):
77 """Validate a file can be read from before using it.
79 def __call__(self, parser, namespace, values, option_string=None):
80 if not os.path.isfile(values):
81 raise argparse.ArgumentTypeError(
82 'the path \'%s\' is not a valid path' % values)
83 elif not os.access(values, os.R_OK):
84 raise argparse.ArgumentTypeError(
85 'the path \'%s\' is not accessible' % values)
87 setattr(namespace, self.dest, values)
89 class _ValidateDirAction(argparse.Action):
90 """Validate a directory can be written to before using it.
92 def __call__(self, parser, namespace, values, option_string=None):
93 if not os.path.isdir(values):
94 raise argparse.ArgumentTypeError(
95 'the path \'%s\' is not a valid path' % values)
96 elif not os.access(values, os.W_OK):
97 raise argparse.ArgumentTypeError(
98 'the path \'%s\' is not accessible' % values)
100 setattr(namespace, self.dest, values)
102 def list_logging_levels():
103 """Give a summary of all available logging levels.
105 :return: List of verbosity level names in decreasing order of
108 return sorted(VERBOSITY_LEVELS.keys(),
109 key=lambda x: VERBOSITY_LEVELS[x])
111 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
112 argparse.ArgumentDefaultsHelpFormatter)
113 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
114 parser.add_argument('--list', '--list-tests', action='store_true',
115 help='list all tests and exit')
116 parser.add_argument('--list-trafficgens', action='store_true',
117 help='list all traffic generators and exit')
118 parser.add_argument('--list-collectors', action='store_true',
119 help='list all system metrics loggers and exit')
120 parser.add_argument('--list-vswitches', action='store_true',
121 help='list all system vswitches and exit')
122 parser.add_argument('--list-vnfs', action='store_true',
123 help='list all system vnfs and exit')
124 parser.add_argument('--list-settings', action='store_true',
125 help='list effective settings configuration and exit')
126 parser.add_argument('test', nargs='*', help='test specification(s)')
128 group = parser.add_argument_group('test selection options')
129 group.add_argument('-f', '--test-spec', help='test specification file')
130 group.add_argument('-d', '--test-dir', help='directory containing tests')
131 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
132 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
133 name contains RFC2544 less those containing "p2p"')
134 group.add_argument('--verbosity', choices=list_logging_levels(),
136 group.add_argument('--trafficgen', help='traffic generator to use')
137 group.add_argument('--vswitch', help='vswitch implementation to use')
138 group.add_argument('--vnf', help='vnf to use')
139 group.add_argument('--sysmetrics', help='system metrics logger to use')
140 group = parser.add_argument_group('test behavior options')
141 group.add_argument('--xunit', action='store_true',
142 help='enable xUnit-formatted output')
143 group.add_argument('--xunit-dir', action=_ValidateDirAction,
144 help='output directory of xUnit-formatted output')
145 group.add_argument('--load-env', action='store_true',
146 help='enable loading of settings from the environment')
147 group.add_argument('--conf-file', action=_ValidateFileAction,
148 help='settings file')
149 group.add_argument('--test-params', action=_SplitTestParamsAction,
150 help='csv list of test parameters: key=val;...')
152 args = vars(parser.parse_args())
157 def configure_logging(level):
158 """Configure logging.
160 log_file_default = os.path.join(
161 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
162 log_file_host_cmds = os.path.join(
163 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
164 log_file_traffic_gen = os.path.join(
165 settings.getValue('LOG_DIR'),
166 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
167 log_file_sys_metrics = os.path.join(
168 settings.getValue('LOG_DIR'),
169 settings.getValue('LOG_FILE_SYS_METRICS'))
171 logger = logging.getLogger()
172 logger.setLevel(logging.DEBUG)
174 stream_logger = logging.StreamHandler(sys.stdout)
175 stream_logger.setLevel(VERBOSITY_LEVELS[level])
176 stream_logger.setFormatter(logging.Formatter(
177 '[%(levelname)s] %(asctime)s : (%(name)s) - %(message)s'))
178 logger.addHandler(stream_logger)
180 file_logger = logging.FileHandler(filename=log_file_default)
181 file_logger.setLevel(logging.DEBUG)
182 logger.addHandler(file_logger)
184 class CommandFilter(logging.Filter):
185 """Filter out strings beginning with 'cmd :'"""
186 def filter(self, record):
187 return record.getMessage().startswith(tasks.CMD_PREFIX)
189 class TrafficGenCommandFilter(logging.Filter):
190 """Filter out strings beginning with 'gencmd :'"""
191 def filter(self, record):
192 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
194 class SystemMetricsCommandFilter(logging.Filter):
195 """Filter out strings beginning with 'gencmd :'"""
196 def filter(self, record):
197 return record.getMessage().startswith(collector.CMD_PREFIX)
199 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
200 cmd_logger.setLevel(logging.DEBUG)
201 cmd_logger.addFilter(CommandFilter())
202 logger.addHandler(cmd_logger)
204 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
205 gen_logger.setLevel(logging.DEBUG)
206 gen_logger.addFilter(TrafficGenCommandFilter())
207 logger.addHandler(gen_logger)
209 metrics_logger = logging.FileHandler(filename=log_file_sys_metrics)
210 metrics_logger.setLevel(logging.DEBUG)
211 metrics_logger.addFilter(SystemMetricsCommandFilter())
212 logger.addHandler(metrics_logger)
215 def apply_filter(tests, tc_filter):
216 """Allow a subset of tests to be conveniently selected
218 :param tests: The list of Tests from which to select.
219 :param tc_filter: A case-insensitive string of comma-separated terms
220 indicating the Tests to select.
221 e.g. 'RFC' - select all tests whose name contains 'RFC'
222 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
224 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
225 or 'burst' and from these remove any containing 'p2p'.
226 e.g. '' - empty string selects all tests.
227 :return: A list of the selected Tests.
230 if tc_filter is None:
233 for term in [x.strip() for x in tc_filter.lower().split(",")]:
234 if not term or term[0] != '!':
235 # Add matching tests from 'tests' into results
236 result.extend([test for test in tests \
237 if test.name.lower().find(term) >= 0])
239 # Term begins with '!' so we remove matching tests
240 result = [test for test in result \
241 if test.name.lower().find(term[1:]) < 0]
246 class MockTestCase(unittest.TestCase):
247 """Allow use of xmlrunner to generate Jenkins compatible output without
248 using xmlrunner to actually run tests.
251 suite = unittest.TestSuite()
252 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
253 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
254 xmlrunner.XMLTestRunner(...).run(suite)
257 def __init__(self, msg, is_pass, test_name):
260 self.is_pass = is_pass
262 #dynamically create a test method with the right name
263 #but point the method at our generic test method
264 setattr(MockTestCase, test_name, self.generic_test)
266 super(MockTestCase, self).__init__(test_name)
268 def generic_test(self):
269 """Provide a generic function that raises or not based
270 on how self.is_pass was set in the constructor"""
271 self.assertTrue(self.is_pass, self.msg)
277 args = parse_arguments()
281 settings.load_from_dir('conf')
283 # load command line parameters first in case there are settings files
285 settings.load_from_dict(args)
287 if args['conf_file']:
288 settings.load_from_file(args['conf_file'])
291 settings.load_from_env()
293 # reload command line parameters since these should take higher priority
294 # than both a settings file and environment variables
295 settings.load_from_dict(args)
297 configure_logging(settings.getValue('VERBOSITY'))
298 logger = logging.getLogger()
300 # configure trafficgens
302 if args['trafficgen']:
303 trafficgens = Loader().get_trafficgens()
304 if args['trafficgen'] not in trafficgens:
305 logging.error('There are no trafficgens matching \'%s\' found in'
306 ' \'%s\'. Exiting...', args['trafficgen'],
307 settings.getValue('TRAFFICGEN_DIR'))
312 vswitches = Loader().get_vswitches()
313 if args['vswitch'] not in vswitches:
314 logging.error('There are no vswitches matching \'%s\' found in'
315 ' \'%s\'. Exiting...', args['vswitch'],
316 settings.getValue('VSWITCH_DIR'))
320 vnfs = Loader().get_vnfs()
321 if args['vnf'] not in vnfs:
322 logging.error('there are no vnfs matching \'%s\' found in'
323 ' \'%s\'. exiting...', args['vnf'],
324 settings.getValue('vnf_dir'))
327 # generate results directory name
328 date = datetime.datetime.fromtimestamp(time.time())
329 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
330 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
333 testcases = settings.getValue('PERFORMANCE_TESTS')
335 for cfg in testcases:
337 all_tests.append(TestCase(cfg, results_path))
338 except (Exception) as _:
339 logger.exception("Failed to create test: %s",
340 cfg.get('Name', '<Name not set>'))
343 # TODO(BOM) Apply filter to select requested tests
344 all_tests = apply_filter(all_tests, args['tests'])
346 # if required, handle list-* operations
349 print("Available Tests:")
351 for test in all_tests:
352 print('* %-18s%s' % ('%s:' % test.name, test.desc))
355 if args['list_trafficgens']:
356 print(Loader().get_trafficgens_printable())
359 if args['list_collectors']:
360 print(Loader().get_collectors_printable())
363 if args['list_vswitches']:
364 print(Loader().get_vswitches_printable())
367 if args['list_vnfs']:
368 print(Loader().get_vnfs_printable())
371 if args['list_settings']:
375 # create results directory
376 if not os.path.exists(results_dir):
377 logger.info("Creating result directory: " + results_path)
378 os.makedirs(results_path)
380 suite = unittest.TestSuite()
383 for test in all_tests:
386 suite.addTest(MockTestCase('', True, test.name))
387 #pylint: disable=broad-except
388 except (Exception) as ex:
389 logger.exception("Failed to run test: %s", test.name)
390 suite.addTest(MockTestCase(str(ex), False, test.name))
391 logger.info("Continuing with next test...")
393 if settings.getValue('XUNIT'):
394 xmlrunner.XMLTestRunner(
395 output=settings.getValue('XUNIT_DIR'), outsuffix="",
396 verbosity=0).run(suite)
398 #remove directory if no result files were created.
399 if os.path.exists(results_path):
400 if os.listdir(results_path) == []:
401 shutil.rmtree(results_path)
403 if __name__ == "__main__":