3 # Copyright 2015 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
30 sys.dont_write_bytecode = True
32 from conf import settings
33 from core.loader import Loader
34 from testcases import TestCase
35 from tools.report import report
36 from tools import tasks
37 from tools.collectors import collector
38 from tools.pkt_gen import trafficgen
39 from vswitches import vswitch
43 'debug': logging.DEBUG,
45 'warning': logging.WARNING,
46 'error': logging.ERROR,
47 'critical': logging.CRITICAL
51 def parse_arguments():
53 Parse command line arguments.
55 class _SplitTestParamsAction(argparse.Action):
57 Parse and split the '--test-params' argument.
59 This expects either 'x=y' or 'x' (implicit true) values.
61 def __call__(self, parser, namespace, values, option_string=None):
64 for value in values.split(';'):
65 result = [key.strip() for key in value.split('=')]
67 results[result[0]] = True
68 elif len(result) == 2:
69 results[result[0]] = result[1]
71 raise argparse.ArgumentTypeError(
72 'expected \'%s\' to be of format \'key=val\' or'
75 setattr(namespace, self.dest, results)
77 class _ValidateFileAction(argparse.Action):
78 """Validate a file can be read from before using it.
80 def __call__(self, parser, namespace, values, option_string=None):
81 if not os.path.isfile(values):
82 raise argparse.ArgumentTypeError(
83 'the path \'%s\' is not a valid path' % values)
84 elif not os.access(values, os.R_OK):
85 raise argparse.ArgumentTypeError(
86 'the path \'%s\' is not accessible' % values)
88 setattr(namespace, self.dest, values)
90 class _ValidateDirAction(argparse.Action):
91 """Validate a directory can be written to before using it.
93 def __call__(self, parser, namespace, values, option_string=None):
94 if not os.path.isdir(values):
95 raise argparse.ArgumentTypeError(
96 'the path \'%s\' is not a valid path' % values)
97 elif not os.access(values, os.W_OK):
98 raise argparse.ArgumentTypeError(
99 'the path \'%s\' is not accessible' % values)
101 setattr(namespace, self.dest, values)
103 def list_logging_levels():
104 """Give a summary of all available logging levels.
106 :return: List of verbosity level names in decreasing order of
109 return sorted(VERBOSITY_LEVELS.keys(),
110 key=lambda x: VERBOSITY_LEVELS[x])
112 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
113 argparse.ArgumentDefaultsHelpFormatter)
114 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
115 parser.add_argument('--list', '--list-tests', action='store_true',
116 help='list all tests and exit')
117 parser.add_argument('--list-trafficgens', action='store_true',
118 help='list all traffic generators and exit')
119 parser.add_argument('--list-collectors', action='store_true',
120 help='list all system metrics loggers and exit')
121 parser.add_argument('--list-vswitches', action='store_true',
122 help='list all system vswitches and exit')
123 parser.add_argument('--list-vnfs', action='store_true',
124 help='list all system vnfs and exit')
125 parser.add_argument('--list-settings', action='store_true',
126 help='list effective settings configuration and exit')
127 parser.add_argument('test', nargs='*', help='test specification(s)')
129 group = parser.add_argument_group('test selection options')
130 group.add_argument('-f', '--test-spec', help='test specification file')
131 group.add_argument('-d', '--test-dir', help='directory containing tests')
132 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
133 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
134 name contains RFC2544 less those containing "p2p"')
135 group.add_argument('--verbosity', choices=list_logging_levels(),
137 group.add_argument('--trafficgen', help='traffic generator to use')
138 group.add_argument('--vswitch', help='vswitch implementation to use')
139 group.add_argument('--vnf', help='vnf to use')
140 group.add_argument('--sysmetrics', help='system metrics logger to use')
141 group = parser.add_argument_group('test behavior options')
142 group.add_argument('--xunit', action='store_true',
143 help='enable xUnit-formatted output')
144 group.add_argument('--xunit-dir', action=_ValidateDirAction,
145 help='output directory of xUnit-formatted output')
146 group.add_argument('--load-env', action='store_true',
147 help='enable loading of settings from the environment')
148 group.add_argument('--conf-file', action=_ValidateFileAction,
149 help='settings file')
150 group.add_argument('--test-params', action=_SplitTestParamsAction,
151 help='csv list of test parameters: key=val;...')
153 args = vars(parser.parse_args())
158 def configure_logging(level):
159 """Configure logging.
161 log_file_default = os.path.join(
162 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
163 log_file_host_cmds = os.path.join(
164 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
165 log_file_traffic_gen = os.path.join(
166 settings.getValue('LOG_DIR'),
167 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
168 log_file_sys_metrics = os.path.join(
169 settings.getValue('LOG_DIR'),
170 settings.getValue('LOG_FILE_SYS_METRICS'))
172 logger = logging.getLogger()
173 logger.setLevel(logging.DEBUG)
175 stream_logger = logging.StreamHandler(sys.stdout)
176 stream_logger.setLevel(VERBOSITY_LEVELS[level])
177 stream_logger.setFormatter(logging.Formatter(
178 '[%(levelname)s] %(asctime)s : (%(name)s) - %(message)s'))
179 logger.addHandler(stream_logger)
181 file_logger = logging.FileHandler(filename=log_file_default)
182 file_logger.setLevel(logging.DEBUG)
183 logger.addHandler(file_logger)
185 class CommandFilter(logging.Filter):
186 """Filter out strings beginning with 'cmd :'"""
187 def filter(self, record):
188 return record.getMessage().startswith(tasks.CMD_PREFIX)
190 class TrafficGenCommandFilter(logging.Filter):
191 """Filter out strings beginning with 'gencmd :'"""
192 def filter(self, record):
193 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
195 class SystemMetricsCommandFilter(logging.Filter):
196 """Filter out strings beginning with 'gencmd :'"""
197 def filter(self, record):
198 return record.getMessage().startswith(collector.CMD_PREFIX)
200 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
201 cmd_logger.setLevel(logging.DEBUG)
202 cmd_logger.addFilter(CommandFilter())
203 logger.addHandler(cmd_logger)
205 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
206 gen_logger.setLevel(logging.DEBUG)
207 gen_logger.addFilter(TrafficGenCommandFilter())
208 logger.addHandler(gen_logger)
210 metrics_logger = logging.FileHandler(filename=log_file_sys_metrics)
211 metrics_logger.setLevel(logging.DEBUG)
212 metrics_logger.addFilter(SystemMetricsCommandFilter())
213 logger.addHandler(metrics_logger)
216 def apply_filter(tests, tc_filter):
217 """Allow a subset of tests to be conveniently selected
219 :param tests: The list of Tests from which to select.
220 :param tc_filter: A case-insensitive string of comma-separated terms
221 indicating the Tests to select.
222 e.g. 'RFC' - select all tests whose name contains 'RFC'
223 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
225 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
226 or 'burst' and from these remove any containing 'p2p'.
227 e.g. '' - empty string selects all tests.
228 :return: A list of the selected Tests.
231 if tc_filter is None:
234 for term in [x.strip() for x in tc_filter.lower().split(",")]:
235 if not term or term[0] != '!':
236 # Add matching tests from 'tests' into results
237 result.extend([test for test in tests \
238 if test.name.lower().find(term) >= 0])
240 # Term begins with '!' so we remove matching tests
241 result = [test for test in result \
242 if test.name.lower().find(term[1:]) < 0]
247 class MockTestCase(unittest.TestCase):
248 """Allow use of xmlrunner to generate Jenkins compatible output without
249 using xmlrunner to actually run tests.
252 suite = unittest.TestSuite()
253 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
254 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
255 xmlrunner.XMLTestRunner(...).run(suite)
258 def __init__(self, msg, is_pass, test_name):
261 self.is_pass = is_pass
263 #dynamically create a test method with the right name
264 #but point the method at our generic test method
265 setattr(MockTestCase, test_name, self.generic_test)
267 super(MockTestCase, self).__init__(test_name)
269 def generic_test(self):
270 """Provide a generic function that raises or not based
271 on how self.is_pass was set in the constructor"""
272 self.assertTrue(self.is_pass, self.msg)
278 args = parse_arguments()
282 settings.load_from_dir('conf')
284 # load command line parameters first in case there are settings files
286 settings.load_from_dict(args)
288 if args['conf_file']:
289 settings.load_from_file(args['conf_file'])
292 settings.load_from_env()
294 # reload command line parameters since these should take higher priority
295 # than both a settings file and environment variables
296 settings.load_from_dict(args)
298 configure_logging(settings.getValue('VERBOSITY'))
299 logger = logging.getLogger()
301 # configure trafficgens
303 if args['trafficgen']:
304 trafficgens = Loader().get_trafficgens()
305 if args['trafficgen'] not in trafficgens:
306 logging.error('There are no trafficgens matching \'%s\' found in'
307 ' \'%s\'. Exiting...', args['trafficgen'],
308 settings.getValue('TRAFFICGEN_DIR'))
313 vswitches = Loader().get_vswitches()
314 if args['vswitch'] not in vswitches:
315 logging.error('There are no vswitches matching \'%s\' found in'
316 ' \'%s\'. Exiting...', args['vswitch'],
317 settings.getValue('VSWITCH_DIR'))
321 vnfs = Loader().get_vnfs()
322 if args['vnf'] not in vnfs:
323 logging.error('there are no vnfs matching \'%s\' found in'
324 ' \'%s\'. exiting...', args['vnf'],
325 settings.getValue('vnf_dir'))
328 # generate results directory name
329 date = datetime.datetime.fromtimestamp(time.time())
330 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
331 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
334 testcases = settings.getValue('PERFORMANCE_TESTS')
336 for cfg in testcases:
338 all_tests.append(TestCase(cfg, results_path))
339 except (Exception) as _:
340 logger.exception("Failed to create test: %s",
341 cfg.get('Name', '<Name not set>'))
344 # TODO(BOM) Apply filter to select requested tests
345 all_tests = apply_filter(all_tests, args['tests'])
347 # if required, handle list-* operations
350 print("Available Tests:")
352 for test in all_tests:
353 print('* %-18s%s' % ('%s:' % test.name, test.desc))
356 if args['list_trafficgens']:
357 print(Loader().get_trafficgens_printable())
360 if args['list_collectors']:
361 print(Loader().get_collectors_printable())
364 if args['list_vswitches']:
365 print(Loader().get_vswitches_printable())
368 if args['list_vnfs']:
369 print(Loader().get_vnfs_printable())
372 if args['list_settings']:
376 # create results directory
377 if not os.path.exists(results_path):
378 logger.info("Creating result directory: " + results_path)
379 os.makedirs(results_path)
381 suite = unittest.TestSuite()
384 for test in all_tests:
387 suite.addTest(MockTestCase('', True, test.name))
388 #pylint: disable=broad-except
389 except (Exception) as ex:
390 logger.exception("Failed to run test: %s", test.name)
391 suite.addTest(MockTestCase(str(ex), False, test.name))
392 logger.info("Continuing with next test...")
394 if settings.getValue('XUNIT'):
395 xmlrunner.XMLTestRunner(
396 output=settings.getValue('XUNIT_DIR'), outsuffix="",
397 verbosity=0).run(suite)
399 #remove directory if no result files were created.
400 if os.path.exists(results_path):
401 files_list = os.listdir(results_path)
403 shutil.rmtree(results_path)
405 for file in files_list:
406 # generate report from all csv files
407 if file[-3:] == 'csv':
408 results_csv = os.path.join(results_path, file)
409 if os.path.isfile(results_csv) and os.access(results_csv, os.R_OK):
410 report.generate(testcases, results_csv)
412 if __name__ == "__main__":