3 # Copyright 2015-2017 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
36 from tabulate import tabulate
37 from conf import merge_spec
38 from conf import settings
39 import core.component_factory as component_factory
40 from core.loader import Loader
41 from testcases import PerformanceTestCase
42 from testcases import IntegrationTestCase
43 from tools import tasks
44 from tools import networkcard
45 from tools import functions
46 from tools.pkt_gen import trafficgen
47 from tools.opnfvdashboard import opnfvdashboard
48 sys.dont_write_bytecode = True
51 'debug': logging.DEBUG,
53 'warning': logging.WARNING,
54 'error': logging.ERROR,
55 'critical': logging.CRITICAL
58 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
60 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
61 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
62 'final' : 'test_report.rst',
63 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
66 _TEMPLATE_MATRIX = "Performance Matrix\n------------------\n\n"\
67 "The following performance matrix was generated with the results of all the\n"\
68 "currently run tests. The metric used for comparison is {}.\n\n{}\n\n"
70 _LOGGER = logging.getLogger()
72 def parse_param_string(values):
74 Parse and split a single '--test-params' argument.
76 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
77 values. For multiple overrides use a ; separated list for
78 e.g. --test-params 'x=z; y=(a,b)'
85 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
90 # values are passed inside string from CLI, so we must retype them accordingly
92 results[param] = ast.literal_eval(value)
94 # for backward compatibility, we have to accept strings without quotes
95 _LOGGER.warning("Adding missing quotes around string value: %s = %s",
97 results[param] = str(value)
103 def parse_arguments():
105 Parse command line arguments.
107 class _SplitTestParamsAction(argparse.Action):
109 Parse and split '--test-params' arguments.
111 This expects either a single list of ; separated overrides
112 as 'x=y', 'x=y,z' or 'x' (implicit true) values.
113 e.g. --test-params 'x=z; y=(a,b)'
114 Or a list of these ; separated lists with overrides for
116 e.g. --test-params "['x=z; y=(a,b)','x=z']"
118 def __call__(self, parser, namespace, values, option_string=None):
120 input_list = ast.literal_eval(values)
122 for test_params in input_list:
123 parameter_list.append(parse_param_string(test_params))
125 parameter_list = parse_param_string(values)
126 results = {'_PARAMS_LIST':parameter_list}
127 setattr(namespace, self.dest, results)
129 class _ValidateFileAction(argparse.Action):
130 """Validate a file can be read from before using it.
132 def __call__(self, parser, namespace, values, option_string=None):
133 if not os.path.isfile(values):
134 raise argparse.ArgumentTypeError(
135 'the path \'%s\' is not a valid path' % values)
136 elif not os.access(values, os.R_OK):
137 raise argparse.ArgumentTypeError(
138 'the path \'%s\' is not accessible' % values)
140 setattr(namespace, self.dest, values)
142 class _ValidateDirAction(argparse.Action):
143 """Validate a directory can be written to before using it.
145 def __call__(self, parser, namespace, values, option_string=None):
146 if not os.path.isdir(values):
147 raise argparse.ArgumentTypeError(
148 'the path \'%s\' is not a valid path' % values)
149 elif not os.access(values, os.W_OK):
150 raise argparse.ArgumentTypeError(
151 'the path \'%s\' is not accessible' % values)
153 setattr(namespace, self.dest, values)
155 def list_logging_levels():
156 """Give a summary of all available logging levels.
158 :return: List of verbosity level names in decreasing order of
161 return sorted(VERBOSITY_LEVELS.keys(),
162 key=lambda x: VERBOSITY_LEVELS[x])
164 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
165 argparse.ArgumentDefaultsHelpFormatter)
166 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
167 parser.add_argument('--list', '--list-tests', action='store_true',
168 help='list all tests and exit')
169 parser.add_argument('--list-trafficgens', action='store_true',
170 help='list all traffic generators and exit')
171 parser.add_argument('--list-collectors', action='store_true',
172 help='list all system metrics loggers and exit')
173 parser.add_argument('--list-vswitches', action='store_true',
174 help='list all system vswitches and exit')
175 parser.add_argument('--list-fwdapps', action='store_true',
176 help='list all system forwarding applications and exit')
177 parser.add_argument('--list-vnfs', action='store_true',
178 help='list all system vnfs and exit')
179 parser.add_argument('--list-loadgens', action='store_true',
180 help='list all background load generators')
181 parser.add_argument('--list-settings', action='store_true',
182 help='list effective settings configuration and exit')
183 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
184 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
185 runs only the two tests with those exact names.\
186 To run all tests omit both positional args and --tests arg.')
188 group = parser.add_argument_group('test selection options')
189 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
190 Values: "normal" - execute vSwitch, VNF and traffic generator;\
191 "trafficgen" - execute only traffic generator; "trafficgen-off" \
192 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
193 and VNF but pause before traffic transmission ', default='normal')
195 group.add_argument('-f', '--test-spec', help='test specification file')
196 group.add_argument('-d', '--test-dir', help='directory containing tests')
197 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
198 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
199 name contains RFC2544 less those containing "p2p"; "!back2back" - \
200 run all tests except those containing back2back')
201 group.add_argument('--verbosity', choices=list_logging_levels(),
203 group.add_argument('--integration', action='store_true', help='execute integration tests')
204 group.add_argument('--trafficgen', help='traffic generator to use')
205 group.add_argument('--vswitch', help='vswitch implementation to use')
206 group.add_argument('--fwdapp', help='packet forwarding application to use')
207 group.add_argument('--vnf', help='vnf to use')
208 group.add_argument('--loadgen', help='loadgen to use')
209 group.add_argument('--sysmetrics', help='system metrics logger to use')
210 group = parser.add_argument_group('test behavior options')
211 group.add_argument('--xunit', action='store_true',
212 help='enable xUnit-formatted output')
213 group.add_argument('--xunit-dir', action=_ValidateDirAction,
214 help='output directory of xUnit-formatted output')
215 group.add_argument('--load-env', action='store_true',
216 help='enable loading of settings from the environment')
217 group.add_argument('--conf-file', action=_ValidateFileAction,
218 help='settings file')
219 group.add_argument('--test-params', action=_SplitTestParamsAction,
220 help='csv list of test parameters: key=val; e.g. '
221 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFFICGEN_DURATION=30; '
222 'GUEST_LOOPBACK=["l2fwd"] ...'
223 ' or a list of csv lists of test parameters: key=val; e.g. '
224 '[\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(128,)\','
225 '\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(64,)\']')
226 group.add_argument('--opnfvpod', help='name of POD in opnfv')
227 group.add_argument('--matrix', help='enable performance matrix analysis',
228 action='store_true', default=False)
230 args = vars(parser.parse_args())
235 def configure_logging(level):
236 """Configure logging.
238 date = datetime.datetime.fromtimestamp(time.time())
239 timestamp = date.strftime('%Y-%m-%d_%H-%M-%S')
240 settings.setValue('LOG_TIMESTAMP', timestamp)
241 name, ext = os.path.splitext(settings.getValue('LOG_FILE_DEFAULT'))
242 rename_default = "{name}_{uid}{ex}".format(name=name, uid=timestamp, ex=ext)
243 log_file_default = os.path.join(
244 settings.getValue('LOG_DIR'), rename_default)
245 log_file_host_cmds = os.path.join(
246 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
247 log_file_traffic_gen = os.path.join(
248 settings.getValue('LOG_DIR'),
249 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
250 metrics_file = (settings.getValue('LOG_FILE_INFRA_METRICS_PFX') +
252 log_file_infra_metrics = os.path.join(settings.getValue('LOG_DIR'),
255 _LOGGER.setLevel(logging.DEBUG)
257 stream_logger = logging.StreamHandler(sys.stdout)
258 stream_logger.setLevel(VERBOSITY_LEVELS[level])
259 stream_logger.setFormatter(logging.Formatter(
260 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
261 _LOGGER.addHandler(stream_logger)
263 file_logger = logging.FileHandler(filename=log_file_default)
264 file_logger.setLevel(logging.DEBUG)
265 file_logger.setFormatter(logging.Formatter(
266 '%(asctime)s : %(message)s'))
267 _LOGGER.addHandler(file_logger)
269 class CommandFilter(logging.Filter):
270 """Filter out strings beginning with 'cmd :'"""
271 def filter(self, record):
272 return record.getMessage().startswith(tasks.CMD_PREFIX)
274 class TrafficGenCommandFilter(logging.Filter):
275 """Filter out strings beginning with 'gencmd :'"""
276 def filter(self, record):
277 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
279 class CollectdMetricsFilter(logging.Filter):
280 """Filter out strings beginning with 'COLLECTD' :'"""
281 def filter(self, record):
282 return record.getMessage().startswith('COLLECTD')
284 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
285 cmd_logger.setLevel(logging.DEBUG)
286 cmd_logger.addFilter(CommandFilter())
287 _LOGGER.addHandler(cmd_logger)
289 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
290 gen_logger.setLevel(logging.DEBUG)
291 gen_logger.addFilter(TrafficGenCommandFilter())
292 _LOGGER.addHandler(gen_logger)
294 if settings.getValue('COLLECTOR') == 'Collectd':
295 met_logger = logging.FileHandler(filename=log_file_infra_metrics)
296 met_logger.setLevel(logging.DEBUG)
297 met_logger.addFilter(CollectdMetricsFilter())
298 _LOGGER.addHandler(met_logger)
301 def apply_filter(tests, tc_filter):
302 """Allow a subset of tests to be conveniently selected
304 :param tests: The list of Tests from which to select.
305 :param tc_filter: A case-insensitive string of comma-separated terms
306 indicating the Tests to select.
307 e.g. 'RFC' - select all tests whose name contains 'RFC'
308 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
310 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
311 or 'burst' and from these remove any containing 'p2p'.
312 e.g. '' - empty string selects all tests.
313 :return: A list of the selected Tests.
315 # if negative filter is first we have to start with full list of tests
316 if tc_filter.strip()[0] == '!':
320 if tc_filter is None:
323 for term in [x.strip() for x in tc_filter.lower().split(",")]:
324 if not term or term[0] != '!':
325 # Add matching tests from 'tests' into results
326 result.extend([test for test in tests \
327 if test['Name'].lower().find(term) >= 0])
329 # Term begins with '!' so we remove matching tests
330 result = [test for test in result \
331 if test['Name'].lower().find(term[1:]) < 0]
336 def check_and_set_locale():
337 """ Function will check locale settings. In case, that it isn't configured
338 properly, then default values specified by DEFAULT_LOCALE will be used.
341 system_locale = locale.getdefaultlocale()
342 if None in system_locale:
343 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
344 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
345 system_locale, locale.getdefaultlocale())
347 def get_vswitch_names(rst_files):
348 """ Function will return a list of vSwitches detected in given ``rst_files``.
350 vswitch_names = set()
353 output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
355 match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
357 vswitch_names.add(match.group(1))
360 return list(vswitch_names)
362 except subprocess.CalledProcessError:
363 _LOGGER.warning('Cannot detect vSwitches used during testing.')
365 # fallback to the default value
369 """ Function will return a Jenkins job ID environment variable.
373 build_tag = os.environ['BUILD_TAG']
376 _LOGGER.warning('Cannot detect Jenkins job ID')
381 def generate_final_report():
382 """ Function will check if partial test results are available
383 and generates final report in rst format.
386 path = settings.getValue('RESULTS_PATH')
387 # check if there are any results in rst format
388 rst_results = glob.glob(os.path.join(path, 'result*rst'))
389 pkt_processors = get_vswitch_names(rst_results)
392 test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
393 # create report caption directly - it is not worth to execute jinja machinery
394 report_caption = '{}\n{} {}\n{}\n\n'.format(
395 '============================================================',
396 'Performance report for',
397 ', '.join(pkt_processors),
398 '============================================================')
400 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
401 file_.write(report_caption)
403 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
404 ' '.join(rst_results), _TEMPLATE_RST['foot'],
405 test_report), shell=True)
406 if retval == 0 and os.path.isfile(test_report):
407 _LOGGER.info('Overall test report written to "%s"', test_report)
409 _LOGGER.error('Generation of overall test report has failed.')
411 # remove temporary file
412 os.remove(_TEMPLATE_RST['tmp'])
414 except subprocess.CalledProcessError:
415 _LOGGER.error('Generatrion of overall test report has failed.')
418 def generate_performance_matrix(selected_tests, results_path):
420 Loads the results of all the currently run tests, compares them
421 based on the MATRIX_METRIC, outputs and saves the generated table.
422 :selected_tests: list of currently run test
423 :results_path: directory path to the results of current tests
425 _LOGGER.info('Performance Matrix:')
428 for test in selected_tests:
429 test_name = test.get('Name', '<Name not set>')
430 test_deployment = test.get('Deployment', '<Deployment not set>')
431 test_list.append({'test_name':test_name, 'test_deployment':test_deployment, 'csv_data':False})
435 all_params = settings.getValue('_PARAMS_LIST')
436 for i in range(len(selected_tests)):
438 if isinstance(all_params, list):
440 if i >= len(all_params):
441 list_index = len(all_params) - 1
442 if settings.getValue('CUMULATIVE_PARAMS') and (i > 0):
443 test_params.update(all_params[list_index])
445 test_params = all_params[list_index]
447 test_params = all_params
448 settings.setValue('TEST_PARAMS', test_params)
449 test['test_params'] = copy.deepcopy(test_params)
451 with open("{}/result_{}_{}_{}.csv".format(results_path, str(i),
452 test['test_name'], test['test_deployment'])) as csvfile:
453 reader = list(csv.DictReader(csvfile))
454 test['csv_data'] = reader[0]
455 # pylint: disable=broad-except
456 except (Exception) as ex:
457 _LOGGER.error("Result file not found: %s", ex)
459 metric = settings.getValue('MATRIX_METRIC')
461 output_header = ("ID", "Name", metric, "Change [%]", "Parameters, "\
462 "CUMULATIVE_PARAMS = {}".format(settings.getValue('CUMULATIVE_PARAMS')))
463 if not test_list[0]['csv_data'] or float(test_list[0]['csv_data'][metric]) == 0:
464 _LOGGER.error("Incorrect format of test results")
466 for i, test in enumerate(test_list):
468 change[i] = float(test['csv_data'][metric])/\
469 (float(test_list[0]['csv_data'][metric]) / 100) - 100
470 output.append([i, test['test_name'], float(test['csv_data'][metric]),
471 change[i], str(test['test_params'])[1:-1]])
474 output.append([i, test['test_name'], "Test Failed", 0, test['test_params']])
475 print(tabulate(output, headers=output_header, tablefmt="grid", floatfmt="0.3f"))
476 with open(results_path + '/result_performance_matrix.rst', 'w+') as output_file:
477 output_file.write(_TEMPLATE_MATRIX.format(metric, tabulate(output, headers=output_header,
478 tablefmt="rst", floatfmt="0.3f")))
479 _LOGGER.info('Performance matrix written to: "%s/result_performance_matrix.rst"', results_path)
481 def enable_sriov(nic_list):
482 """ Enable SRIOV for given enhanced PCI IDs
484 :param nic_list: A list of enhanced PCI IDs
486 # detect if sriov is required
489 if networkcard.is_sriov_nic(nic):
490 tmp_nic = nic.split('|')
491 if tmp_nic[0] in sriov_nic:
492 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
493 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
495 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
497 # sriov is required for some NICs
499 for nic in sriov_nic:
500 # check if SRIOV is supported and enough virt interfaces are available
501 if not networkcard.is_sriov_supported(nic) \
502 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
503 # if not, enable and set appropriate number of VFs
504 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
505 raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
507 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
509 # ensure that path to the bind tool is valid
510 functions.settings_update_paths()
512 # WORKAROUND: it has been observed with IXGBE(VF) driver,
513 # that NIC doesn't correclty dispatch traffic to VFs based
514 # on their MAC address. Unbind and bind to the same driver
516 networkcard.reinit_vfs(nic)
518 # After SRIOV is enabled it takes some time until network drivers
519 # properly initialize all cards.
520 # Wait also in case, that SRIOV was already configured as it can be
521 # configured automatically just before vsperf execution.
529 def disable_sriov(nic_list):
530 """ Disable SRIOV for given PCI IDs
532 :param nic_list: A list of enhanced PCI IDs
535 if networkcard.is_sriov_nic(nic):
536 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
537 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
539 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
542 def handle_list_options(args):
543 """ Process --list cli arguments if needed
545 :param args: A dictionary with all CLI arguments
547 if args['list_trafficgens']:
548 print(Loader().get_trafficgens_printable())
551 if args['list_collectors']:
552 print(Loader().get_collectors_printable())
555 if args['list_vswitches']:
556 print(Loader().get_vswitches_printable())
559 if args['list_vnfs']:
560 print(Loader().get_vnfs_printable())
563 if args['list_fwdapps']:
564 print(Loader().get_pktfwds_printable())
567 if args['list_loadgens']:
568 print(Loader().get_loadgens_printable())
571 if args['list_settings']:
580 def list_testcases(args):
581 """ Print list of testcases requested by --list CLI argument
583 :param args: A dictionary with all CLI arguments
586 if args['integration']:
587 testcases = settings.getValue('INTEGRATION_TESTS')
589 testcases = settings.getValue('PERFORMANCE_TESTS')
591 print("Available Tests:")
592 print("================")
594 for test in testcases:
595 description = functions.format_description(test['Description'], 70)
596 if len(test['Name']) < 40:
597 print('* {:40} {}'.format('{}:'.format(test['Name']), description[0]))
599 print('* {}'.format('{}:'.format(test['Name'])))
600 print(' {:40} {}'.format('', description[0]))
601 for i in range(1, len(description)):
602 print(' {:40} {}'.format('', description[i]))
605 def vsperf_finalize():
606 """ Clean up before exit
608 # remove directory if no result files were created
610 results_path = settings.getValue('RESULTS_PATH')
611 if os.path.exists(results_path):
612 files_list = os.listdir(results_path)
614 _LOGGER.info("Removing empty result directory: %s", results_path)
615 shutil.rmtree(results_path)
616 except AttributeError:
617 # skip it if parameter doesn't exist
620 # disable SRIOV if needed
622 if settings.getValue('SRIOV_ENABLED'):
623 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
624 except AttributeError:
625 # skip it if parameter doesn't exist
629 class MockTestCase(unittest.TestCase):
630 """Allow use of xmlrunner to generate Jenkins compatible output without
631 using xmlrunner to actually run tests.
634 suite = unittest.TestSuite()
635 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
636 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
637 xmlrunner.XMLTestRunner(...).run(suite)
640 def __init__(self, msg, is_pass, test_name):
643 self.is_pass = is_pass
645 #dynamically create a test method with the right name
646 #but point the method at our generic test method
647 setattr(MockTestCase, test_name, self.generic_test)
649 super(MockTestCase, self).__init__(test_name)
651 def generic_test(self):
652 """Provide a generic function that raises or not based
653 on how self.is_pass was set in the constructor"""
654 self.assertTrue(self.is_pass, self.msg)
656 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
660 args = parse_arguments()
664 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
666 # Load non performance/integration tests
667 if args['integration']:
668 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
670 # load command line parameters first in case there are settings files
672 settings.load_from_dict(args)
674 if args['conf_file']:
675 settings.load_from_file(args['conf_file'])
678 settings.load_from_env()
680 # reload command line parameters since these should take higher priority
681 # than both a settings file and environment variables
682 settings.load_from_dict(args)
684 settings.setValue('mode', args['mode'])
686 # update paths to trafficgens if required
687 if settings.getValue('mode') == 'trafficgen':
688 functions.settings_update_paths()
690 # if required, handle list-* operations
691 handle_list_options(args)
693 configure_logging(settings.getValue('VERBOSITY'))
695 # check and fix locale
696 check_and_set_locale()
698 # configure trafficgens
699 if args['trafficgen']:
700 trafficgens = Loader().get_trafficgens()
701 if args['trafficgen'] not in trafficgens:
702 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
703 ' \'%s\'. Exiting...', args['trafficgen'],
704 settings.getValue('TRAFFICGEN_DIR'))
707 # configuration validity checks
709 vswitch_none = args['vswitch'].strip().lower() == 'none'
711 settings.setValue('VSWITCH', 'none')
713 vswitches = Loader().get_vswitches()
714 if args['vswitch'] not in vswitches:
715 _LOGGER.error('There are no vswitches matching \'%s\' found in'
716 ' \'%s\'. Exiting...', args['vswitch'],
717 settings.getValue('VSWITCH_DIR'))
721 settings.setValue('PKTFWD', args['fwdapp'])
722 fwdapps = Loader().get_pktfwds()
723 if args['fwdapp'] not in fwdapps:
724 _LOGGER.error('There are no forwarding application'
725 ' matching \'%s\' found in'
726 ' \'%s\'. Exiting...', args['fwdapp'],
727 settings.getValue('PKTFWD_DIR'))
731 vnfs = Loader().get_vnfs()
732 if args['vnf'] not in vnfs:
733 _LOGGER.error('there are no vnfs matching \'%s\' found in'
734 ' \'%s\'. exiting...', args['vnf'],
735 settings.getValue('VNF_DIR'))
739 loadgens = Loader().get_loadgens()
740 if args['loadgen'] not in loadgens:
741 _LOGGER.error('There are no loadgens matching \'%s\' found in'
742 ' \'%s\'. Exiting...', args['loadgen'],
743 settings.getValue('LOADGEN_DIR'))
746 if args['exact_test_name'] and args['tests']:
747 _LOGGER.error("Cannot specify tests with both positional args and --test.")
750 # modify NIC configuration to decode enhanced PCI IDs
751 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
752 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
754 # sriov handling is performed on checked/expanded PCI IDs
755 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
758 for nic in wl_nics_orig:
759 tmp_nic = networkcard.get_nic_info(nic)
761 nic_list.append({'pci' : tmp_nic,
762 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
763 'mac' : networkcard.get_mac(tmp_nic),
764 'driver' : networkcard.get_driver(tmp_nic),
765 'device' : networkcard.get_device_name(tmp_nic)})
768 raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
770 settings.setValue('NICS', nic_list)
771 # for backward compatibility
772 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
774 # generate results directory name
775 date = datetime.datetime.fromtimestamp(time.time())
776 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
777 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
778 settings.setValue('RESULTS_PATH', results_path)
780 # create results directory
781 if not os.path.exists(results_path):
782 _LOGGER.info("Creating result directory: %s", results_path)
783 os.makedirs(results_path)
784 # pylint: disable=too-many-nested-blocks
785 if settings.getValue('mode') == 'trafficgen':
786 # execute only traffic generator
787 _LOGGER.debug("Executing traffic generator:")
789 # set traffic details, so they can be passed to traffic ctl
790 traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
791 traffic = functions.check_traffic(traffic)
793 traffic_ctl = component_factory.create_traffic(
794 traffic['traffic_type'],
795 loader.get_trafficgen_class())
797 traffic_ctl.send_traffic(traffic)
798 _LOGGER.debug("Traffic Results:")
799 traffic_ctl.print_results()
801 # write results into CSV file
802 result_file = os.path.join(results_path, "result.csv")
803 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
806 if args['integration']:
807 testcases = settings.getValue('INTEGRATION_TESTS')
809 testcases = settings.getValue('PERFORMANCE_TESTS')
811 if args['exact_test_name']:
812 exact_names = args['exact_test_name']
813 # positional args => exact matches only
815 for test_name in exact_names:
816 for test in testcases:
817 if test['Name'] == test_name:
818 selected_tests.append(test)
820 # --tests => apply filter to select requested tests
821 selected_tests = apply_filter(testcases, args['tests'])
823 # Default - run all tests
824 selected_tests = testcases
826 if not selected_tests:
827 _LOGGER.error("No tests matched --tests option or positional args. Done.")
831 suite = unittest.TestSuite()
832 settings_snapshot = copy.deepcopy(settings.__dict__)
834 for i, cfg in enumerate(selected_tests):
835 settings.setValue('_TEST_INDEX', i)
836 test_name = cfg.get('Name', '<Name not set>')
838 test_params = settings.getValue('_PARAMS_LIST')
839 if isinstance(test_params, list):
841 if i >= len(test_params):
842 list_index = len(test_params) - 1
843 test_params = test_params[list_index]
844 if settings.getValue('CUMULATIVE_PARAMS'):
845 test_params = merge_spec(settings.getValue('TEST_PARAMS'), test_params)
846 settings.setValue('TEST_PARAMS', test_params)
848 if args['integration']:
849 test = IntegrationTestCase(cfg)
851 test = PerformanceTestCase(cfg)
854 suite.addTest(MockTestCase('', True, test.name))
856 # pylint: disable=broad-except
857 except (Exception) as ex:
858 _LOGGER.exception("Failed to run test: %s", test_name)
859 suite.addTest(MockTestCase(str(ex), False, test_name))
860 _LOGGER.info("Continuing with next test...")
862 if not settings.getValue('CUMULATIVE_PARAMS'):
863 settings.restore_from_dict(settings_snapshot)
865 settings.restore_from_dict(settings_snapshot)
868 # Generate and printout Performance Matrix
870 generate_performance_matrix(selected_tests, results_path)
872 # generate final rst report with results of all executed TCs
873 generate_final_report()
877 if settings.getValue('XUNIT'):
878 xmlrunner.XMLTestRunner(
879 output=settings.getValue('XUNIT_DIR'), outsuffix="",
880 verbosity=0).run(suite)
883 pod_name = args['opnfvpod']
884 installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()
885 opnfv_url = settings.getValue('OPNFV_URL')
886 pkg_list = settings.getValue('PACKAGE_LIST')
888 int_data = {'pod': pod_name,
889 'build_tag': get_build_tag(),
890 'installer': installer_name,
891 'pkg_list': pkg_list,
893 # pass vswitch name from configuration to be used for failed
894 # TCs; In case of successful TCs it is safer to use vswitch
895 # name from CSV as TC can override global configuration
896 'vswitch': str(settings.getValue('VSWITCH')).lower()}
897 tc_names = [tc['Name'] for tc in selected_tests]
898 opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
900 # cleanup before exit
903 if __name__ == "__main__":