3 # Copyright 2015-2017 Intel Corporation.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """VSPERF main script.
36 from tabulate import tabulate
37 from conf import merge_spec
38 from conf import settings
39 import core.component_factory as component_factory
40 from core.loader import Loader
41 from testcases import PerformanceTestCase
42 from testcases import IntegrationTestCase
43 from tools import tasks
44 from tools import networkcard
45 from tools import functions
46 from tools.pkt_gen import trafficgen
47 from tools.opnfvdashboard import opnfvdashboard
48 sys.dont_write_bytecode = True
51 'debug': logging.DEBUG,
53 'warning': logging.WARNING,
54 'error': logging.ERROR,
55 'critical': logging.CRITICAL
58 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
60 _TEMPLATE_RST = {'head' : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
61 'foot' : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
62 'final' : 'test_report.rst',
63 'tmp' : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
66 _TEMPLATE_MATRIX = "Performance Matrix\n------------------\n\n"\
67 "The following performance matrix was generated with the results of all the\n"\
68 "currently run tests. The metric used for comparison is {}.\n\n{}\n\n"
70 _LOGGER = logging.getLogger()
72 def parse_param_string(values):
74 Parse and split a single '--test-params' argument.
76 This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
77 values. For multiple overrides use a ; separated list for
78 e.g. --test-params 'x=z; y=(a,b)'
85 for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
90 # values are passed inside string from CLI, so we must retype them accordingly
92 results[param] = ast.literal_eval(value)
94 # for backward compatibility, we have to accept strings without quotes
95 _LOGGER.warning("Adding missing quotes around string value: %s = %s",
97 results[param] = str(value)
103 def parse_arguments():
105 Parse command line arguments.
107 class _SplitTestParamsAction(argparse.Action):
109 Parse and split '--test-params' arguments.
111 This expects either a single list of ; separated overrides
112 as 'x=y', 'x=y,z' or 'x' (implicit true) values.
113 e.g. --test-params 'x=z; y=(a,b)'
114 Or a list of these ; separated lists with overrides for
116 e.g. --test-params "['x=z; y=(a,b)','x=z']"
118 def __call__(self, parser, namespace, values, option_string=None):
120 input_list = ast.literal_eval(values)
122 for test_params in input_list:
123 parameter_list.append(parse_param_string(test_params))
125 parameter_list = parse_param_string(values)
126 results = {'_PARAMS_LIST':parameter_list}
127 setattr(namespace, self.dest, results)
129 class _ValidateFileAction(argparse.Action):
130 """Validate a file can be read from before using it.
132 def __call__(self, parser, namespace, values, option_string=None):
133 if not os.path.isfile(values):
134 raise argparse.ArgumentTypeError(
135 'the path \'%s\' is not a valid path' % values)
136 elif not os.access(values, os.R_OK):
137 raise argparse.ArgumentTypeError(
138 'the path \'%s\' is not accessible' % values)
140 setattr(namespace, self.dest, values)
142 class _ValidateDirAction(argparse.Action):
143 """Validate a directory can be written to before using it.
145 def __call__(self, parser, namespace, values, option_string=None):
146 if not os.path.isdir(values):
147 raise argparse.ArgumentTypeError(
148 'the path \'%s\' is not a valid path' % values)
149 elif not os.access(values, os.W_OK):
150 raise argparse.ArgumentTypeError(
151 'the path \'%s\' is not accessible' % values)
153 setattr(namespace, self.dest, values)
155 def list_logging_levels():
156 """Give a summary of all available logging levels.
158 :return: List of verbosity level names in decreasing order of
161 return sorted(VERBOSITY_LEVELS.keys(),
162 key=lambda x: VERBOSITY_LEVELS[x])
164 parser = argparse.ArgumentParser(prog=__file__, formatter_class=
165 argparse.ArgumentDefaultsHelpFormatter)
166 parser.add_argument('--version', action='version', version='%(prog)s 0.2')
167 parser.add_argument('--list', '--list-tests', action='store_true',
168 help='list all tests and exit')
169 parser.add_argument('--list-trafficgens', action='store_true',
170 help='list all traffic generators and exit')
171 parser.add_argument('--list-collectors', action='store_true',
172 help='list all system metrics loggers and exit')
173 parser.add_argument('--list-vswitches', action='store_true',
174 help='list all system vswitches and exit')
175 parser.add_argument('--list-fwdapps', action='store_true',
176 help='list all system forwarding applications and exit')
177 parser.add_argument('--list-vnfs', action='store_true',
178 help='list all system vnfs and exit')
179 parser.add_argument('--list-loadgens', action='store_true',
180 help='list all background load generators')
181 parser.add_argument('--list-settings', action='store_true',
182 help='list effective settings configuration and exit')
183 parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
184 tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
185 runs only the two tests with those exact names.\
186 To run all tests omit both positional args and --tests arg.')
188 group = parser.add_argument_group('test selection options')
189 group.add_argument('-m', '--mode', help='vsperf mode of operation;\
190 Values: "normal" - execute vSwitch, VNF and traffic generator;\
191 "trafficgen" - execute only traffic generator; "trafficgen-off" \
192 - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
193 and VNF but pause before traffic transmission ', default='normal')
195 group.add_argument('-f', '--test-spec', help='test specification file')
196 group.add_argument('-d', '--test-dir', help='directory containing tests')
197 group.add_argument('-t', '--tests', help='Comma-separated list of terms \
198 indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
199 name contains RFC2544 less those containing "p2p"; "!back2back" - \
200 run all tests except those containing back2back')
201 group.add_argument('--verbosity', choices=list_logging_levels(),
203 group.add_argument('--integration', action='store_true', help='execute integration tests')
204 group.add_argument('--trafficgen', help='traffic generator to use')
205 group.add_argument('--vswitch', help='vswitch implementation to use')
206 group.add_argument('--fwdapp', help='packet forwarding application to use')
207 group.add_argument('--vnf', help='vnf to use')
208 group.add_argument('--loadgen', help='loadgen to use')
209 group.add_argument('--sysmetrics', help='system metrics logger to use')
210 group = parser.add_argument_group('test behavior options')
211 group.add_argument('--xunit', action='store_true',
212 help='enable xUnit-formatted output')
213 group.add_argument('--xunit-dir', action=_ValidateDirAction,
214 help='output directory of xUnit-formatted output')
215 group.add_argument('--load-env', action='store_true',
216 help='enable loading of settings from the environment')
217 group.add_argument('--conf-file', action=_ValidateFileAction,
218 help='settings file')
219 group.add_argument('--test-params', action=_SplitTestParamsAction,
220 help='csv list of test parameters: key=val; e.g. '
221 'TRAFFICGEN_PKT_SIZES=(64,128);TRAFFICGEN_DURATION=30; '
222 'GUEST_LOOPBACK=["l2fwd"] ...'
223 ' or a list of csv lists of test parameters: key=val; e.g. '
224 '[\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(128,)\','
225 '\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(64,)\']')
226 group.add_argument('--opnfvpod', help='name of POD in opnfv')
227 group.add_argument('--matrix', help='enable performance matrix analysis',
228 action='store_true', default=False)
230 args = vars(parser.parse_args())
235 def configure_logging(level):
236 """Configure logging.
238 log_file_default = os.path.join(
239 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_DEFAULT'))
240 log_file_host_cmds = os.path.join(
241 settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_HOST_CMDS'))
242 log_file_traffic_gen = os.path.join(
243 settings.getValue('LOG_DIR'),
244 settings.getValue('LOG_FILE_TRAFFIC_GEN'))
246 _LOGGER.setLevel(logging.DEBUG)
248 stream_logger = logging.StreamHandler(sys.stdout)
249 stream_logger.setLevel(VERBOSITY_LEVELS[level])
250 stream_logger.setFormatter(logging.Formatter(
251 '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s'))
252 _LOGGER.addHandler(stream_logger)
254 file_logger = logging.FileHandler(filename=log_file_default)
255 file_logger.setLevel(logging.DEBUG)
256 _LOGGER.addHandler(file_logger)
258 class CommandFilter(logging.Filter):
259 """Filter out strings beginning with 'cmd :'"""
260 def filter(self, record):
261 return record.getMessage().startswith(tasks.CMD_PREFIX)
263 class TrafficGenCommandFilter(logging.Filter):
264 """Filter out strings beginning with 'gencmd :'"""
265 def filter(self, record):
266 return record.getMessage().startswith(trafficgen.CMD_PREFIX)
268 cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
269 cmd_logger.setLevel(logging.DEBUG)
270 cmd_logger.addFilter(CommandFilter())
271 _LOGGER.addHandler(cmd_logger)
273 gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
274 gen_logger.setLevel(logging.DEBUG)
275 gen_logger.addFilter(TrafficGenCommandFilter())
276 _LOGGER.addHandler(gen_logger)
279 def apply_filter(tests, tc_filter):
280 """Allow a subset of tests to be conveniently selected
282 :param tests: The list of Tests from which to select.
283 :param tc_filter: A case-insensitive string of comma-separated terms
284 indicating the Tests to select.
285 e.g. 'RFC' - select all tests whose name contains 'RFC'
286 e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
288 e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
289 or 'burst' and from these remove any containing 'p2p'.
290 e.g. '' - empty string selects all tests.
291 :return: A list of the selected Tests.
293 # if negative filter is first we have to start with full list of tests
294 if tc_filter.strip()[0] == '!':
298 if tc_filter is None:
301 for term in [x.strip() for x in tc_filter.lower().split(",")]:
302 if not term or term[0] != '!':
303 # Add matching tests from 'tests' into results
304 result.extend([test for test in tests \
305 if test['Name'].lower().find(term) >= 0])
307 # Term begins with '!' so we remove matching tests
308 result = [test for test in result \
309 if test['Name'].lower().find(term[1:]) < 0]
314 def check_and_set_locale():
315 """ Function will check locale settings. In case, that it isn't configured
316 properly, then default values specified by DEFAULT_LOCALE will be used.
319 system_locale = locale.getdefaultlocale()
320 if None in system_locale:
321 os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
322 _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
323 system_locale, locale.getdefaultlocale())
325 def get_vswitch_names(rst_files):
326 """ Function will return a list of vSwitches detected in given ``rst_files``.
328 vswitch_names = set()
331 output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
333 match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
335 vswitch_names.add(match.group(1))
338 return list(vswitch_names)
340 except subprocess.CalledProcessError:
341 _LOGGER.warning('Cannot detect vSwitches used during testing.')
343 # fallback to the default value
347 """ Function will return a Jenkins job ID environment variable.
351 build_tag = os.environ['BUILD_TAG']
354 _LOGGER.warning('Cannot detect Jenkins job ID')
359 def generate_final_report():
360 """ Function will check if partial test results are available
361 and generates final report in rst format.
364 path = settings.getValue('RESULTS_PATH')
365 # check if there are any results in rst format
366 rst_results = glob.glob(os.path.join(path, 'result*rst'))
367 pkt_processors = get_vswitch_names(rst_results)
370 test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
371 # create report caption directly - it is not worth to execute jinja machinery
372 report_caption = '{}\n{} {}\n{}\n\n'.format(
373 '============================================================',
374 'Performance report for',
375 ', '.join(pkt_processors),
376 '============================================================')
378 with open(_TEMPLATE_RST['tmp'], 'w') as file_:
379 file_.write(report_caption)
381 retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
382 ' '.join(rst_results), _TEMPLATE_RST['foot'],
383 test_report), shell=True)
384 if retval == 0 and os.path.isfile(test_report):
385 _LOGGER.info('Overall test report written to "%s"', test_report)
387 _LOGGER.error('Generation of overall test report has failed.')
389 # remove temporary file
390 os.remove(_TEMPLATE_RST['tmp'])
392 except subprocess.CalledProcessError:
393 _LOGGER.error('Generatrion of overall test report has failed.')
396 def generate_performance_matrix(selected_tests, results_path):
398 Loads the results of all the currently run tests, compares them
399 based on the MATRIX_METRIC, outputs and saves the generated table.
400 :selected_tests: list of currently run test
401 :results_path: directory path to the results of current tests
403 _LOGGER.info('Performance Matrix:')
406 for test in selected_tests:
407 test_name = test.get('Name', '<Name not set>')
408 test_deployment = test.get('Deployment', '<Deployment not set>')
409 test_list.append({'test_name':test_name, 'test_deployment':test_deployment, 'csv_data':False})
413 all_params = settings.getValue('_PARAMS_LIST')
414 for i in range(len(selected_tests)):
416 if isinstance(all_params, list):
418 if i >= len(all_params):
419 list_index = len(all_params) - 1
420 if settings.getValue('CUMULATIVE_PARAMS') and (i > 0):
421 test_params.update(all_params[list_index])
423 test_params = all_params[list_index]
425 test_params = all_params
426 settings.setValue('TEST_PARAMS', test_params)
427 test['test_params'] = copy.deepcopy(test_params)
429 with open("{}/result_{}_{}_{}.csv".format(results_path, str(i),
430 test['test_name'], test['test_deployment'])) as csvfile:
431 reader = list(csv.DictReader(csvfile))
432 test['csv_data'] = reader[0]
433 # pylint: disable=broad-except
434 except (Exception) as ex:
435 _LOGGER.error("Result file not found: %s", ex)
437 metric = settings.getValue('MATRIX_METRIC')
439 output_header = ("ID", "Name", metric, "Change [%]", "Parameters, "\
440 "CUMULATIVE_PARAMS = {}".format(settings.getValue('CUMULATIVE_PARAMS')))
441 if not test_list[0]['csv_data'] or float(test_list[0]['csv_data'][metric]) == 0:
442 _LOGGER.error("Incorrect format of test results")
444 for i, test in enumerate(test_list):
446 change[i] = float(test['csv_data'][metric])/\
447 (float(test_list[0]['csv_data'][metric]) / 100) - 100
448 output.append([i, test['test_name'], float(test['csv_data'][metric]),
449 change[i], str(test['test_params'])[1:-1]])
452 output.append([i, test['test_name'], "Test Failed", 0, test['test_params']])
453 print(tabulate(output, headers=output_header, tablefmt="grid", floatfmt="0.3f"))
454 with open(results_path + '/result_performance_matrix.rst', 'w+') as output_file:
455 output_file.write(_TEMPLATE_MATRIX.format(metric, tabulate(output, headers=output_header,
456 tablefmt="rst", floatfmt="0.3f")))
457 _LOGGER.info('Performance matrix written to: "%s/result_performance_matrix.rst"', results_path)
459 def enable_sriov(nic_list):
460 """ Enable SRIOV for given enhanced PCI IDs
462 :param nic_list: A list of enhanced PCI IDs
464 # detect if sriov is required
467 if networkcard.is_sriov_nic(nic):
468 tmp_nic = nic.split('|')
469 if tmp_nic[0] in sriov_nic:
470 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
471 sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
473 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
475 # sriov is required for some NICs
477 for nic in sriov_nic:
478 # check if SRIOV is supported and enough virt interfaces are available
479 if not networkcard.is_sriov_supported(nic) \
480 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
481 # if not, enable and set appropriate number of VFs
482 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
483 raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
485 _LOGGER.debug("SRIOV enabled for NIC %s", nic)
487 # ensure that path to the bind tool is valid
488 functions.settings_update_paths()
490 # WORKAROUND: it has been observed with IXGBE(VF) driver,
491 # that NIC doesn't correclty dispatch traffic to VFs based
492 # on their MAC address. Unbind and bind to the same driver
494 networkcard.reinit_vfs(nic)
496 # After SRIOV is enabled it takes some time until network drivers
497 # properly initialize all cards.
498 # Wait also in case, that SRIOV was already configured as it can be
499 # configured automatically just before vsperf execution.
507 def disable_sriov(nic_list):
508 """ Disable SRIOV for given PCI IDs
510 :param nic_list: A list of enhanced PCI IDs
513 if networkcard.is_sriov_nic(nic):
514 if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
515 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
517 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
520 def handle_list_options(args):
521 """ Process --list cli arguments if needed
523 :param args: A dictionary with all CLI arguments
525 if args['list_trafficgens']:
526 print(Loader().get_trafficgens_printable())
529 if args['list_collectors']:
530 print(Loader().get_collectors_printable())
533 if args['list_vswitches']:
534 print(Loader().get_vswitches_printable())
537 if args['list_vnfs']:
538 print(Loader().get_vnfs_printable())
541 if args['list_fwdapps']:
542 print(Loader().get_pktfwds_printable())
545 if args['list_loadgens']:
546 print(Loader().get_loadgens_printable())
549 if args['list_settings']:
558 def list_testcases(args):
559 """ Print list of testcases requested by --list CLI argument
561 :param args: A dictionary with all CLI arguments
564 if args['integration']:
565 testcases = settings.getValue('INTEGRATION_TESTS')
567 testcases = settings.getValue('PERFORMANCE_TESTS')
569 print("Available Tests:")
570 print("================")
572 for test in testcases:
573 description = functions.format_description(test['Description'], 70)
574 if len(test['Name']) < 40:
575 print('* {:40} {}'.format('{}:'.format(test['Name']), description[0]))
577 print('* {}'.format('{}:'.format(test['Name'])))
578 print(' {:40} {}'.format('', description[0]))
579 for i in range(1, len(description)):
580 print(' {:40} {}'.format('', description[i]))
583 def vsperf_finalize():
584 """ Clean up before exit
586 # remove directory if no result files were created
588 results_path = settings.getValue('RESULTS_PATH')
589 if os.path.exists(results_path):
590 files_list = os.listdir(results_path)
592 _LOGGER.info("Removing empty result directory: %s", results_path)
593 shutil.rmtree(results_path)
594 except AttributeError:
595 # skip it if parameter doesn't exist
598 # disable SRIOV if needed
600 if settings.getValue('SRIOV_ENABLED'):
601 disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
602 except AttributeError:
603 # skip it if parameter doesn't exist
607 class MockTestCase(unittest.TestCase):
608 """Allow use of xmlrunner to generate Jenkins compatible output without
609 using xmlrunner to actually run tests.
612 suite = unittest.TestSuite()
613 suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
614 suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
615 xmlrunner.XMLTestRunner(...).run(suite)
618 def __init__(self, msg, is_pass, test_name):
621 self.is_pass = is_pass
623 #dynamically create a test method with the right name
624 #but point the method at our generic test method
625 setattr(MockTestCase, test_name, self.generic_test)
627 super(MockTestCase, self).__init__(test_name)
629 def generic_test(self):
630 """Provide a generic function that raises or not based
631 on how self.is_pass was set in the constructor"""
632 self.assertTrue(self.is_pass, self.msg)
634 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
638 args = parse_arguments()
642 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
644 # Load non performance/integration tests
645 if args['integration']:
646 settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
648 # load command line parameters first in case there are settings files
650 settings.load_from_dict(args)
652 if args['conf_file']:
653 settings.load_from_file(args['conf_file'])
656 settings.load_from_env()
658 # reload command line parameters since these should take higher priority
659 # than both a settings file and environment variables
660 settings.load_from_dict(args)
662 settings.setValue('mode', args['mode'])
664 # update paths to trafficgens if required
665 if settings.getValue('mode') == 'trafficgen':
666 functions.settings_update_paths()
668 # if required, handle list-* operations
669 handle_list_options(args)
671 configure_logging(settings.getValue('VERBOSITY'))
673 # check and fix locale
674 check_and_set_locale()
676 # configure trafficgens
677 if args['trafficgen']:
678 trafficgens = Loader().get_trafficgens()
679 if args['trafficgen'] not in trafficgens:
680 _LOGGER.error('There are no trafficgens matching \'%s\' found in'
681 ' \'%s\'. Exiting...', args['trafficgen'],
682 settings.getValue('TRAFFICGEN_DIR'))
685 # configuration validity checks
687 vswitch_none = args['vswitch'].strip().lower() == 'none'
689 settings.setValue('VSWITCH', 'none')
691 vswitches = Loader().get_vswitches()
692 if args['vswitch'] not in vswitches:
693 _LOGGER.error('There are no vswitches matching \'%s\' found in'
694 ' \'%s\'. Exiting...', args['vswitch'],
695 settings.getValue('VSWITCH_DIR'))
699 settings.setValue('PKTFWD', args['fwdapp'])
700 fwdapps = Loader().get_pktfwds()
701 if args['fwdapp'] not in fwdapps:
702 _LOGGER.error('There are no forwarding application'
703 ' matching \'%s\' found in'
704 ' \'%s\'. Exiting...', args['fwdapp'],
705 settings.getValue('PKTFWD_DIR'))
709 vnfs = Loader().get_vnfs()
710 if args['vnf'] not in vnfs:
711 _LOGGER.error('there are no vnfs matching \'%s\' found in'
712 ' \'%s\'. exiting...', args['vnf'],
713 settings.getValue('VNF_DIR'))
717 loadgens = Loader().get_loadgens()
718 if args['loadgen'] not in loadgens:
719 _LOGGER.error('There are no loadgens matching \'%s\' found in'
720 ' \'%s\'. Exiting...', args['loadgen'],
721 settings.getValue('LOADGEN_DIR'))
724 if args['exact_test_name'] and args['tests']:
725 _LOGGER.error("Cannot specify tests with both positional args and --test.")
728 # modify NIC configuration to decode enhanced PCI IDs
729 wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
730 settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
732 # sriov handling is performed on checked/expanded PCI IDs
733 settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
736 for nic in wl_nics_orig:
737 tmp_nic = networkcard.get_nic_info(nic)
739 nic_list.append({'pci' : tmp_nic,
740 'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
741 'mac' : networkcard.get_mac(tmp_nic),
742 'driver' : networkcard.get_driver(tmp_nic),
743 'device' : networkcard.get_device_name(tmp_nic)})
746 raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
748 settings.setValue('NICS', nic_list)
749 # for backward compatibility
750 settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
752 # generate results directory name
753 date = datetime.datetime.fromtimestamp(time.time())
754 results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
755 results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
756 settings.setValue('RESULTS_PATH', results_path)
758 # create results directory
759 if not os.path.exists(results_path):
760 _LOGGER.info("Creating result directory: %s", results_path)
761 os.makedirs(results_path)
762 # pylint: disable=too-many-nested-blocks
763 if settings.getValue('mode') == 'trafficgen':
764 # execute only traffic generator
765 _LOGGER.debug("Executing traffic generator:")
767 # set traffic details, so they can be passed to traffic ctl
768 traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
769 traffic = functions.check_traffic(traffic)
771 traffic_ctl = component_factory.create_traffic(
772 traffic['traffic_type'],
773 loader.get_trafficgen_class())
775 traffic_ctl.send_traffic(traffic)
776 _LOGGER.debug("Traffic Results:")
777 traffic_ctl.print_results()
779 # write results into CSV file
780 result_file = os.path.join(results_path, "result.csv")
781 PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
784 if args['integration']:
785 testcases = settings.getValue('INTEGRATION_TESTS')
787 testcases = settings.getValue('PERFORMANCE_TESTS')
789 if args['exact_test_name']:
790 exact_names = args['exact_test_name']
791 # positional args => exact matches only
793 for test_name in exact_names:
794 for test in testcases:
795 if test['Name'] == test_name:
796 selected_tests.append(test)
798 # --tests => apply filter to select requested tests
799 selected_tests = apply_filter(testcases, args['tests'])
801 # Default - run all tests
802 selected_tests = testcases
804 if not selected_tests:
805 _LOGGER.error("No tests matched --tests option or positional args. Done.")
809 suite = unittest.TestSuite()
810 settings_snapshot = copy.deepcopy(settings.__dict__)
812 for i, cfg in enumerate(selected_tests):
813 settings.setValue('_TEST_INDEX', i)
814 test_name = cfg.get('Name', '<Name not set>')
816 test_params = settings.getValue('_PARAMS_LIST')
817 if isinstance(test_params, list):
819 if i >= len(test_params):
820 list_index = len(test_params) - 1
821 test_params = test_params[list_index]
822 if settings.getValue('CUMULATIVE_PARAMS'):
823 test_params = merge_spec(settings.getValue('TEST_PARAMS'), test_params)
824 settings.setValue('TEST_PARAMS', test_params)
826 if args['integration']:
827 test = IntegrationTestCase(cfg)
829 test = PerformanceTestCase(cfg)
832 suite.addTest(MockTestCase('', True, test.name))
834 # pylint: disable=broad-except
835 except (Exception) as ex:
836 _LOGGER.exception("Failed to run test: %s", test_name)
837 suite.addTest(MockTestCase(str(ex), False, test_name))
838 _LOGGER.info("Continuing with next test...")
840 if not settings.getValue('CUMULATIVE_PARAMS'):
841 settings.restore_from_dict(settings_snapshot)
843 settings.restore_from_dict(settings_snapshot)
846 # Generate and printout Performance Matrix
848 generate_performance_matrix(selected_tests, results_path)
850 # generate final rst report with results of all executed TCs
851 generate_final_report()
855 if settings.getValue('XUNIT'):
856 xmlrunner.XMLTestRunner(
857 output=settings.getValue('XUNIT_DIR'), outsuffix="",
858 verbosity=0).run(suite)
861 pod_name = args['opnfvpod']
862 installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()
863 opnfv_url = settings.getValue('OPNFV_URL')
864 pkg_list = settings.getValue('PACKAGE_LIST')
866 int_data = {'pod': pod_name,
867 'build_tag': get_build_tag(),
868 'installer': installer_name,
869 'pkg_list': pkg_list,
871 # pass vswitch name from configuration to be used for failed
872 # TCs; In case of successful TCs it is safer to use vswitch
873 # name from CSV as TC can override global configuration
874 'vswitch': str(settings.getValue('VSWITCH')).lower()}
875 tc_names = [tc['Name'] for tc in selected_tests]
876 opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
878 # cleanup before exit
881 if __name__ == "__main__":