vsperf

   1 #!/usr/bin/env python3
   2
   3 # Copyright 2015-2017 Intel Corporation.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """VSPERF main script.
  18 """
  19
  20 import logging
  21 import os
  22 import sys
  23 import argparse
  24 import re
  25 import time
  26 import csv
  27 import datetime
  28 import shutil
  29 import unittest
  30 import locale
  31 import copy
  32 import glob
  33 import subprocess
  34 import ast
  35 import xmlrunner
  36 from tabulate import tabulate
  37 from conf import merge_spec
  38 from conf import settings
  39 import core.component_factory as component_factory
  40 from core.loader import Loader
  41 from testcases import PerformanceTestCase
  42 from testcases import IntegrationTestCase
  43 from tools import tasks
  44 from tools import networkcard
  45 from tools import functions
  46 from tools.pkt_gen import trafficgen
  47 from tools.opnfvdashboard import opnfvdashboard
  48 sys.dont_write_bytecode = True
  49
  50 VERBOSITY_LEVELS = {
  51     'debug': logging.DEBUG,
  52     'info': logging.INFO,
  53     'warning': logging.WARNING,
  54     'error': logging.ERROR,
  55     'critical': logging.CRITICAL
  56 }
  57
  58 _CURR_DIR = os.path.dirname(os.path.realpath(__file__))
  59
  60 _TEMPLATE_RST = {'head'  : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
  61                  'foot'  : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),
  62                  'final' : 'test_report.rst',
  63                  'tmp'   : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')
  64                 }
  65
  66 _TEMPLATE_MATRIX = "Performance Matrix\n------------------\n\n"\
  67                    "The following performance matrix was generated with the results of all the\n"\
  68                    "currently run tests. The metric used for comparison is {}.\n\n{}\n\n"
  69
  70 _LOGGER = logging.getLogger()
  71
  72 def parse_param_string(values):
  73     """
  74     Parse and split a single '--test-params' argument.
  75
  76     This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
  77     values. For multiple overrides use a ; separated list for
  78     e.g. --test-params 'x=z; y=(a,b)'
  79     """
  80     results = {}
  81
  82     if values == '':
  83         return {}
  84
  85     for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
  86         param = param.strip()
  87         value = value.strip()
  88         if param:
  89             if value:
  90                 # values are passed inside string from CLI, so we must retype them accordingly
  91                 try:
  92                     results[param] = ast.literal_eval(value)
  93                 except ValueError:
  94                     # for backward compatibility, we have to accept strings without quotes
  95                     _LOGGER.warning("Adding missing quotes around string value: %s = %s",
  96                                     param, str(value))
  97                     results[param] = str(value)
  98             else:
  99                 results[param] = True
 100     return results
 101
 102
 103 def parse_arguments():
 104     """
 105     Parse command line arguments.
 106     """
 107     class _SplitTestParamsAction(argparse.Action):
 108         """
 109         Parse and split '--test-params' arguments.
 110
 111         This expects either a single list of ; separated overrides
 112         as 'x=y', 'x=y,z' or 'x' (implicit true) values.
 113         e.g. --test-params 'x=z; y=(a,b)'
 114         Or a list of these ; separated lists with overrides for
 115         multiple tests.
 116         e.g. --test-params "['x=z; y=(a,b)','x=z']"
 117         """
 118         def __call__(self, parser, namespace, values, option_string=None):
 119             if values[0] == '[':
 120                 input_list = ast.literal_eval(values)
 121                 parameter_list = []
 122                 for test_params in input_list:
 123                     parameter_list.append(parse_param_string(test_params))
 124             else:
 125                 parameter_list = parse_param_string(values)
 126             results = {'_PARAMS_LIST':parameter_list}
 127             setattr(namespace, self.dest, results)
 128
 129     class _ValidateFileAction(argparse.Action):
 130         """Validate a file can be read from before using it.
 131         """
 132         def __call__(self, parser, namespace, values, option_string=None):
 133             if not os.path.isfile(values):
 134                 raise argparse.ArgumentTypeError(
 135                     'the path \'%s\' is not a valid path' % values)
 136             elif not os.access(values, os.R_OK):
 137                 raise argparse.ArgumentTypeError(
 138                     'the path \'%s\' is not accessible' % values)
 139
 140             setattr(namespace, self.dest, values)
 141
 142     class _ValidateDirAction(argparse.Action):
 143         """Validate a directory can be written to before using it.
 144         """
 145         def __call__(self, parser, namespace, values, option_string=None):
 146             if not os.path.isdir(values):
 147                 raise argparse.ArgumentTypeError(
 148                     'the path \'%s\' is not a valid path' % values)
 149             elif not os.access(values, os.W_OK):
 150                 raise argparse.ArgumentTypeError(
 151                     'the path \'%s\' is not accessible' % values)
 152
 153             setattr(namespace, self.dest, values)
 154
 155     def list_logging_levels():
 156         """Give a summary of all available logging levels.
 157
 158         :return: List of verbosity level names in decreasing order of
 159             verbosity
 160         """
 161         return sorted(VERBOSITY_LEVELS.keys(),
 162                       key=lambda x: VERBOSITY_LEVELS[x])
 163
 164     parser = argparse.ArgumentParser(prog=__file__, formatter_class=
 165                                      argparse.ArgumentDefaultsHelpFormatter)
 166     parser.add_argument('--version', action='version', version='%(prog)s 0.2')
 167     parser.add_argument('--list', '--list-tests', action='store_true',
 168                         help='list all tests and exit')
 169     parser.add_argument('--list-trafficgens', action='store_true',
 170                         help='list all traffic generators and exit')
 171     parser.add_argument('--list-collectors', action='store_true',
 172                         help='list all system metrics loggers and exit')
 173     parser.add_argument('--list-vswitches', action='store_true',
 174                         help='list all system vswitches and exit')
 175     parser.add_argument('--list-fwdapps', action='store_true',
 176                         help='list all system forwarding applications and exit')
 177     parser.add_argument('--list-vnfs', action='store_true',
 178                         help='list all system vnfs and exit')
 179     parser.add_argument('--list-loadgens', action='store_true',
 180                         help='list all background load generators')
 181     parser.add_argument('--list-settings', action='store_true',
 182                         help='list effective settings configuration and exit')
 183     parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
 184             tests to run. E.g "vsperf phy2phy_tput phy2phy_cont"\
 185             runs only the two tests with those exact names.\
 186             To run all tests omit both positional args and --tests arg.')
 187
 188     group = parser.add_argument_group('test selection options')
 189     group.add_argument('-m', '--mode', help='vsperf mode of operation;\
 190             Values: "normal" - execute vSwitch, VNF and traffic generator;\
 191             "trafficgen" - execute only traffic generator; "trafficgen-off" \
 192             - execute vSwitch and VNF; trafficgen-pause - execute vSwitch \
 193             and VNF but pause before traffic transmission ', default='normal')
 194
 195     group.add_argument('-f', '--test-spec', help='test specification file')
 196     group.add_argument('-d', '--test-dir', help='directory containing tests')
 197     group.add_argument('-t', '--tests', help='Comma-separated list of terms \
 198             indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
 199             name contains RFC2544 less those containing "p2p"; "!back2back" - \
 200             run all tests except those containing back2back')
 201     group.add_argument('--verbosity', choices=list_logging_levels(),
 202                        help='debug level')
 203     group.add_argument('--integration', action='store_true', help='execute integration tests')
 204     group.add_argument('--trafficgen', help='traffic generator to use')
 205     group.add_argument('--vswitch', help='vswitch implementation to use')
 206     group.add_argument('--fwdapp', help='packet forwarding application to use')
 207     group.add_argument('--vnf', help='vnf to use')
 208     group.add_argument('--loadgen', help='loadgen to use')
 209     group.add_argument('--sysmetrics', help='system metrics logger to use')
 210     group = parser.add_argument_group('test behavior options')
 211     group.add_argument('--xunit', action='store_true',
 212                        help='enable xUnit-formatted output')
 213     group.add_argument('--xunit-dir', action=_ValidateDirAction,
 214                        help='output directory of xUnit-formatted output')
 215     group.add_argument('--load-env', action='store_true',
 216                        help='enable loading of settings from the environment')
 217     group.add_argument('--conf-file', action=_ValidateFileAction,
 218                        help='settings file')
 219     group.add_argument('--test-params', action=_SplitTestParamsAction,
 220                        help='csv list of test parameters: key=val; e.g. '
 221                        'TRAFFICGEN_PKT_SIZES=(64,128);TRAFFICGEN_DURATION=30; '
 222                        'GUEST_LOOPBACK=["l2fwd"] ...'
 223                        ' or a list of csv lists of test parameters: key=val; e.g. '
 224                        '[\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(128,)\','
 225                        '\'TRAFFICGEN_DURATION=10;TRAFFICGEN_PKT_SIZES=(64,)\']')
 226     group.add_argument('--opnfvpod', help='name of POD in opnfv')
 227     group.add_argument('--matrix', help='enable performance matrix analysis',
 228                        action='store_true', default=False)
 229
 230     args = vars(parser.parse_args())
 231
 232     return args
 233
 234
 235 def configure_logging(level):
 236     """Configure logging.
 237     """
 238     name, ext = os.path.splitext(settings.getValue('LOG_FILE_DEFAULT'))
 239     rename_default = "{name}_{uid}{ex}".format(name=name,
 240                                                uid=settings.getValue(
 241                                                    'LOG_TIMESTAMP'),
 242                                                ex=ext)
 243     log_file_default = os.path.join(
 244         settings.getValue('LOG_DIR'), rename_default)
 245     name, ext = os.path.splitext(settings.getValue('LOG_FILE_HOST_CMDS'))
 246     rename_hostcmd = "{name}_{uid}{ex}".format(name=name,
 247                                                uid=settings.getValue(
 248                                                    'LOG_TIMESTAMP'),
 249                                                ex=ext)
 250     log_file_host_cmds = os.path.join(
 251         settings.getValue('LOG_DIR'), rename_hostcmd)
 252     name, ext = os.path.splitext(settings.getValue('LOG_FILE_TRAFFIC_GEN'))
 253     rename_traffic = "{name}_{uid}{ex}".format(name=name,
 254                                                uid=settings.getValue(
 255                                                    'LOG_TIMESTAMP'),
 256                                                ex=ext)
 257     log_file_traffic_gen = os.path.join(
 258         settings.getValue('LOG_DIR'), rename_traffic)
 259     metrics_file = (settings.getValue('LOG_FILE_INFRA_METRICS_PFX') +
 260                     settings.getValue('LOG_TIMESTAMP') + '.log')
 261     log_file_infra_metrics = os.path.join(settings.getValue('LOG_DIR'),
 262                                           metrics_file)
 263
 264     _LOGGER.setLevel(logging.DEBUG)
 265
 266     stream_logger = logging.StreamHandler(sys.stdout)
 267     stream_logger.setLevel(VERBOSITY_LEVELS[level])
 268     stream_logger.setFormatter(logging.Formatter(
 269         '[%(levelname)-5s]  %(asctime)s : (%(name)s) - %(message)s'))
 270     _LOGGER.addHandler(stream_logger)
 271
 272     file_logger = logging.FileHandler(filename=log_file_default)
 273     file_logger.setLevel(logging.DEBUG)
 274     file_logger.setFormatter(logging.Formatter(
 275         '%(asctime)s : %(message)s'))
 276     _LOGGER.addHandler(file_logger)
 277
 278     class CommandFilter(logging.Filter):
 279         """Filter out strings beginning with 'cmd :'"""
 280         def filter(self, record):
 281             return record.getMessage().startswith(tasks.CMD_PREFIX)
 282
 283     class TrafficGenCommandFilter(logging.Filter):
 284         """Filter out strings beginning with 'gencmd :'"""
 285         def filter(self, record):
 286             return record.getMessage().startswith(trafficgen.CMD_PREFIX)
 287
 288     class CollectdMetricsFilter(logging.Filter):
 289         """Filter out strings beginning with 'COLLECTD' :'"""
 290         def filter(self, record):
 291             return record.getMessage().startswith('COLLECTD')
 292
 293     cmd_logger = logging.FileHandler(filename=log_file_host_cmds)
 294     cmd_logger.setLevel(logging.DEBUG)
 295     cmd_logger.addFilter(CommandFilter())
 296     _LOGGER.addHandler(cmd_logger)
 297
 298     gen_logger = logging.FileHandler(filename=log_file_traffic_gen)
 299     gen_logger.setLevel(logging.DEBUG)
 300     gen_logger.addFilter(TrafficGenCommandFilter())
 301     _LOGGER.addHandler(gen_logger)
 302
 303     if settings.getValue('COLLECTOR') == 'Collectd':
 304         met_logger = logging.FileHandler(filename=log_file_infra_metrics)
 305         met_logger.setLevel(logging.DEBUG)
 306         met_logger.addFilter(CollectdMetricsFilter())
 307         _LOGGER.addHandler(met_logger)
 308
 309
 310 def apply_filter(tests, tc_filter):
 311     """Allow a subset of tests to be conveniently selected
 312
 313     :param tests: The list of Tests from which to select.
 314     :param tc_filter: A case-insensitive string of comma-separated terms
 315         indicating the Tests to select.
 316         e.g. 'RFC' - select all tests whose name contains 'RFC'
 317         e.g. 'RFC,burst' - select all tests whose name contains 'RFC' or
 318             'burst'
 319         e.g. 'RFC,burst,!p2p' - select all tests whose name contains 'RFC'
 320             or 'burst' and from these remove any containing 'p2p'.
 321         e.g. '' - empty string selects all tests.
 322     :return: A list of the selected Tests.
 323     """
 324     # if negative filter is first we have to start with full list of tests
 325     if tc_filter.strip()[0] == '!':
 326         result = tests
 327     else:
 328         result = []
 329     if tc_filter is None:
 330         tc_filter = ""
 331
 332     for term in [x.strip() for x in tc_filter.lower().split(",")]:
 333         if not term or term[0] != '!':
 334             # Add matching tests from 'tests' into results
 335             result.extend([test for test in tests \
 336                 if test['Name'].lower().find(term) >= 0])
 337         else:
 338             # Term begins with '!' so we remove matching tests
 339             result = [test for test in result \
 340                 if test['Name'].lower().find(term[1:]) < 0]
 341
 342     return result
 343
 344
 345 def check_and_set_locale():
 346     """ Function will check locale settings. In case, that it isn't configured
 347     properly, then default values specified by DEFAULT_LOCALE will be used.
 348     """
 349
 350     system_locale = locale.getdefaultlocale()
 351     if None in system_locale:
 352         os.environ['LC_ALL'] = settings.getValue('DEFAULT_LOCALE')
 353         _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
 354                         system_locale, locale.getdefaultlocale())
 355
 356 def get_vswitch_names(rst_files):
 357     """ Function will return a list of vSwitches detected in given ``rst_files``.
 358     """
 359     vswitch_names = set()
 360     if rst_files:
 361         try:
 362             output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
 363             for line in output:
 364                 match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
 365                 if match:
 366                     vswitch_names.add(match.group(1))
 367
 368             if vswitch_names:
 369                 return list(vswitch_names)
 370
 371         except subprocess.CalledProcessError:
 372             _LOGGER.warning('Cannot detect vSwitches used during testing.')
 373
 374     # fallback to the default value
 375     return ['vSwitch']
 376
 377 def get_build_tag():
 378     """ Function will return a Jenkins job ID environment variable.
 379     """
 380
 381     try:
 382         build_tag = os.environ['BUILD_TAG']
 383
 384     except KeyError:
 385         _LOGGER.warning('Cannot detect Jenkins job ID')
 386         build_tag = "none"
 387
 388     return build_tag
 389
 390 def generate_final_report():
 391     """ Function will check if partial test results are available
 392     and generates final report in rst format.
 393     """
 394
 395     path = settings.getValue('RESULTS_PATH')
 396     # check if there are any results in rst format
 397     rst_results = glob.glob(os.path.join(path, 'result*rst'))
 398     pkt_processors = get_vswitch_names(rst_results)
 399     if rst_results:
 400         try:
 401             test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))
 402             # create report caption directly - it is not worth to execute jinja machinery
 403             report_caption = '{}\n{} {}\n{}\n\n'.format(
 404                 '============================================================',
 405                 'Performance report for',
 406                 ', '.join(pkt_processors),
 407                 '============================================================')
 408
 409             with open(_TEMPLATE_RST['tmp'], 'w') as file_:
 410                 file_.write(report_caption)
 411
 412             retval = subprocess.call('cat {} {} {} {} > {}'.format(_TEMPLATE_RST['tmp'], _TEMPLATE_RST['head'],
 413                                                                    ' '.join(rst_results), _TEMPLATE_RST['foot'],
 414                                                                    test_report), shell=True)
 415             if retval == 0 and os.path.isfile(test_report):
 416                 _LOGGER.info('Overall test report written to "%s"', test_report)
 417             else:
 418                 _LOGGER.error('Generation of overall test report has failed.')
 419
 420             # remove temporary file
 421             os.remove(_TEMPLATE_RST['tmp'])
 422
 423         except subprocess.CalledProcessError:
 424             _LOGGER.error('Generatrion of overall test report has failed.')
 425
 426
 427 def generate_performance_matrix(selected_tests, results_path):
 428     """
 429     Loads the results of all the currently run tests, compares them
 430     based on the MATRIX_METRIC, outputs and saves the generated table.
 431     :selected_tests: list of currently run test
 432     :results_path: directory path to the results of current tests
 433     """
 434     _LOGGER.info('Performance Matrix:')
 435     test_list = []
 436
 437     for test in selected_tests:
 438         test_name = test.get('Name', '<Name not set>')
 439         test_deployment = test.get('Deployment', '<Deployment not set>')
 440         test_list.append({'test_name':test_name, 'test_deployment':test_deployment, 'csv_data':False})
 441
 442     test_params = {}
 443     output = []
 444     all_params = settings.getValue('_PARAMS_LIST')
 445     for i in range(len(selected_tests)):
 446         test = test_list[i]
 447         if isinstance(all_params, list):
 448             list_index = i
 449             if i >= len(all_params):
 450                 list_index = len(all_params) - 1
 451             if settings.getValue('CUMULATIVE_PARAMS') and (i > 0):
 452                 test_params.update(all_params[list_index])
 453             else:
 454                 test_params = all_params[list_index]
 455         else:
 456             test_params = all_params
 457         settings.setValue('TEST_PARAMS', test_params)
 458         test['test_params'] = copy.deepcopy(test_params)
 459         try:
 460             with open("{}/result_{}_{}_{}.csv".format(results_path, str(i),
 461                                                       test['test_name'], test['test_deployment'])) as csvfile:
 462                 reader = list(csv.DictReader(csvfile))
 463                 test['csv_data'] = reader[0]
 464         # pylint: disable=broad-except
 465         except (Exception) as ex:
 466             _LOGGER.error("Result file not found: %s", ex)
 467
 468     metric = settings.getValue('MATRIX_METRIC')
 469     change = {}
 470     output_header = ("ID", "Name", metric, "Change [%]", "Parameters, "\
 471                      "CUMULATIVE_PARAMS = {}".format(settings.getValue('CUMULATIVE_PARAMS')))
 472     if not test_list[0]['csv_data'] or float(test_list[0]['csv_data'][metric]) == 0:
 473         _LOGGER.error("Incorrect format of test results")
 474         return
 475     for i, test in enumerate(test_list):
 476         if test['csv_data']:
 477             change[i] = float(test['csv_data'][metric])/\
 478                         (float(test_list[0]['csv_data'][metric]) / 100) - 100
 479             output.append([i, test['test_name'], float(test['csv_data'][metric]),
 480                            change[i], str(test['test_params'])[1:-1]])
 481         else:
 482             change[i] = 0
 483             output.append([i, test['test_name'], "Test Failed", 0, test['test_params']])
 484     print(tabulate(output, headers=output_header, tablefmt="grid", floatfmt="0.3f"))
 485     with open(results_path + '/result_performance_matrix.rst', 'w+') as output_file:
 486         output_file.write(_TEMPLATE_MATRIX.format(metric, tabulate(output, headers=output_header,
 487                                                                    tablefmt="rst", floatfmt="0.3f")))
 488         _LOGGER.info('Performance matrix written to: "%s/result_performance_matrix.rst"', results_path)
 489
 490 def enable_sriov(nic_list):
 491     """ Enable SRIOV for given enhanced PCI IDs
 492
 493     :param nic_list: A list of enhanced PCI IDs
 494     """
 495     # detect if sriov is required
 496     sriov_nic = {}
 497     for nic in nic_list:
 498         if networkcard.is_sriov_nic(nic):
 499             tmp_nic = nic.split('|')
 500             if tmp_nic[0] in sriov_nic:
 501                 if int(tmp_nic[1][2:]) > sriov_nic[tmp_nic[0]]:
 502                     sriov_nic[tmp_nic[0]] = int(tmp_nic[1][2:])
 503             else:
 504                 sriov_nic.update({tmp_nic[0] : int(tmp_nic[1][2:])})
 505
 506     # sriov is required for some NICs
 507     if sriov_nic:
 508         for nic in sriov_nic:
 509             # check if SRIOV is supported and enough virt interfaces are available
 510             if not networkcard.is_sriov_supported(nic) \
 511                 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
 512                 # if not, enable and set appropriate number of VFs
 513                 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
 514                     raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))
 515                 else:
 516                     _LOGGER.debug("SRIOV enabled for NIC %s", nic)
 517
 518                 # ensure that path to the bind tool is valid
 519                 functions.settings_update_paths()
 520
 521                 # WORKAROUND: it has been observed with IXGBE(VF) driver,
 522                 # that NIC doesn't correclty dispatch traffic to VFs based
 523                 # on their MAC address. Unbind and bind to the same driver
 524                 # solves this issue.
 525                 networkcard.reinit_vfs(nic)
 526
 527         # After SRIOV is enabled it takes some time until network drivers
 528         # properly initialize all cards.
 529         # Wait also in case, that SRIOV was already configured as it can be
 530         # configured automatically just before vsperf execution.
 531         time.sleep(2)
 532
 533         return True
 534
 535     return False
 536
 537
 538 def disable_sriov(nic_list):
 539     """ Disable SRIOV for given PCI IDs
 540
 541     :param nic_list: A list of enhanced PCI IDs
 542     """
 543     for nic in nic_list:
 544         if networkcard.is_sriov_nic(nic):
 545             if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
 546                 raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))
 547             else:
 548                 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
 549
 550
 551 def handle_list_options(args):
 552     """ Process --list cli arguments if needed
 553
 554     :param args: A dictionary with all CLI arguments
 555     """
 556     if args['list_trafficgens']:
 557         print(Loader().get_trafficgens_printable())
 558         sys.exit(0)
 559
 560     if args['list_collectors']:
 561         print(Loader().get_collectors_printable())
 562         sys.exit(0)
 563
 564     if args['list_vswitches']:
 565         print(Loader().get_vswitches_printable())
 566         sys.exit(0)
 567
 568     if args['list_vnfs']:
 569         print(Loader().get_vnfs_printable())
 570         sys.exit(0)
 571
 572     if args['list_fwdapps']:
 573         print(Loader().get_pktfwds_printable())
 574         sys.exit(0)
 575
 576     if args['list_loadgens']:
 577         print(Loader().get_loadgens_printable())
 578         sys.exit(0)
 579
 580     if args['list_settings']:
 581         print(str(settings))
 582         sys.exit(0)
 583
 584     if args['list']:
 585         list_testcases(args)
 586         sys.exit(0)
 587
 588
 589 def list_testcases(args):
 590     """ Print list of testcases requested by --list CLI argument
 591
 592     :param args: A dictionary with all CLI arguments
 593     """
 594     # configure tests
 595     if args['integration']:
 596         testcases = settings.getValue('INTEGRATION_TESTS')
 597     else:
 598         testcases = settings.getValue('PERFORMANCE_TESTS')
 599
 600     print("Available Tests:")
 601     print("================")
 602
 603     for test in testcases:
 604         description = functions.format_description(test['Description'], 70)
 605         if len(test['Name']) < 40:
 606             print('* {:40} {}'.format('{}:'.format(test['Name']), description[0]))
 607         else:
 608             print('* {}'.format('{}:'.format(test['Name'])))
 609             print('  {:40} {}'.format('', description[0]))
 610         for i in range(1, len(description)):
 611             print('  {:40} {}'.format('', description[i]))
 612
 613
 614 def vsperf_finalize():
 615     """ Clean up before exit
 616     """
 617     # remove directory if no result files were created
 618     try:
 619         results_path = settings.getValue('RESULTS_PATH')
 620         if os.path.exists(results_path):
 621             files_list = os.listdir(results_path)
 622             if files_list == []:
 623                 _LOGGER.info("Removing empty result directory: %s", results_path)
 624                 shutil.rmtree(results_path)
 625     except AttributeError:
 626         # skip it if parameter doesn't exist
 627         pass
 628
 629     # disable SRIOV if needed
 630     try:
 631         if settings.getValue('SRIOV_ENABLED'):
 632             disable_sriov(settings.getValue('WHITELIST_NICS_ORIG'))
 633     except AttributeError:
 634         # skip it if parameter doesn't exist
 635         pass
 636
 637
 638 class MockTestCase(unittest.TestCase):
 639     """Allow use of xmlrunner to generate Jenkins compatible output without
 640     using xmlrunner to actually run tests.
 641
 642     Usage:
 643         suite = unittest.TestSuite()
 644         suite.addTest(MockTestCase('Test1 passed ', True, 'Test1'))
 645         suite.addTest(MockTestCase('Test2 failed because...', False, 'Test2'))
 646         xmlrunner.XMLTestRunner(...).run(suite)
 647     """
 648
 649     def __init__(self, msg, is_pass, test_name):
 650         #remember the things
 651         self.msg = msg
 652         self.is_pass = is_pass
 653
 654         #dynamically create a test method with the right name
 655         #but point the method at our generic test method
 656         setattr(MockTestCase, test_name, self.generic_test)
 657
 658         super(MockTestCase, self).__init__(test_name)
 659
 660     def generic_test(self):
 661         """Provide a generic function that raises or not based
 662         on how self.is_pass was set in the constructor"""
 663         self.assertTrue(self.is_pass, self.msg)
 664
 665 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 666 def main():
 667     """Main function.
 668     """
 669     args = parse_arguments()
 670
 671     # configure settings
 672
 673     settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))
 674
 675     # Define the timestamp to be used by logs and results
 676     date = datetime.datetime.fromtimestamp(time.time())
 677     timestamp = date.strftime('%Y-%m-%d_%H-%M-%S')
 678     settings.setValue('LOG_TIMESTAMP', timestamp)
 679
 680     # Load non performance/integration tests
 681     if args['integration']:
 682         settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))
 683
 684     # load command line parameters first in case there are settings files
 685     # to be used
 686     settings.load_from_dict(args)
 687
 688     if args['conf_file']:
 689         settings.load_from_file(args['conf_file'])
 690
 691     if args['load_env']:
 692         settings.load_from_env()
 693
 694     # reload command line parameters since these should take higher priority
 695     # than both a settings file and environment variables
 696     settings.load_from_dict(args)
 697
 698     settings.setValue('mode', args['mode'])
 699
 700     # update paths to trafficgens if required
 701     if settings.getValue('mode') == 'trafficgen':
 702         functions.settings_update_paths()
 703
 704     # if required, handle list-* operations
 705     handle_list_options(args)
 706
 707     configure_logging(settings.getValue('VERBOSITY'))
 708
 709     # check and fix locale
 710     check_and_set_locale()
 711
 712     # configure trafficgens
 713     if args['trafficgen']:
 714         trafficgens = Loader().get_trafficgens()
 715         if args['trafficgen'] not in trafficgens:
 716             _LOGGER.error('There are no trafficgens matching \'%s\' found in'
 717                           ' \'%s\'. Exiting...', args['trafficgen'],
 718                           settings.getValue('TRAFFICGEN_DIR'))
 719             sys.exit(1)
 720
 721     # configuration validity checks
 722     if args['vswitch']:
 723         vswitch_none = args['vswitch'].strip().lower() == 'none'
 724         if vswitch_none:
 725             settings.setValue('VSWITCH', 'none')
 726         else:
 727             vswitches = Loader().get_vswitches()
 728             if args['vswitch'] not in vswitches:
 729                 _LOGGER.error('There are no vswitches matching \'%s\' found in'
 730                               ' \'%s\'. Exiting...', args['vswitch'],
 731                               settings.getValue('VSWITCH_DIR'))
 732                 sys.exit(1)
 733
 734     if args['fwdapp']:
 735         settings.setValue('PKTFWD', args['fwdapp'])
 736         fwdapps = Loader().get_pktfwds()
 737         if args['fwdapp'] not in fwdapps:
 738             _LOGGER.error('There are no forwarding application'
 739                           ' matching \'%s\' found in'
 740                           ' \'%s\'. Exiting...', args['fwdapp'],
 741                           settings.getValue('PKTFWD_DIR'))
 742             sys.exit(1)
 743
 744     if args['vnf']:
 745         vnfs = Loader().get_vnfs()
 746         if args['vnf'] not in vnfs:
 747             _LOGGER.error('there are no vnfs matching \'%s\' found in'
 748                           ' \'%s\'. exiting...', args['vnf'],
 749                           settings.getValue('VNF_DIR'))
 750             sys.exit(1)
 751
 752     if args['loadgen']:
 753         loadgens = Loader().get_loadgens()
 754         if args['loadgen'] not in loadgens:
 755             _LOGGER.error('There are no loadgens matching \'%s\' found in'
 756                           ' \'%s\'. Exiting...', args['loadgen'],
 757                           settings.getValue('LOADGEN_DIR'))
 758             sys.exit(1)
 759
 760     if args['exact_test_name'] and args['tests']:
 761         _LOGGER.error("Cannot specify tests with both positional args and --test.")
 762         sys.exit(1)
 763
 764     # modify NIC configuration to decode enhanced PCI IDs
 765     wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
 766     settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
 767
 768     # sriov handling is performed on checked/expanded PCI IDs
 769     settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
 770
 771     nic_list = []
 772     for nic in wl_nics_orig:
 773         tmp_nic = networkcard.get_nic_info(nic)
 774         if tmp_nic:
 775             nic_list.append({'pci' : tmp_nic,
 776                              'type' : 'vf' if networkcard.get_sriov_pf(tmp_nic) else 'pf',
 777                              'mac' : networkcard.get_mac(tmp_nic),
 778                              'driver' : networkcard.get_driver(tmp_nic),
 779                              'device' : networkcard.get_device_name(tmp_nic)})
 780         else:
 781             vsperf_finalize()
 782             raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))
 783
 784     settings.setValue('NICS', nic_list)
 785     # for backward compatibility
 786     settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
 787
 788     # generate results directory name
 789     # date = datetime.datetime.fromtimestamp(time.time())
 790     results_dir = "results_" + timestamp
 791     results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir)
 792     settings.setValue('RESULTS_PATH', results_path)
 793
 794     # create results directory
 795     if not os.path.exists(results_path):
 796         _LOGGER.info("Creating result directory: %s", results_path)
 797         os.makedirs(results_path)
 798     # pylint: disable=too-many-nested-blocks
 799     if settings.getValue('mode') == 'trafficgen':
 800         # execute only traffic generator
 801         _LOGGER.debug("Executing traffic generator:")
 802         loader = Loader()
 803         # set traffic details, so they can be passed to traffic ctl
 804         traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
 805         traffic = functions.check_traffic(traffic)
 806
 807         traffic_ctl = component_factory.create_traffic(
 808             traffic['traffic_type'],
 809             loader.get_trafficgen_class())
 810         with traffic_ctl:
 811             traffic_ctl.send_traffic(traffic)
 812         _LOGGER.debug("Traffic Results:")
 813         traffic_ctl.print_results()
 814
 815         # write results into CSV file
 816         result_file = os.path.join(results_path, "result.csv")
 817         PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)
 818     else:
 819         # configure tests
 820         if args['integration']:
 821             testcases = settings.getValue('INTEGRATION_TESTS')
 822         else:
 823             testcases = settings.getValue('PERFORMANCE_TESTS')
 824
 825         if args['exact_test_name']:
 826             exact_names = args['exact_test_name']
 827             # positional args => exact matches only
 828             selected_tests = []
 829             for test_name in exact_names:
 830                 for test in testcases:
 831                     if test['Name'] == test_name:
 832                         selected_tests.append(test)
 833         elif args['tests']:
 834             # --tests => apply filter to select requested tests
 835             selected_tests = apply_filter(testcases, args['tests'])
 836         else:
 837             # Default - run all tests
 838             selected_tests = testcases
 839
 840         if not selected_tests:
 841             _LOGGER.error("No tests matched --tests option or positional args. Done.")
 842             vsperf_finalize()
 843             sys.exit(1)
 844
 845         suite = unittest.TestSuite()
 846         settings_snapshot = copy.deepcopy(settings.__dict__)
 847
 848         for i, cfg in enumerate(selected_tests):
 849             settings.setValue('_TEST_INDEX', i)
 850             test_name = cfg.get('Name', '<Name not set>')
 851             try:
 852                 test_params = settings.getValue('_PARAMS_LIST')
 853                 if isinstance(test_params, list):
 854                     list_index = i
 855                     if i >= len(test_params):
 856                         list_index = len(test_params) - 1
 857                     test_params = test_params[list_index]
 858                 if settings.getValue('CUMULATIVE_PARAMS'):
 859                     test_params = merge_spec(settings.getValue('TEST_PARAMS'), test_params)
 860                 settings.setValue('TEST_PARAMS', test_params)
 861
 862                 if args['integration']:
 863                     test = IntegrationTestCase(cfg)
 864                 else:
 865                     test = PerformanceTestCase(cfg)
 866
 867                 test.run()
 868                 suite.addTest(MockTestCase('', True, test.name))
 869
 870             # pylint: disable=broad-except
 871             except (Exception) as ex:
 872                 _LOGGER.exception("Failed to run test: %s", test_name)
 873                 suite.addTest(MockTestCase(str(ex), False, test_name))
 874                 _LOGGER.info("Continuing with next test...")
 875             finally:
 876                 if not settings.getValue('CUMULATIVE_PARAMS'):
 877                     settings.restore_from_dict(settings_snapshot)
 878
 879         settings.restore_from_dict(settings_snapshot)
 880
 881
 882         # Generate and printout Performance Matrix
 883         if args['matrix']:
 884             generate_performance_matrix(selected_tests, results_path)
 885
 886         # generate final rst report with results of all executed TCs
 887         generate_final_report()
 888
 889
 890
 891         if settings.getValue('XUNIT'):
 892             xmlrunner.XMLTestRunner(
 893                 output=settings.getValue('XUNIT_DIR'), outsuffix="",
 894                 verbosity=0).run(suite)
 895
 896         if args['opnfvpod']:
 897             pod_name = args['opnfvpod']
 898             installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()
 899             opnfv_url = settings.getValue('OPNFV_URL')
 900             pkg_list = settings.getValue('PACKAGE_LIST')
 901
 902             int_data = {'pod': pod_name,
 903                         'build_tag': get_build_tag(),
 904                         'installer': installer_name,
 905                         'pkg_list': pkg_list,
 906                         'db_url': opnfv_url,
 907                         # pass vswitch name from configuration to be used for failed
 908                         # TCs; In case of successful TCs it is safer to use vswitch
 909                         # name from CSV as TC can override global configuration
 910                         'vswitch': str(settings.getValue('VSWITCH')).lower()}
 911             tc_names = [tc['Name'] for tc in selected_tests]
 912             opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
 913
 914     # cleanup before exit
 915     vsperf_finalize()
 916
 917 if __name__ == "__main__":
 918     main()