Merge "paths: Modify algorithm for PATHS verification"

[vswitchperf.git] / vsperf

diff --git a/vsperf b/vsperf
index 57d6899..6f248f8 100755 (executable)
--- a/vsperf
+++ b/vsperf
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
 #!/usr/bin/env python3
 

-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.

 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.


@@ -21,29 +21,29 @@ import logging
 import os
 import sys
 import argparse
 import os
 import sys
 import argparse

+import re

 import time
 import datetime
 import shutil
 import unittest
 import time
 import datetime
 import shutil
 import unittest

-import xmlrunner

 import locale
 import copy
 import glob
 import subprocess
 import locale
 import copy
 import glob
 import subprocess

-
-sys.dont_write_bytecode = True
-
+import ast
+import xmlrunner

 from conf import settings
 from conf import settings

-from conf import get_test_param
+import core.component_factory as component_factory

 from core.loader import Loader
 from testcases import PerformanceTestCase
 from testcases import IntegrationTestCase
 from tools import tasks
 from tools import networkcard
 from core.loader import Loader
 from testcases import PerformanceTestCase
 from testcases import IntegrationTestCase
 from tools import tasks
 from tools import networkcard

+from tools import functions

 from tools.pkt_gen import trafficgen
 from tools.opnfvdashboard import opnfvdashboard
 from tools.pkt_gen import trafficgen
 from tools.opnfvdashboard import opnfvdashboard

-from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
-import core.component_factory as component_factory
+
+sys.dont_write_bytecode = True

 
 VERBOSITY_LEVELS = {
     'debug': logging.DEBUG,
 
 VERBOSITY_LEVELS = {
     'debug': logging.DEBUG,


@@ -53,12 +53,15 @@ VERBOSITY_LEVELS = {
     'critical': logging.CRITICAL
 }
 
     'critical': logging.CRITICAL
 }
 

-_TEMPLATE_RST = {'head'  : 'tools/report/report_head.rst',
-                 'foot'  : 'tools/report/report_foot.rst',
+_CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+
+_TEMPLATE_RST = {'head'  : os.path.join(_CURR_DIR, 'tools/report/report_head.rst'),
+                 'foot'  : os.path.join(_CURR_DIR, 'tools/report/report_foot.rst'),

                  'final' : 'test_report.rst',
                  'final' : 'test_report.rst',

-                 'tmp'   : 'tools/report/report_tmp_caption.rst'
+                 'tmp'   : os.path.join(_CURR_DIR, 'tools/report/report_tmp_caption.rst')

                 }
 
                 }
 

+

 _LOGGER = logging.getLogger()
 
 def parse_arguments():
 _LOGGER = logging.getLogger()
 
 def parse_arguments():


@@ -71,21 +74,26 @@ def parse_arguments():
 
         This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
         values. For multiple overrides use a ; separated list for
 
         This expects either 'x=y', 'x=y,z' or 'x' (implicit true)
         values. For multiple overrides use a ; separated list for

-        e.g. --test-params 'x=z; y=a,b'
+        e.g. --test-params 'x=z; y=(a,b)'

         """
         def __call__(self, parser, namespace, values, option_string=None):
             results = {}
 
         """
         def __call__(self, parser, namespace, values, option_string=None):
             results = {}
 

-            for value in values.split(';'):
-                result = [key.strip() for key in value.split('=')]
-                if len(result) == 1:
-                    results[result[0]] = True
-                elif len(result) == 2:
-                    results[result[0]] = result[1]
-                else:
-                    raise argparse.ArgumentTypeError(
-                        'expected \'%s\' to be of format \'key=val\' or'
-                        ' \'key\'' % result)
+            for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values):
+                param = param.strip()
+                value = value.strip()
+                if len(param):
+                    if len(value):
+                        # values are passed inside string from CLI, so we must retype them accordingly
+                        try:
+                            results[param] = ast.literal_eval(value)
+                        except ValueError:
+                            # for backward compatibility, we have to accept strings without quotes
+                            _LOGGER.warning("Adding missing quotes around string value: %s = %s",
+                                            param, str(value))
+                            results[param] = str(value)
+                    else:
+                        results[param] = True

 
             setattr(namespace, self.dest, results)
 
 
             setattr(namespace, self.dest, results)
 


@@ -139,6 +147,8 @@ def parse_arguments():
                         help='list all system forwarding applications and exit')
     parser.add_argument('--list-vnfs', action='store_true',
                         help='list all system vnfs and exit')
                         help='list all system forwarding applications and exit')
     parser.add_argument('--list-vnfs', action='store_true',
                         help='list all system vnfs and exit')

+    parser.add_argument('--list-loadgens', action='store_true',
+                        help='list all background load generators')

     parser.add_argument('--list-settings', action='store_true',
                         help='list effective settings configuration and exit')
     parser.add_argument('exact_test_name', nargs='*', help='Exact names of\
     parser.add_argument('--list-settings', action='store_true',
                         help='list effective settings configuration and exit')
     parser.add_argument('exact_test_name', nargs='*', help='Exact names of\


@@ -157,7 +167,8 @@ def parse_arguments():
     group.add_argument('-d', '--test-dir', help='directory containing tests')
     group.add_argument('-t', '--tests', help='Comma-separated list of terms \
             indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
     group.add_argument('-d', '--test-dir', help='directory containing tests')
     group.add_argument('-t', '--tests', help='Comma-separated list of terms \
             indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\

-            name contains RFC2544 less those containing "p2p"')
+            name contains RFC2544 less those containing "p2p"; "!back2back" - \
+            run all tests except those containing back2back')

     group.add_argument('--verbosity', choices=list_logging_levels(),
                        help='debug level')
     group.add_argument('--integration', action='store_true', help='execute integration tests')
     group.add_argument('--verbosity', choices=list_logging_levels(),
                        help='debug level')
     group.add_argument('--integration', action='store_true', help='execute integration tests')


@@ -165,6 +176,7 @@ def parse_arguments():
     group.add_argument('--vswitch', help='vswitch implementation to use')
     group.add_argument('--fwdapp', help='packet forwarding application to use')
     group.add_argument('--vnf', help='vnf to use')
     group.add_argument('--vswitch', help='vswitch implementation to use')
     group.add_argument('--fwdapp', help='packet forwarding application to use')
     group.add_argument('--vnf', help='vnf to use')

+    group.add_argument('--loadgen', help='loadgen to use')

     group.add_argument('--sysmetrics', help='system metrics logger to use')
     group = parser.add_argument_group('test behavior options')
     group.add_argument('--xunit', action='store_true',
     group.add_argument('--sysmetrics', help='system metrics logger to use')
     group = parser.add_argument_group('test behavior options')
     group.add_argument('--xunit', action='store_true',


@@ -176,9 +188,9 @@ def parse_arguments():
     group.add_argument('--conf-file', action=_ValidateFileAction,
                        help='settings file')
     group.add_argument('--test-params', action=_SplitTestParamsAction,
     group.add_argument('--conf-file', action=_ValidateFileAction,
                        help='settings file')
     group.add_argument('--test-params', action=_SplitTestParamsAction,

-                       help='csv list of test parameters: key=val; e.g.'
-                       'including pkt_sizes=x,y; duration=x; '
-                       'rfc2544_trials=x ...')
+                       help='csv list of test parameters: key=val; e.g. '
+                       'TRAFFICGEN_PKT_SIZES=(64,128);TRAFICGEN_DURATION=30; '
+                       'GUEST_LOOPBACK=["l2fwd"] ...')

     group.add_argument('--opnfvpod', help='name of POD in opnfv')
 
     args = vars(parser.parse_args())
     group.add_argument('--opnfvpod', help='name of POD in opnfv')
 
     args = vars(parser.parse_args())


@@ -244,7 +256,11 @@ def apply_filter(tests, tc_filter):
         e.g. '' - empty string selects all tests.
     :return: A list of the selected Tests.
     """
         e.g. '' - empty string selects all tests.
     :return: A list of the selected Tests.
     """

-    result = []
+    # if negative filter is first we have to start with full list of tests
+    if tc_filter.strip()[0] == '!':
+        result = tests
+    else:
+        result = []

     if tc_filter is None:
         tc_filter = ""
 
     if tc_filter is None:
         tc_filter = ""
 


@@ -252,11 +268,11 @@ def apply_filter(tests, tc_filter):
         if not term or term[0] != '!':
             # Add matching tests from 'tests' into results
             result.extend([test for test in tests \
         if not term or term[0] != '!':
             # Add matching tests from 'tests' into results
             result.extend([test for test in tests \

-                if test.name.lower().find(term) >= 0])
+                if test['Name'].lower().find(term) >= 0])

         else:
             # Term begins with '!' so we remove matching tests
             result = [test for test in result \
         else:
             # Term begins with '!' so we remove matching tests
             result = [test for test in result \

-                if test.name.lower().find(term[1:]) < 0]
+                if test['Name'].lower().find(term[1:]) < 0]

 
     return result
 
 
     return result
 


@@ -272,6 +288,39 @@ def check_and_set_locale():
         _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
                         system_locale, locale.getdefaultlocale())
 
         _LOGGER.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
                         system_locale, locale.getdefaultlocale())
 

+def get_vswitch_names(rst_files):
+    """ Function will return a list of vSwitches detected in given ``rst_files``.
+    """
+    vswitch_names = set()
+    if len(rst_files):
+        try:
+            output = subprocess.check_output(['grep', '-h', '^* vSwitch'] + rst_files).decode().splitlines()
+            for line in output:
+                match = re.search(r'^\* vSwitch: ([^,]+)', str(line))
+                if match:
+                    vswitch_names.add(match.group(1))
+
+            if len(vswitch_names):
+                return list(vswitch_names)
+
+        except subprocess.CalledProcessError:
+            _LOGGER.warning('Cannot detect vSwitches used during testing.')
+
+    # fallback to the default value
+    return ['vSwitch']
+
+def get_build_tag():
+    """ Function will return a Jenkins job ID environment variable.
+    """
+
+    try:
+        build_tag = os.environ['BUILD_TAG']
+
+    except KeyError:
+        _LOGGER.warning('Cannot detect Jenkins job ID')
+        build_tag = "none"
+
+    return build_tag

 
 def generate_final_report():
     """ Function will check if partial test results are available
 
 def generate_final_report():
     """ Function will check if partial test results are available


@@ -281,18 +330,15 @@ def generate_final_report():
     path = settings.getValue('RESULTS_PATH')
     # check if there are any results in rst format
     rst_results = glob.glob(os.path.join(path, 'result*rst'))
     path = settings.getValue('RESULTS_PATH')
     # check if there are any results in rst format
     rst_results = glob.glob(os.path.join(path, 'result*rst'))

+    pkt_processors = get_vswitch_names(rst_results)

     if len(rst_results):
         try:
     if len(rst_results):
         try:

-            test_report = os.path.join(path, '{}_{}'.format(settings.getValue('VSWITCH'), _TEMPLATE_RST['final']))
+            test_report = os.path.join(path, '{}_{}'.format('_'.join(pkt_processors), _TEMPLATE_RST['final']))

             # create report caption directly - it is not worth to execute jinja machinery
             # create report caption directly - it is not worth to execute jinja machinery

-            if settings.getValue('VSWITCH').lower() != 'none':
-                pkt_processor = Loader().get_vswitches()[settings.getValue('VSWITCH')].__doc__.strip().split('\n')[0]
-            else:
-                pkt_processor = Loader().get_pktfwds()[settings.getValue('PKTFWD')].__doc__.strip().split('\n')[0]

             report_caption = '{}\n{} {}\n{}\n\n'.format(
                 '============================================================',
                 'Performance report for',
             report_caption = '{}\n{} {}\n{}\n\n'.format(
                 '============================================================',
                 'Performance report for',

-                pkt_processor,
+                ', '.join(pkt_processors),

                 '============================================================')
 
             with open(_TEMPLATE_RST['tmp'], 'w') as file_:
                 '============================================================')
 
             with open(_TEMPLATE_RST['tmp'], 'w') as file_:


@@ -304,7 +350,7 @@ def generate_final_report():
             if retval == 0 and os.path.isfile(test_report):
                 _LOGGER.info('Overall test report written to "%s"', test_report)
             else:
             if retval == 0 and os.path.isfile(test_report):
                 _LOGGER.info('Overall test report written to "%s"', test_report)
             else:

-                _LOGGER.error('Generatrion of overall test report has failed.')
+                _LOGGER.error('Generation of overall test report has failed.')

 
             # remove temporary file
             os.remove(_TEMPLATE_RST['tmp'])
 
             # remove temporary file
             os.remove(_TEMPLATE_RST['tmp'])


@@ -337,8 +383,7 @@ def enable_sriov(nic_list):
                 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
                 # if not, enable and set appropriate number of VFs
                 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):
                 or networkcard.get_sriov_numvfs(nic) <= sriov_nic[nic]:
                 # if not, enable and set appropriate number of VFs
                 if not networkcard.set_sriov_numvfs(nic, sriov_nic[nic] + 1):

-                    _LOGGER.error("SRIOV cannot be enabled for NIC %s", nic)
-                    raise
+                    raise RuntimeError('SRIOV cannot be enabled for NIC {}'.format(nic))

                 else:
                     _LOGGER.debug("SRIOV enabled for NIC %s", nic)
 
                 else:
                     _LOGGER.debug("SRIOV enabled for NIC %s", nic)
 


@@ -367,8 +412,7 @@ def disable_sriov(nic_list):
     for nic in nic_list:
         if networkcard.is_sriov_nic(nic):
             if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):
     for nic in nic_list:
         if networkcard.is_sriov_nic(nic):
             if not networkcard.set_sriov_numvfs(nic.split('|')[0], 0):

-                _LOGGER.error("SRIOV cannot be disabled for NIC %s", nic)
-                raise
+                raise RuntimeError('SRIOV cannot be disabled for NIC {}'.format(nic))

             else:
                 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
 
             else:
                 _LOGGER.debug("SRIOV disabled for NIC %s", nic.split('|')[0])
 


@@ -398,23 +442,45 @@ def handle_list_options(args):
         print(Loader().get_pktfwds_printable())
         sys.exit(0)
 
         print(Loader().get_pktfwds_printable())
         sys.exit(0)
 

+    if args['list_loadgens']:
+        print(Loader().get_loadgens_printable())
+        sys.exit(0)
+

     if args['list_settings']:
         print(str(settings))
         sys.exit(0)
 
     if args['list']:
     if args['list_settings']:
         print(str(settings))
         sys.exit(0)
 
     if args['list']:

-        # configure tests
-        if args['integration']:
-            testcases = settings.getValue('INTEGRATION_TESTS')
+        list_testcases(args)
+        sys.exit(0)
+
+
+def list_testcases(args):
+    """ Print list of testcases requested by --list CLI argument
+
+    :param args: A dictionary with all CLI arguments
+    """
+    # configure tests
+    if args['integration']:
+        testcases = settings.getValue('INTEGRATION_TESTS')
+    else:
+        testcases = settings.getValue('PERFORMANCE_TESTS')
+
+    print("Available Tests:")
+    print("================")
+
+    for test in testcases:
+        description = functions.format_description(test['Description'], 70)
+        if len(test['Name']) < 40:
+            print('* {:40} {}'.format('{}:'.format(test['Name']), description[0]))

         else:
         else:

-            testcases = settings.getValue('PERFORMANCE_TESTS')
+            print('* {}'.format('{}:'.format(test['Name'])))
+            print('  {:40} {}'.format('', description[0]))
+        for i in range(1, len(description)):
+            print('  {:40} {}'.format('', description[i]))
+

 
 

-        print("Available Tests:")
-        print("================")

 
 

-        for test in testcases:
-            print('* %-30s %s' % ('%s:' % test['Name'], test['Description']))
-        sys.exit(0)

 
 
 def vsperf_finalize():
 
 
 def vsperf_finalize():


@@ -468,7 +534,7 @@ class MockTestCase(unittest.TestCase):
         on how self.is_pass was set in the constructor"""
         self.assertTrue(self.is_pass, self.msg)
 
         on how self.is_pass was set in the constructor"""
         self.assertTrue(self.is_pass, self.msg)
 

-
+# pylint: disable=too-many-locals, too-many-branches, too-many-statements

 def main():
     """Main function.
     """
 def main():
     """Main function.
     """


@@ -476,11 +542,11 @@ def main():
 
     # configure settings
 
 
     # configure settings
 

-    settings.load_from_dir('conf')
+    settings.load_from_dir(os.path.join(_CURR_DIR, 'conf'))

 
     # Load non performance/integration tests
     if args['integration']:
 
     # Load non performance/integration tests
     if args['integration']:

-        settings.load_from_dir('conf/integration')
+        settings.load_from_dir(os.path.join(_CURR_DIR, 'conf/integration'))

 
     # load command line parameters first in case there are settings files
     # to be used
 
     # load command line parameters first in case there are settings files
     # to be used


@@ -496,26 +562,11 @@ def main():
     # than both a settings file and environment variables
     settings.load_from_dict(args)
 
     # than both a settings file and environment variables
     settings.load_from_dict(args)
 

-    vswitch_none = False
-    # set dpdk and ovs paths accorfing to VNF and VSWITCH
-    if settings.getValue('VSWITCH').endswith('Vanilla'):
-        # settings paths for Vanilla
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
-    elif settings.getValue('VSWITCH').endswith('Vhost'):
-        if settings.getValue('VNF').endswith('Cuse'):
-            # settings paths for Cuse
-            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
-            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
-        else:
-            # settings paths for VhostUser
-            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
-            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
-    else:
-        # default - set to VHOST USER but can be changed during enhancement
-        settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
-        if 'none' == settings.getValue('VSWITCH').strip().lower():
-            vswitch_none = True
+    settings.setValue('mode', args['mode'])
+
+    # update paths to trafficgens if required
+    if settings.getValue('mode') == 'trafficgen':
+        functions.settings_update_paths()

 
     # if required, handle list-* operations
     handle_list_options(args)
 
     # if required, handle list-* operations
     handle_list_options(args)


@@ -536,7 +587,7 @@ def main():
 
     # configuration validity checks
     if args['vswitch']:
 
     # configuration validity checks
     if args['vswitch']:

-        vswitch_none = 'none' == args['vswitch'].strip().lower()
+        vswitch_none = args['vswitch'].strip().lower() == 'none'

         if vswitch_none:
             settings.setValue('VSWITCH', 'none')
         else:
         if vswitch_none:
             settings.setValue('VSWITCH', 'none')
         else:


@@ -562,20 +613,28 @@ def main():
         if args['vnf'] not in vnfs:
             _LOGGER.error('there are no vnfs matching \'%s\' found in'
                           ' \'%s\'. exiting...', args['vnf'],
         if args['vnf'] not in vnfs:
             _LOGGER.error('there are no vnfs matching \'%s\' found in'
                           ' \'%s\'. exiting...', args['vnf'],

-                          settings.getValue('vnf_dir'))
+                          settings.getValue('VNF_DIR'))
+            sys.exit(1)
+
+    if args['loadgen']:
+        loadgens = Loader().get_loadgens()
+        if args['loadgen'] not in loadgens:
+            _LOGGER.error('There are no loadgens matching \'%s\' found in'
+                          ' \'%s\'. Exiting...', args['loadgen'],
+                          settings.getValue('LOADGEN_DIR'))

             sys.exit(1)
 
     if args['exact_test_name'] and args['tests']:
         _LOGGER.error("Cannot specify tests with both positional args and --test.")
         sys.exit(1)
 
             sys.exit(1)
 
     if args['exact_test_name'] and args['tests']:
         _LOGGER.error("Cannot specify tests with both positional args and --test.")
         sys.exit(1)
 

-    # sriov handling
-    settings.setValue('SRIOV_ENABLED', enable_sriov(settings.getValue('WHITELIST_NICS')))
-

     # modify NIC configuration to decode enhanced PCI IDs
     wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
     settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
 
     # modify NIC configuration to decode enhanced PCI IDs
     wl_nics_orig = list(networkcard.check_pci(pci) for pci in settings.getValue('WHITELIST_NICS'))
     settings.setValue('WHITELIST_NICS_ORIG', wl_nics_orig)
 

+    # sriov handling is performed on checked/expanded PCI IDs
+    settings.setValue('SRIOV_ENABLED', enable_sriov(wl_nics_orig))
+

     nic_list = []
     for nic in wl_nics_orig:
         tmp_nic = networkcard.get_nic_info(nic)
     nic_list = []
     for nic in wl_nics_orig:
         tmp_nic = networkcard.get_nic_info(nic)


@@ -586,24 +645,13 @@ def main():
                              'driver' : networkcard.get_driver(tmp_nic),
                              'device' : networkcard.get_device_name(tmp_nic)})
         else:
                              'driver' : networkcard.get_driver(tmp_nic),
                              'device' : networkcard.get_device_name(tmp_nic)})
         else:

-            _LOGGER.error("Invalid network card PCI ID: '%s'", nic)

             vsperf_finalize()
             vsperf_finalize()

-            raise
+            raise RuntimeError("Invalid network card PCI ID: '{}'".format(nic))

 
     settings.setValue('NICS', nic_list)
     # for backward compatibility
     settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
 
 
     settings.setValue('NICS', nic_list)
     # for backward compatibility
     settings.setValue('WHITELIST_NICS', list(nic['pci'] for nic in nic_list))
 

-    # update global settings
-    guest_loopback = get_test_param('guest_loopback', None)
-    if guest_loopback:
-        tmp_gl = []
-        for dummy_i in range(len(settings.getValue('GUEST_LOOPBACK'))):
-            tmp_gl.append(guest_loopback)
-        settings.setValue('GUEST_LOOPBACK', tmp_gl)
-
-    settings.setValue('mode', args['mode'])
-

     # generate results directory name
     date = datetime.datetime.fromtimestamp(time.time())
     results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')
     # generate results directory name
     date = datetime.datetime.fromtimestamp(time.time())
     results_dir = "results_" + date.strftime('%Y-%m-%d_%H-%M-%S')


@@ -620,12 +668,9 @@ def main():
         _LOGGER.debug("Executing traffic generator:")
         loader = Loader()
         # set traffic details, so they can be passed to traffic ctl
         _LOGGER.debug("Executing traffic generator:")
         loader = Loader()
         # set traffic details, so they can be passed to traffic ctl

-        traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
-        traffic.update({'traffic_type': get_test_param('traffic_type', 'rfc2544'),
-                        'bidir': get_test_param('bidirectional', False),
-                        'multistream': int(get_test_param('multistream', 0)),
-                        'stream_type': get_test_param('stream_type', 'L4'),
-                        'frame_rate': int(get_test_param('iload', 100))})
+        traffic = copy.deepcopy(settings.getValue('TRAFFIC'))
+
+        traffic = functions.check_traffic(traffic)

 
         traffic_ctl = component_factory.create_traffic(
             traffic['traffic_type'],
 
         traffic_ctl = component_factory.create_traffic(
             traffic['traffic_type'],


@@ -634,6 +679,10 @@ def main():
             traffic_ctl.send_traffic(traffic)
         _LOGGER.debug("Traffic Results:")
         traffic_ctl.print_results()
             traffic_ctl.send_traffic(traffic)
         _LOGGER.debug("Traffic Results:")
         traffic_ctl.print_results()

+
+        # write results into CSV file
+        result_file = os.path.join(results_path, "result.csv")
+        PerformanceTestCase.write_result_to_file(traffic_ctl.get_results(), result_file)

     else:
         # configure tests
         if args['integration']:
     else:
         # configure tests
         if args['integration']:


@@ -641,47 +690,44 @@ def main():
         else:
             testcases = settings.getValue('PERFORMANCE_TESTS')
 
         else:
             testcases = settings.getValue('PERFORMANCE_TESTS')
 

-        all_tests = []
-        for cfg in testcases:
-            try:
-                if args['integration']:
-                    all_tests.append(IntegrationTestCase(cfg))
-                else:
-                    all_tests.append(PerformanceTestCase(cfg))
-            except (Exception) as _:
-                _LOGGER.exception("Failed to create test: %s",
-                                  cfg.get('Name', '<Name not set>'))
-                vsperf_finalize()
-                raise
-
-        # select requested tests

         if args['exact_test_name']:
             exact_names = args['exact_test_name']
             # positional args => exact matches only
         if args['exact_test_name']:
             exact_names = args['exact_test_name']
             # positional args => exact matches only

-            selected_tests = [test for test in all_tests if test.name in exact_names]
+            selected_tests = [test for test in testcases if test['Name'] in exact_names]

         elif args['tests']:
             # --tests => apply filter to select requested tests
         elif args['tests']:
             # --tests => apply filter to select requested tests

-            selected_tests = apply_filter(all_tests, args['tests'])
+            selected_tests = apply_filter(testcases, args['tests'])

         else:
             # Default - run all tests
         else:
             # Default - run all tests

-            selected_tests = all_tests
+            selected_tests = testcases

 
 

-        if not selected_tests:
-            _LOGGER.error("No tests matched --test option or positional args. Done.")
+        if not len(selected_tests):
+            _LOGGER.error("No tests matched --tests option or positional args. Done.")

             vsperf_finalize()
             sys.exit(1)
 
         # run tests
             vsperf_finalize()
             sys.exit(1)
 
         # run tests

+        # Add pylint exception: Redefinition of test type from
+        # testcases.integration.IntegrationTestCase to testcases.performance.PerformanceTestCase
+        # pylint: disable=redefined-variable-type

         suite = unittest.TestSuite()
         suite = unittest.TestSuite()

-        for test in selected_tests:
+        settings_snapshot = copy.deepcopy(settings.__dict__)
+        for cfg in selected_tests:
+            test_name = cfg.get('Name', '<Name not set>')

             try:
             try:

+                if args['integration']:
+                    test = IntegrationTestCase(cfg)
+                else:
+                    test = PerformanceTestCase(cfg)

                 test.run()
                 suite.addTest(MockTestCase('', True, test.name))
                 test.run()
                 suite.addTest(MockTestCase('', True, test.name))

-            #pylint: disable=broad-except
+            # pylint: disable=broad-except

             except (Exception) as ex:
             except (Exception) as ex:

-                _LOGGER.exception("Failed to run test: %s", test.name)
-                suite.addTest(MockTestCase(str(ex), False, test.name))
+                _LOGGER.exception("Failed to run test: %s", test_name)
+                suite.addTest(MockTestCase(str(ex), False, test_name))

                 _LOGGER.info("Continuing with next test...")
                 _LOGGER.info("Continuing with next test...")

+            finally:
+                settings.restore_from_dict(settings_snapshot)

 
         # generate final rst report with results of all executed TCs
         generate_final_report()
 
         # generate final rst report with results of all executed TCs
         generate_final_report()


@@ -693,25 +739,24 @@ def main():
 
         if args['opnfvpod']:
             pod_name = args['opnfvpod']
 
         if args['opnfvpod']:
             pod_name = args['opnfvpod']

-            installer_name = settings.getValue('OPNFV_INSTALLER')
+            installer_name = str(settings.getValue('OPNFV_INSTALLER')).lower()

             opnfv_url = settings.getValue('OPNFV_URL')
             pkg_list = settings.getValue('PACKAGE_LIST')
 
             opnfv_url = settings.getValue('OPNFV_URL')
             pkg_list = settings.getValue('PACKAGE_LIST')
 

-            int_data = {'cuse': False,
-                        'vanilla': False,
-                        'pod': pod_name,
+            int_data = {'pod': pod_name,
+                        'build_tag': get_build_tag(),

                         'installer': installer_name,
                         'pkg_list': pkg_list,
                         'installer': installer_name,
                         'pkg_list': pkg_list,

-                        'db_url': opnfv_url}
-            if settings.getValue('VSWITCH').endswith('Vanilla'):
-                int_data['vanilla'] = True
-            if settings.getValue('VNF').endswith('Cuse'):
-                int_data['cuse'] = True
-            opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
+                        'db_url': opnfv_url,
+                        # pass vswitch name from configuration to be used for failed
+                        # TCs; In case of successful TCs it is safer to use vswitch
+                        # name from CSV as TC can override global configuration
+                        'vswitch': str(settings.getValue('VSWITCH')).lower()}
+            tc_names = [tc['Name'] for tc in selected_tests]
+            opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)

 
     # cleanup before exit
     vsperf_finalize()
 
 if __name__ == "__main__":
     main()
 
     # cleanup before exit
     vsperf_finalize()
 
 if __name__ == "__main__":
     main()

-