testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import get_test_param, merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39
  40 CHECK_PREFIX = 'validate_'
  41
  42 # pylint: disable=too-many-instance-attributes
  43 class TestCase(object):
  44     """TestCase base class
  45
  46     In this basic form runs RFC2544 throughput test
  47     """
  48     # pylint: disable=too-many-statements
  49     def __init__(self, test_cfg):
  50         """Pull out fields from test config
  51
  52         :param test_cfg: A dictionary of string-value pairs describing the test
  53             configuration. Both the key and values strings use well-known
  54             values.
  55         :param results_dir: Where the csv formatted results are written.
  56         """
  57         # make a local copy of test configuration to avoid modification of
  58         # original content used in vsperf main script
  59         cfg = copy.deepcopy(test_cfg)
  60
  61         self._testcase_start_time = time.time()
  62         self._testcase_stop_time = self._testcase_start_time
  63         self._hugepages_mounted = False
  64         self._traffic_ctl = None
  65         self._vnf_ctl = None
  66         self._vswitch_ctl = None
  67         self._collector = None
  68         self._loadgen = None
  69         self._output_file = None
  70         self._tc_results = None
  71         self._settings_paths_modified = False
  72         self._testcast_run_time = None
  73         self._versions = []
  74         # initialization of step driven specific members
  75         self._step_check = False    # by default don't check result for step driven testcases
  76         self._step_vnf_list = {}
  77         self._step_result = []
  78         self._step_result_mapping = {}
  79         self._step_status = None
  80         self._step_send_traffic = False # indication if send_traffic was called within test steps
  81         self._vnf_list = []
  82         self._testcase_run_time = None
  83
  84         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  85         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  86         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  87         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  88         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  89         test_params = merge_spec(test_params, tc_test_params)
  90         S.setValue('TEST_PARAMS', test_params)
  91         S.check_test_params()
  92
  93         # override all redefined GUEST_ values to have them expanded correctly
  94         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  95         for key in tmp_test_params:
  96             if key.startswith('GUEST_'):
  97                 S.setValue(key, S.getValue(key))
  98                 S.getValue('TEST_PARAMS').pop(key)
  99
 100         # update global settings
 101         functions.settings_update_paths()
 102
 103         # set test parameters; CLI options take precedence to testcase settings
 104         self._logger = logging.getLogger(__name__)
 105         self.name = cfg['Name']
 106         self.desc = cfg.get('Description', 'No description given.')
 107         self.test = cfg.get('TestSteps', None)
 108
 109         # log testcase name and details
 110         tmp_desc = functions.format_description(self.desc, 50)
 111         self._logger.info('############################################################')
 112         self._logger.info('# Test:    %s', self.name)
 113         self._logger.info('# Details: %s', tmp_desc[0])
 114         for i in range(1, len(tmp_desc)):
 115             self._logger.info('#          %s', tmp_desc[i])
 116         self._logger.info('############################################################')
 117
 118         bidirectional = S.getValue('TRAFFIC')['bidir']
 119         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 120             raise TypeError(
 121                 'Bi-dir value must be of type string')
 122         bidirectional = bidirectional.title()  # Keep things consistent
 123
 124         self.deployment = cfg['Deployment']
 125         self._frame_mod = cfg.get('Frame Modification', None)
 126
 127         self._tunnel_type = None
 128         self._tunnel_operation = None
 129
 130         if self.deployment == 'op2p':
 131             self._tunnel_operation = cfg['Tunnel Operation']
 132
 133             if 'Tunnel Type' in cfg:
 134                 self._tunnel_type = cfg['Tunnel Type']
 135                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 136                                                    self._tunnel_type)
 137
 138         # check if test requires background load and which generator it uses
 139         self._load_cfg = cfg.get('Load', None)
 140
 141         if self._frame_mod:
 142             self._frame_mod = self._frame_mod.lower()
 143         self._results_dir = S.getValue('RESULTS_PATH')
 144
 145         # set traffic details, so they can be passed to vswitch and traffic ctls
 146         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 147         self._traffic.update({'bidir': bidirectional,
 148                               'tunnel_type': self._tunnel_type,})
 149
 150         self._traffic = functions.check_traffic(self._traffic)
 151
 152         # Packet Forwarding mode
 153         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 154
 155         # trafficgen configuration required for tests of tunneling protocols
 156         if self.deployment == "op2p":
 157             self._traffic['l2'].update({'srcmac':
 158                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 159                                         'dstmac':
 160                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 161
 162             self._traffic['l3'].update({'srcip':
 163                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 164                                         'dstip':
 165                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 166
 167             if self._tunnel_operation == "decapsulation":
 168                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 169                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 170                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 171                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 172         elif len(S.getValue('NICS')) >= 2 and \
 173              (S.getValue('NICS')[0]['type'] == 'vf' or
 174               S.getValue('NICS')[1]['type'] == 'vf'):
 175             mac1 = S.getValue('NICS')[0]['mac']
 176             mac2 = S.getValue('NICS')[1]['mac']
 177             if mac1 and mac2:
 178                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 179             else:
 180                 self._logger.debug("MAC addresses can not be read")
 181
 182         # count how many VNFs are involved in TestSteps
 183         if self.test:
 184             for step in self.test:
 185                 if step[0].startswith('vnf'):
 186                     self._step_vnf_list[step[0]] = None
 187
 188     def run_initialize(self):
 189         """ Prepare test execution environment
 190         """
 191         # mount hugepages if needed
 192         self._mount_hugepages()
 193
 194         self._logger.debug("Controllers:")
 195         loader = Loader()
 196         self._traffic_ctl = component_factory.create_traffic(
 197             self._traffic['traffic_type'],
 198             loader.get_trafficgen_class())
 199
 200         self._vnf_ctl = component_factory.create_vnf(
 201             self.deployment,
 202             loader.get_vnf_class(),
 203             len(self._step_vnf_list))
 204
 205         self._vnf_list = self._vnf_ctl.get_vnfs()
 206
 207         # verify enough hugepages are free to run the testcase
 208         if not self._check_for_enough_hugepages():
 209             raise RuntimeError('Not enough hugepages free to run test.')
 210
 211         # perform guest related handling
 212         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 213         if tmp_vm_count:
 214             # copy sources of l2 forwarding tools into VM shared dir if needed
 215             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 216
 217             # in case of multi VM in parallel, set the number of streams to the number of VMs
 218             if self.deployment.startswith('pvpv'):
 219                 # for each VM NIC pair we need an unique stream
 220                 streams = 0
 221                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 222                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 223                 self._logger.debug("VMs with parallel connection were detected. "
 224                                    "Thus Number of streams was set to %s", streams)
 225                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 226                 # user can define a proper stream count manually
 227                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 228                     self._traffic.update({'multistream': streams})
 229
 230             # OVS Vanilla requires guest VM MAC address and IPs to work
 231             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 232                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 233                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 234                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 235                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 236
 237         if self._vswitch_none:
 238             self._vswitch_ctl = component_factory.create_pktfwd(
 239                 self.deployment,
 240                 loader.get_pktfwd_class())
 241         else:
 242             self._vswitch_ctl = component_factory.create_vswitch(
 243                 self.deployment,
 244                 loader.get_vswitch_class(),
 245                 self._traffic,
 246                 self._tunnel_operation)
 247
 248         self._collector = component_factory.create_collector(
 249             loader.get_collector_class(),
 250             self._results_dir, self.name)
 251         self._loadgen = component_factory.create_loadgen(
 252             loader.get_loadgen_class(),
 253             self._load_cfg)
 254
 255         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 256                                          "_" + self.deployment + ".csv")
 257
 258         self._step_status = {'status' : True, 'details' : ''}
 259
 260         self._logger.debug("Setup:")
 261
 262     def run_finalize(self):
 263         """ Tear down test execution environment and record test results
 264         """
 265         # Stop all VNFs started by TestSteps in case that something went wrong
 266         self.step_stop_vnfs()
 267
 268         # umount hugepages if mounted
 269         self._umount_hugepages()
 270
 271         # cleanup any namespaces created
 272         if os.path.isdir('/tmp/namespaces'):
 273             namespace_list = os.listdir('/tmp/namespaces')
 274             if len(namespace_list):
 275                 self._logger.info('Cleaning up namespaces')
 276             for name in namespace_list:
 277                 namespace.delete_namespace(name)
 278             os.rmdir('/tmp/namespaces')
 279         # cleanup any veth ports created
 280         if os.path.isdir('/tmp/veth'):
 281             veth_list = os.listdir('/tmp/veth')
 282             if len(veth_list):
 283                 self._logger.info('Cleaning up veth ports')
 284             for eth in veth_list:
 285                 port1, port2 = eth.split('-')
 286                 veth.del_veth_port(port1, port2)
 287             os.rmdir('/tmp/veth')
 288
 289     def run_report(self):
 290         """ Report test results
 291         """
 292         self._logger.debug("self._collector Results:")
 293         self._collector.print_results()
 294
 295         results = self._traffic_ctl.get_results()
 296         if results:
 297             self._logger.debug("Traffic Results:")
 298             self._traffic_ctl.print_results()
 299
 300         if self._tc_results is None:
 301             self._tc_results = self._append_results(results)
 302         else:
 303             # integration step driven tests have their status and possible
 304             # failure details stored inside self._tc_results
 305             results = self._append_results(results)
 306             if len(self._tc_results) < len(results):
 307                 if len(self._tc_results) > 1:
 308                     raise RuntimeError('Testcase results do not match:'
 309                                        'results: {}\n'
 310                                        'trafficgen results: {}\n',
 311                                        self._tc_results,
 312                                        results)
 313                 else:
 314                     tmp_results = copy.deepcopy(self._tc_results[0])
 315                     self._tc_results = []
 316                     for res in results:
 317                         tmp_res = copy.deepcopy(tmp_results)
 318                         tmp_res.update(res)
 319                         self._tc_results.append(tmp_res)
 320             else:
 321                 for i, result in enumerate(results):
 322                     self._tc_results[i].update(result)
 323
 324         TestCase.write_result_to_file(self._tc_results, self._output_file)
 325
 326     def run(self):
 327         """Run the test
 328
 329         All setup and teardown through controllers is included.
 330         """
 331         # prepare test execution environment
 332         self.run_initialize()
 333
 334         try:
 335             with self._vswitch_ctl, self._loadgen:
 336                 with self._vnf_ctl, self._collector:
 337                     if not self._vswitch_none:
 338                         self._add_flows()
 339
 340                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 341
 342                     with self._traffic_ctl:
 343                         # execute test based on TestSteps definition if needed...
 344                         if self.step_run():
 345                             # ...and continue with traffic generation, but keep
 346                             # in mind, that clean deployment does not configure
 347                             # OVS nor executes the traffic
 348                             if self.deployment != 'clean' and not self._step_send_traffic:
 349                                 self._traffic_ctl.send_traffic(self._traffic)
 350
 351                         # dump vswitch flows before they are affected by VNF termination
 352                         if not self._vswitch_none:
 353                             self._vswitch_ctl.dump_vswitch_flows()
 354
 355                     # garbage collection for case that TestSteps modify existing deployment
 356                     self.step_stop_vnfs()
 357
 358         finally:
 359             # tear down test execution environment and log results
 360             self.run_finalize()
 361
 362         self._testcase_stop_time = time.time()
 363         self._testcase_run_time = time.strftime("%H:%M:%S",
 364                                                 time.gmtime(self._testcase_stop_time -
 365                                                             self._testcase_start_time))
 366         logging.info("Testcase execution time: " + self._testcase_run_time)
 367         # report test results
 368         self.run_report()
 369
 370     def _append_results(self, results):
 371         """
 372         Method appends mandatory Test Case results to list of dictionaries.
 373
 374         :param results: list of dictionaries which contains results from
 375                 traffic generator.
 376
 377         :returns: modified list of dictionaries.
 378         """
 379         for item in results:
 380             item[ResultsConstants.ID] = self.name
 381             item[ResultsConstants.DEPLOYMENT] = self.deployment
 382             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 383             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 384             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 385             # convert timestamps to human readable format
 386             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 387                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 388             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 389                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 390             if self._traffic['multistream']:
 391                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 392                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 393                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 394             if self._vnf_ctl.get_vnfs_number():
 395                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 396             if self._tunnel_type:
 397                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 398         return results
 399
 400     def _copy_fwd_tools_for_all_guests(self, vm_count):
 401         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 402         """
 403         # consider only VNFs involved in the test
 404         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 405             self._copy_fwd_tools_for_guest(guest_dir)
 406
 407     def _copy_fwd_tools_for_guest(self, guest_dir):
 408         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 409
 410         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 411         """
 412         # remove shared dir if it exists to avoid issues with file consistency
 413         if os.path.exists(guest_dir):
 414             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 415                            'Removing content of shared directory...', True)
 416
 417         # directory to share files between host and guest
 418         os.makedirs(guest_dir)
 419
 420         # copy sources into shared dir only if neccessary
 421         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 422         if 'testpmd' in guest_loopback:
 423             try:
 424                 # exclude whole .git/ subdirectory and all o-files;
 425                 # It is assumed, that the same RTE_TARGET is used in both host
 426                 # and VMs; This simplification significantly speeds up testpmd
 427                 # build. If we will need a different RTE_TARGET in VM,
 428                 # then we have to build whole DPDK from the scratch in VM.
 429                 # In that case we can copy just DPDK sources (e.g. by excluding
 430                 # all items obtained by git status -unormal --porcelain).
 431                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 432                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 433                 exclude = []
 434                 exclude.append(r'--exclude=.git/')
 435                 exclude.append(r'--exclude=*.o')
 436                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 437                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 438                                 os.path.join(guest_dir, 'DPDK')],
 439                                self._logger,
 440                                'Copying DPDK to shared directory...',
 441                                True)
 442             except subprocess.CalledProcessError:
 443                 self._logger.error('Unable to copy DPDK to shared directory')
 444                 raise
 445         if 'l2fwd' in guest_loopback:
 446             try:
 447                 tasks.run_task(['rsync', '-a', '-r', '-l',
 448                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 449                                 os.path.join(guest_dir, 'l2fwd')],
 450                                self._logger,
 451                                'Copying l2fwd to shared directory...',
 452                                True)
 453             except subprocess.CalledProcessError:
 454                 self._logger.error('Unable to copy l2fwd to shared directory')
 455                 raise
 456
 457     def _mount_hugepages(self):
 458         """Mount hugepages if usage of DPDK or Qemu is detected
 459         """
 460         # pylint: disable=too-many-boolean-expressions
 461         # hugepages are needed by DPDK and Qemu
 462         if not self._hugepages_mounted and \
 463             (self.deployment.count('v') or \
 464              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 465              self._vswitch_none or \
 466              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 467             hugepages.mount_hugepages()
 468             self._hugepages_mounted = True
 469
 470     def _umount_hugepages(self):
 471         """Umount hugepages if they were mounted before
 472         """
 473         if self._hugepages_mounted:
 474             hugepages.umount_hugepages()
 475             self._hugepages_mounted = False
 476
 477     def _check_for_enough_hugepages(self):
 478         """Check to make sure enough hugepages are free to satisfy the
 479         test environment.
 480         """
 481         hugepages_needed = 0
 482         hugepage_size = hugepages.get_hugepage_size()
 483         # get hugepage amounts per guest involved in the test
 484         for guest in range(self._vnf_ctl.get_vnfs_number()):
 485             hugepages_needed += math.ceil((int(S.getValue(
 486                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 487
 488         # get hugepage amounts for each socket on dpdk
 489         sock0_mem, sock1_mem = 0, 0
 490
 491         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 492             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 493             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 494                                     int(sock_mem[1]) * 1024 / hugepage_size)
 495
 496         # If hugepages needed, verify the amounts are free
 497         if any([hugepages_needed, sock0_mem, sock1_mem]):
 498             free_hugepages = hugepages.get_free_hugepages()
 499             if hugepages_needed:
 500                 logging.info('Need %s hugepages free for guests',
 501                              hugepages_needed)
 502                 result1 = free_hugepages >= hugepages_needed
 503                 free_hugepages -= hugepages_needed
 504             else:
 505                 result1 = True
 506
 507             if sock0_mem:
 508                 logging.info('Need %s hugepages free for dpdk socket 0',
 509                              sock0_mem)
 510                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 511                 free_hugepages -= sock0_mem
 512             else:
 513                 result2 = True
 514
 515             if sock1_mem:
 516                 logging.info('Need %s hugepages free for dpdk socket 1',
 517                              sock1_mem)
 518                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 519                 free_hugepages -= sock1_mem
 520             else:
 521                 result3 = True
 522
 523             logging.info('Need a total of %s total hugepages',
 524                          hugepages_needed + sock1_mem + sock0_mem)
 525
 526             # The only drawback here is sometimes dpdk doesn't release
 527             # its hugepages on a test failure. This could cause a test
 528             # to fail when dpdk would be OK to start because it will just
 529             # use the previously allocated hugepages.
 530             result4 = True if free_hugepages >= 0 else False
 531
 532             return all([result1, result2, result3, result4])
 533         else:
 534             return True
 535
 536     @staticmethod
 537     def write_result_to_file(results, output):
 538         """Write list of dictionaries to a CSV file.
 539
 540         Each element on list will create separate row in output file.
 541         If output file already exists, data will be appended at the end,
 542         otherwise it will be created.
 543
 544         :param results: list of dictionaries.
 545         :param output: path to output file.
 546         """
 547         with open(output, 'a') as csvfile:
 548
 549             logging.info("Write results to file: " + output)
 550             fieldnames = TestCase._get_unique_keys(results)
 551
 552             writer = csv.DictWriter(csvfile, fieldnames)
 553
 554             if not csvfile.tell():  # file is now empty
 555                 writer.writeheader()
 556
 557             for result in results:
 558                 writer.writerow(result)
 559
 560     @staticmethod
 561     def _get_unique_keys(list_of_dicts):
 562         """Gets unique key values as ordered list of strings in given dicts
 563
 564         :param list_of_dicts: list of dictionaries.
 565
 566         :returns: list of unique keys(strings).
 567         """
 568         result = OrderedDict()
 569         for item in list_of_dicts:
 570             for key in item.keys():
 571                 result[key] = ''
 572
 573         return list(result.keys())
 574
 575     def _add_flows(self):
 576         """Add flows to the vswitch
 577         """
 578         vswitch = self._vswitch_ctl.get_vswitch()
 579         # NOTE BOM 15-08-07 the frame mod code assumes that the
 580         # physical ports are ports 1 & 2. The actual numbers
 581         # need to be retrived from the vSwitch and the metadata value
 582         # updated accordingly.
 583         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 584         if self._frame_mod == "vlan":
 585             # 0x8100 => VLAN ethertype
 586             self._logger.debug(" ****   VLAN   ***** ")
 587             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 588                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 589             vswitch.add_flow(bridge, flow)
 590             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 591                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 592             vswitch.add_flow(bridge, flow)
 593         elif self._frame_mod == "mpls":
 594             # 0x8847 => MPLS unicast ethertype
 595             self._logger.debug(" ****   MPLS  ***** ")
 596             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 597                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 598             vswitch.add_flow(bridge, flow)
 599             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 600                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 601             vswitch.add_flow(bridge, flow)
 602         elif self._frame_mod == "mac":
 603             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 604                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 605                                 'goto_table:3']}
 606             vswitch.add_flow(bridge, flow)
 607             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 608                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 609                                 'goto_table:3']}
 610             vswitch.add_flow(bridge, flow)
 611         elif self._frame_mod == "dscp":
 612             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 613             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 614                     'dl_type':'0x0800',
 615                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 616             vswitch.add_flow(bridge, flow)
 617             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 618                     'dl_type':'0x0800',
 619                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 620             vswitch.add_flow(bridge, flow)
 621         elif self._frame_mod == "ttl":
 622             # 251 and 241 are the highest prime numbers < 255
 623             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 624                     'dl_type':'0x0800',
 625                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 626             vswitch.add_flow(bridge, flow)
 627             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 628                     'dl_type':'0x0800',
 629                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 630             vswitch.add_flow(bridge, flow)
 631         elif self._frame_mod == "ip_addr":
 632             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 633                     'dl_type':'0x0800',
 634                     'actions': ['mod_nw_src:10.10.10.10',
 635                                 'mod_nw_dst:20.20.20.20',
 636                                 'goto_table:3']}
 637             vswitch.add_flow(bridge, flow)
 638             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 639                     'dl_type':'0x0800',
 640                     'actions': ['mod_nw_src:20.20.20.20',
 641                                 'mod_nw_dst:10.10.10.10',
 642                                 'goto_table:3']}
 643             vswitch.add_flow(bridge, flow)
 644         elif self._frame_mod == "ip_port":
 645             # NOTE BOM 15-08-27 The traffic generated is assumed
 646             # to be UDP (nw_proto 17d) which is the default case but
 647             # we will need to pick up the actual traffic params in use.
 648             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 649                     'dl_type':'0x0800', 'nw_proto':'17',
 650                     'actions': ['mod_tp_src:44444',
 651                                 'mod_tp_dst:44444', 'goto_table:3']}
 652             vswitch.add_flow(bridge, flow)
 653             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 654                     'dl_type':'0x0800', 'nw_proto':'17',
 655                     'actions': ['mod_tp_src:44444',
 656                                 'mod_tp_dst:44444', 'goto_table:3']}
 657             vswitch.add_flow(bridge, flow)
 658         else:
 659             pass
 660
 661
 662     #
 663     # TestSteps realted methods
 664     #
 665     def step_report_status(self, label, status):
 666         """ Log status of test step
 667         """
 668         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 669
 670     def step_stop_vnfs(self):
 671         """ Stop all VNFs started by TestSteps
 672         """
 673         for vnf in self._step_vnf_list:
 674             if self._step_vnf_list[vnf]:
 675                 self._step_vnf_list[vnf].stop()
 676
 677     def step_eval_param(self, param, step_result):
 678         """ Helper function for #STEP macro evaluation
 679         """
 680         if isinstance(param, str):
 681             # evaluate every #STEP reference inside parameter itself
 682             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 683
 684             if macros:
 685                 for macro in macros:
 686                     if macro[1] in self._step_result_mapping:
 687                         key = self._step_result_mapping[macro[1]]
 688                     else:
 689                         key = macro[1]
 690                     # pylint: disable=eval-used
 691                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 692                     param = param.replace(macro[0], tmp_val)
 693
 694             # evaluate references to vsperf configuration options
 695             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 696             if macros:
 697                 for macro in macros:
 698                     # pylint: disable=eval-used
 699                     try:
 700                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 701                         param = param.replace('${}'.format(macro[0]), tmp_val)
 702                     # ignore that required option can't be evaluated
 703                     except (IndexError, KeyError, AttributeError):
 704                         self._logger.debug("Skipping %s as it isn't a configuration "
 705                                            "parameter.", '${}'.format(macro[0]))
 706             return param
 707         elif isinstance(param, list) or isinstance(param, tuple):
 708             tmp_list = []
 709             for item in param:
 710                 tmp_list.append(self.step_eval_param(item, step_result))
 711             return tmp_list
 712         elif isinstance(param, dict):
 713             tmp_dict = {}
 714             for (key, value) in param.items():
 715                 tmp_dict[key] = self.step_eval_param(value, step_result)
 716             return tmp_dict
 717         else:
 718             return param
 719
 720     def step_eval_params(self, params, step_result):
 721         """ Evaluates referrences to results from previous steps
 722         """
 723         eval_params = []
 724         # evaluate all parameters if needed
 725         for param in params:
 726             eval_params.append(self.step_eval_param(param, step_result))
 727         return eval_params
 728
 729     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 730     def step_run(self):
 731         """ Execute actions specified by TestSteps list
 732
 733         :return: False if any error was detected
 734                  True otherwise
 735         """
 736         # anything to do?
 737         if not self.test:
 738             return True
 739
 740         # required for VNFs initialization
 741         loader = Loader()
 742         # initialize list with results
 743         self._step_result = [None] * len(self.test)
 744
 745         # We have to suppress pylint report, because test_object has to be set according
 746         # to the test step definition
 747         # pylint: disable=redefined-variable-type
 748         # run test step by step...
 749         for i, step in enumerate(self.test):
 750             step_ok = not self._step_check
 751             step_check = self._step_check
 752             regex = None
 753             # configure step result mapping if step alias/label is detected
 754             if step[0].startswith('#'):
 755                 key = step[0][1:]
 756                 if key.isdigit():
 757                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 758                 if key in self._step_result_mapping:
 759                     raise RuntimeError('Step alias {} has been used already for step '
 760                                        '{}'.format(key, self._step_result_mapping[key]))
 761                 self._step_result_mapping[step[0][1:]] = i
 762                 step = step[1:]
 763
 764             # store regex filter if it is specified
 765             if isinstance(step[-1], str) and step[-1].startswith('|'):
 766                 # evalute macros and variables used in regex
 767                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 768                 step = step[:-1]
 769
 770             # check if step verification should be suppressed
 771             if step[0].startswith('!'):
 772                 step_check = False
 773                 step_ok = True
 774                 step[0] = step[0][1:]
 775             if step[0] == 'vswitch':
 776                 test_object = self._vswitch_ctl.get_vswitch()
 777             elif step[0] == 'namespace':
 778                 test_object = namespace
 779             elif step[0] == 'veth':
 780                 test_object = veth
 781             elif step[0] == 'settings':
 782                 test_object = S
 783             elif step[0] == 'tools':
 784                 test_object = TestStepsTools()
 785                 step[1] = step[1].title()
 786             elif step[0] == 'trafficgen':
 787                 test_object = self._traffic_ctl
 788                 # in case of send_traffic or send_traffic_async methods, ensure
 789                 # that specified traffic values are merged with existing self._traffic
 790                 if step[1].startswith('send_traffic'):
 791                     tmp_traffic = copy.deepcopy(self._traffic)
 792                     tmp_traffic.update(step[2])
 793                     step[2] = tmp_traffic
 794                     # store indication that traffic has been sent
 795                     # so it is not sent again after the execution of teststeps
 796                     self._step_send_traffic = True
 797             elif step[0].startswith('vnf'):
 798                 # use vnf started within TestSteps
 799                 if not self._step_vnf_list[step[0]]:
 800                     # initialize new VM
 801                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 802                 test_object = self._step_vnf_list[step[0]]
 803             elif step[0].startswith('VNF'):
 804                 if step[1] in ('start', 'stop'):
 805                     raise RuntimeError("Cannot execute start() or stop() method of "
 806                                        "VNF deployed automatically by scenario.")
 807                 # use vnf started by scenario deployment (e.g. pvp)
 808                 vnf_index = int(step[0][3:])
 809                 try:
 810                     test_object = self._vnf_list[vnf_index]
 811                 except IndexError:
 812                     raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
 813             elif step[0] == 'wait':
 814                 input(os.linesep + "Step {}: Press Enter to continue with "
 815                       "the next step...".format(i) + os.linesep + os.linesep)
 816                 continue
 817             elif step[0] == 'sleep':
 818                 self._logger.debug("Sleep %s seconds", step[1])
 819                 time.sleep(int(step[1]))
 820                 continue
 821             elif step[0] == 'log':
 822                 test_object = self._logger
 823                 # there isn't a need for validation of log entry
 824                 step_check = False
 825                 step_ok = True
 826             elif step[0] == 'pdb':
 827                 import pdb
 828                 pdb.set_trace()
 829                 continue
 830             else:
 831                 self._logger.error("Unsupported test object %s", step[0])
 832                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 833                 self.step_report_status("Step '{}'".format(' '.join(step)),
 834                                         self._step_status['status'])
 835                 return False
 836
 837             test_method = getattr(test_object, step[1])
 838             if step_check:
 839                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 840             else:
 841                 test_method_check = None
 842
 843             step_params = []
 844             try:
 845                 # eval parameters, but use only valid step_results
 846                 # to support negative indexes
 847                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 848                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 849                 step_log += ' filter "{}"'.format(regex) if regex else ''
 850                 self._logger.debug("Step %s '%s' start", i, step_log)
 851                 self._step_result[i] = test_method(*step_params)
 852                 if regex:
 853                     # apply regex to step output
 854                     self._step_result[i] = functions.filter_output(
 855                         self._step_result[i], regex)
 856
 857                 self._logger.debug("Step %s '%s' results '%s'", i,
 858                                    step_log, self._step_result[i])
 859                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 860                 if step_check:
 861                     step_ok = test_method_check(self._step_result[i], *step_params)
 862             except (AssertionError, AttributeError, IndexError) as ex:
 863                 step_ok = False
 864                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 865
 866             if step_check:
 867                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 868
 869             if not step_ok:
 870                 self._step_status = {'status' : False, 'details' : step_log}
 871                 # Stop all VNFs started by TestSteps
 872                 self.step_stop_vnfs()
 873                 return False
 874
 875         # all steps processed without any issue
 876         return True
 877
 878     #
 879     # get methods for TestCase members, which needs to be publicly available
 880     #
 881     def get_output_file(self):
 882         """Return content of self._output_file member
 883         """
 884         return self._output_file
 885
 886     def get_desc(self):
 887         """Return content of self.desc member
 888         """
 889         return self.desc
 890
 891     def get_versions(self):
 892         """Return content of self.versions member
 893         """
 894         return self._versions
 895
 896     def get_traffic(self):
 897         """Return content of self._traffic member
 898         """
 899         return self._traffic
 900
 901     def get_tc_results(self):
 902         """Return content of self._tc_results member
 903         """
 904         return self._tc_results
 905
 906     def get_collector(self):
 907         """Return content of self._collector member
 908         """
 909         return self._collector