testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from conf import settings as S
  28 from conf import get_test_param, merge_spec
  29 import core.component_factory as component_factory
  30 from core.loader import Loader
  31 from core.results.results_constants import ResultsConstants
  32 from tools import tasks
  33 from tools import hugepages
  34 from tools import functions
  35 from tools import namespace
  36 from tools import veth
  37 from tools.teststepstools import TestStepsTools
  38
  39 CHECK_PREFIX = 'validate_'
  40
  41 # pylint: disable=too-many-instance-attributes
  42 class TestCase(object):
  43     """TestCase base class
  44
  45     In this basic form runs RFC2544 throughput test
  46     """
  47     # pylint: disable=too-many-statements
  48     def __init__(self, cfg):
  49         """Pull out fields from test config
  50
  51         :param cfg: A dictionary of string-value pairs describing the test
  52             configuration. Both the key and values strings use well-known
  53             values.
  54         :param results_dir: Where the csv formatted results are written.
  55         """
  56         self._testcase_start_time = time.time()
  57         self._hugepages_mounted = False
  58         self._traffic_ctl = None
  59         self._vnf_ctl = None
  60         self._vswitch_ctl = None
  61         self._collector = None
  62         self._loadgen = None
  63         self._output_file = None
  64         self._tc_results = None
  65         self._settings_original = {}
  66         self._settings_paths_modified = False
  67         self._testcast_run_time = None
  68         self._versions = []
  69         # initialization of step driven specific members
  70         self._step_check = False    # by default don't check result for step driven testcases
  71         self._step_vnf_list = {}
  72         self._step_result = []
  73         self._step_status = None
  74         self._testcase_run_time = None
  75
  76         # store all GUEST_ specific settings to keep original values before their expansion
  77         for key in S.__dict__:
  78             if key.startswith('GUEST_'):
  79                 self._settings_original[key] = S.getValue(key)
  80
  81         self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  82         self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
  83         self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  84         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  85         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  86         test_params = merge_spec(test_params, tc_test_params)
  87         self._update_settings('TEST_PARAMS', test_params)
  88         S.check_test_params()
  89
  90         # override all redefined GUEST_ values to have them expanded correctly
  91         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  92         for key in tmp_test_params:
  93             if key.startswith('GUEST_'):
  94                 S.setValue(key, S.getValue(key))
  95                 S.getValue('TEST_PARAMS').pop(key)
  96
  97         # update global settings
  98         functions.settings_update_paths()
  99
 100         # set test parameters; CLI options take precedence to testcase settings
 101         self._logger = logging.getLogger(__name__)
 102         self.name = cfg['Name']
 103         self.desc = cfg.get('Description', 'No description given.')
 104         self.test = cfg.get('TestSteps', None)
 105
 106         bidirectional = S.getValue('TRAFFIC')['bidir']
 107         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 108             raise TypeError(
 109                 'Bi-dir value must be of type string')
 110         bidirectional = bidirectional.title()  # Keep things consistent
 111
 112         self.deployment = cfg['Deployment']
 113         self._frame_mod = cfg.get('Frame Modification', None)
 114
 115         self._tunnel_type = None
 116         self._tunnel_operation = None
 117
 118         if self.deployment == 'op2p':
 119             self._tunnel_operation = cfg['Tunnel Operation']
 120
 121             if 'Tunnel Type' in cfg:
 122                 self._tunnel_type = cfg['Tunnel Type']
 123                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 124                                                    self._tunnel_type)
 125
 126         # check if test requires background load and which generator it uses
 127         self._load_cfg = cfg.get('Load', None)
 128         if self._load_cfg and 'tool' in self._load_cfg:
 129             self._loadgen = self._load_cfg['tool']
 130         else:
 131             # background load is not requested, so use dummy implementation
 132             self._loadgen = "Dummy"
 133
 134         if self._frame_mod:
 135             self._frame_mod = self._frame_mod.lower()
 136         self._results_dir = S.getValue('RESULTS_PATH')
 137
 138         # set traffic details, so they can be passed to vswitch and traffic ctls
 139         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 140         self._traffic.update({'bidir': bidirectional,
 141                               'tunnel_type': self._tunnel_type,})
 142
 143         self._traffic = functions.check_traffic(self._traffic)
 144
 145         # Packet Forwarding mode
 146         self._vswitch_none = S.getValue('VSWITCH').strip().lower() == 'none'
 147
 148         # trafficgen configuration required for tests of tunneling protocols
 149         if self.deployment == "op2p":
 150             self._traffic['l2'].update({'srcmac':
 151                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 152                                         'dstmac':
 153                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 154
 155             self._traffic['l3'].update({'srcip':
 156                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 157                                         'dstip':
 158                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 159
 160             if self._tunnel_operation == "decapsulation":
 161                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 162                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 163                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 164         elif len(S.getValue('NICS')) and \
 165              (S.getValue('NICS')[0]['type'] == 'vf' or
 166               S.getValue('NICS')[1]['type'] == 'vf'):
 167             mac1 = S.getValue('NICS')[0]['mac']
 168             mac2 = S.getValue('NICS')[1]['mac']
 169             if mac1 and mac2:
 170                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 171             else:
 172                 self._logger.debug("MAC addresses can not be read")
 173
 174         # count how many VNFs are involved in TestSteps
 175         if self.test:
 176             for step in self.test:
 177                 if step[0].startswith('vnf'):
 178                     self._step_vnf_list[step[0]] = None
 179
 180     def run_initialize(self):
 181         """ Prepare test execution environment
 182         """
 183         self._logger.debug(self.name)
 184
 185         # mount hugepages if needed
 186         self._mount_hugepages()
 187
 188         self._logger.debug("Controllers:")
 189         loader = Loader()
 190         self._traffic_ctl = component_factory.create_traffic(
 191             self._traffic['traffic_type'],
 192             loader.get_trafficgen_class())
 193
 194         self._vnf_ctl = component_factory.create_vnf(
 195             self.deployment,
 196             loader.get_vnf_class(),
 197             len(self._step_vnf_list))
 198
 199         # verify enough hugepages are free to run the testcase
 200         if not self._check_for_enough_hugepages():
 201             raise RuntimeError('Not enough hugepages free to run test.')
 202
 203         # perform guest related handling
 204         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 205         if tmp_vm_count:
 206             # copy sources of l2 forwarding tools into VM shared dir if needed
 207             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 208
 209             # in case of multi VM in parallel, set the number of streams to the number of VMs
 210             if self.deployment.startswith('pvpv'):
 211                 # for each VM NIC pair we need an unique stream
 212                 streams = 0
 213                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 214                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 215                 self._logger.debug("VMs with parallel connection were detected. "
 216                                    "Thus Number of streams was set to %s", streams)
 217                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 218                 # user can define a proper stream count manually
 219                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 220                     self._traffic.update({'multistream': streams})
 221
 222             # OVS Vanilla requires guest VM MAC address and IPs to work
 223             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 224                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 225                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 226                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 227                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 228
 229         if self._vswitch_none:
 230             self._vswitch_ctl = component_factory.create_pktfwd(
 231                 self.deployment,
 232                 loader.get_pktfwd_class())
 233         else:
 234             self._vswitch_ctl = component_factory.create_vswitch(
 235                 self.deployment,
 236                 loader.get_vswitch_class(),
 237                 self._traffic,
 238                 self._tunnel_operation)
 239
 240         self._collector = component_factory.create_collector(
 241             loader.get_collector_class(),
 242             self._results_dir, self.name)
 243         self._loadgen = component_factory.create_loadgen(
 244             self._loadgen,
 245             self._load_cfg)
 246
 247         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 248                                          "_" + self.deployment + ".csv")
 249
 250         self._step_status = {'status' : True, 'details' : ''}
 251
 252         self._logger.debug("Setup:")
 253
 254     def run_finalize(self):
 255         """ Tear down test execution environment and record test results
 256         """
 257         # Stop all VNFs started by TestSteps in case that something went wrong
 258         self.step_stop_vnfs()
 259
 260         # umount hugepages if mounted
 261         self._umount_hugepages()
 262
 263         # cleanup any namespaces created
 264         if os.path.isdir('/tmp/namespaces'):
 265             namespace_list = os.listdir('/tmp/namespaces')
 266             if len(namespace_list):
 267                 self._logger.info('Cleaning up namespaces')
 268             for name in namespace_list:
 269                 namespace.delete_namespace(name)
 270             os.rmdir('/tmp/namespaces')
 271         # cleanup any veth ports created
 272         if os.path.isdir('/tmp/veth'):
 273             veth_list = os.listdir('/tmp/veth')
 274             if len(veth_list):
 275                 self._logger.info('Cleaning up veth ports')
 276             for eth in veth_list:
 277                 port1, port2 = eth.split('-')
 278                 veth.del_veth_port(port1, port2)
 279             os.rmdir('/tmp/veth')
 280
 281     def run_report(self):
 282         """ Report test results
 283         """
 284         self._logger.debug("self._collector Results:")
 285         self._collector.print_results()
 286
 287         results = self._traffic_ctl.get_results()
 288         if results:
 289             self._logger.debug("Traffic Results:")
 290             self._traffic_ctl.print_results()
 291
 292             self._tc_results = self._append_results(results)
 293             TestCase.write_result_to_file(self._tc_results, self._output_file)
 294
 295     def run(self):
 296         """Run the test
 297
 298         All setup and teardown through controllers is included.
 299         """
 300         # prepare test execution environment
 301         self.run_initialize()
 302
 303         try:
 304             with self._vswitch_ctl, self._loadgen:
 305                 with self._vnf_ctl, self._collector:
 306                     if not self._vswitch_none:
 307                         self._add_flows()
 308
 309                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 310
 311                     with self._traffic_ctl:
 312                         # execute test based on TestSteps definition if needed...
 313                         if self.step_run():
 314                             # ...and continue with traffic generation, but keep
 315                             # in mind, that clean deployment does not configure
 316                             # OVS nor executes the traffic
 317                             if self.deployment != 'clean':
 318                                 self._traffic_ctl.send_traffic(self._traffic)
 319
 320                         # dump vswitch flows before they are affected by VNF termination
 321                         if not self._vswitch_none:
 322                             self._vswitch_ctl.dump_vswitch_flows()
 323
 324                     # garbage collection for case that TestSteps modify existing deployment
 325                     self.step_stop_vnfs()
 326
 327         finally:
 328             # tear down test execution environment and log results
 329             self.run_finalize()
 330
 331         self._testcase_run_time = time.strftime("%H:%M:%S",
 332                                                 time.gmtime(time.time() -
 333                                                             self._testcase_start_time))
 334         logging.info("Testcase execution time: " + self._testcase_run_time)
 335         # report test results
 336         self.run_report()
 337
 338         # restore original settings
 339         S.load_from_dict(self._settings_original)
 340
 341     def _update_settings(self, param, value):
 342         """ Check value of given configuration parameter
 343         In case that new value is different, then testcase
 344         specific settings is updated and original value stored
 345
 346         :param param: Name of parameter inside settings
 347         :param value: Disired parameter value
 348         """
 349         orig_value = S.getValue(param)
 350         if orig_value != value:
 351             self._settings_original[param] = copy.deepcopy(orig_value)
 352             S.setValue(param, value)
 353
 354     def _append_results(self, results):
 355         """
 356         Method appends mandatory Test Case results to list of dictionaries.
 357
 358         :param results: list of dictionaries which contains results from
 359                 traffic generator.
 360
 361         :returns: modified list of dictionaries.
 362         """
 363         for item in results:
 364             item[ResultsConstants.ID] = self.name
 365             item[ResultsConstants.DEPLOYMENT] = self.deployment
 366             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 367             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 368             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 369             if self._traffic['multistream']:
 370                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 371                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 372                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 373             if self._vnf_ctl.get_vnfs_number():
 374                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 375             if self._tunnel_type:
 376                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 377         return results
 378
 379     def _copy_fwd_tools_for_all_guests(self, vm_count):
 380         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 381         """
 382         # consider only VNFs involved in the test
 383         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 384             self._copy_fwd_tools_for_guest(guest_dir)
 385
 386     def _copy_fwd_tools_for_guest(self, guest_dir):
 387         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 388
 389         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 390         """
 391         # remove shared dir if it exists to avoid issues with file consistency
 392         if os.path.exists(guest_dir):
 393             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 394                            'Removing content of shared directory...', True)
 395
 396         # directory to share files between host and guest
 397         os.makedirs(guest_dir)
 398
 399         # copy sources into shared dir only if neccessary
 400         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 401         if 'testpmd' in guest_loopback:
 402             try:
 403                 # exclude whole .git/ subdirectory and all o-files;
 404                 # It is assumed, that the same RTE_TARGET is used in both host
 405                 # and VMs; This simplification significantly speeds up testpmd
 406                 # build. If we will need a different RTE_TARGET in VM,
 407                 # then we have to build whole DPDK from the scratch in VM.
 408                 # In that case we can copy just DPDK sources (e.g. by excluding
 409                 # all items obtained by git status -unormal --porcelain).
 410                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 411                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 412                 exclude = []
 413                 exclude.append(r'--exclude=.git/')
 414                 exclude.append(r'--exclude=*.o')
 415                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 416                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 417                                 os.path.join(guest_dir, 'DPDK')],
 418                                self._logger,
 419                                'Copying DPDK to shared directory...',
 420                                True)
 421             except subprocess.CalledProcessError:
 422                 self._logger.error('Unable to copy DPDK to shared directory')
 423                 raise
 424         if 'l2fwd' in guest_loopback:
 425             try:
 426                 tasks.run_task(['rsync', '-a', '-r', '-l',
 427                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 428                                 os.path.join(guest_dir, 'l2fwd')],
 429                                self._logger,
 430                                'Copying l2fwd to shared directory...',
 431                                True)
 432             except subprocess.CalledProcessError:
 433                 self._logger.error('Unable to copy l2fwd to shared directory')
 434                 raise
 435
 436     def _mount_hugepages(self):
 437         """Mount hugepages if usage of DPDK or Qemu is detected
 438         """
 439         # pylint: disable=too-many-boolean-expressions
 440         # hugepages are needed by DPDK and Qemu
 441         if not self._hugepages_mounted and \
 442             (self.deployment.count('v') or \
 443              S.getValue('VSWITCH').lower().count('dpdk') or \
 444              self._vswitch_none or \
 445              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 446             hugepages.mount_hugepages()
 447             self._hugepages_mounted = True
 448
 449     def _umount_hugepages(self):
 450         """Umount hugepages if they were mounted before
 451         """
 452         if self._hugepages_mounted:
 453             hugepages.umount_hugepages()
 454             self._hugepages_mounted = False
 455
 456     def _check_for_enough_hugepages(self):
 457         """Check to make sure enough hugepages are free to satisfy the
 458         test environment.
 459         """
 460         hugepages_needed = 0
 461         hugepage_size = hugepages.get_hugepage_size()
 462         # get hugepage amounts per guest involved in the test
 463         for guest in range(self._vnf_ctl.get_vnfs_number()):
 464             hugepages_needed += math.ceil((int(S.getValue(
 465                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 466
 467         # get hugepage amounts for each socket on dpdk
 468         sock0_mem, sock1_mem = 0, 0
 469
 470         if S.getValue('VSWITCH').lower().count('dpdk'):
 471             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 472             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 473                                     int(sock_mem[1]) * 1024 / hugepage_size)
 474
 475         # If hugepages needed, verify the amounts are free
 476         if any([hugepages_needed, sock0_mem, sock1_mem]):
 477             free_hugepages = hugepages.get_free_hugepages()
 478             if hugepages_needed:
 479                 logging.info('Need %s hugepages free for guests',
 480                              hugepages_needed)
 481                 result1 = free_hugepages >= hugepages_needed
 482                 free_hugepages -= hugepages_needed
 483             else:
 484                 result1 = True
 485
 486             if sock0_mem:
 487                 logging.info('Need %s hugepages free for dpdk socket 0',
 488                              sock0_mem)
 489                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 490                 free_hugepages -= sock0_mem
 491             else:
 492                 result2 = True
 493
 494             if sock1_mem:
 495                 logging.info('Need %s hugepages free for dpdk socket 1',
 496                              sock1_mem)
 497                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 498                 free_hugepages -= sock1_mem
 499             else:
 500                 result3 = True
 501
 502             logging.info('Need a total of %s total hugepages',
 503                          hugepages_needed + sock1_mem + sock0_mem)
 504
 505             # The only drawback here is sometimes dpdk doesn't release
 506             # its hugepages on a test failure. This could cause a test
 507             # to fail when dpdk would be OK to start because it will just
 508             # use the previously allocated hugepages.
 509             result4 = True if free_hugepages >= 0 else False
 510
 511             return all([result1, result2, result3, result4])
 512         else:
 513             return True
 514
 515     @staticmethod
 516     def write_result_to_file(results, output):
 517         """Write list of dictionaries to a CSV file.
 518
 519         Each element on list will create separate row in output file.
 520         If output file already exists, data will be appended at the end,
 521         otherwise it will be created.
 522
 523         :param results: list of dictionaries.
 524         :param output: path to output file.
 525         """
 526         with open(output, 'a') as csvfile:
 527
 528             logging.info("Write results to file: " + output)
 529             fieldnames = TestCase._get_unique_keys(results)
 530
 531             writer = csv.DictWriter(csvfile, fieldnames)
 532
 533             if not csvfile.tell():  # file is now empty
 534                 writer.writeheader()
 535
 536             for result in results:
 537                 writer.writerow(result)
 538
 539     @staticmethod
 540     def _get_unique_keys(list_of_dicts):
 541         """Gets unique key values as ordered list of strings in given dicts
 542
 543         :param list_of_dicts: list of dictionaries.
 544
 545         :returns: list of unique keys(strings).
 546         """
 547         result = OrderedDict()
 548         for item in list_of_dicts:
 549             for key in item.keys():
 550                 result[key] = ''
 551
 552         return list(result.keys())
 553
 554     def _add_flows(self):
 555         """Add flows to the vswitch
 556         """
 557         vswitch = self._vswitch_ctl.get_vswitch()
 558         # NOTE BOM 15-08-07 the frame mod code assumes that the
 559         # physical ports are ports 1 & 2. The actual numbers
 560         # need to be retrived from the vSwitch and the metadata value
 561         # updated accordingly.
 562         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 563         if self._frame_mod == "vlan":
 564             # 0x8100 => VLAN ethertype
 565             self._logger.debug(" ****   VLAN   ***** ")
 566             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 567                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 568             vswitch.add_flow(bridge, flow)
 569             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 570                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 571             vswitch.add_flow(bridge, flow)
 572         elif self._frame_mod == "mpls":
 573             # 0x8847 => MPLS unicast ethertype
 574             self._logger.debug(" ****   MPLS  ***** ")
 575             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 576                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 577             vswitch.add_flow(bridge, flow)
 578             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 579                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 580             vswitch.add_flow(bridge, flow)
 581         elif self._frame_mod == "mac":
 582             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 583                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 584                                 'goto_table:3']}
 585             vswitch.add_flow(bridge, flow)
 586             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 587                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 588                                 'goto_table:3']}
 589             vswitch.add_flow(bridge, flow)
 590         elif self._frame_mod == "dscp":
 591             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 592             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 593                     'dl_type':'0x0800',
 594                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 595             vswitch.add_flow(bridge, flow)
 596             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 597                     'dl_type':'0x0800',
 598                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 599             vswitch.add_flow(bridge, flow)
 600         elif self._frame_mod == "ttl":
 601             # 251 and 241 are the highest prime numbers < 255
 602             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 603                     'dl_type':'0x0800',
 604                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 605             vswitch.add_flow(bridge, flow)
 606             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 607                     'dl_type':'0x0800',
 608                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 609             vswitch.add_flow(bridge, flow)
 610         elif self._frame_mod == "ip_addr":
 611             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 612                     'dl_type':'0x0800',
 613                     'actions': ['mod_nw_src:10.10.10.10',
 614                                 'mod_nw_dst:20.20.20.20',
 615                                 'goto_table:3']}
 616             vswitch.add_flow(bridge, flow)
 617             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 618                     'dl_type':'0x0800',
 619                     'actions': ['mod_nw_src:20.20.20.20',
 620                                 'mod_nw_dst:10.10.10.10',
 621                                 'goto_table:3']}
 622             vswitch.add_flow(bridge, flow)
 623         elif self._frame_mod == "ip_port":
 624             # NOTE BOM 15-08-27 The traffic generated is assumed
 625             # to be UDP (nw_proto 17d) which is the default case but
 626             # we will need to pick up the actual traffic params in use.
 627             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 628                     'dl_type':'0x0800', 'nw_proto':'17',
 629                     'actions': ['mod_tp_src:44444',
 630                                 'mod_tp_dst:44444', 'goto_table:3']}
 631             vswitch.add_flow(bridge, flow)
 632             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 633                     'dl_type':'0x0800', 'nw_proto':'17',
 634                     'actions': ['mod_tp_src:44444',
 635                                 'mod_tp_dst:44444', 'goto_table:3']}
 636             vswitch.add_flow(bridge, flow)
 637         else:
 638             pass
 639
 640
 641     #
 642     # TestSteps realted methods
 643     #
 644     def step_report_status(self, label, status):
 645         """ Log status of test step
 646         """
 647         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 648
 649     def step_stop_vnfs(self):
 650         """ Stop all VNFs started by TestSteps
 651         """
 652         for vnf in self._step_vnf_list:
 653             if self._step_vnf_list[vnf]:
 654                 self._step_vnf_list[vnf].stop()
 655
 656     @staticmethod
 657     def step_eval_param(param, STEP):
 658         # pylint: disable=invalid-name
 659         """ Helper function for #STEP macro evaluation
 660         """
 661         if isinstance(param, str):
 662             # evaluate every #STEP reference inside parameter itself
 663             macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
 664             if macros:
 665                 for macro in macros:
 666                     # pylint: disable=eval-used
 667                     tmp_val = str(eval(macro[1:]))
 668                     param = param.replace(macro, tmp_val)
 669             return param
 670         elif isinstance(param, list) or isinstance(param, tuple):
 671             tmp_list = []
 672             for item in param:
 673                 tmp_list.append(TestCase.step_eval_param(item, STEP))
 674             return tmp_list
 675         elif isinstance(param, dict):
 676             tmp_dict = {}
 677             for (key, value) in param.items():
 678                 tmp_dict[key] = TestCase.step_eval_param(value, STEP)
 679             return tmp_dict
 680         else:
 681             return param
 682
 683     @staticmethod
 684     def step_eval_params(params, step_result):
 685         """ Evaluates referrences to results from previous steps
 686         """
 687         eval_params = []
 688         # evaluate all parameters if needed
 689         for param in params:
 690             eval_params.append(TestCase.step_eval_param(param, step_result))
 691         return eval_params
 692
 693     def step_run(self):
 694         """ Execute actions specified by TestSteps list
 695
 696         :return: False if any error was detected
 697                  True otherwise
 698         """
 699         # anything to do?
 700         if not self.test:
 701             return True
 702
 703         # required for VNFs initialization
 704         loader = Loader()
 705         # initialize list with results
 706         self._step_result = [None] * len(self.test)
 707
 708         # We have to suppress pylint report, because test_object has to be set according
 709         # to the test step definition
 710         # pylint: disable=redefined-variable-type
 711         # run test step by step...
 712         for i, step in enumerate(self.test):
 713             step_ok = not self._step_check
 714             if step[0] == 'vswitch':
 715                 test_object = self._vswitch_ctl.get_vswitch()
 716             elif step[0] == 'namespace':
 717                 test_object = namespace
 718             elif step[0] == 'veth':
 719                 test_object = veth
 720             elif step[0] == 'settings':
 721                 test_object = S
 722             elif step[0] == 'tools':
 723                 test_object = TestStepsTools()
 724                 step[1] = step[1].title()
 725             elif step[0] == 'trafficgen':
 726                 test_object = self._traffic_ctl
 727                 # in case of send_traffic or send_traffic_async methods, ensure
 728                 # that specified traffic values are merged with existing self._traffic
 729                 if step[1].startswith('send_traffic'):
 730                     tmp_traffic = copy.deepcopy(self._traffic)
 731                     tmp_traffic.update(step[2])
 732                     step[2] = tmp_traffic
 733             elif step[0].startswith('vnf'):
 734                 if not self._step_vnf_list[step[0]]:
 735                     # initialize new VM
 736                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 737                 test_object = self._step_vnf_list[step[0]]
 738             elif step[0] == 'wait':
 739                 input(os.linesep + "Step {}: Press Enter to continue with "
 740                       "the next step...".format(i) + os.linesep + os.linesep)
 741                 continue
 742             else:
 743                 self._logger.error("Unsupported test object %s", step[0])
 744                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 745                 self.step_report_status("Step '{}'".format(' '.join(step)),
 746                                         self._step_status['status'])
 747                 return False
 748
 749             test_method = getattr(test_object, step[1])
 750             if self._step_check:
 751                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 752             else:
 753                 test_method_check = None
 754
 755             step_params = []
 756             try:
 757                 # eval parameters, but use only valid step_results
 758                 # to support negative indexes
 759                 step_params = TestCase.step_eval_params(step[2:], self._step_result[:i])
 760                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 761                 self._logger.debug("Step %s '%s' start", i, step_log)
 762                 self._step_result[i] = test_method(*step_params)
 763                 self._logger.debug("Step %s '%s' results '%s'", i,
 764                                    step_log, self._step_result[i])
 765                 time.sleep(5)
 766                 if self._step_check:
 767                     step_ok = test_method_check(self._step_result[i], *step_params)
 768             except (AssertionError, AttributeError, IndexError) as ex:
 769                 step_ok = False
 770                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 771
 772             if self._step_check:
 773                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 774
 775             if not step_ok:
 776                 self._step_status = {'status' : False, 'details' : step_log}
 777                 # Stop all VNFs started by TestSteps
 778                 self.step_stop_vnfs()
 779                 return False
 780
 781         # all steps processed without any issue
 782         return True
 783
 784     #
 785     # get methods for TestCase members, which needs to be publicly available
 786     #
 787     def get_output_file(self):
 788         """Return content of self._output_file member
 789         """
 790         return self._output_file
 791
 792     def get_desc(self):
 793         """Return content of self.desc member
 794         """
 795         return self.desc
 796
 797     def get_versions(self):
 798         """Return content of self.versions member
 799         """
 800         return self._versions
 801
 802     def get_traffic(self):
 803         """Return content of self._traffic member
 804         """
 805         return self._traffic
 806
 807     def get_tc_results(self):
 808         """Return content of self._tc_results member
 809         """
 810         return self._tc_results
 811
 812     def get_collector(self):
 813         """Return content of self._collector member
 814         """
 815         return self._collector