testcases/testcase.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from conf import settings as S
  28 from conf import get_test_param, merge_spec
  29 import core.component_factory as component_factory
  30 from core.loader import Loader
  31 from core.results.results_constants import ResultsConstants
  32 from tools import tasks
  33 from tools import hugepages
  34 from tools import functions
  35 from tools import namespace
  36 from tools import veth
  37 from tools.teststepstools import TestStepsTools
  38
  39 CHECK_PREFIX = 'validate_'
  40
  41 class TestCase(object):
  42     """TestCase base class
  43
  44     In this basic form runs RFC2544 throughput test
  45     """
  46     def __init__(self, cfg):
  47         """Pull out fields from test config
  48
  49         :param cfg: A dictionary of string-value pairs describing the test
  50             configuration. Both the key and values strings use well-known
  51             values.
  52         :param results_dir: Where the csv formatted results are written.
  53         """
  54         self._testcase_start_time = time.time()
  55         self._hugepages_mounted = False
  56         self._traffic_ctl = None
  57         self._vnf_ctl = None
  58         self._vswitch_ctl = None
  59         self._collector = None
  60         self._loadgen = None
  61         self._output_file = None
  62         self._tc_results = None
  63         self._settings_original = {}
  64         self._settings_paths_modified = False
  65         self._testcast_run_time = None
  66         # initialization of step driven specific members
  67         self._step_check = False    # by default don't check result for step driven testcases
  68         self._step_vnf_list = {}
  69         self._step_result = []
  70         self._step_status = None
  71         self._testcase_run_time = None
  72
  73         # store all GUEST_ specific settings to keep original values before their expansion
  74         for key in S.__dict__:
  75             if key.startswith('GUEST_'):
  76                 self._settings_original[key] = S.getValue(key)
  77
  78         self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  79         self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
  80         self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  81         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  82         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  83         test_params = merge_spec(test_params, tc_test_params)
  84         self._update_settings('TEST_PARAMS', test_params)
  85         S.check_test_params()
  86
  87         # override all redefined GUEST_ values to have them expanded correctly
  88         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  89         for key in tmp_test_params:
  90             if key.startswith('GUEST_'):
  91                 S.setValue(key, S.getValue(key))
  92                 S.getValue('TEST_PARAMS').pop(key)
  93
  94         # update global settings
  95         functions.settings_update_paths()
  96
  97         # set test parameters; CLI options take precedence to testcase settings
  98         self._logger = logging.getLogger(__name__)
  99         self.name = cfg['Name']
 100         self.desc = cfg.get('Description', 'No description given.')
 101         self.test = cfg.get('TestSteps', None)
 102
 103         bidirectional = S.getValue('TRAFFIC')['bidir']
 104         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 105             raise TypeError(
 106                 'Bi-dir value must be of type string')
 107         bidirectional = bidirectional.title()  # Keep things consistent
 108
 109         self.deployment = cfg['Deployment']
 110         self._frame_mod = cfg.get('Frame Modification', None)
 111
 112         self._tunnel_type = None
 113         self._tunnel_operation = None
 114
 115         if self.deployment == 'op2p':
 116             self._tunnel_operation = cfg['Tunnel Operation']
 117
 118             if 'Tunnel Type' in cfg:
 119                 self._tunnel_type = cfg['Tunnel Type']
 120                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 121                                                    self._tunnel_type)
 122
 123         # check if test requires background load and which generator it uses
 124         self._load_cfg = cfg.get('Load', None)
 125         if self._load_cfg and 'tool' in self._load_cfg:
 126             self._loadgen = self._load_cfg['tool']
 127         else:
 128             # background load is not requested, so use dummy implementation
 129             self._loadgen = "Dummy"
 130
 131         if self._frame_mod:
 132             self._frame_mod = self._frame_mod.lower()
 133         self._results_dir = S.getValue('RESULTS_PATH')
 134
 135         # set traffic details, so they can be passed to vswitch and traffic ctls
 136         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 137         self._traffic.update({'bidir': bidirectional,
 138                               'tunnel_type': self._tunnel_type,})
 139
 140         # Packet Forwarding mode
 141         self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower()
 142
 143         # trafficgen configuration required for tests of tunneling protocols
 144         if self.deployment == "op2p":
 145             self._traffic['l2'].update({'srcmac':
 146                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 147                                         'dstmac':
 148                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 149
 150             self._traffic['l3'].update({'srcip':
 151                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 152                                         'dstip':
 153                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 154
 155             if self._tunnel_operation == "decapsulation":
 156                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 157                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 158                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 159         elif len(S.getValue('NICS')) and \
 160              (S.getValue('NICS')[0]['type'] == 'vf' or
 161               S.getValue('NICS')[1]['type'] == 'vf'):
 162             mac1 = S.getValue('NICS')[0]['mac']
 163             mac2 = S.getValue('NICS')[1]['mac']
 164             if mac1 and mac2:
 165                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 166             else:
 167                 self._logger.debug("MAC addresses can not be read")
 168
 169         # count how many VNFs are involved in TestSteps
 170         if self.test:
 171             for step in self.test:
 172                 if step[0].startswith('vnf'):
 173                     self._step_vnf_list[step[0]] = None
 174
 175     def run_initialize(self):
 176         """ Prepare test execution environment
 177         """
 178         self._logger.debug(self.name)
 179
 180         # mount hugepages if needed
 181         self._mount_hugepages()
 182
 183         self._logger.debug("Controllers:")
 184         loader = Loader()
 185         self._traffic_ctl = component_factory.create_traffic(
 186             self._traffic['traffic_type'],
 187             loader.get_trafficgen_class())
 188
 189         self._vnf_ctl = component_factory.create_vnf(
 190             self.deployment,
 191             loader.get_vnf_class(),
 192             len(self._step_vnf_list))
 193
 194         # verify enough hugepages are free to run the testcase
 195         if not self._check_for_enough_hugepages():
 196             raise RuntimeError('Not enough hugepages free to run test.')
 197
 198         # perform guest related handling
 199         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 200         if tmp_vm_count:
 201             # copy sources of l2 forwarding tools into VM shared dir if needed
 202             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 203
 204             # in case of multi VM in parallel, set the number of streams to the number of VMs
 205             if self.deployment.startswith('pvpv'):
 206                 # for each VM NIC pair we need an unique stream
 207                 streams = 0
 208                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 209                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 210                 self._logger.debug("VMs with parallel connection were detected. "
 211                                    "Thus Number of streams was set to %s", streams)
 212                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 213                 # user can define a proper stream count manually
 214                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 215                     self._traffic.update({'multistream': streams})
 216
 217             # OVS Vanilla requires guest VM MAC address and IPs to work
 218             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 219                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 220                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 221                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 222                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 223
 224         if self._vswitch_none:
 225             self._vswitch_ctl = component_factory.create_pktfwd(
 226                 self.deployment,
 227                 loader.get_pktfwd_class())
 228         else:
 229             self._vswitch_ctl = component_factory.create_vswitch(
 230                 self.deployment,
 231                 loader.get_vswitch_class(),
 232                 self._traffic,
 233                 self._tunnel_operation)
 234
 235         self._collector = component_factory.create_collector(
 236             loader.get_collector_class(),
 237             self._results_dir, self.name)
 238         self._loadgen = component_factory.create_loadgen(
 239             self._loadgen,
 240             self._load_cfg)
 241
 242         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 243                                          "_" + self.deployment + ".csv")
 244
 245         self._step_status = {'status' : True, 'details' : ''}
 246
 247         self._logger.debug("Setup:")
 248
 249     def run_finalize(self):
 250         """ Tear down test execution environment and record test results
 251         """
 252         # Stop all VNFs started by TestSteps in case that something went wrong
 253         self.step_stop_vnfs()
 254
 255         # umount hugepages if mounted
 256         self._umount_hugepages()
 257
 258         # restore original settings
 259         S.load_from_dict(self._settings_original)
 260
 261         # cleanup any namespaces created
 262         if os.path.isdir('/tmp/namespaces'):
 263             namespace_list = os.listdir('/tmp/namespaces')
 264             if len(namespace_list):
 265                 self._logger.info('Cleaning up namespaces')
 266             for name in namespace_list:
 267                 namespace.delete_namespace(name)
 268             os.rmdir('/tmp/namespaces')
 269         # cleanup any veth ports created
 270         if os.path.isdir('/tmp/veth'):
 271             veth_list = os.listdir('/tmp/veth')
 272             if len(veth_list):
 273                 self._logger.info('Cleaning up veth ports')
 274             for eth in veth_list:
 275                 port1, port2 = eth.split('-')
 276                 veth.del_veth_port(port1, port2)
 277             os.rmdir('/tmp/veth')
 278
 279     def run_report(self):
 280         """ Report test results
 281         """
 282         self._logger.debug("self._collector Results:")
 283         self._collector.print_results()
 284
 285         results = self._traffic_ctl.get_results()
 286         if results:
 287             self._logger.debug("Traffic Results:")
 288             self._traffic_ctl.print_results()
 289
 290             self._tc_results = self._append_results(results)
 291             TestCase.write_result_to_file(self._tc_results, self._output_file)
 292
 293     def run(self):
 294         """Run the test
 295
 296         All setup and teardown through controllers is included.
 297         """
 298         # prepare test execution environment
 299         self.run_initialize()
 300
 301         try:
 302             with self._vswitch_ctl, self._loadgen:
 303                 with self._vnf_ctl, self._collector:
 304                     if not self._vswitch_none:
 305                         self._add_flows()
 306
 307                     with self._traffic_ctl:
 308                         # execute test based on TestSteps definition if needed...
 309                         if self.step_run():
 310                             # ...and continue with traffic generation, but keep
 311                             # in mind, that clean deployment does not configure
 312                             # OVS nor executes the traffic
 313                             if self.deployment != 'clean':
 314                                 self._traffic_ctl.send_traffic(self._traffic)
 315
 316                         # dump vswitch flows before they are affected by VNF termination
 317                         if not self._vswitch_none:
 318                             self._vswitch_ctl.dump_vswitch_flows()
 319
 320                     # garbage collection for case that TestSteps modify existing deployment
 321                     self.step_stop_vnfs()
 322
 323         finally:
 324             # tear down test execution environment and log results
 325             self.run_finalize()
 326
 327         self._testcase_run_time = time.strftime("%H:%M:%S",
 328                                                 time.gmtime(time.time() -
 329                                                             self._testcase_start_time))
 330         logging.info("Testcase execution time: " + self._testcase_run_time)
 331         # report test results
 332         self.run_report()
 333
 334     def _update_settings(self, param, value):
 335         """ Check value of given configuration parameter
 336         In case that new value is different, then testcase
 337         specific settings is updated and original value stored
 338
 339         :param param: Name of parameter inside settings
 340         :param value: Disired parameter value
 341         """
 342         orig_value = S.getValue(param)
 343         if orig_value != value:
 344             self._settings_original[param] = copy.deepcopy(orig_value)
 345             S.setValue(param, value)
 346
 347     def _append_results(self, results):
 348         """
 349         Method appends mandatory Test Case results to list of dictionaries.
 350
 351         :param results: list of dictionaries which contains results from
 352                 traffic generator.
 353
 354         :returns: modified list of dictionaries.
 355         """
 356         for item in results:
 357             item[ResultsConstants.ID] = self.name
 358             item[ResultsConstants.DEPLOYMENT] = self.deployment
 359             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 360             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 361             if self._traffic['multistream']:
 362                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 363                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 364                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 365             if self._vnf_ctl.get_vnfs_number():
 366                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 367             if self._tunnel_type:
 368                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 369         return results
 370
 371     def _copy_fwd_tools_for_all_guests(self, vm_count):
 372         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 373         """
 374         # consider only VNFs involved in the test
 375         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 376             self._copy_fwd_tools_for_guest(guest_dir)
 377
 378     def _copy_fwd_tools_for_guest(self, guest_dir):
 379         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 380
 381         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 382         """
 383         # remove shared dir if it exists to avoid issues with file consistency
 384         if os.path.exists(guest_dir):
 385             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 386                            'Removing content of shared directory...', True)
 387
 388         # directory to share files between host and guest
 389         os.makedirs(guest_dir)
 390
 391         # copy sources into shared dir only if neccessary
 392         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 393         if 'testpmd' in guest_loopback:
 394             try:
 395                 # exclude whole .git/ subdirectory and all o-files;
 396                 # It is assumed, that the same RTE_TARGET is used in both host
 397                 # and VMs; This simplification significantly speeds up testpmd
 398                 # build. If we will need a different RTE_TARGET in VM,
 399                 # then we have to build whole DPDK from the scratch in VM.
 400                 # In that case we can copy just DPDK sources (e.g. by excluding
 401                 # all items obtained by git status -unormal --porcelain).
 402                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 403                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 404                 exclude = []
 405                 exclude.append(r'--exclude=.git/')
 406                 exclude.append(r'--exclude=*.o')
 407                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 408                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 409                                 os.path.join(guest_dir, 'DPDK')],
 410                                self._logger,
 411                                'Copying DPDK to shared directory...',
 412                                True)
 413             except subprocess.CalledProcessError:
 414                 self._logger.error('Unable to copy DPDK to shared directory')
 415                 raise
 416         if 'l2fwd' in guest_loopback:
 417             try:
 418                 tasks.run_task(['rsync', '-a', '-r', '-l',
 419                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 420                                 os.path.join(guest_dir, 'l2fwd')],
 421                                self._logger,
 422                                'Copying l2fwd to shared directory...',
 423                                True)
 424             except subprocess.CalledProcessError:
 425                 self._logger.error('Unable to copy l2fwd to shared directory')
 426                 raise
 427
 428     def _mount_hugepages(self):
 429         """Mount hugepages if usage of DPDK or Qemu is detected
 430         """
 431         # hugepages are needed by DPDK and Qemu
 432         if not self._hugepages_mounted and \
 433             (self.deployment.count('v') or \
 434              S.getValue('VSWITCH').lower().count('dpdk') or \
 435              self._vswitch_none or \
 436              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 437             hugepages.mount_hugepages()
 438             self._hugepages_mounted = True
 439
 440     def _umount_hugepages(self):
 441         """Umount hugepages if they were mounted before
 442         """
 443         if self._hugepages_mounted:
 444             hugepages.umount_hugepages()
 445             self._hugepages_mounted = False
 446
 447     def _check_for_enough_hugepages(self):
 448         """Check to make sure enough hugepages are free to satisfy the
 449         test environment.
 450         """
 451         hugepages_needed = 0
 452         hugepage_size = hugepages.get_hugepage_size()
 453         # get hugepage amounts per guest involved in the test
 454         for guest in range(self._vnf_ctl.get_vnfs_number()):
 455             hugepages_needed += math.ceil((int(S.getValue(
 456                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 457
 458         # get hugepage amounts for each socket on dpdk
 459         sock0_mem, sock1_mem = 0, 0
 460         if S.getValue('VSWITCH').lower().count('dpdk'):
 461             # the import below needs to remain here and not put into the module
 462             # imports because of an exception due to settings not yet loaded
 463             from vswitches import ovs_dpdk_vhost
 464             if ovs_dpdk_vhost.OvsDpdkVhost.old_dpdk_config():
 465                 match = re.search(
 466                     r'-socket-mem\s+(\d+),(\d+)',
 467                     ''.join(S.getValue('VSWITCHD_DPDK_ARGS')))
 468                 if match:
 469                     sock0_mem, sock1_mem = (int(match.group(1)) * 1024 / hugepage_size,
 470                                             int(match.group(2)) * 1024 / hugepage_size)
 471                 else:
 472                     logging.info(
 473                         'Could not parse socket memory config in dpdk params.')
 474             else:
 475                 sock0_mem, sock1_mem = (
 476                     S.getValue(
 477                         'VSWITCHD_DPDK_CONFIG')['dpdk-socket-mem'].split(','))
 478                 sock0_mem, sock1_mem = (int(sock0_mem) * 1024 / hugepage_size,
 479                                         int(sock1_mem) * 1024 / hugepage_size)
 480
 481         # If hugepages needed, verify the amounts are free
 482         if any([hugepages_needed, sock0_mem, sock1_mem]):
 483             free_hugepages = hugepages.get_free_hugepages()
 484             if hugepages_needed:
 485                 logging.info('Need %s hugepages free for guests',
 486                              hugepages_needed)
 487                 result1 = free_hugepages >= hugepages_needed
 488                 free_hugepages -= hugepages_needed
 489             else:
 490                 result1 = True
 491
 492             if sock0_mem:
 493                 logging.info('Need %s hugepages free for dpdk socket 0',
 494                              sock0_mem)
 495                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 496                 free_hugepages -= sock0_mem
 497             else:
 498                 result2 = True
 499
 500             if sock1_mem:
 501                 logging.info('Need %s hugepages free for dpdk socket 1',
 502                              sock1_mem)
 503                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 504                 free_hugepages -= sock1_mem
 505             else:
 506                 result3 = True
 507
 508             logging.info('Need a total of {} total hugepages'.format(
 509                 hugepages_needed + sock1_mem + sock0_mem))
 510
 511             # The only drawback here is sometimes dpdk doesn't release
 512             # its hugepages on a test failure. This could cause a test
 513             # to fail when dpdk would be OK to start because it will just
 514             # use the previously allocated hugepages.
 515             result4 = True if free_hugepages >= 0 else False
 516
 517             return all([result1, result2, result3, result4])
 518         else:
 519             return True
 520
 521     @staticmethod
 522     def write_result_to_file(results, output):
 523         """Write list of dictionaries to a CSV file.
 524
 525         Each element on list will create separate row in output file.
 526         If output file already exists, data will be appended at the end,
 527         otherwise it will be created.
 528
 529         :param results: list of dictionaries.
 530         :param output: path to output file.
 531         """
 532         with open(output, 'a') as csvfile:
 533
 534             logging.info("Write results to file: " + output)
 535             fieldnames = TestCase._get_unique_keys(results)
 536
 537             writer = csv.DictWriter(csvfile, fieldnames)
 538
 539             if not csvfile.tell():  # file is now empty
 540                 writer.writeheader()
 541
 542             for result in results:
 543                 writer.writerow(result)
 544
 545     @staticmethod
 546     def _get_unique_keys(list_of_dicts):
 547         """Gets unique key values as ordered list of strings in given dicts
 548
 549         :param list_of_dicts: list of dictionaries.
 550
 551         :returns: list of unique keys(strings).
 552         """
 553         result = OrderedDict()
 554         for item in list_of_dicts:
 555             for key in item.keys():
 556                 result[key] = ''
 557
 558         return list(result.keys())
 559
 560     def _add_flows(self):
 561         """Add flows to the vswitch
 562         """
 563         vswitch = self._vswitch_ctl.get_vswitch()
 564         # TODO BOM 15-08-07 the frame mod code assumes that the
 565         # physical ports are ports 1 & 2. The actual numbers
 566         # need to be retrived from the vSwitch and the metadata value
 567         # updated accordingly.
 568         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 569         if self._frame_mod == "vlan":
 570             # 0x8100 => VLAN ethertype
 571             self._logger.debug(" ****   VLAN   ***** ")
 572             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 573                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 574             vswitch.add_flow(bridge, flow)
 575             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 576                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 577             vswitch.add_flow(bridge, flow)
 578         elif self._frame_mod == "mpls":
 579             # 0x8847 => MPLS unicast ethertype
 580             self._logger.debug(" ****   MPLS  ***** ")
 581             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 582                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 583             vswitch.add_flow(bridge, flow)
 584             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 585                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 586             vswitch.add_flow(bridge, flow)
 587         elif self._frame_mod == "mac":
 588             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 589                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 590                                 'goto_table:3']}
 591             vswitch.add_flow(bridge, flow)
 592             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 593                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 594                                 'goto_table:3']}
 595             vswitch.add_flow(bridge, flow)
 596         elif self._frame_mod == "dscp":
 597             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 598             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 599                     'dl_type':'0x0800',
 600                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 601             vswitch.add_flow(bridge, flow)
 602             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 603                     'dl_type':'0x0800',
 604                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 605             vswitch.add_flow(bridge, flow)
 606         elif self._frame_mod == "ttl":
 607             # 251 and 241 are the highest prime numbers < 255
 608             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 609                     'dl_type':'0x0800',
 610                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 611             vswitch.add_flow(bridge, flow)
 612             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 613                     'dl_type':'0x0800',
 614                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 615             vswitch.add_flow(bridge, flow)
 616         elif self._frame_mod == "ip_addr":
 617             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 618                     'dl_type':'0x0800',
 619                     'actions': ['mod_nw_src:10.10.10.10',
 620                                 'mod_nw_dst:20.20.20.20',
 621                                 'goto_table:3']}
 622             vswitch.add_flow(bridge, flow)
 623             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 624                     'dl_type':'0x0800',
 625                     'actions': ['mod_nw_src:20.20.20.20',
 626                                 'mod_nw_dst:10.10.10.10',
 627                                 'goto_table:3']}
 628             vswitch.add_flow(bridge, flow)
 629         elif self._frame_mod == "ip_port":
 630             # TODO BOM 15-08-27 The traffic generated is assumed
 631             # to be UDP (nw_proto 17d) which is the default case but
 632             # we will need to pick up the actual traffic params in use.
 633             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 634                     'dl_type':'0x0800', 'nw_proto':'17',
 635                     'actions': ['mod_tp_src:44444',
 636                                 'mod_tp_dst:44444', 'goto_table:3']}
 637             vswitch.add_flow(bridge, flow)
 638             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 639                     'dl_type':'0x0800', 'nw_proto':'17',
 640                     'actions': ['mod_tp_src:44444',
 641                                 'mod_tp_dst:44444', 'goto_table:3']}
 642             vswitch.add_flow(bridge, flow)
 643         else:
 644             pass
 645
 646
 647     #
 648     # TestSteps realted methods
 649     #
 650     def step_report_status(self, label, status):
 651         """ Log status of test step
 652         """
 653         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 654
 655     def step_stop_vnfs(self):
 656         """ Stop all VNFs started by TestSteps
 657         """
 658         for vnf in self._step_vnf_list:
 659             if self._step_vnf_list[vnf]:
 660                 self._step_vnf_list[vnf].stop()
 661
 662     @staticmethod
 663     def step_eval_param(param, STEP):
 664         # pylint: disable=invalid-name
 665         """ Helper function for #STEP macro evaluation
 666         """
 667         if isinstance(param, str):
 668             # evaluate every #STEP reference inside parameter itself
 669             macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
 670             if macros:
 671                 for macro in macros:
 672                     # pylint: disable=eval-used
 673                     tmp_val = str(eval(macro[1:]))
 674                     param = param.replace(macro, tmp_val)
 675             return param
 676         elif isinstance(param, list) or isinstance(param, tuple):
 677             tmp_list = []
 678             for item in param:
 679                 tmp_list.append(TestCase.step_eval_param(item, STEP))
 680             return tmp_list
 681         elif isinstance(param, dict):
 682             tmp_dict = {}
 683             for (key, value) in param.items():
 684                 tmp_dict[key] = TestCase.step_eval_param(value, STEP)
 685             return tmp_dict
 686         else:
 687             return param
 688
 689     @staticmethod
 690     def step_eval_params(params, step_result):
 691         """ Evaluates referrences to results from previous steps
 692         """
 693         eval_params = []
 694         # evaluate all parameters if needed
 695         for param in params:
 696             eval_params.append(TestCase.step_eval_param(param, step_result))
 697         return eval_params
 698
 699     def step_run(self):
 700         """ Execute actions specified by TestSteps list
 701
 702         :return: False if any error was detected
 703                  True otherwise
 704         """
 705         # anything to do?
 706         if not self.test:
 707             return True
 708
 709         # required for VNFs initialization
 710         loader = Loader()
 711         # initialize list with results
 712         self._step_result = [None] * len(self.test)
 713
 714         # run test step by step...
 715         for i, step in enumerate(self.test):
 716             step_ok = not self._step_check
 717             if step[0] == 'vswitch':
 718                 test_object = self._vswitch_ctl.get_vswitch()
 719             elif step[0] == 'namespace':
 720                 test_object = namespace
 721             elif step[0] == 'veth':
 722                 test_object = veth
 723             elif step[0] == 'settings':
 724                 test_object = S
 725             elif step[0] == 'tools':
 726                 test_object = TestStepsTools()
 727                 step[1] = step[1].title()
 728             elif step[0] == 'trafficgen':
 729                 test_object = self._traffic_ctl
 730                 # in case of send_traffic or send_traffic_async methods, ensure
 731                 # that specified traffic values are merged with existing self._traffic
 732                 if step[1].startswith('send_traffic'):
 733                     tmp_traffic = copy.deepcopy(self._traffic)
 734                     tmp_traffic.update(step[2])
 735                     step[2] = tmp_traffic
 736             elif step[0].startswith('vnf'):
 737                 if not self._step_vnf_list[step[0]]:
 738                     # initialize new VM
 739                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 740                 test_object = self._step_vnf_list[step[0]]
 741             elif step[0] == 'wait':
 742                 input(os.linesep + "Step {}: Press Enter to continue with "
 743                       "the next step...".format(i) + os.linesep + os.linesep)
 744                 continue
 745             else:
 746                 self._logger.error("Unsupported test object %s", step[0])
 747                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 748                 self.step_report_status("Step '{}'".format(' '.join(step)),
 749                                         self._step_status['status'])
 750                 return False
 751
 752             test_method = getattr(test_object, step[1])
 753             if self._step_check:
 754                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 755             else:
 756                 test_method_check = None
 757
 758             step_params = []
 759             try:
 760                 # eval parameters, but use only valid step_results
 761                 # to support negative indexes
 762                 step_params = TestCase.step_eval_params(step[2:], self._step_result[:i])
 763                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 764                 self._logger.debug("Step %s '%s' start", i, step_log)
 765                 self._step_result[i] = test_method(*step_params)
 766                 self._logger.debug("Step %s '%s' results '%s'", i,
 767                                    step_log, self._step_result[i])
 768                 time.sleep(5)
 769                 if self._step_check:
 770                     step_ok = test_method_check(self._step_result[i], *step_params)
 771             except (AssertionError, AttributeError, IndexError) as ex:
 772                 step_ok = False
 773                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 774
 775             if self._step_check:
 776                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 777
 778             if not step_ok:
 779                 self._step_status = {'status' : False, 'details' : step_log}
 780                 # Stop all VNFs started by TestSteps
 781                 self.step_stop_vnfs()
 782                 return False
 783
 784         # all steps processed without any issue
 785         return True