testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import get_test_param, merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39
  40 CHECK_PREFIX = 'validate_'
  41
  42 # pylint: disable=too-many-instance-attributes
  43 class TestCase(object):
  44     """TestCase base class
  45
  46     In this basic form runs RFC2544 throughput test
  47     """
  48     # pylint: disable=too-many-statements
  49     def __init__(self, test_cfg):
  50         """Pull out fields from test config
  51
  52         :param test_cfg: A dictionary of string-value pairs describing the test
  53             configuration. Both the key and values strings use well-known
  54             values.
  55         :param results_dir: Where the csv formatted results are written.
  56         """
  57         # make a local copy of test configuration to avoid modification of
  58         # original content used in vsperf main script
  59         cfg = copy.deepcopy(test_cfg)
  60
  61         self._testcase_start_time = time.time()
  62         self._testcase_stop_time = self._testcase_start_time
  63         self._hugepages_mounted = False
  64         self._traffic_ctl = None
  65         self._vnf_ctl = None
  66         self._vswitch_ctl = None
  67         self._collector = None
  68         self._loadgen = None
  69         self._output_file = None
  70         self._tc_results = None
  71         self._settings_original = {}
  72         self._settings_paths_modified = False
  73         self._testcast_run_time = None
  74         self._versions = []
  75         # initialization of step driven specific members
  76         self._step_check = False    # by default don't check result for step driven testcases
  77         self._step_vnf_list = {}
  78         self._step_result = []
  79         self._step_status = None
  80         self._testcase_run_time = None
  81
  82         # store all GUEST_ specific settings to keep original values before their expansion
  83         for key in S.__dict__:
  84             if key.startswith('GUEST_'):
  85                 self._settings_original[key] = S.getValue(key)
  86
  87         self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  88         self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
  89         self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  90         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  91         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  92         test_params = merge_spec(test_params, tc_test_params)
  93         self._update_settings('TEST_PARAMS', test_params)
  94         S.check_test_params()
  95
  96         # override all redefined GUEST_ values to have them expanded correctly
  97         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  98         for key in tmp_test_params:
  99             if key.startswith('GUEST_'):
 100                 S.setValue(key, S.getValue(key))
 101                 S.getValue('TEST_PARAMS').pop(key)
 102
 103         # update global settings
 104         functions.settings_update_paths()
 105
 106         # set test parameters; CLI options take precedence to testcase settings
 107         self._logger = logging.getLogger(__name__)
 108         self.name = cfg['Name']
 109         self.desc = cfg.get('Description', 'No description given.')
 110         self.test = cfg.get('TestSteps', None)
 111
 112         bidirectional = S.getValue('TRAFFIC')['bidir']
 113         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 114             raise TypeError(
 115                 'Bi-dir value must be of type string')
 116         bidirectional = bidirectional.title()  # Keep things consistent
 117
 118         self.deployment = cfg['Deployment']
 119         self._frame_mod = cfg.get('Frame Modification', None)
 120
 121         self._tunnel_type = None
 122         self._tunnel_operation = None
 123
 124         if self.deployment == 'op2p':
 125             self._tunnel_operation = cfg['Tunnel Operation']
 126
 127             if 'Tunnel Type' in cfg:
 128                 self._tunnel_type = cfg['Tunnel Type']
 129                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 130                                                    self._tunnel_type)
 131
 132         # check if test requires background load and which generator it uses
 133         self._load_cfg = cfg.get('Load', None)
 134
 135         if self._frame_mod:
 136             self._frame_mod = self._frame_mod.lower()
 137         self._results_dir = S.getValue('RESULTS_PATH')
 138
 139         # set traffic details, so they can be passed to vswitch and traffic ctls
 140         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 141         self._traffic.update({'bidir': bidirectional,
 142                               'tunnel_type': self._tunnel_type,})
 143
 144         self._traffic = functions.check_traffic(self._traffic)
 145
 146         # Packet Forwarding mode
 147         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 148
 149         # trafficgen configuration required for tests of tunneling protocols
 150         if self.deployment == "op2p":
 151             self._traffic['l2'].update({'srcmac':
 152                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 153                                         'dstmac':
 154                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 155
 156             self._traffic['l3'].update({'srcip':
 157                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 158                                         'dstip':
 159                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 160
 161             if self._tunnel_operation == "decapsulation":
 162                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 163                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 164                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 165         elif len(S.getValue('NICS')) and \
 166              (S.getValue('NICS')[0]['type'] == 'vf' or
 167               S.getValue('NICS')[1]['type'] == 'vf'):
 168             mac1 = S.getValue('NICS')[0]['mac']
 169             mac2 = S.getValue('NICS')[1]['mac']
 170             if mac1 and mac2:
 171                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 172             else:
 173                 self._logger.debug("MAC addresses can not be read")
 174
 175         # count how many VNFs are involved in TestSteps
 176         if self.test:
 177             for step in self.test:
 178                 if step[0].startswith('vnf'):
 179                     self._step_vnf_list[step[0]] = None
 180
 181     def run_initialize(self):
 182         """ Prepare test execution environment
 183         """
 184         self._logger.debug(self.name)
 185
 186         # mount hugepages if needed
 187         self._mount_hugepages()
 188
 189         self._logger.debug("Controllers:")
 190         loader = Loader()
 191         self._traffic_ctl = component_factory.create_traffic(
 192             self._traffic['traffic_type'],
 193             loader.get_trafficgen_class())
 194
 195         self._vnf_ctl = component_factory.create_vnf(
 196             self.deployment,
 197             loader.get_vnf_class(),
 198             len(self._step_vnf_list))
 199
 200         # verify enough hugepages are free to run the testcase
 201         if not self._check_for_enough_hugepages():
 202             raise RuntimeError('Not enough hugepages free to run test.')
 203
 204         # perform guest related handling
 205         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 206         if tmp_vm_count:
 207             # copy sources of l2 forwarding tools into VM shared dir if needed
 208             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 209
 210             # in case of multi VM in parallel, set the number of streams to the number of VMs
 211             if self.deployment.startswith('pvpv'):
 212                 # for each VM NIC pair we need an unique stream
 213                 streams = 0
 214                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 215                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 216                 self._logger.debug("VMs with parallel connection were detected. "
 217                                    "Thus Number of streams was set to %s", streams)
 218                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 219                 # user can define a proper stream count manually
 220                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 221                     self._traffic.update({'multistream': streams})
 222
 223             # OVS Vanilla requires guest VM MAC address and IPs to work
 224             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 225                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 226                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 227                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 228                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 229
 230         if self._vswitch_none:
 231             self._vswitch_ctl = component_factory.create_pktfwd(
 232                 self.deployment,
 233                 loader.get_pktfwd_class())
 234         else:
 235             self._vswitch_ctl = component_factory.create_vswitch(
 236                 self.deployment,
 237                 loader.get_vswitch_class(),
 238                 self._traffic,
 239                 self._tunnel_operation)
 240
 241         self._collector = component_factory.create_collector(
 242             loader.get_collector_class(),
 243             self._results_dir, self.name)
 244         self._loadgen = component_factory.create_loadgen(
 245             loader.get_loadgen_class(),
 246             self._load_cfg)
 247
 248         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 249                                          "_" + self.deployment + ".csv")
 250
 251         self._step_status = {'status' : True, 'details' : ''}
 252
 253         self._logger.debug("Setup:")
 254
 255     def run_finalize(self):
 256         """ Tear down test execution environment and record test results
 257         """
 258         # Stop all VNFs started by TestSteps in case that something went wrong
 259         self.step_stop_vnfs()
 260
 261         # umount hugepages if mounted
 262         self._umount_hugepages()
 263
 264         # cleanup any namespaces created
 265         if os.path.isdir('/tmp/namespaces'):
 266             namespace_list = os.listdir('/tmp/namespaces')
 267             if len(namespace_list):
 268                 self._logger.info('Cleaning up namespaces')
 269             for name in namespace_list:
 270                 namespace.delete_namespace(name)
 271             os.rmdir('/tmp/namespaces')
 272         # cleanup any veth ports created
 273         if os.path.isdir('/tmp/veth'):
 274             veth_list = os.listdir('/tmp/veth')
 275             if len(veth_list):
 276                 self._logger.info('Cleaning up veth ports')
 277             for eth in veth_list:
 278                 port1, port2 = eth.split('-')
 279                 veth.del_veth_port(port1, port2)
 280             os.rmdir('/tmp/veth')
 281
 282     def run_report(self):
 283         """ Report test results
 284         """
 285         self._logger.debug("self._collector Results:")
 286         self._collector.print_results()
 287
 288         results = self._traffic_ctl.get_results()
 289         if results:
 290             self._logger.debug("Traffic Results:")
 291             self._traffic_ctl.print_results()
 292
 293         if self._tc_results is None:
 294             self._tc_results = self._append_results(results)
 295         else:
 296             # integration step driven tests have their status and possible
 297             # failure details stored inside self._tc_results
 298             results = self._append_results(results)
 299             if len(self._tc_results) < len(results):
 300                 if len(self._tc_results) > 1:
 301                     raise RuntimeError('Testcase results do not match:'
 302                                        'results: {}\n'
 303                                        'trafficgen results: {}\n',
 304                                        self._tc_results,
 305                                        results)
 306                 else:
 307                     tmp_results = copy.deepcopy(self._tc_results[0])
 308                     self._tc_results = []
 309                     for res in results:
 310                         tmp_res = copy.deepcopy(tmp_results)
 311                         tmp_res.update(res)
 312                         self._tc_results.append(tmp_res)
 313             else:
 314                 for i, result in enumerate(results):
 315                     self._tc_results[i].update(result)
 316
 317         TestCase.write_result_to_file(self._tc_results, self._output_file)
 318
 319     def run(self):
 320         """Run the test
 321
 322         All setup and teardown through controllers is included.
 323         """
 324         # prepare test execution environment
 325         self.run_initialize()
 326
 327         try:
 328             with self._vswitch_ctl, self._loadgen:
 329                 with self._vnf_ctl, self._collector:
 330                     if not self._vswitch_none:
 331                         self._add_flows()
 332
 333                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 334
 335                     with self._traffic_ctl:
 336                         # execute test based on TestSteps definition if needed...
 337                         if self.step_run():
 338                             # ...and continue with traffic generation, but keep
 339                             # in mind, that clean deployment does not configure
 340                             # OVS nor executes the traffic
 341                             if self.deployment != 'clean':
 342                                 self._traffic_ctl.send_traffic(self._traffic)
 343
 344                         # dump vswitch flows before they are affected by VNF termination
 345                         if not self._vswitch_none:
 346                             self._vswitch_ctl.dump_vswitch_flows()
 347
 348                     # garbage collection for case that TestSteps modify existing deployment
 349                     self.step_stop_vnfs()
 350
 351         finally:
 352             # tear down test execution environment and log results
 353             self.run_finalize()
 354
 355         self._testcase_stop_time = time.time()
 356         self._testcase_run_time = time.strftime("%H:%M:%S",
 357                                                 time.gmtime(self._testcase_stop_time -
 358                                                             self._testcase_start_time))
 359         logging.info("Testcase execution time: " + self._testcase_run_time)
 360         # report test results
 361         self.run_report()
 362
 363         # restore original settings
 364         for key in self._settings_original:
 365             S.setValue(key, self._settings_original[key])
 366
 367     def _update_settings(self, param, value):
 368         """ Check value of given configuration parameter
 369         In case that new value is different, then testcase
 370         specific settings is updated and original value stored
 371
 372         :param param: Name of parameter inside settings
 373         :param value: Disired parameter value
 374         """
 375         orig_value = S.getValue(param)
 376         if orig_value != value:
 377             self._settings_original[param] = copy.deepcopy(orig_value)
 378             S.setValue(param, value)
 379
 380     def _append_results(self, results):
 381         """
 382         Method appends mandatory Test Case results to list of dictionaries.
 383
 384         :param results: list of dictionaries which contains results from
 385                 traffic generator.
 386
 387         :returns: modified list of dictionaries.
 388         """
 389         for item in results:
 390             item[ResultsConstants.ID] = self.name
 391             item[ResultsConstants.DEPLOYMENT] = self.deployment
 392             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 393             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 394             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 395             # convert timestamps to human readable format
 396             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 397                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 398             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 399                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 400             if self._traffic['multistream']:
 401                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 402                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 403                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 404             if self._vnf_ctl.get_vnfs_number():
 405                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 406             if self._tunnel_type:
 407                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 408         return results
 409
 410     def _copy_fwd_tools_for_all_guests(self, vm_count):
 411         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 412         """
 413         # consider only VNFs involved in the test
 414         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 415             self._copy_fwd_tools_for_guest(guest_dir)
 416
 417     def _copy_fwd_tools_for_guest(self, guest_dir):
 418         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 419
 420         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 421         """
 422         # remove shared dir if it exists to avoid issues with file consistency
 423         if os.path.exists(guest_dir):
 424             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 425                            'Removing content of shared directory...', True)
 426
 427         # directory to share files between host and guest
 428         os.makedirs(guest_dir)
 429
 430         # copy sources into shared dir only if neccessary
 431         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 432         if 'testpmd' in guest_loopback:
 433             try:
 434                 # exclude whole .git/ subdirectory and all o-files;
 435                 # It is assumed, that the same RTE_TARGET is used in both host
 436                 # and VMs; This simplification significantly speeds up testpmd
 437                 # build. If we will need a different RTE_TARGET in VM,
 438                 # then we have to build whole DPDK from the scratch in VM.
 439                 # In that case we can copy just DPDK sources (e.g. by excluding
 440                 # all items obtained by git status -unormal --porcelain).
 441                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 442                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 443                 exclude = []
 444                 exclude.append(r'--exclude=.git/')
 445                 exclude.append(r'--exclude=*.o')
 446                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 447                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 448                                 os.path.join(guest_dir, 'DPDK')],
 449                                self._logger,
 450                                'Copying DPDK to shared directory...',
 451                                True)
 452             except subprocess.CalledProcessError:
 453                 self._logger.error('Unable to copy DPDK to shared directory')
 454                 raise
 455         if 'l2fwd' in guest_loopback:
 456             try:
 457                 tasks.run_task(['rsync', '-a', '-r', '-l',
 458                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 459                                 os.path.join(guest_dir, 'l2fwd')],
 460                                self._logger,
 461                                'Copying l2fwd to shared directory...',
 462                                True)
 463             except subprocess.CalledProcessError:
 464                 self._logger.error('Unable to copy l2fwd to shared directory')
 465                 raise
 466
 467     def _mount_hugepages(self):
 468         """Mount hugepages if usage of DPDK or Qemu is detected
 469         """
 470         # pylint: disable=too-many-boolean-expressions
 471         # hugepages are needed by DPDK and Qemu
 472         if not self._hugepages_mounted and \
 473             (self.deployment.count('v') or \
 474              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 475              self._vswitch_none or \
 476              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 477             hugepages.mount_hugepages()
 478             self._hugepages_mounted = True
 479
 480     def _umount_hugepages(self):
 481         """Umount hugepages if they were mounted before
 482         """
 483         if self._hugepages_mounted:
 484             hugepages.umount_hugepages()
 485             self._hugepages_mounted = False
 486
 487     def _check_for_enough_hugepages(self):
 488         """Check to make sure enough hugepages are free to satisfy the
 489         test environment.
 490         """
 491         hugepages_needed = 0
 492         hugepage_size = hugepages.get_hugepage_size()
 493         # get hugepage amounts per guest involved in the test
 494         for guest in range(self._vnf_ctl.get_vnfs_number()):
 495             hugepages_needed += math.ceil((int(S.getValue(
 496                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 497
 498         # get hugepage amounts for each socket on dpdk
 499         sock0_mem, sock1_mem = 0, 0
 500
 501         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 502             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 503             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 504                                     int(sock_mem[1]) * 1024 / hugepage_size)
 505
 506         # If hugepages needed, verify the amounts are free
 507         if any([hugepages_needed, sock0_mem, sock1_mem]):
 508             free_hugepages = hugepages.get_free_hugepages()
 509             if hugepages_needed:
 510                 logging.info('Need %s hugepages free for guests',
 511                              hugepages_needed)
 512                 result1 = free_hugepages >= hugepages_needed
 513                 free_hugepages -= hugepages_needed
 514             else:
 515                 result1 = True
 516
 517             if sock0_mem:
 518                 logging.info('Need %s hugepages free for dpdk socket 0',
 519                              sock0_mem)
 520                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 521                 free_hugepages -= sock0_mem
 522             else:
 523                 result2 = True
 524
 525             if sock1_mem:
 526                 logging.info('Need %s hugepages free for dpdk socket 1',
 527                              sock1_mem)
 528                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 529                 free_hugepages -= sock1_mem
 530             else:
 531                 result3 = True
 532
 533             logging.info('Need a total of %s total hugepages',
 534                          hugepages_needed + sock1_mem + sock0_mem)
 535
 536             # The only drawback here is sometimes dpdk doesn't release
 537             # its hugepages on a test failure. This could cause a test
 538             # to fail when dpdk would be OK to start because it will just
 539             # use the previously allocated hugepages.
 540             result4 = True if free_hugepages >= 0 else False
 541
 542             return all([result1, result2, result3, result4])
 543         else:
 544             return True
 545
 546     @staticmethod
 547     def write_result_to_file(results, output):
 548         """Write list of dictionaries to a CSV file.
 549
 550         Each element on list will create separate row in output file.
 551         If output file already exists, data will be appended at the end,
 552         otherwise it will be created.
 553
 554         :param results: list of dictionaries.
 555         :param output: path to output file.
 556         """
 557         with open(output, 'a') as csvfile:
 558
 559             logging.info("Write results to file: " + output)
 560             fieldnames = TestCase._get_unique_keys(results)
 561
 562             writer = csv.DictWriter(csvfile, fieldnames)
 563
 564             if not csvfile.tell():  # file is now empty
 565                 writer.writeheader()
 566
 567             for result in results:
 568                 writer.writerow(result)
 569
 570     @staticmethod
 571     def _get_unique_keys(list_of_dicts):
 572         """Gets unique key values as ordered list of strings in given dicts
 573
 574         :param list_of_dicts: list of dictionaries.
 575
 576         :returns: list of unique keys(strings).
 577         """
 578         result = OrderedDict()
 579         for item in list_of_dicts:
 580             for key in item.keys():
 581                 result[key] = ''
 582
 583         return list(result.keys())
 584
 585     def _add_flows(self):
 586         """Add flows to the vswitch
 587         """
 588         vswitch = self._vswitch_ctl.get_vswitch()
 589         # NOTE BOM 15-08-07 the frame mod code assumes that the
 590         # physical ports are ports 1 & 2. The actual numbers
 591         # need to be retrived from the vSwitch and the metadata value
 592         # updated accordingly.
 593         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 594         if self._frame_mod == "vlan":
 595             # 0x8100 => VLAN ethertype
 596             self._logger.debug(" ****   VLAN   ***** ")
 597             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 598                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 599             vswitch.add_flow(bridge, flow)
 600             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 601                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 602             vswitch.add_flow(bridge, flow)
 603         elif self._frame_mod == "mpls":
 604             # 0x8847 => MPLS unicast ethertype
 605             self._logger.debug(" ****   MPLS  ***** ")
 606             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 607                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 608             vswitch.add_flow(bridge, flow)
 609             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 610                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 611             vswitch.add_flow(bridge, flow)
 612         elif self._frame_mod == "mac":
 613             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 614                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 615                                 'goto_table:3']}
 616             vswitch.add_flow(bridge, flow)
 617             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 618                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 619                                 'goto_table:3']}
 620             vswitch.add_flow(bridge, flow)
 621         elif self._frame_mod == "dscp":
 622             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 623             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 624                     'dl_type':'0x0800',
 625                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 626             vswitch.add_flow(bridge, flow)
 627             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 628                     'dl_type':'0x0800',
 629                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 630             vswitch.add_flow(bridge, flow)
 631         elif self._frame_mod == "ttl":
 632             # 251 and 241 are the highest prime numbers < 255
 633             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 634                     'dl_type':'0x0800',
 635                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 636             vswitch.add_flow(bridge, flow)
 637             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 638                     'dl_type':'0x0800',
 639                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 640             vswitch.add_flow(bridge, flow)
 641         elif self._frame_mod == "ip_addr":
 642             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 643                     'dl_type':'0x0800',
 644                     'actions': ['mod_nw_src:10.10.10.10',
 645                                 'mod_nw_dst:20.20.20.20',
 646                                 'goto_table:3']}
 647             vswitch.add_flow(bridge, flow)
 648             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 649                     'dl_type':'0x0800',
 650                     'actions': ['mod_nw_src:20.20.20.20',
 651                                 'mod_nw_dst:10.10.10.10',
 652                                 'goto_table:3']}
 653             vswitch.add_flow(bridge, flow)
 654         elif self._frame_mod == "ip_port":
 655             # NOTE BOM 15-08-27 The traffic generated is assumed
 656             # to be UDP (nw_proto 17d) which is the default case but
 657             # we will need to pick up the actual traffic params in use.
 658             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 659                     'dl_type':'0x0800', 'nw_proto':'17',
 660                     'actions': ['mod_tp_src:44444',
 661                                 'mod_tp_dst:44444', 'goto_table:3']}
 662             vswitch.add_flow(bridge, flow)
 663             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 664                     'dl_type':'0x0800', 'nw_proto':'17',
 665                     'actions': ['mod_tp_src:44444',
 666                                 'mod_tp_dst:44444', 'goto_table:3']}
 667             vswitch.add_flow(bridge, flow)
 668         else:
 669             pass
 670
 671
 672     #
 673     # TestSteps realted methods
 674     #
 675     def step_report_status(self, label, status):
 676         """ Log status of test step
 677         """
 678         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 679
 680     def step_stop_vnfs(self):
 681         """ Stop all VNFs started by TestSteps
 682         """
 683         for vnf in self._step_vnf_list:
 684             if self._step_vnf_list[vnf]:
 685                 self._step_vnf_list[vnf].stop()
 686
 687     def step_eval_param(self, param, STEP):
 688         # pylint: disable=invalid-name
 689         """ Helper function for #STEP macro evaluation
 690         """
 691         if isinstance(param, str):
 692             # evaluate every #STEP reference inside parameter itself
 693             macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
 694             if macros:
 695                 for macro in macros:
 696                     # pylint: disable=eval-used
 697                     tmp_val = str(eval(macro[1:]))
 698                     param = param.replace(macro, tmp_val)
 699
 700             # evaluate references to vsperf configuration options
 701             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 702             if macros:
 703                 for macro in macros:
 704                     # pylint: disable=eval-used
 705                     try:
 706                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 707                         param = param.replace('${}'.format(macro[0]), tmp_val)
 708                     # ignore that required option can't be evaluated
 709                     except (IndexError, KeyError, AttributeError):
 710                         self._logger.debug("Skipping %s as it isn't a configuration "
 711                                            "parameter.", '${}'.format(macro[0]))
 712             return param
 713         elif isinstance(param, list) or isinstance(param, tuple):
 714             tmp_list = []
 715             for item in param:
 716                 tmp_list.append(self.step_eval_param(item, STEP))
 717             return tmp_list
 718         elif isinstance(param, dict):
 719             tmp_dict = {}
 720             for (key, value) in param.items():
 721                 tmp_dict[key] = self.step_eval_param(value, STEP)
 722             return tmp_dict
 723         else:
 724             return param
 725
 726     def step_eval_params(self, params, step_result):
 727         """ Evaluates referrences to results from previous steps
 728         """
 729         eval_params = []
 730         # evaluate all parameters if needed
 731         for param in params:
 732             eval_params.append(self.step_eval_param(param, step_result))
 733         return eval_params
 734
 735     def step_run(self):
 736         """ Execute actions specified by TestSteps list
 737
 738         :return: False if any error was detected
 739                  True otherwise
 740         """
 741         # anything to do?
 742         if not self.test:
 743             return True
 744
 745         # required for VNFs initialization
 746         loader = Loader()
 747         # initialize list with results
 748         self._step_result = [None] * len(self.test)
 749
 750         # We have to suppress pylint report, because test_object has to be set according
 751         # to the test step definition
 752         # pylint: disable=redefined-variable-type
 753         # run test step by step...
 754         for i, step in enumerate(self.test):
 755             step_ok = not self._step_check
 756             if step[0] == 'vswitch':
 757                 test_object = self._vswitch_ctl.get_vswitch()
 758             elif step[0] == 'namespace':
 759                 test_object = namespace
 760             elif step[0] == 'veth':
 761                 test_object = veth
 762             elif step[0] == 'settings':
 763                 test_object = S
 764             elif step[0] == 'tools':
 765                 test_object = TestStepsTools()
 766                 step[1] = step[1].title()
 767             elif step[0] == 'trafficgen':
 768                 test_object = self._traffic_ctl
 769                 # in case of send_traffic or send_traffic_async methods, ensure
 770                 # that specified traffic values are merged with existing self._traffic
 771                 if step[1].startswith('send_traffic'):
 772                     tmp_traffic = copy.deepcopy(self._traffic)
 773                     tmp_traffic.update(step[2])
 774                     step[2] = tmp_traffic
 775             elif step[0].startswith('vnf'):
 776                 if not self._step_vnf_list[step[0]]:
 777                     # initialize new VM
 778                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 779                 test_object = self._step_vnf_list[step[0]]
 780             elif step[0] == 'wait':
 781                 input(os.linesep + "Step {}: Press Enter to continue with "
 782                       "the next step...".format(i) + os.linesep + os.linesep)
 783                 continue
 784             elif step[0] == 'sleep':
 785                 self._logger.debug("Sleep %s seconds", step[1])
 786                 time.sleep(int(step[1]))
 787                 continue
 788             else:
 789                 self._logger.error("Unsupported test object %s", step[0])
 790                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 791                 self.step_report_status("Step '{}'".format(' '.join(step)),
 792                                         self._step_status['status'])
 793                 return False
 794
 795             test_method = getattr(test_object, step[1])
 796             if self._step_check:
 797                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 798             else:
 799                 test_method_check = None
 800
 801             step_params = []
 802             try:
 803                 # eval parameters, but use only valid step_results
 804                 # to support negative indexes
 805                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 806                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 807                 self._logger.debug("Step %s '%s' start", i, step_log)
 808                 self._step_result[i] = test_method(*step_params)
 809                 self._logger.debug("Step %s '%s' results '%s'", i,
 810                                    step_log, self._step_result[i])
 811                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 812                 if self._step_check:
 813                     step_ok = test_method_check(self._step_result[i], *step_params)
 814             except (AssertionError, AttributeError, IndexError) as ex:
 815                 step_ok = False
 816                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 817
 818             if self._step_check:
 819                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 820
 821             if not step_ok:
 822                 self._step_status = {'status' : False, 'details' : step_log}
 823                 # Stop all VNFs started by TestSteps
 824                 self.step_stop_vnfs()
 825                 return False
 826
 827         # all steps processed without any issue
 828         return True
 829
 830     #
 831     # get methods for TestCase members, which needs to be publicly available
 832     #
 833     def get_output_file(self):
 834         """Return content of self._output_file member
 835         """
 836         return self._output_file
 837
 838     def get_desc(self):
 839         """Return content of self.desc member
 840         """
 841         return self.desc
 842
 843     def get_versions(self):
 844         """Return content of self.versions member
 845         """
 846         return self._versions
 847
 848     def get_traffic(self):
 849         """Return content of self._traffic member
 850         """
 851         return self._traffic
 852
 853     def get_tc_results(self):
 854         """Return content of self._tc_results member
 855         """
 856         return self._tc_results
 857
 858     def get_collector(self):
 859         """Return content of self._collector member
 860         """
 861         return self._collector