testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import get_test_param, merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39
  40 CHECK_PREFIX = 'validate_'
  41
  42 # pylint: disable=too-many-instance-attributes
  43 class TestCase(object):
  44     """TestCase base class
  45
  46     In this basic form runs RFC2544 throughput test
  47     """
  48     # pylint: disable=too-many-statements
  49     def __init__(self, test_cfg):
  50         """Pull out fields from test config
  51
  52         :param test_cfg: A dictionary of string-value pairs describing the test
  53             configuration. Both the key and values strings use well-known
  54             values.
  55         :param results_dir: Where the csv formatted results are written.
  56         """
  57         # make a local copy of test configuration to avoid modification of
  58         # original content used in vsperf main script
  59         cfg = copy.deepcopy(test_cfg)
  60
  61         self._testcase_start_time = time.time()
  62         self._testcase_stop_time = self._testcase_start_time
  63         self._hugepages_mounted = False
  64         self._traffic_ctl = None
  65         self._vnf_ctl = None
  66         self._vswitch_ctl = None
  67         self._collector = None
  68         self._loadgen = None
  69         self._output_file = None
  70         self._tc_results = None
  71         self._settings_paths_modified = False
  72         self._testcast_run_time = None
  73         self._versions = []
  74         # initialization of step driven specific members
  75         self._step_check = False    # by default don't check result for step driven testcases
  76         self._step_vnf_list = {}
  77         self._step_result = []
  78         self._step_result_mapping = {}
  79         self._step_status = None
  80         self._step_send_traffic = False # indication if send_traffic was called within test steps
  81         self._testcase_run_time = None
  82
  83         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  84         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  85         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  86         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  87         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  88         test_params = merge_spec(test_params, tc_test_params)
  89         S.setValue('TEST_PARAMS', test_params)
  90         S.check_test_params()
  91
  92         # override all redefined GUEST_ values to have them expanded correctly
  93         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  94         for key in tmp_test_params:
  95             if key.startswith('GUEST_'):
  96                 S.setValue(key, S.getValue(key))
  97                 S.getValue('TEST_PARAMS').pop(key)
  98
  99         # update global settings
 100         functions.settings_update_paths()
 101
 102         # set test parameters; CLI options take precedence to testcase settings
 103         self._logger = logging.getLogger(__name__)
 104         self.name = cfg['Name']
 105         self.desc = cfg.get('Description', 'No description given.')
 106         self.test = cfg.get('TestSteps', None)
 107
 108         # log testcase name and details
 109         tmp_desc = functions.format_description(self.desc, 50)
 110         self._logger.info('############################################################')
 111         self._logger.info('# Test:    %s', self.name)
 112         self._logger.info('# Details: %s', tmp_desc[0])
 113         for i in range(1, len(tmp_desc)):
 114             self._logger.info('#          %s', tmp_desc[i])
 115         self._logger.info('############################################################')
 116
 117         bidirectional = S.getValue('TRAFFIC')['bidir']
 118         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 119             raise TypeError(
 120                 'Bi-dir value must be of type string')
 121         bidirectional = bidirectional.title()  # Keep things consistent
 122
 123         self.deployment = cfg['Deployment']
 124         self._frame_mod = cfg.get('Frame Modification', None)
 125
 126         self._tunnel_type = None
 127         self._tunnel_operation = None
 128
 129         if self.deployment == 'op2p':
 130             self._tunnel_operation = cfg['Tunnel Operation']
 131
 132             if 'Tunnel Type' in cfg:
 133                 self._tunnel_type = cfg['Tunnel Type']
 134                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 135                                                    self._tunnel_type)
 136
 137         # check if test requires background load and which generator it uses
 138         self._load_cfg = cfg.get('Load', None)
 139
 140         if self._frame_mod:
 141             self._frame_mod = self._frame_mod.lower()
 142         self._results_dir = S.getValue('RESULTS_PATH')
 143
 144         # set traffic details, so they can be passed to vswitch and traffic ctls
 145         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 146         self._traffic.update({'bidir': bidirectional,
 147                               'tunnel_type': self._tunnel_type,})
 148
 149         self._traffic = functions.check_traffic(self._traffic)
 150
 151         # Packet Forwarding mode
 152         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 153
 154         # trafficgen configuration required for tests of tunneling protocols
 155         if self.deployment == "op2p":
 156             self._traffic['l2'].update({'srcmac':
 157                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 158                                         'dstmac':
 159                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 160
 161             self._traffic['l3'].update({'srcip':
 162                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 163                                         'dstip':
 164                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 165
 166             if self._tunnel_operation == "decapsulation":
 167                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 168                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 169                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 170                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 171         elif len(S.getValue('NICS')) and \
 172              (S.getValue('NICS')[0]['type'] == 'vf' or
 173               S.getValue('NICS')[1]['type'] == 'vf'):
 174             mac1 = S.getValue('NICS')[0]['mac']
 175             mac2 = S.getValue('NICS')[1]['mac']
 176             if mac1 and mac2:
 177                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 178             else:
 179                 self._logger.debug("MAC addresses can not be read")
 180
 181         # count how many VNFs are involved in TestSteps
 182         if self.test:
 183             for step in self.test:
 184                 if step[0].startswith('vnf'):
 185                     self._step_vnf_list[step[0]] = None
 186
 187     def run_initialize(self):
 188         """ Prepare test execution environment
 189         """
 190         # mount hugepages if needed
 191         self._mount_hugepages()
 192
 193         self._logger.debug("Controllers:")
 194         loader = Loader()
 195         self._traffic_ctl = component_factory.create_traffic(
 196             self._traffic['traffic_type'],
 197             loader.get_trafficgen_class())
 198
 199         self._vnf_ctl = component_factory.create_vnf(
 200             self.deployment,
 201             loader.get_vnf_class(),
 202             len(self._step_vnf_list))
 203
 204         # verify enough hugepages are free to run the testcase
 205         if not self._check_for_enough_hugepages():
 206             raise RuntimeError('Not enough hugepages free to run test.')
 207
 208         # perform guest related handling
 209         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 210         if tmp_vm_count:
 211             # copy sources of l2 forwarding tools into VM shared dir if needed
 212             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 213
 214             # in case of multi VM in parallel, set the number of streams to the number of VMs
 215             if self.deployment.startswith('pvpv'):
 216                 # for each VM NIC pair we need an unique stream
 217                 streams = 0
 218                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 219                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 220                 self._logger.debug("VMs with parallel connection were detected. "
 221                                    "Thus Number of streams was set to %s", streams)
 222                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 223                 # user can define a proper stream count manually
 224                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 225                     self._traffic.update({'multistream': streams})
 226
 227             # OVS Vanilla requires guest VM MAC address and IPs to work
 228             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 229                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 230                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 231                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 232                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 233
 234         if self._vswitch_none:
 235             self._vswitch_ctl = component_factory.create_pktfwd(
 236                 self.deployment,
 237                 loader.get_pktfwd_class())
 238         else:
 239             self._vswitch_ctl = component_factory.create_vswitch(
 240                 self.deployment,
 241                 loader.get_vswitch_class(),
 242                 self._traffic,
 243                 self._tunnel_operation)
 244
 245         self._collector = component_factory.create_collector(
 246             loader.get_collector_class(),
 247             self._results_dir, self.name)
 248         self._loadgen = component_factory.create_loadgen(
 249             loader.get_loadgen_class(),
 250             self._load_cfg)
 251
 252         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 253                                          "_" + self.deployment + ".csv")
 254
 255         self._step_status = {'status' : True, 'details' : ''}
 256
 257         self._logger.debug("Setup:")
 258
 259     def run_finalize(self):
 260         """ Tear down test execution environment and record test results
 261         """
 262         # Stop all VNFs started by TestSteps in case that something went wrong
 263         self.step_stop_vnfs()
 264
 265         # umount hugepages if mounted
 266         self._umount_hugepages()
 267
 268         # cleanup any namespaces created
 269         if os.path.isdir('/tmp/namespaces'):
 270             namespace_list = os.listdir('/tmp/namespaces')
 271             if len(namespace_list):
 272                 self._logger.info('Cleaning up namespaces')
 273             for name in namespace_list:
 274                 namespace.delete_namespace(name)
 275             os.rmdir('/tmp/namespaces')
 276         # cleanup any veth ports created
 277         if os.path.isdir('/tmp/veth'):
 278             veth_list = os.listdir('/tmp/veth')
 279             if len(veth_list):
 280                 self._logger.info('Cleaning up veth ports')
 281             for eth in veth_list:
 282                 port1, port2 = eth.split('-')
 283                 veth.del_veth_port(port1, port2)
 284             os.rmdir('/tmp/veth')
 285
 286     def run_report(self):
 287         """ Report test results
 288         """
 289         self._logger.debug("self._collector Results:")
 290         self._collector.print_results()
 291
 292         results = self._traffic_ctl.get_results()
 293         if results:
 294             self._logger.debug("Traffic Results:")
 295             self._traffic_ctl.print_results()
 296
 297         if self._tc_results is None:
 298             self._tc_results = self._append_results(results)
 299         else:
 300             # integration step driven tests have their status and possible
 301             # failure details stored inside self._tc_results
 302             results = self._append_results(results)
 303             if len(self._tc_results) < len(results):
 304                 if len(self._tc_results) > 1:
 305                     raise RuntimeError('Testcase results do not match:'
 306                                        'results: {}\n'
 307                                        'trafficgen results: {}\n',
 308                                        self._tc_results,
 309                                        results)
 310                 else:
 311                     tmp_results = copy.deepcopy(self._tc_results[0])
 312                     self._tc_results = []
 313                     for res in results:
 314                         tmp_res = copy.deepcopy(tmp_results)
 315                         tmp_res.update(res)
 316                         self._tc_results.append(tmp_res)
 317             else:
 318                 for i, result in enumerate(results):
 319                     self._tc_results[i].update(result)
 320
 321         TestCase.write_result_to_file(self._tc_results, self._output_file)
 322
 323     def run(self):
 324         """Run the test
 325
 326         All setup and teardown through controllers is included.
 327         """
 328         # prepare test execution environment
 329         self.run_initialize()
 330
 331         try:
 332             with self._vswitch_ctl, self._loadgen:
 333                 with self._vnf_ctl, self._collector:
 334                     if not self._vswitch_none:
 335                         self._add_flows()
 336
 337                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 338
 339                     with self._traffic_ctl:
 340                         # execute test based on TestSteps definition if needed...
 341                         if self.step_run():
 342                             # ...and continue with traffic generation, but keep
 343                             # in mind, that clean deployment does not configure
 344                             # OVS nor executes the traffic
 345                             if self.deployment != 'clean' and not self._step_send_traffic:
 346                                 self._traffic_ctl.send_traffic(self._traffic)
 347
 348                         # dump vswitch flows before they are affected by VNF termination
 349                         if not self._vswitch_none:
 350                             self._vswitch_ctl.dump_vswitch_flows()
 351
 352                     # garbage collection for case that TestSteps modify existing deployment
 353                     self.step_stop_vnfs()
 354
 355         finally:
 356             # tear down test execution environment and log results
 357             self.run_finalize()
 358
 359         self._testcase_stop_time = time.time()
 360         self._testcase_run_time = time.strftime("%H:%M:%S",
 361                                                 time.gmtime(self._testcase_stop_time -
 362                                                             self._testcase_start_time))
 363         logging.info("Testcase execution time: " + self._testcase_run_time)
 364         # report test results
 365         self.run_report()
 366
 367     def _append_results(self, results):
 368         """
 369         Method appends mandatory Test Case results to list of dictionaries.
 370
 371         :param results: list of dictionaries which contains results from
 372                 traffic generator.
 373
 374         :returns: modified list of dictionaries.
 375         """
 376         for item in results:
 377             item[ResultsConstants.ID] = self.name
 378             item[ResultsConstants.DEPLOYMENT] = self.deployment
 379             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 380             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 381             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 382             # convert timestamps to human readable format
 383             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 384                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 385             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 386                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 387             if self._traffic['multistream']:
 388                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 389                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 390                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 391             if self._vnf_ctl.get_vnfs_number():
 392                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 393             if self._tunnel_type:
 394                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 395         return results
 396
 397     def _copy_fwd_tools_for_all_guests(self, vm_count):
 398         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 399         """
 400         # consider only VNFs involved in the test
 401         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 402             self._copy_fwd_tools_for_guest(guest_dir)
 403
 404     def _copy_fwd_tools_for_guest(self, guest_dir):
 405         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 406
 407         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 408         """
 409         # remove shared dir if it exists to avoid issues with file consistency
 410         if os.path.exists(guest_dir):
 411             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 412                            'Removing content of shared directory...', True)
 413
 414         # directory to share files between host and guest
 415         os.makedirs(guest_dir)
 416
 417         # copy sources into shared dir only if neccessary
 418         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 419         if 'testpmd' in guest_loopback:
 420             try:
 421                 # exclude whole .git/ subdirectory and all o-files;
 422                 # It is assumed, that the same RTE_TARGET is used in both host
 423                 # and VMs; This simplification significantly speeds up testpmd
 424                 # build. If we will need a different RTE_TARGET in VM,
 425                 # then we have to build whole DPDK from the scratch in VM.
 426                 # In that case we can copy just DPDK sources (e.g. by excluding
 427                 # all items obtained by git status -unormal --porcelain).
 428                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 429                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 430                 exclude = []
 431                 exclude.append(r'--exclude=.git/')
 432                 exclude.append(r'--exclude=*.o')
 433                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 434                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 435                                 os.path.join(guest_dir, 'DPDK')],
 436                                self._logger,
 437                                'Copying DPDK to shared directory...',
 438                                True)
 439             except subprocess.CalledProcessError:
 440                 self._logger.error('Unable to copy DPDK to shared directory')
 441                 raise
 442         if 'l2fwd' in guest_loopback:
 443             try:
 444                 tasks.run_task(['rsync', '-a', '-r', '-l',
 445                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 446                                 os.path.join(guest_dir, 'l2fwd')],
 447                                self._logger,
 448                                'Copying l2fwd to shared directory...',
 449                                True)
 450             except subprocess.CalledProcessError:
 451                 self._logger.error('Unable to copy l2fwd to shared directory')
 452                 raise
 453
 454     def _mount_hugepages(self):
 455         """Mount hugepages if usage of DPDK or Qemu is detected
 456         """
 457         # pylint: disable=too-many-boolean-expressions
 458         # hugepages are needed by DPDK and Qemu
 459         if not self._hugepages_mounted and \
 460             (self.deployment.count('v') or \
 461              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 462              self._vswitch_none or \
 463              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 464             hugepages.mount_hugepages()
 465             self._hugepages_mounted = True
 466
 467     def _umount_hugepages(self):
 468         """Umount hugepages if they were mounted before
 469         """
 470         if self._hugepages_mounted:
 471             hugepages.umount_hugepages()
 472             self._hugepages_mounted = False
 473
 474     def _check_for_enough_hugepages(self):
 475         """Check to make sure enough hugepages are free to satisfy the
 476         test environment.
 477         """
 478         hugepages_needed = 0
 479         hugepage_size = hugepages.get_hugepage_size()
 480         # get hugepage amounts per guest involved in the test
 481         for guest in range(self._vnf_ctl.get_vnfs_number()):
 482             hugepages_needed += math.ceil((int(S.getValue(
 483                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 484
 485         # get hugepage amounts for each socket on dpdk
 486         sock0_mem, sock1_mem = 0, 0
 487
 488         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 489             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 490             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 491                                     int(sock_mem[1]) * 1024 / hugepage_size)
 492
 493         # If hugepages needed, verify the amounts are free
 494         if any([hugepages_needed, sock0_mem, sock1_mem]):
 495             free_hugepages = hugepages.get_free_hugepages()
 496             if hugepages_needed:
 497                 logging.info('Need %s hugepages free for guests',
 498                              hugepages_needed)
 499                 result1 = free_hugepages >= hugepages_needed
 500                 free_hugepages -= hugepages_needed
 501             else:
 502                 result1 = True
 503
 504             if sock0_mem:
 505                 logging.info('Need %s hugepages free for dpdk socket 0',
 506                              sock0_mem)
 507                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 508                 free_hugepages -= sock0_mem
 509             else:
 510                 result2 = True
 511
 512             if sock1_mem:
 513                 logging.info('Need %s hugepages free for dpdk socket 1',
 514                              sock1_mem)
 515                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 516                 free_hugepages -= sock1_mem
 517             else:
 518                 result3 = True
 519
 520             logging.info('Need a total of %s total hugepages',
 521                          hugepages_needed + sock1_mem + sock0_mem)
 522
 523             # The only drawback here is sometimes dpdk doesn't release
 524             # its hugepages on a test failure. This could cause a test
 525             # to fail when dpdk would be OK to start because it will just
 526             # use the previously allocated hugepages.
 527             result4 = True if free_hugepages >= 0 else False
 528
 529             return all([result1, result2, result3, result4])
 530         else:
 531             return True
 532
 533     @staticmethod
 534     def write_result_to_file(results, output):
 535         """Write list of dictionaries to a CSV file.
 536
 537         Each element on list will create separate row in output file.
 538         If output file already exists, data will be appended at the end,
 539         otherwise it will be created.
 540
 541         :param results: list of dictionaries.
 542         :param output: path to output file.
 543         """
 544         with open(output, 'a') as csvfile:
 545
 546             logging.info("Write results to file: " + output)
 547             fieldnames = TestCase._get_unique_keys(results)
 548
 549             writer = csv.DictWriter(csvfile, fieldnames)
 550
 551             if not csvfile.tell():  # file is now empty
 552                 writer.writeheader()
 553
 554             for result in results:
 555                 writer.writerow(result)
 556
 557     @staticmethod
 558     def _get_unique_keys(list_of_dicts):
 559         """Gets unique key values as ordered list of strings in given dicts
 560
 561         :param list_of_dicts: list of dictionaries.
 562
 563         :returns: list of unique keys(strings).
 564         """
 565         result = OrderedDict()
 566         for item in list_of_dicts:
 567             for key in item.keys():
 568                 result[key] = ''
 569
 570         return list(result.keys())
 571
 572     def _add_flows(self):
 573         """Add flows to the vswitch
 574         """
 575         vswitch = self._vswitch_ctl.get_vswitch()
 576         # NOTE BOM 15-08-07 the frame mod code assumes that the
 577         # physical ports are ports 1 & 2. The actual numbers
 578         # need to be retrived from the vSwitch and the metadata value
 579         # updated accordingly.
 580         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 581         if self._frame_mod == "vlan":
 582             # 0x8100 => VLAN ethertype
 583             self._logger.debug(" ****   VLAN   ***** ")
 584             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 585                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 586             vswitch.add_flow(bridge, flow)
 587             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 588                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 589             vswitch.add_flow(bridge, flow)
 590         elif self._frame_mod == "mpls":
 591             # 0x8847 => MPLS unicast ethertype
 592             self._logger.debug(" ****   MPLS  ***** ")
 593             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 594                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 595             vswitch.add_flow(bridge, flow)
 596             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 597                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 598             vswitch.add_flow(bridge, flow)
 599         elif self._frame_mod == "mac":
 600             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 601                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 602                                 'goto_table:3']}
 603             vswitch.add_flow(bridge, flow)
 604             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 605                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 606                                 'goto_table:3']}
 607             vswitch.add_flow(bridge, flow)
 608         elif self._frame_mod == "dscp":
 609             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 610             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 611                     'dl_type':'0x0800',
 612                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 613             vswitch.add_flow(bridge, flow)
 614             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 615                     'dl_type':'0x0800',
 616                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 617             vswitch.add_flow(bridge, flow)
 618         elif self._frame_mod == "ttl":
 619             # 251 and 241 are the highest prime numbers < 255
 620             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 621                     'dl_type':'0x0800',
 622                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 623             vswitch.add_flow(bridge, flow)
 624             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 625                     'dl_type':'0x0800',
 626                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 627             vswitch.add_flow(bridge, flow)
 628         elif self._frame_mod == "ip_addr":
 629             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 630                     'dl_type':'0x0800',
 631                     'actions': ['mod_nw_src:10.10.10.10',
 632                                 'mod_nw_dst:20.20.20.20',
 633                                 'goto_table:3']}
 634             vswitch.add_flow(bridge, flow)
 635             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 636                     'dl_type':'0x0800',
 637                     'actions': ['mod_nw_src:20.20.20.20',
 638                                 'mod_nw_dst:10.10.10.10',
 639                                 'goto_table:3']}
 640             vswitch.add_flow(bridge, flow)
 641         elif self._frame_mod == "ip_port":
 642             # NOTE BOM 15-08-27 The traffic generated is assumed
 643             # to be UDP (nw_proto 17d) which is the default case but
 644             # we will need to pick up the actual traffic params in use.
 645             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 646                     'dl_type':'0x0800', 'nw_proto':'17',
 647                     'actions': ['mod_tp_src:44444',
 648                                 'mod_tp_dst:44444', 'goto_table:3']}
 649             vswitch.add_flow(bridge, flow)
 650             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 651                     'dl_type':'0x0800', 'nw_proto':'17',
 652                     'actions': ['mod_tp_src:44444',
 653                                 'mod_tp_dst:44444', 'goto_table:3']}
 654             vswitch.add_flow(bridge, flow)
 655         else:
 656             pass
 657
 658
 659     #
 660     # TestSteps realted methods
 661     #
 662     def step_report_status(self, label, status):
 663         """ Log status of test step
 664         """
 665         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 666
 667     def step_stop_vnfs(self):
 668         """ Stop all VNFs started by TestSteps
 669         """
 670         for vnf in self._step_vnf_list:
 671             if self._step_vnf_list[vnf]:
 672                 self._step_vnf_list[vnf].stop()
 673
 674     def step_eval_param(self, param, step_result):
 675         """ Helper function for #STEP macro evaluation
 676         """
 677         if isinstance(param, str):
 678             # evaluate every #STEP reference inside parameter itself
 679             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 680
 681             if macros:
 682                 for macro in macros:
 683                     if macro[1] in self._step_result_mapping:
 684                         key = self._step_result_mapping[macro[1]]
 685                     else:
 686                         key = macro[1]
 687                     # pylint: disable=eval-used
 688                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 689                     param = param.replace(macro[0], tmp_val)
 690
 691             # evaluate references to vsperf configuration options
 692             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 693             if macros:
 694                 for macro in macros:
 695                     # pylint: disable=eval-used
 696                     try:
 697                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 698                         param = param.replace('${}'.format(macro[0]), tmp_val)
 699                     # ignore that required option can't be evaluated
 700                     except (IndexError, KeyError, AttributeError):
 701                         self._logger.debug("Skipping %s as it isn't a configuration "
 702                                            "parameter.", '${}'.format(macro[0]))
 703             return param
 704         elif isinstance(param, list) or isinstance(param, tuple):
 705             tmp_list = []
 706             for item in param:
 707                 tmp_list.append(self.step_eval_param(item, step_result))
 708             return tmp_list
 709         elif isinstance(param, dict):
 710             tmp_dict = {}
 711             for (key, value) in param.items():
 712                 tmp_dict[key] = self.step_eval_param(value, step_result)
 713             return tmp_dict
 714         else:
 715             return param
 716
 717     def step_eval_params(self, params, step_result):
 718         """ Evaluates referrences to results from previous steps
 719         """
 720         eval_params = []
 721         # evaluate all parameters if needed
 722         for param in params:
 723             eval_params.append(self.step_eval_param(param, step_result))
 724         return eval_params
 725
 726     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 727     def step_run(self):
 728         """ Execute actions specified by TestSteps list
 729
 730         :return: False if any error was detected
 731                  True otherwise
 732         """
 733         # anything to do?
 734         if not self.test:
 735             return True
 736
 737         # required for VNFs initialization
 738         loader = Loader()
 739         # initialize list with results
 740         self._step_result = [None] * len(self.test)
 741
 742         # We have to suppress pylint report, because test_object has to be set according
 743         # to the test step definition
 744         # pylint: disable=redefined-variable-type
 745         # run test step by step...
 746         for i, step in enumerate(self.test):
 747             step_ok = not self._step_check
 748             step_check = self._step_check
 749             regex = None
 750             # configure step result mapping if step alias/label is detected
 751             if step[0].startswith('#'):
 752                 key = step[0][1:]
 753                 if key.isdigit():
 754                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 755                 if key in self._step_result_mapping:
 756                     raise RuntimeError('Step alias {} has been used already for step '
 757                                        '{}'.format(key, self._step_result_mapping[key]))
 758                 self._step_result_mapping[step[0][1:]] = i
 759                 step = step[1:]
 760
 761             # store regex filter if it is specified
 762             if isinstance(step[-1], str) and step[-1].startswith('|'):
 763                 # evalute macros and variables used in regex
 764                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 765                 step = step[:-1]
 766
 767             # check if step verification should be suppressed
 768             if step[0].startswith('!'):
 769                 step_check = False
 770                 step_ok = True
 771                 step[0] = step[0][1:]
 772             if step[0] == 'vswitch':
 773                 test_object = self._vswitch_ctl.get_vswitch()
 774             elif step[0] == 'namespace':
 775                 test_object = namespace
 776             elif step[0] == 'veth':
 777                 test_object = veth
 778             elif step[0] == 'settings':
 779                 test_object = S
 780             elif step[0] == 'tools':
 781                 test_object = TestStepsTools()
 782                 step[1] = step[1].title()
 783             elif step[0] == 'trafficgen':
 784                 test_object = self._traffic_ctl
 785                 # in case of send_traffic or send_traffic_async methods, ensure
 786                 # that specified traffic values are merged with existing self._traffic
 787                 if step[1].startswith('send_traffic'):
 788                     tmp_traffic = copy.deepcopy(self._traffic)
 789                     tmp_traffic.update(step[2])
 790                     step[2] = tmp_traffic
 791                     # store indication that traffic has been sent
 792                     # so it is not sent again after the execution of teststeps
 793                     self._step_send_traffic = True
 794             elif step[0].startswith('vnf'):
 795                 if not self._step_vnf_list[step[0]]:
 796                     # initialize new VM
 797                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 798                 test_object = self._step_vnf_list[step[0]]
 799             elif step[0] == 'wait':
 800                 input(os.linesep + "Step {}: Press Enter to continue with "
 801                       "the next step...".format(i) + os.linesep + os.linesep)
 802                 continue
 803             elif step[0] == 'sleep':
 804                 self._logger.debug("Sleep %s seconds", step[1])
 805                 time.sleep(int(step[1]))
 806                 continue
 807             elif step[0] == 'log':
 808                 test_object = self._logger
 809                 # there isn't a need for validation of log entry
 810                 step_check = False
 811                 step_ok = True
 812             elif step[0] == 'pdb':
 813                 import pdb
 814                 pdb.set_trace()
 815                 continue
 816             else:
 817                 self._logger.error("Unsupported test object %s", step[0])
 818                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 819                 self.step_report_status("Step '{}'".format(' '.join(step)),
 820                                         self._step_status['status'])
 821                 return False
 822
 823             test_method = getattr(test_object, step[1])
 824             if step_check:
 825                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 826             else:
 827                 test_method_check = None
 828
 829             step_params = []
 830             try:
 831                 # eval parameters, but use only valid step_results
 832                 # to support negative indexes
 833                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 834                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 835                 step_log += ' filter "{}"'.format(regex) if regex else ''
 836                 self._logger.debug("Step %s '%s' start", i, step_log)
 837                 self._step_result[i] = test_method(*step_params)
 838                 if regex:
 839                     # apply regex to step output
 840                     self._step_result[i] = functions.filter_output(
 841                         self._step_result[i], regex)
 842
 843                 self._logger.debug("Step %s '%s' results '%s'", i,
 844                                    step_log, self._step_result[i])
 845                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 846                 if step_check:
 847                     step_ok = test_method_check(self._step_result[i], *step_params)
 848             except (AssertionError, AttributeError, IndexError) as ex:
 849                 step_ok = False
 850                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 851
 852             if step_check:
 853                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 854
 855             if not step_ok:
 856                 self._step_status = {'status' : False, 'details' : step_log}
 857                 # Stop all VNFs started by TestSteps
 858                 self.step_stop_vnfs()
 859                 return False
 860
 861         # all steps processed without any issue
 862         return True
 863
 864     #
 865     # get methods for TestCase members, which needs to be publicly available
 866     #
 867     def get_output_file(self):
 868         """Return content of self._output_file member
 869         """
 870         return self._output_file
 871
 872     def get_desc(self):
 873         """Return content of self.desc member
 874         """
 875         return self.desc
 876
 877     def get_versions(self):
 878         """Return content of self.versions member
 879         """
 880         return self._versions
 881
 882     def get_traffic(self):
 883         """Return content of self._traffic member
 884         """
 885         return self._traffic
 886
 887     def get_tc_results(self):
 888         """Return content of self._tc_results member
 889         """
 890         return self._tc_results
 891
 892     def get_collector(self):
 893         """Return content of self._collector member
 894         """
 895         return self._collector