testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import get_test_param, merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39
  40 CHECK_PREFIX = 'validate_'
  41
  42 # pylint: disable=too-many-instance-attributes
  43 class TestCase(object):
  44     """TestCase base class
  45
  46     In this basic form runs RFC2544 throughput test
  47     """
  48     # pylint: disable=too-many-statements
  49     def __init__(self, test_cfg):
  50         """Pull out fields from test config
  51
  52         :param test_cfg: A dictionary of string-value pairs describing the test
  53             configuration. Both the key and values strings use well-known
  54             values.
  55         :param results_dir: Where the csv formatted results are written.
  56         """
  57         # make a local copy of test configuration to avoid modification of
  58         # original content used in vsperf main script
  59         cfg = copy.deepcopy(test_cfg)
  60
  61         self._testcase_start_time = time.time()
  62         self._testcase_stop_time = self._testcase_start_time
  63         self._hugepages_mounted = False
  64         self._traffic_ctl = None
  65         self._vnf_ctl = None
  66         self._vswitch_ctl = None
  67         self._collector = None
  68         self._loadgen = None
  69         self._output_file = None
  70         self._tc_results = None
  71         self._settings_original = {}
  72         self._settings_paths_modified = False
  73         self._testcast_run_time = None
  74         self._versions = []
  75         # initialization of step driven specific members
  76         self._step_check = False    # by default don't check result for step driven testcases
  77         self._step_vnf_list = {}
  78         self._step_result = []
  79         self._step_status = None
  80         self._testcase_run_time = None
  81
  82         # store all GUEST_ specific settings to keep original values before their expansion
  83         for key in S.__dict__:
  84             if key.startswith('GUEST_'):
  85                 self._settings_original[key] = S.getValue(key)
  86
  87         self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  88         self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
  89         self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  90         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  91         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  92         test_params = merge_spec(test_params, tc_test_params)
  93         self._update_settings('TEST_PARAMS', test_params)
  94         S.check_test_params()
  95
  96         # override all redefined GUEST_ values to have them expanded correctly
  97         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  98         for key in tmp_test_params:
  99             if key.startswith('GUEST_'):
 100                 S.setValue(key, S.getValue(key))
 101                 S.getValue('TEST_PARAMS').pop(key)
 102
 103         # update global settings
 104         functions.settings_update_paths()
 105
 106         # set test parameters; CLI options take precedence to testcase settings
 107         self._logger = logging.getLogger(__name__)
 108         self.name = cfg['Name']
 109         self.desc = cfg.get('Description', 'No description given.')
 110         self.test = cfg.get('TestSteps', None)
 111
 112         bidirectional = S.getValue('TRAFFIC')['bidir']
 113         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 114             raise TypeError(
 115                 'Bi-dir value must be of type string')
 116         bidirectional = bidirectional.title()  # Keep things consistent
 117
 118         self.deployment = cfg['Deployment']
 119         self._frame_mod = cfg.get('Frame Modification', None)
 120
 121         self._tunnel_type = None
 122         self._tunnel_operation = None
 123
 124         if self.deployment == 'op2p':
 125             self._tunnel_operation = cfg['Tunnel Operation']
 126
 127             if 'Tunnel Type' in cfg:
 128                 self._tunnel_type = cfg['Tunnel Type']
 129                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 130                                                    self._tunnel_type)
 131
 132         # check if test requires background load and which generator it uses
 133         self._load_cfg = cfg.get('Load', None)
 134         if self._load_cfg and 'tool' in self._load_cfg:
 135             self._loadgen = self._load_cfg['tool']
 136         else:
 137             # background load is not requested, so use dummy implementation
 138             self._loadgen = "Dummy"
 139
 140         if self._frame_mod:
 141             self._frame_mod = self._frame_mod.lower()
 142         self._results_dir = S.getValue('RESULTS_PATH')
 143
 144         # set traffic details, so they can be passed to vswitch and traffic ctls
 145         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 146         self._traffic.update({'bidir': bidirectional,
 147                               'tunnel_type': self._tunnel_type,})
 148
 149         self._traffic = functions.check_traffic(self._traffic)
 150
 151         # Packet Forwarding mode
 152         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 153
 154         # trafficgen configuration required for tests of tunneling protocols
 155         if self.deployment == "op2p":
 156             self._traffic['l2'].update({'srcmac':
 157                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 158                                         'dstmac':
 159                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 160
 161             self._traffic['l3'].update({'srcip':
 162                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 163                                         'dstip':
 164                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 165
 166             if self._tunnel_operation == "decapsulation":
 167                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 168                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 169                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 170         elif len(S.getValue('NICS')) and \
 171              (S.getValue('NICS')[0]['type'] == 'vf' or
 172               S.getValue('NICS')[1]['type'] == 'vf'):
 173             mac1 = S.getValue('NICS')[0]['mac']
 174             mac2 = S.getValue('NICS')[1]['mac']
 175             if mac1 and mac2:
 176                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 177             else:
 178                 self._logger.debug("MAC addresses can not be read")
 179
 180         # count how many VNFs are involved in TestSteps
 181         if self.test:
 182             for step in self.test:
 183                 if step[0].startswith('vnf'):
 184                     self._step_vnf_list[step[0]] = None
 185
 186     def run_initialize(self):
 187         """ Prepare test execution environment
 188         """
 189         self._logger.debug(self.name)
 190
 191         # mount hugepages if needed
 192         self._mount_hugepages()
 193
 194         self._logger.debug("Controllers:")
 195         loader = Loader()
 196         self._traffic_ctl = component_factory.create_traffic(
 197             self._traffic['traffic_type'],
 198             loader.get_trafficgen_class())
 199
 200         self._vnf_ctl = component_factory.create_vnf(
 201             self.deployment,
 202             loader.get_vnf_class(),
 203             len(self._step_vnf_list))
 204
 205         # verify enough hugepages are free to run the testcase
 206         if not self._check_for_enough_hugepages():
 207             raise RuntimeError('Not enough hugepages free to run test.')
 208
 209         # perform guest related handling
 210         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 211         if tmp_vm_count:
 212             # copy sources of l2 forwarding tools into VM shared dir if needed
 213             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 214
 215             # in case of multi VM in parallel, set the number of streams to the number of VMs
 216             if self.deployment.startswith('pvpv'):
 217                 # for each VM NIC pair we need an unique stream
 218                 streams = 0
 219                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 220                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 221                 self._logger.debug("VMs with parallel connection were detected. "
 222                                    "Thus Number of streams was set to %s", streams)
 223                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 224                 # user can define a proper stream count manually
 225                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 226                     self._traffic.update({'multistream': streams})
 227
 228             # OVS Vanilla requires guest VM MAC address and IPs to work
 229             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 230                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 231                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 232                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 233                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 234
 235         if self._vswitch_none:
 236             self._vswitch_ctl = component_factory.create_pktfwd(
 237                 self.deployment,
 238                 loader.get_pktfwd_class())
 239         else:
 240             self._vswitch_ctl = component_factory.create_vswitch(
 241                 self.deployment,
 242                 loader.get_vswitch_class(),
 243                 self._traffic,
 244                 self._tunnel_operation)
 245
 246         self._collector = component_factory.create_collector(
 247             loader.get_collector_class(),
 248             self._results_dir, self.name)
 249         self._loadgen = component_factory.create_loadgen(
 250             self._loadgen,
 251             self._load_cfg)
 252
 253         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 254                                          "_" + self.deployment + ".csv")
 255
 256         self._step_status = {'status' : True, 'details' : ''}
 257
 258         self._logger.debug("Setup:")
 259
 260     def run_finalize(self):
 261         """ Tear down test execution environment and record test results
 262         """
 263         # Stop all VNFs started by TestSteps in case that something went wrong
 264         self.step_stop_vnfs()
 265
 266         # umount hugepages if mounted
 267         self._umount_hugepages()
 268
 269         # cleanup any namespaces created
 270         if os.path.isdir('/tmp/namespaces'):
 271             namespace_list = os.listdir('/tmp/namespaces')
 272             if len(namespace_list):
 273                 self._logger.info('Cleaning up namespaces')
 274             for name in namespace_list:
 275                 namespace.delete_namespace(name)
 276             os.rmdir('/tmp/namespaces')
 277         # cleanup any veth ports created
 278         if os.path.isdir('/tmp/veth'):
 279             veth_list = os.listdir('/tmp/veth')
 280             if len(veth_list):
 281                 self._logger.info('Cleaning up veth ports')
 282             for eth in veth_list:
 283                 port1, port2 = eth.split('-')
 284                 veth.del_veth_port(port1, port2)
 285             os.rmdir('/tmp/veth')
 286
 287     def run_report(self):
 288         """ Report test results
 289         """
 290         self._logger.debug("self._collector Results:")
 291         self._collector.print_results()
 292
 293         results = self._traffic_ctl.get_results()
 294         if results:
 295             self._logger.debug("Traffic Results:")
 296             self._traffic_ctl.print_results()
 297
 298         if self._tc_results is None:
 299             self._tc_results = self._append_results(results)
 300         else:
 301             # integration step driven tests have their status and possible
 302             # failure details stored inside self._tc_results
 303             results = self._append_results(results)
 304             if len(self._tc_results) < len(results):
 305                 if len(self._tc_results) > 1:
 306                     raise RuntimeError('Testcase results do not match:'
 307                                        'results: {}\n'
 308                                        'trafficgen results: {}\n',
 309                                        self._tc_results,
 310                                        results)
 311                 else:
 312                     tmp_results = copy.deepcopy(self._tc_results[0])
 313                     self._tc_results = []
 314                     for res in results:
 315                         tmp_res = copy.deepcopy(tmp_results)
 316                         tmp_res.update(res)
 317                         self._tc_results.append(tmp_res)
 318             else:
 319                 for i, result in enumerate(results):
 320                     self._tc_results[i].update(result)
 321
 322         TestCase.write_result_to_file(self._tc_results, self._output_file)
 323
 324     def run(self):
 325         """Run the test
 326
 327         All setup and teardown through controllers is included.
 328         """
 329         # prepare test execution environment
 330         self.run_initialize()
 331
 332         try:
 333             with self._vswitch_ctl, self._loadgen:
 334                 with self._vnf_ctl, self._collector:
 335                     if not self._vswitch_none:
 336                         self._add_flows()
 337
 338                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 339
 340                     with self._traffic_ctl:
 341                         # execute test based on TestSteps definition if needed...
 342                         if self.step_run():
 343                             # ...and continue with traffic generation, but keep
 344                             # in mind, that clean deployment does not configure
 345                             # OVS nor executes the traffic
 346                             if self.deployment != 'clean':
 347                                 self._traffic_ctl.send_traffic(self._traffic)
 348
 349                         # dump vswitch flows before they are affected by VNF termination
 350                         if not self._vswitch_none:
 351                             self._vswitch_ctl.dump_vswitch_flows()
 352
 353                     # garbage collection for case that TestSteps modify existing deployment
 354                     self.step_stop_vnfs()
 355
 356         finally:
 357             # tear down test execution environment and log results
 358             self.run_finalize()
 359
 360         self._testcase_stop_time = time.time()
 361         self._testcase_run_time = time.strftime("%H:%M:%S",
 362                                                 time.gmtime(self._testcase_stop_time -
 363                                                             self._testcase_start_time))
 364         logging.info("Testcase execution time: " + self._testcase_run_time)
 365         # report test results
 366         self.run_report()
 367
 368         # restore original settings
 369         for key in self._settings_original:
 370             S.setValue(key, self._settings_original[key])
 371
 372     def _update_settings(self, param, value):
 373         """ Check value of given configuration parameter
 374         In case that new value is different, then testcase
 375         specific settings is updated and original value stored
 376
 377         :param param: Name of parameter inside settings
 378         :param value: Disired parameter value
 379         """
 380         orig_value = S.getValue(param)
 381         if orig_value != value:
 382             self._settings_original[param] = copy.deepcopy(orig_value)
 383             S.setValue(param, value)
 384
 385     def _append_results(self, results):
 386         """
 387         Method appends mandatory Test Case results to list of dictionaries.
 388
 389         :param results: list of dictionaries which contains results from
 390                 traffic generator.
 391
 392         :returns: modified list of dictionaries.
 393         """
 394         for item in results:
 395             item[ResultsConstants.ID] = self.name
 396             item[ResultsConstants.DEPLOYMENT] = self.deployment
 397             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 398             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 399             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 400             # convert timestamps to human readable format
 401             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 402                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 403             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 404                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 405             if self._traffic['multistream']:
 406                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 407                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 408                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 409             if self._vnf_ctl.get_vnfs_number():
 410                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 411             if self._tunnel_type:
 412                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 413         return results
 414
 415     def _copy_fwd_tools_for_all_guests(self, vm_count):
 416         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 417         """
 418         # consider only VNFs involved in the test
 419         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 420             self._copy_fwd_tools_for_guest(guest_dir)
 421
 422     def _copy_fwd_tools_for_guest(self, guest_dir):
 423         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 424
 425         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 426         """
 427         # remove shared dir if it exists to avoid issues with file consistency
 428         if os.path.exists(guest_dir):
 429             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 430                            'Removing content of shared directory...', True)
 431
 432         # directory to share files between host and guest
 433         os.makedirs(guest_dir)
 434
 435         # copy sources into shared dir only if neccessary
 436         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 437         if 'testpmd' in guest_loopback:
 438             try:
 439                 # exclude whole .git/ subdirectory and all o-files;
 440                 # It is assumed, that the same RTE_TARGET is used in both host
 441                 # and VMs; This simplification significantly speeds up testpmd
 442                 # build. If we will need a different RTE_TARGET in VM,
 443                 # then we have to build whole DPDK from the scratch in VM.
 444                 # In that case we can copy just DPDK sources (e.g. by excluding
 445                 # all items obtained by git status -unormal --porcelain).
 446                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 447                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 448                 exclude = []
 449                 exclude.append(r'--exclude=.git/')
 450                 exclude.append(r'--exclude=*.o')
 451                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 452                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 453                                 os.path.join(guest_dir, 'DPDK')],
 454                                self._logger,
 455                                'Copying DPDK to shared directory...',
 456                                True)
 457             except subprocess.CalledProcessError:
 458                 self._logger.error('Unable to copy DPDK to shared directory')
 459                 raise
 460         if 'l2fwd' in guest_loopback:
 461             try:
 462                 tasks.run_task(['rsync', '-a', '-r', '-l',
 463                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 464                                 os.path.join(guest_dir, 'l2fwd')],
 465                                self._logger,
 466                                'Copying l2fwd to shared directory...',
 467                                True)
 468             except subprocess.CalledProcessError:
 469                 self._logger.error('Unable to copy l2fwd to shared directory')
 470                 raise
 471
 472     def _mount_hugepages(self):
 473         """Mount hugepages if usage of DPDK or Qemu is detected
 474         """
 475         # pylint: disable=too-many-boolean-expressions
 476         # hugepages are needed by DPDK and Qemu
 477         if not self._hugepages_mounted and \
 478             (self.deployment.count('v') or \
 479              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 480              self._vswitch_none or \
 481              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 482             hugepages.mount_hugepages()
 483             self._hugepages_mounted = True
 484
 485     def _umount_hugepages(self):
 486         """Umount hugepages if they were mounted before
 487         """
 488         if self._hugepages_mounted:
 489             hugepages.umount_hugepages()
 490             self._hugepages_mounted = False
 491
 492     def _check_for_enough_hugepages(self):
 493         """Check to make sure enough hugepages are free to satisfy the
 494         test environment.
 495         """
 496         hugepages_needed = 0
 497         hugepage_size = hugepages.get_hugepage_size()
 498         # get hugepage amounts per guest involved in the test
 499         for guest in range(self._vnf_ctl.get_vnfs_number()):
 500             hugepages_needed += math.ceil((int(S.getValue(
 501                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 502
 503         # get hugepage amounts for each socket on dpdk
 504         sock0_mem, sock1_mem = 0, 0
 505
 506         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 507             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 508             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 509                                     int(sock_mem[1]) * 1024 / hugepage_size)
 510
 511         # If hugepages needed, verify the amounts are free
 512         if any([hugepages_needed, sock0_mem, sock1_mem]):
 513             free_hugepages = hugepages.get_free_hugepages()
 514             if hugepages_needed:
 515                 logging.info('Need %s hugepages free for guests',
 516                              hugepages_needed)
 517                 result1 = free_hugepages >= hugepages_needed
 518                 free_hugepages -= hugepages_needed
 519             else:
 520                 result1 = True
 521
 522             if sock0_mem:
 523                 logging.info('Need %s hugepages free for dpdk socket 0',
 524                              sock0_mem)
 525                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 526                 free_hugepages -= sock0_mem
 527             else:
 528                 result2 = True
 529
 530             if sock1_mem:
 531                 logging.info('Need %s hugepages free for dpdk socket 1',
 532                              sock1_mem)
 533                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 534                 free_hugepages -= sock1_mem
 535             else:
 536                 result3 = True
 537
 538             logging.info('Need a total of %s total hugepages',
 539                          hugepages_needed + sock1_mem + sock0_mem)
 540
 541             # The only drawback here is sometimes dpdk doesn't release
 542             # its hugepages on a test failure. This could cause a test
 543             # to fail when dpdk would be OK to start because it will just
 544             # use the previously allocated hugepages.
 545             result4 = True if free_hugepages >= 0 else False
 546
 547             return all([result1, result2, result3, result4])
 548         else:
 549             return True
 550
 551     @staticmethod
 552     def write_result_to_file(results, output):
 553         """Write list of dictionaries to a CSV file.
 554
 555         Each element on list will create separate row in output file.
 556         If output file already exists, data will be appended at the end,
 557         otherwise it will be created.
 558
 559         :param results: list of dictionaries.
 560         :param output: path to output file.
 561         """
 562         with open(output, 'a') as csvfile:
 563
 564             logging.info("Write results to file: " + output)
 565             fieldnames = TestCase._get_unique_keys(results)
 566
 567             writer = csv.DictWriter(csvfile, fieldnames)
 568
 569             if not csvfile.tell():  # file is now empty
 570                 writer.writeheader()
 571
 572             for result in results:
 573                 writer.writerow(result)
 574
 575     @staticmethod
 576     def _get_unique_keys(list_of_dicts):
 577         """Gets unique key values as ordered list of strings in given dicts
 578
 579         :param list_of_dicts: list of dictionaries.
 580
 581         :returns: list of unique keys(strings).
 582         """
 583         result = OrderedDict()
 584         for item in list_of_dicts:
 585             for key in item.keys():
 586                 result[key] = ''
 587
 588         return list(result.keys())
 589
 590     def _add_flows(self):
 591         """Add flows to the vswitch
 592         """
 593         vswitch = self._vswitch_ctl.get_vswitch()
 594         # NOTE BOM 15-08-07 the frame mod code assumes that the
 595         # physical ports are ports 1 & 2. The actual numbers
 596         # need to be retrived from the vSwitch and the metadata value
 597         # updated accordingly.
 598         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 599         if self._frame_mod == "vlan":
 600             # 0x8100 => VLAN ethertype
 601             self._logger.debug(" ****   VLAN   ***** ")
 602             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 603                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 604             vswitch.add_flow(bridge, flow)
 605             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 606                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 607             vswitch.add_flow(bridge, flow)
 608         elif self._frame_mod == "mpls":
 609             # 0x8847 => MPLS unicast ethertype
 610             self._logger.debug(" ****   MPLS  ***** ")
 611             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 612                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 613             vswitch.add_flow(bridge, flow)
 614             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 615                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 616             vswitch.add_flow(bridge, flow)
 617         elif self._frame_mod == "mac":
 618             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 619                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 620                                 'goto_table:3']}
 621             vswitch.add_flow(bridge, flow)
 622             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 623                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 624                                 'goto_table:3']}
 625             vswitch.add_flow(bridge, flow)
 626         elif self._frame_mod == "dscp":
 627             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 628             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 629                     'dl_type':'0x0800',
 630                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 631             vswitch.add_flow(bridge, flow)
 632             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 633                     'dl_type':'0x0800',
 634                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 635             vswitch.add_flow(bridge, flow)
 636         elif self._frame_mod == "ttl":
 637             # 251 and 241 are the highest prime numbers < 255
 638             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 639                     'dl_type':'0x0800',
 640                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 641             vswitch.add_flow(bridge, flow)
 642             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 643                     'dl_type':'0x0800',
 644                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 645             vswitch.add_flow(bridge, flow)
 646         elif self._frame_mod == "ip_addr":
 647             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 648                     'dl_type':'0x0800',
 649                     'actions': ['mod_nw_src:10.10.10.10',
 650                                 'mod_nw_dst:20.20.20.20',
 651                                 'goto_table:3']}
 652             vswitch.add_flow(bridge, flow)
 653             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 654                     'dl_type':'0x0800',
 655                     'actions': ['mod_nw_src:20.20.20.20',
 656                                 'mod_nw_dst:10.10.10.10',
 657                                 'goto_table:3']}
 658             vswitch.add_flow(bridge, flow)
 659         elif self._frame_mod == "ip_port":
 660             # NOTE BOM 15-08-27 The traffic generated is assumed
 661             # to be UDP (nw_proto 17d) which is the default case but
 662             # we will need to pick up the actual traffic params in use.
 663             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 664                     'dl_type':'0x0800', 'nw_proto':'17',
 665                     'actions': ['mod_tp_src:44444',
 666                                 'mod_tp_dst:44444', 'goto_table:3']}
 667             vswitch.add_flow(bridge, flow)
 668             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 669                     'dl_type':'0x0800', 'nw_proto':'17',
 670                     'actions': ['mod_tp_src:44444',
 671                                 'mod_tp_dst:44444', 'goto_table:3']}
 672             vswitch.add_flow(bridge, flow)
 673         else:
 674             pass
 675
 676
 677     #
 678     # TestSteps realted methods
 679     #
 680     def step_report_status(self, label, status):
 681         """ Log status of test step
 682         """
 683         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 684
 685     def step_stop_vnfs(self):
 686         """ Stop all VNFs started by TestSteps
 687         """
 688         for vnf in self._step_vnf_list:
 689             if self._step_vnf_list[vnf]:
 690                 self._step_vnf_list[vnf].stop()
 691
 692     def step_eval_param(self, param, STEP):
 693         # pylint: disable=invalid-name
 694         """ Helper function for #STEP macro evaluation
 695         """
 696         if isinstance(param, str):
 697             # evaluate every #STEP reference inside parameter itself
 698             macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
 699             if macros:
 700                 for macro in macros:
 701                     # pylint: disable=eval-used
 702                     tmp_val = str(eval(macro[1:]))
 703                     param = param.replace(macro, tmp_val)
 704
 705             # evaluate references to vsperf configuration options
 706             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 707             if macros:
 708                 for macro in macros:
 709                     # pylint: disable=eval-used
 710                     try:
 711                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 712                         param = param.replace('${}'.format(macro[0]), tmp_val)
 713                     # ignore that required option can't be evaluated
 714                     except (IndexError, KeyError, AttributeError):
 715                         self._logger.debug("Skipping %s as it isn't a configuration "
 716                                            "parameter.", '${}'.format(macro[0]))
 717             return param
 718         elif isinstance(param, list) or isinstance(param, tuple):
 719             tmp_list = []
 720             for item in param:
 721                 tmp_list.append(self.step_eval_param(item, STEP))
 722             return tmp_list
 723         elif isinstance(param, dict):
 724             tmp_dict = {}
 725             for (key, value) in param.items():
 726                 tmp_dict[key] = self.step_eval_param(value, STEP)
 727             return tmp_dict
 728         else:
 729             return param
 730
 731     def step_eval_params(self, params, step_result):
 732         """ Evaluates referrences to results from previous steps
 733         """
 734         eval_params = []
 735         # evaluate all parameters if needed
 736         for param in params:
 737             eval_params.append(self.step_eval_param(param, step_result))
 738         return eval_params
 739
 740     def step_run(self):
 741         """ Execute actions specified by TestSteps list
 742
 743         :return: False if any error was detected
 744                  True otherwise
 745         """
 746         # anything to do?
 747         if not self.test:
 748             return True
 749
 750         # required for VNFs initialization
 751         loader = Loader()
 752         # initialize list with results
 753         self._step_result = [None] * len(self.test)
 754
 755         # We have to suppress pylint report, because test_object has to be set according
 756         # to the test step definition
 757         # pylint: disable=redefined-variable-type
 758         # run test step by step...
 759         for i, step in enumerate(self.test):
 760             step_ok = not self._step_check
 761             if step[0] == 'vswitch':
 762                 test_object = self._vswitch_ctl.get_vswitch()
 763             elif step[0] == 'namespace':
 764                 test_object = namespace
 765             elif step[0] == 'veth':
 766                 test_object = veth
 767             elif step[0] == 'settings':
 768                 test_object = S
 769             elif step[0] == 'tools':
 770                 test_object = TestStepsTools()
 771                 step[1] = step[1].title()
 772             elif step[0] == 'trafficgen':
 773                 test_object = self._traffic_ctl
 774                 # in case of send_traffic or send_traffic_async methods, ensure
 775                 # that specified traffic values are merged with existing self._traffic
 776                 if step[1].startswith('send_traffic'):
 777                     tmp_traffic = copy.deepcopy(self._traffic)
 778                     tmp_traffic.update(step[2])
 779                     step[2] = tmp_traffic
 780             elif step[0].startswith('vnf'):
 781                 if not self._step_vnf_list[step[0]]:
 782                     # initialize new VM
 783                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 784                 test_object = self._step_vnf_list[step[0]]
 785             elif step[0] == 'wait':
 786                 input(os.linesep + "Step {}: Press Enter to continue with "
 787                       "the next step...".format(i) + os.linesep + os.linesep)
 788                 continue
 789             elif step[0] == 'sleep':
 790                 self._logger.debug("Sleep %s seconds", step[1])
 791                 time.sleep(int(step[1]))
 792                 continue
 793             else:
 794                 self._logger.error("Unsupported test object %s", step[0])
 795                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 796                 self.step_report_status("Step '{}'".format(' '.join(step)),
 797                                         self._step_status['status'])
 798                 return False
 799
 800             test_method = getattr(test_object, step[1])
 801             if self._step_check:
 802                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 803             else:
 804                 test_method_check = None
 805
 806             step_params = []
 807             try:
 808                 # eval parameters, but use only valid step_results
 809                 # to support negative indexes
 810                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 811                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 812                 self._logger.debug("Step %s '%s' start", i, step_log)
 813                 self._step_result[i] = test_method(*step_params)
 814                 self._logger.debug("Step %s '%s' results '%s'", i,
 815                                    step_log, self._step_result[i])
 816                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 817                 if self._step_check:
 818                     step_ok = test_method_check(self._step_result[i], *step_params)
 819             except (AssertionError, AttributeError, IndexError) as ex:
 820                 step_ok = False
 821                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 822
 823             if self._step_check:
 824                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 825
 826             if not step_ok:
 827                 self._step_status = {'status' : False, 'details' : step_log}
 828                 # Stop all VNFs started by TestSteps
 829                 self.step_stop_vnfs()
 830                 return False
 831
 832         # all steps processed without any issue
 833         return True
 834
 835     #
 836     # get methods for TestCase members, which needs to be publicly available
 837     #
 838     def get_output_file(self):
 839         """Return content of self._output_file member
 840         """
 841         return self._output_file
 842
 843     def get_desc(self):
 844         """Return content of self.desc member
 845         """
 846         return self.desc
 847
 848     def get_versions(self):
 849         """Return content of self.versions member
 850         """
 851         return self._versions
 852
 853     def get_traffic(self):
 854         """Return content of self._traffic member
 855         """
 856         return self._traffic
 857
 858     def get_tc_results(self):
 859         """Return content of self._tc_results member
 860         """
 861         return self._tc_results
 862
 863     def get_collector(self):
 864         """Return content of self._collector member
 865         """
 866         return self._collector