testcases/testcase.py

   1 # Copyright 2015-2018 Intel Corporation, Tieto and others.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39 from tools.llc_management import rmd
  40
  41 CHECK_PREFIX = 'validate_'
  42
  43 # pylint: disable=too-many-instance-attributes
  44 class TestCase(object):
  45     """TestCase base class
  46
  47     In this basic form runs RFC2544 throughput test
  48     """
  49     # pylint: disable=too-many-statements
  50     def __init__(self, test_cfg):
  51         """Pull out fields from test config
  52
  53         :param test_cfg: A dictionary of string-value pairs describing the test
  54             configuration. Both the key and values strings use well-known
  55             values.
  56         :param results_dir: Where the csv formatted results are written.
  57         """
  58         # make a local copy of test configuration to avoid modification of
  59         # original content used in vsperf main script
  60         cfg = copy.deepcopy(test_cfg)
  61
  62         self._testcase_start_time = time.time()
  63         self._testcase_stop_time = self._testcase_start_time
  64         self._hugepages_mounted = False
  65         self._traffic_ctl = None
  66         self._vnf_ctl = None
  67         self._vswitch_ctl = None
  68         self._collector = None
  69         self._loadgen = None
  70         self._output_file = None
  71         self._tc_results = None
  72         self._settings_paths_modified = False
  73         self._testcast_run_time = None
  74         self._versions = []
  75         # initialization of step driven specific members
  76         self._step_check = False    # by default don't check result for step driven testcases
  77         self._step_vnf_list = {}
  78         self._step_result = []
  79         self._step_result_mapping = {}
  80         self._step_status = None
  81         self._step_send_traffic = False # indication if send_traffic was called within test steps
  82         self._vnf_list = []
  83         self._testcase_run_time = None
  84
  85         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  86         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  87         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  88         S.setValue('TUNNEL_TYPE', cfg.get('Tunnel Type', S.getValue('TUNNEL_TYPE')))
  89         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  90         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  91         test_params = merge_spec(test_params, tc_test_params)
  92         S.setValue('TEST_PARAMS', test_params)
  93         S.check_test_params()
  94
  95         # override all redefined GUEST_ values to have them expanded correctly
  96         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  97         for key in tmp_test_params:
  98             if key.startswith('GUEST_'):
  99                 S.setValue(key, S.getValue(key))
 100                 S.getValue('TEST_PARAMS').pop(key)
 101
 102         # update global settings
 103         functions.settings_update_paths()
 104
 105         # set test parameters; CLI options take precedence to testcase settings
 106         self._logger = logging.getLogger(__name__)
 107         self.name = cfg['Name']
 108         self.desc = cfg.get('Description', 'No description given.')
 109         self.test = cfg.get('TestSteps', None)
 110
 111         # log testcase name and details
 112         tmp_desc = functions.format_description(self.desc, 50)
 113         self._logger.info('############################################################')
 114         self._logger.info('# Test:    %s', self.name)
 115         self._logger.info('# Details: %s', tmp_desc[0])
 116         for i in range(1, len(tmp_desc)):
 117             self._logger.info('#          %s', tmp_desc[i])
 118         self._logger.info('############################################################')
 119
 120         bidirectional = S.getValue('TRAFFIC')['bidir']
 121         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 122             raise TypeError(
 123                 'Bi-dir value must be of type string')
 124         bidirectional = bidirectional.title()  # Keep things consistent
 125
 126         self.deployment = cfg['Deployment']
 127         self._frame_mod = cfg.get('Frame Modification', None)
 128
 129         self._tunnel_operation = cfg.get('Tunnel Operation', None)
 130
 131         # check if test requires background load and which generator it uses
 132         self._load_cfg = cfg.get('Load', None)
 133
 134         if self._frame_mod:
 135             self._frame_mod = self._frame_mod.lower()
 136         self._results_dir = S.getValue('RESULTS_PATH')
 137
 138         # set traffic details, so they can be passed to vswitch and traffic ctls
 139         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 140         self._traffic.update({'bidir': bidirectional})
 141
 142         # Packet Forwarding mode
 143         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 144
 145         # trafficgen configuration required for tests of tunneling protocols
 146         if self._tunnel_operation:
 147             self._traffic.update({'tunnel_type': S.getValue('TUNNEL_TYPE')})
 148             self._traffic['l2'].update({'srcmac':
 149                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 150                                         'dstmac':
 151                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 152
 153             self._traffic['l3'].update({'srcip':
 154                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 155                                         'dstip':
 156                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 157
 158             if self._tunnel_operation == "decapsulation":
 159                 self._traffic['l2'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L2'))
 160                 self._traffic['l3'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L3'))
 161                 self._traffic['l4'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L4'))
 162                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 163         elif len(S.getValue('NICS')) >= 2 and \
 164              (S.getValue('NICS')[0]['type'] == 'vf' or
 165               S.getValue('NICS')[1]['type'] == 'vf'):
 166             mac1 = S.getValue('NICS')[0]['mac']
 167             mac2 = S.getValue('NICS')[1]['mac']
 168             if mac1 and mac2:
 169                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 170             else:
 171                 self._logger.debug("MAC addresses can not be read")
 172
 173         self._traffic = functions.check_traffic(self._traffic)
 174
 175         # count how many VNFs are involved in TestSteps
 176         if self.test:
 177             for step in self.test:
 178                 if step[0].startswith('vnf'):
 179                     self._step_vnf_list[step[0]] = None
 180
 181         # if llc allocation is required, initialize it.
 182         if S.getValue('LLC_ALLOCATION'):
 183             self._rmd = rmd.CacheAllocator()
 184
 185     def run_initialize(self):
 186         """ Prepare test execution environment
 187         """
 188         # mount hugepages if needed
 189         self._mount_hugepages()
 190
 191         self._logger.debug("Controllers:")
 192         loader = Loader()
 193         self._traffic_ctl = component_factory.create_traffic(
 194             self._traffic['traffic_type'],
 195             loader.get_trafficgen_class())
 196
 197         self._vnf_ctl = component_factory.create_vnf(
 198             self.deployment,
 199             loader.get_vnf_class(),
 200             len(self._step_vnf_list))
 201
 202         self._vnf_list = self._vnf_ctl.get_vnfs()
 203
 204         # verify enough hugepages are free to run the testcase
 205         if not self._check_for_enough_hugepages():
 206             raise RuntimeError('Not enough hugepages free to run test.')
 207
 208         # perform guest related handling
 209         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 210         if tmp_vm_count:
 211             # copy sources of l2 forwarding tools into VM shared dir if needed
 212             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 213
 214             # in case of multi VM in parallel, set the number of streams to the number of VMs
 215             if self.deployment.startswith('pvpv'):
 216                 # for each VM NIC pair we need an unique stream
 217                 streams = 0
 218                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 219                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 220                 self._logger.debug("VMs with parallel connection were detected. "
 221                                    "Thus Number of streams was set to %s", streams)
 222                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 223                 # user can define a proper stream count manually
 224                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 225                     self._traffic.update({'multistream': streams})
 226
 227             # OVS Vanilla requires guest VM MAC address and IPs to work
 228             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 229                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 230                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 231                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 232                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 233
 234         if self._vswitch_none:
 235             self._vswitch_ctl = component_factory.create_pktfwd(
 236                 self.deployment,
 237                 loader.get_pktfwd_class())
 238         else:
 239             self._vswitch_ctl = component_factory.create_vswitch(
 240                 self.deployment,
 241                 loader.get_vswitch_class(),
 242                 self._traffic,
 243                 self._tunnel_operation)
 244
 245         self._collector = component_factory.create_collector(
 246             loader.get_collector_class(),
 247             self._results_dir, self.name)
 248         self._loadgen = component_factory.create_loadgen(
 249             loader.get_loadgen_class(),
 250             self._load_cfg)
 251
 252         self._output_file = os.path.join(self._results_dir, "result_{}_{}_{}.csv".format(
 253             str(S.getValue('_TEST_INDEX')), self.name, self.deployment))
 254
 255         self._step_status = {'status' : True, 'details' : ''}
 256
 257         # Perform LLC-allocations
 258         if S.getValue('LLC_ALLOCATION'):
 259             self._rmd.setup_llc_allocation()
 260
 261         self._logger.debug("Setup:")
 262
 263     def run_finalize(self):
 264         """ Tear down test execution environment and record test results
 265         """
 266         # Stop all VNFs started by TestSteps in case that something went wrong
 267         self.step_stop_vnfs()
 268
 269         # Cleanup any LLC-allocations
 270         if S.getValue('LLC_ALLOCATION'):
 271             self._rmd.cleanup_llc_allocation()
 272
 273         # Stop all processes executed by testcase
 274         tasks.terminate_all_tasks(self._logger)
 275
 276         # umount hugepages if mounted
 277         self._umount_hugepages()
 278
 279         # cleanup any namespaces created
 280         if os.path.isdir('/tmp/namespaces'):
 281             namespace_list = os.listdir('/tmp/namespaces')
 282             if namespace_list:
 283                 self._logger.info('Cleaning up namespaces')
 284             for name in namespace_list:
 285                 namespace.delete_namespace(name)
 286             os.rmdir('/tmp/namespaces')
 287         # cleanup any veth ports created
 288         if os.path.isdir('/tmp/veth'):
 289             veth_list = os.listdir('/tmp/veth')
 290             if veth_list:
 291                 self._logger.info('Cleaning up veth ports')
 292             for eth in veth_list:
 293                 port1, port2 = eth.split('-')
 294                 veth.del_veth_port(port1, port2)
 295             os.rmdir('/tmp/veth')
 296
 297     def run_report(self):
 298         """ Report test results
 299         """
 300         self._logger.debug("self._collector Results:")
 301         self._collector.print_results()
 302
 303         results = self._traffic_ctl.get_results()
 304         if results:
 305             self._logger.debug("Traffic Results:")
 306             self._traffic_ctl.print_results()
 307
 308         if self._tc_results is None:
 309             self._tc_results = self._append_results(results)
 310         else:
 311             # integration step driven tests have their status and possible
 312             # failure details stored inside self._tc_results
 313             results = self._append_results(results)
 314             if len(self._tc_results) < len(results):
 315                 if len(self._tc_results) > 1:
 316                     raise RuntimeError('Testcase results do not match:'
 317                                        'results: %s\n'
 318                                        'trafficgen results: %s\n' %
 319                                        self._tc_results,
 320                                        results)
 321                 else:
 322                     tmp_results = copy.deepcopy(self._tc_results[0])
 323                     self._tc_results = []
 324                     for res in results:
 325                         tmp_res = copy.deepcopy(tmp_results)
 326                         tmp_res.update(res)
 327                         self._tc_results.append(tmp_res)
 328             else:
 329                 for i, result in enumerate(results):
 330                     self._tc_results[i].update(result)
 331
 332         TestCase.write_result_to_file(self._tc_results, self._output_file)
 333
 334     def run(self):
 335         """Run the test
 336
 337         All setup and teardown through controllers is included.
 338         """
 339         # prepare test execution environment
 340         self.run_initialize()
 341
 342         try:
 343             with self._vswitch_ctl:
 344                 with self._vnf_ctl, self._collector, self._loadgen:
 345                     if not self._vswitch_none:
 346                         self._add_flows()
 347
 348                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 349
 350                     with self._traffic_ctl:
 351                         # execute test based on TestSteps definition if needed...
 352                         if self.step_run():
 353                             # ...and continue with traffic generation, but keep
 354                             # in mind, that clean deployment does not configure
 355                             # OVS nor executes the traffic
 356                             if self.deployment != 'clean' and not self._step_send_traffic:
 357                                 self._traffic_ctl.send_traffic(self._traffic)
 358
 359                         # dump vswitch flows before they are affected by VNF termination
 360                         if not self._vswitch_none:
 361                             self._vswitch_ctl.dump_vswitch_flows()
 362
 363                     # garbage collection for case that TestSteps modify existing deployment
 364                     self.step_stop_vnfs()
 365
 366         finally:
 367             # tear down test execution environment and log results
 368             self.run_finalize()
 369
 370         self._testcase_stop_time = time.time()
 371         self._testcase_run_time = time.strftime("%H:%M:%S",
 372                                                 time.gmtime(self._testcase_stop_time -
 373                                                             self._testcase_start_time))
 374         logging.info("Testcase execution time: %s", self._testcase_run_time)
 375         # report test results
 376         self.run_report()
 377
 378     def _append_results(self, results):
 379         """
 380         Method appends mandatory Test Case results to list of dictionaries.
 381
 382         :param results: list of dictionaries which contains results from
 383                 traffic generator.
 384
 385         :returns: modified list of dictionaries.
 386         """
 387         for item in results:
 388             item[ResultsConstants.ID] = self.name
 389             item[ResultsConstants.DEPLOYMENT] = self.deployment
 390             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 391             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 392             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 393             # convert timestamps to human readable format
 394             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 395                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 396             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 397                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 398             if self._traffic['multistream']:
 399                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 400                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 401                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 402             if self._vnf_ctl.get_vnfs_number():
 403                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 404             if self._tunnel_operation:
 405                 item[ResultsConstants.TUNNEL_TYPE] = S.getValue('TUNNEL_TYPE')
 406         return results
 407
 408     def _copy_fwd_tools_for_all_guests(self, vm_count):
 409         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 410         """
 411         # consider only VNFs involved in the test
 412         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 413             self._copy_fwd_tools_for_guest(guest_dir)
 414
 415     def _copy_fwd_tools_for_guest(self, guest_dir):
 416         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 417
 418         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 419         """
 420         # remove shared dir if it exists to avoid issues with file consistency
 421         if os.path.exists(guest_dir):
 422             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 423                            'Removing content of shared directory...', True)
 424
 425         # directory to share files between host and guest
 426         os.makedirs(guest_dir)
 427
 428         # copy sources into shared dir only if neccessary
 429         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 430         if 'testpmd' in guest_loopback:
 431             try:
 432                 # exclude whole .git/ subdirectory and all o-files;
 433                 # It is assumed, that the same RTE_TARGET is used in both host
 434                 # and VMs; This simplification significantly speeds up testpmd
 435                 # build. If we will need a different RTE_TARGET in VM,
 436                 # then we have to build whole DPDK from the scratch in VM.
 437                 # In that case we can copy just DPDK sources (e.g. by excluding
 438                 # all items obtained by git status -unormal --porcelain).
 439                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 440                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 441                 exclude = []
 442                 exclude.append(r'--exclude=.git/')
 443                 exclude.append(r'--exclude=*.o')
 444                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 445                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 446                                 os.path.join(guest_dir, 'DPDK')],
 447                                self._logger,
 448                                'Copying DPDK to shared directory...',
 449                                True)
 450             except subprocess.CalledProcessError:
 451                 self._logger.error('Unable to copy DPDK to shared directory')
 452                 raise
 453         if 'l2fwd' in guest_loopback:
 454             try:
 455                 tasks.run_task(['rsync', '-a', '-r', '-l',
 456                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 457                                 os.path.join(guest_dir, 'l2fwd')],
 458                                self._logger,
 459                                'Copying l2fwd to shared directory...',
 460                                True)
 461             except subprocess.CalledProcessError:
 462                 self._logger.error('Unable to copy l2fwd to shared directory')
 463                 raise
 464
 465     def _mount_hugepages(self):
 466         """Mount hugepages if usage of DPDK or Qemu is detected
 467         """
 468         # pylint: disable=too-many-boolean-expressions
 469         # hugepages are needed by DPDK and Qemu
 470         if not self._hugepages_mounted and \
 471             (self.deployment.count('v') or \
 472              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 473              self._vswitch_none or \
 474              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 475             hugepages.mount_hugepages()
 476             self._hugepages_mounted = True
 477
 478     def _umount_hugepages(self):
 479         """Umount hugepages if they were mounted before
 480         """
 481         if self._hugepages_mounted:
 482             hugepages.umount_hugepages()
 483             self._hugepages_mounted = False
 484
 485     def _check_for_enough_hugepages(self):
 486         """Check to make sure enough hugepages are free to satisfy the
 487         test environment.
 488         """
 489         hugepages_needed = 0
 490         hugepage_size = hugepages.get_hugepage_size()
 491         # get hugepage amounts per guest involved in the test
 492         for guest in range(self._vnf_ctl.get_vnfs_number()):
 493             hugepages_needed += math.ceil((int(S.getValue(
 494                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 495
 496         # get hugepage amounts for each socket on dpdk
 497         sock0_mem, sock1_mem = 0, 0
 498
 499         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 500             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 501             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 502                                     int(sock_mem[1]) * 1024 / hugepage_size)
 503
 504         # If hugepages needed, verify the amounts are free
 505         if any([hugepages_needed, sock0_mem, sock1_mem]):
 506             free_hugepages = hugepages.get_free_hugepages()
 507             if hugepages_needed:
 508                 logging.info('Need %s hugepages free for guests',
 509                              hugepages_needed)
 510                 result1 = free_hugepages >= hugepages_needed
 511                 free_hugepages -= hugepages_needed
 512             else:
 513                 result1 = True
 514
 515             if sock0_mem:
 516                 logging.info('Need %s hugepages free for dpdk socket 0',
 517                              sock0_mem)
 518                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 519                 free_hugepages -= sock0_mem
 520             else:
 521                 result2 = True
 522
 523             if sock1_mem:
 524                 logging.info('Need %s hugepages free for dpdk socket 1',
 525                              sock1_mem)
 526                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 527                 free_hugepages -= sock1_mem
 528             else:
 529                 result3 = True
 530
 531             logging.info('Need a total of %s total hugepages',
 532                          hugepages_needed + sock1_mem + sock0_mem)
 533
 534             # The only drawback here is sometimes dpdk doesn't release
 535             # its hugepages on a test failure. This could cause a test
 536             # to fail when dpdk would be OK to start because it will just
 537             # use the previously allocated hugepages.
 538             result4 = True if free_hugepages >= 0 else False
 539
 540             return all([result1, result2, result3, result4])
 541         else:
 542             return True
 543
 544     @staticmethod
 545     def write_result_to_file(results, output):
 546         """Write list of dictionaries to a CSV file.
 547
 548         Each element on list will create separate row in output file.
 549         If output file already exists, data will be appended at the end,
 550         otherwise it will be created.
 551
 552         :param results: list of dictionaries.
 553         :param output: path to output file.
 554         """
 555         with open(output, 'a') as csvfile:
 556
 557             logging.info("Write results to file: %s", output)
 558             fieldnames = TestCase._get_unique_keys(results)
 559
 560             writer = csv.DictWriter(csvfile, fieldnames)
 561
 562             if not csvfile.tell():  # file is now empty
 563                 writer.writeheader()
 564
 565             for result in results:
 566                 writer.writerow(result)
 567
 568     @staticmethod
 569     def _get_unique_keys(list_of_dicts):
 570         """Gets unique key values as ordered list of strings in given dicts
 571
 572         :param list_of_dicts: list of dictionaries.
 573
 574         :returns: list of unique keys(strings).
 575         """
 576         result = OrderedDict()
 577         for item in list_of_dicts:
 578             for key in item.keys():
 579                 result[key] = ''
 580
 581         return list(result.keys())
 582
 583     def _add_flows(self):
 584         """Add flows to the vswitch
 585         """
 586         vswitch = self._vswitch_ctl.get_vswitch()
 587         # NOTE BOM 15-08-07 the frame mod code assumes that the
 588         # physical ports are ports 1 & 2. The actual numbers
 589         # need to be retrived from the vSwitch and the metadata value
 590         # updated accordingly.
 591         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 592         if self._frame_mod == "vlan":
 593             # 0x8100 => VLAN ethertype
 594             self._logger.debug(" ****   VLAN   ***** ")
 595             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 596                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 597             vswitch.add_flow(bridge, flow)
 598             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 599                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 600             vswitch.add_flow(bridge, flow)
 601         elif self._frame_mod == "mpls":
 602             # 0x8847 => MPLS unicast ethertype
 603             self._logger.debug(" ****   MPLS  ***** ")
 604             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 605                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 606             vswitch.add_flow(bridge, flow)
 607             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 608                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 609             vswitch.add_flow(bridge, flow)
 610         elif self._frame_mod == "mac":
 611             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 612                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 613                                 'goto_table:3']}
 614             vswitch.add_flow(bridge, flow)
 615             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 616                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 617                                 'goto_table:3']}
 618             vswitch.add_flow(bridge, flow)
 619         elif self._frame_mod == "dscp":
 620             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 621             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 622                     'dl_type':'0x0800',
 623                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 624             vswitch.add_flow(bridge, flow)
 625             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 626                     'dl_type':'0x0800',
 627                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 628             vswitch.add_flow(bridge, flow)
 629         elif self._frame_mod == "ttl":
 630             # 251 and 241 are the highest prime numbers < 255
 631             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 632                     'dl_type':'0x0800',
 633                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 634             vswitch.add_flow(bridge, flow)
 635             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 636                     'dl_type':'0x0800',
 637                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 638             vswitch.add_flow(bridge, flow)
 639         elif self._frame_mod == "ip_addr":
 640             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 641                     'dl_type':'0x0800',
 642                     'actions': ['mod_nw_src:10.10.10.10',
 643                                 'mod_nw_dst:20.20.20.20',
 644                                 'goto_table:3']}
 645             vswitch.add_flow(bridge, flow)
 646             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 647                     'dl_type':'0x0800',
 648                     'actions': ['mod_nw_src:20.20.20.20',
 649                                 'mod_nw_dst:10.10.10.10',
 650                                 'goto_table:3']}
 651             vswitch.add_flow(bridge, flow)
 652         elif self._frame_mod == "ip_port":
 653             # NOTE BOM 15-08-27 The traffic generated is assumed
 654             # to be UDP (nw_proto 17d) which is the default case but
 655             # we will need to pick up the actual traffic params in use.
 656             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 657                     'dl_type':'0x0800', 'nw_proto':'17',
 658                     'actions': ['mod_tp_src:44444',
 659                                 'mod_tp_dst:44444', 'goto_table:3']}
 660             vswitch.add_flow(bridge, flow)
 661             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 662                     'dl_type':'0x0800', 'nw_proto':'17',
 663                     'actions': ['mod_tp_src:44444',
 664                                 'mod_tp_dst:44444', 'goto_table:3']}
 665             vswitch.add_flow(bridge, flow)
 666         else:
 667             pass
 668
 669
 670     #
 671     # TestSteps realted methods
 672     #
 673     def step_report_status(self, label, status):
 674         """ Log status of test step
 675         """
 676         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 677
 678     def step_stop_vnfs(self):
 679         """ Stop all VNFs started by TestSteps
 680         """
 681         for vnf in self._step_vnf_list:
 682             if self._step_vnf_list[vnf]:
 683                 self._step_vnf_list[vnf].stop()
 684
 685     def step_eval_param(self, param, step_result):
 686         """ Helper function for #STEP macro evaluation
 687         """
 688         if isinstance(param, str):
 689             # evaluate every #STEP reference inside parameter itself
 690             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 691
 692             if macros:
 693                 for macro in macros:
 694                     if macro[1] in self._step_result_mapping:
 695                         key = self._step_result_mapping[macro[1]]
 696                     else:
 697                         key = macro[1]
 698                     # pylint: disable=eval-used
 699                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 700                     param = param.replace(macro[0], tmp_val)
 701
 702             # evaluate references to vsperf configuration options
 703             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 704             if macros:
 705                 for macro in macros:
 706                     # pylint: disable=eval-used
 707                     try:
 708                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 709                         param = param.replace('${}'.format(macro[0]), tmp_val)
 710                     # ignore that required option can't be evaluated
 711                     except (IndexError, KeyError, AttributeError):
 712                         self._logger.debug("Skipping %s as it isn't a configuration "
 713                                            "parameter.", '${}'.format(macro[0]))
 714             return param
 715         elif isinstance(param, (list, tuple)):
 716             tmp_list = []
 717             for item in param:
 718                 tmp_list.append(self.step_eval_param(item, step_result))
 719             return tmp_list
 720         elif isinstance(param, dict):
 721             tmp_dict = {}
 722             for (key, value) in param.items():
 723                 tmp_dict[key] = self.step_eval_param(value, step_result)
 724             return tmp_dict
 725         else:
 726             return param
 727
 728     def step_eval_params(self, params, step_result):
 729         """ Evaluates referrences to results from previous steps
 730         """
 731         eval_params = []
 732         # evaluate all parameters if needed
 733         for param in params:
 734             eval_params.append(self.step_eval_param(param, step_result))
 735         return eval_params
 736
 737     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 738     def step_run(self):
 739         """ Execute actions specified by TestSteps list
 740
 741         :return: False if any error was detected
 742                  True otherwise
 743         """
 744         # anything to do?
 745         if not self.test:
 746             return True
 747
 748         # required for VNFs initialization
 749         loader = Loader()
 750         # initialize list with results
 751         self._step_result = [None] * len(self.test)
 752
 753         # run test step by step...
 754         for i, step in enumerate(self.test):
 755             step_ok = not self._step_check
 756             step_check = self._step_check
 757             regex = None
 758             # configure step result mapping if step alias/label is detected
 759             if step[0].startswith('#'):
 760                 key = step[0][1:]
 761                 if key.isdigit():
 762                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 763                 if key in self._step_result_mapping:
 764                     raise RuntimeError('Step alias {} has been used already for step '
 765                                        '{}'.format(key, self._step_result_mapping[key]))
 766                 self._step_result_mapping[step[0][1:]] = i
 767                 step = step[1:]
 768
 769             # store regex filter if it is specified
 770             if isinstance(step[-1], str) and step[-1].startswith('|'):
 771                 # evalute macros and variables used in regex
 772                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 773                 step = step[:-1]
 774
 775             # check if step verification should be suppressed
 776             if step[0].startswith('!'):
 777                 step_check = False
 778                 step_ok = True
 779                 step[0] = step[0][1:]
 780             if step[0] == 'vswitch':
 781                 test_object = self._vswitch_ctl.get_vswitch()
 782             elif step[0] == 'namespace':
 783                 test_object = namespace
 784             elif step[0] == 'veth':
 785                 test_object = veth
 786             elif step[0] == 'settings':
 787                 test_object = S
 788             elif step[0] == 'tools':
 789                 test_object = TestStepsTools()
 790                 step[1] = step[1].title()
 791             elif step[0] == 'trafficgen':
 792                 test_object = self._traffic_ctl
 793                 # in case of send_traffic or send_traffic_async methods, ensure
 794                 # that specified traffic values are merged with existing self._traffic
 795                 if step[1].startswith('send_traffic'):
 796                     tmp_traffic = copy.deepcopy(self._traffic)
 797                     tmp_traffic.update(step[2])
 798                     step[2] = tmp_traffic
 799                     # store indication that traffic has been sent
 800                     # so it is not sent again after the execution of teststeps
 801                     self._step_send_traffic = True
 802             elif step[0].startswith('vnf'):
 803                 # use vnf started within TestSteps
 804                 if not self._step_vnf_list[step[0]]:
 805                     # initialize new VM
 806                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 807                 test_object = self._step_vnf_list[step[0]]
 808             elif step[0].startswith('VNF'):
 809                 if step[1] in ('start', 'stop'):
 810                     raise RuntimeError("Cannot execute start() or stop() method of "
 811                                        "VNF deployed automatically by scenario.")
 812                 # use vnf started by scenario deployment (e.g. pvp)
 813                 vnf_index = int(step[0][3:])
 814                 try:
 815                     test_object = self._vnf_list[vnf_index]
 816                 except IndexError:
 817                     raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
 818             elif step[0] == 'wait':
 819                 input(os.linesep + "Step {}: Press Enter to continue with "
 820                       "the next step...".format(i) + os.linesep + os.linesep)
 821                 continue
 822             elif step[0] == 'sleep':
 823                 self._logger.debug("Sleep %s seconds", step[1])
 824                 time.sleep(int(step[1]))
 825                 continue
 826             elif step[0] == 'log':
 827                 test_object = self._logger
 828                 # there isn't a need for validation of log entry
 829                 step_check = False
 830                 step_ok = True
 831             elif step[0] == 'pdb':
 832                 import pdb
 833                 pdb.set_trace()
 834                 continue
 835             else:
 836                 self._logger.error("Unsupported test object %s", step[0])
 837                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 838                 self.step_report_status("Step '{}'".format(' '.join(step)),
 839                                         self._step_status['status'])
 840                 return False
 841
 842             test_method = getattr(test_object, step[1])
 843             if step_check:
 844                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 845             else:
 846                 test_method_check = None
 847
 848             step_params = []
 849             try:
 850                 # eval parameters, but use only valid step_results
 851                 # to support negative indexes
 852                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 853                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 854                 step_log += ' filter "{}"'.format(regex) if regex else ''
 855                 self._logger.debug("Step %s '%s' start", i, step_log)
 856                 self._step_result[i] = test_method(*step_params)
 857                 if regex:
 858                     # apply regex to step output
 859                     self._step_result[i] = functions.filter_output(
 860                         self._step_result[i], regex)
 861
 862                 self._logger.debug("Step %s '%s' results '%s'", i,
 863                                    step_log, self._step_result[i])
 864                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 865                 if step_check:
 866                     step_ok = test_method_check(self._step_result[i], *step_params)
 867             except (AssertionError, AttributeError, IndexError) as ex:
 868                 step_ok = False
 869                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 870
 871             if step_check:
 872                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 873
 874             if not step_ok:
 875                 self._step_status = {'status' : False, 'details' : step_log}
 876                 # Stop all VNFs started by TestSteps
 877                 self.step_stop_vnfs()
 878                 return False
 879
 880         # all steps processed without any issue
 881         return True
 882
 883     #
 884     # get methods for TestCase members, which needs to be publicly available
 885     #
 886     def get_output_file(self):
 887         """Return content of self._output_file member
 888         """
 889         return self._output_file
 890
 891     def get_desc(self):
 892         """Return content of self.desc member
 893         """
 894         return self.desc
 895
 896     def get_versions(self):
 897         """Return content of self.versions member
 898         """
 899         return self._versions
 900
 901     def get_traffic(self):
 902         """Return content of self._traffic member
 903         """
 904         return self._traffic
 905
 906     def get_tc_results(self):
 907         """Return content of self._tc_results member
 908         """
 909         return self._tc_results
 910
 911     def get_collector(self):
 912         """Return content of self._collector member
 913         """
 914         return self._collector