testcases/testcase.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import get_test_param, merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39 from tools.llc_management import rmd
  40
  41 CHECK_PREFIX = 'validate_'
  42
  43 # pylint: disable=too-many-instance-attributes
  44 class TestCase(object):
  45     """TestCase base class
  46
  47     In this basic form runs RFC2544 throughput test
  48     """
  49     # pylint: disable=too-many-statements
  50     def __init__(self, test_cfg):
  51         """Pull out fields from test config
  52
  53         :param test_cfg: A dictionary of string-value pairs describing the test
  54             configuration. Both the key and values strings use well-known
  55             values.
  56         :param results_dir: Where the csv formatted results are written.
  57         """
  58         # make a local copy of test configuration to avoid modification of
  59         # original content used in vsperf main script
  60         cfg = copy.deepcopy(test_cfg)
  61
  62         self._testcase_start_time = time.time()
  63         self._testcase_stop_time = self._testcase_start_time
  64         self._hugepages_mounted = False
  65         self._traffic_ctl = None
  66         self._vnf_ctl = None
  67         self._vswitch_ctl = None
  68         self._collector = None
  69         self._loadgen = None
  70         self._output_file = None
  71         self._tc_results = None
  72         self._settings_paths_modified = False
  73         self._testcast_run_time = None
  74         self._versions = []
  75         # initialization of step driven specific members
  76         self._step_check = False    # by default don't check result for step driven testcases
  77         self._step_vnf_list = {}
  78         self._step_result = []
  79         self._step_result_mapping = {}
  80         self._step_status = None
  81         self._step_send_traffic = False # indication if send_traffic was called within test steps
  82         self._vnf_list = []
  83         self._testcase_run_time = None
  84
  85         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  86         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  87         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  88         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  89         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
  90         test_params = merge_spec(test_params, tc_test_params)
  91         S.setValue('TEST_PARAMS', test_params)
  92         S.check_test_params()
  93
  94         # override all redefined GUEST_ values to have them expanded correctly
  95         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  96         for key in tmp_test_params:
  97             if key.startswith('GUEST_'):
  98                 S.setValue(key, S.getValue(key))
  99                 S.getValue('TEST_PARAMS').pop(key)
 100
 101         # update global settings
 102         functions.settings_update_paths()
 103
 104         # set test parameters; CLI options take precedence to testcase settings
 105         self._logger = logging.getLogger(__name__)
 106         self.name = cfg['Name']
 107         self.desc = cfg.get('Description', 'No description given.')
 108         self.test = cfg.get('TestSteps', None)
 109
 110         # log testcase name and details
 111         tmp_desc = functions.format_description(self.desc, 50)
 112         self._logger.info('############################################################')
 113         self._logger.info('# Test:    %s', self.name)
 114         self._logger.info('# Details: %s', tmp_desc[0])
 115         for i in range(1, len(tmp_desc)):
 116             self._logger.info('#          %s', tmp_desc[i])
 117         self._logger.info('############################################################')
 118
 119         bidirectional = S.getValue('TRAFFIC')['bidir']
 120         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 121             raise TypeError(
 122                 'Bi-dir value must be of type string')
 123         bidirectional = bidirectional.title()  # Keep things consistent
 124
 125         self.deployment = cfg['Deployment']
 126         self._frame_mod = cfg.get('Frame Modification', None)
 127
 128         self._tunnel_type = None
 129         self._tunnel_operation = None
 130
 131         if self.deployment == 'op2p':
 132             self._tunnel_operation = cfg['Tunnel Operation']
 133
 134             if 'Tunnel Type' in cfg:
 135                 self._tunnel_type = cfg['Tunnel Type']
 136                 self._tunnel_type = get_test_param('TUNNEL_TYPE',
 137                                                    self._tunnel_type)
 138
 139         # check if test requires background load and which generator it uses
 140         self._load_cfg = cfg.get('Load', None)
 141
 142         if self._frame_mod:
 143             self._frame_mod = self._frame_mod.lower()
 144         self._results_dir = S.getValue('RESULTS_PATH')
 145
 146         # set traffic details, so they can be passed to vswitch and traffic ctls
 147         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 148         self._traffic.update({'bidir': bidirectional,
 149                               'tunnel_type': self._tunnel_type,})
 150
 151         self._traffic = functions.check_traffic(self._traffic)
 152
 153         # Packet Forwarding mode
 154         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 155
 156         # trafficgen configuration required for tests of tunneling protocols
 157         if self.deployment == "op2p":
 158             self._traffic['l2'].update({'srcmac':
 159                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 160                                         'dstmac':
 161                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 162
 163             self._traffic['l3'].update({'srcip':
 164                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 165                                         'dstip':
 166                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 167
 168             if self._tunnel_operation == "decapsulation":
 169                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 170                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 171                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 172                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 173         elif len(S.getValue('NICS')) >= 2 and \
 174              (S.getValue('NICS')[0]['type'] == 'vf' or
 175               S.getValue('NICS')[1]['type'] == 'vf'):
 176             mac1 = S.getValue('NICS')[0]['mac']
 177             mac2 = S.getValue('NICS')[1]['mac']
 178             if mac1 and mac2:
 179                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 180             else:
 181                 self._logger.debug("MAC addresses can not be read")
 182
 183         # count how many VNFs are involved in TestSteps
 184         if self.test:
 185             for step in self.test:
 186                 if step[0].startswith('vnf'):
 187                     self._step_vnf_list[step[0]] = None
 188
 189         # if llc allocation is required, initialize it.
 190         if S.getValue('LLC_ALLOCATION'):
 191             self._rmd = rmd.CacheAllocator()
 192
 193     def run_initialize(self):
 194         """ Prepare test execution environment
 195         """
 196         # mount hugepages if needed
 197         self._mount_hugepages()
 198
 199         self._logger.debug("Controllers:")
 200         loader = Loader()
 201         self._traffic_ctl = component_factory.create_traffic(
 202             self._traffic['traffic_type'],
 203             loader.get_trafficgen_class())
 204
 205         self._vnf_ctl = component_factory.create_vnf(
 206             self.deployment,
 207             loader.get_vnf_class(),
 208             len(self._step_vnf_list))
 209
 210         self._vnf_list = self._vnf_ctl.get_vnfs()
 211
 212         # verify enough hugepages are free to run the testcase
 213         if not self._check_for_enough_hugepages():
 214             raise RuntimeError('Not enough hugepages free to run test.')
 215
 216         # perform guest related handling
 217         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 218         if tmp_vm_count:
 219             # copy sources of l2 forwarding tools into VM shared dir if needed
 220             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 221
 222             # in case of multi VM in parallel, set the number of streams to the number of VMs
 223             if self.deployment.startswith('pvpv'):
 224                 # for each VM NIC pair we need an unique stream
 225                 streams = 0
 226                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 227                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 228                 self._logger.debug("VMs with parallel connection were detected. "
 229                                    "Thus Number of streams was set to %s", streams)
 230                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 231                 # user can define a proper stream count manually
 232                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 233                     self._traffic.update({'multistream': streams})
 234
 235             # OVS Vanilla requires guest VM MAC address and IPs to work
 236             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 237                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 238                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 239                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 240                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 241
 242         if self._vswitch_none:
 243             self._vswitch_ctl = component_factory.create_pktfwd(
 244                 self.deployment,
 245                 loader.get_pktfwd_class())
 246         else:
 247             self._vswitch_ctl = component_factory.create_vswitch(
 248                 self.deployment,
 249                 loader.get_vswitch_class(),
 250                 self._traffic,
 251                 self._tunnel_operation)
 252
 253         self._collector = component_factory.create_collector(
 254             loader.get_collector_class(),
 255             self._results_dir, self.name)
 256         self._loadgen = component_factory.create_loadgen(
 257             loader.get_loadgen_class(),
 258             self._load_cfg)
 259
 260         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 261                                          "_" + self.deployment + ".csv")
 262
 263         self._step_status = {'status' : True, 'details' : ''}
 264
 265         # Perform LLC-allocations
 266         if S.getValue('LLC_ALLOCATION'):
 267             self._rmd.setup_llc_allocation()
 268
 269         self._logger.debug("Setup:")
 270
 271     def run_finalize(self):
 272         """ Tear down test execution environment and record test results
 273         """
 274         # Stop all VNFs started by TestSteps in case that something went wrong
 275         self.step_stop_vnfs()
 276
 277         # Cleanup any LLC-allocations
 278         if S.getValue('LLC_ALLOCATION'):
 279             self._rmd.cleanup_llc_allocation()
 280
 281         # Stop all processes executed by testcase
 282         tasks.terminate_all_tasks(self._logger)
 283
 284         # umount hugepages if mounted
 285         self._umount_hugepages()
 286
 287         # cleanup any namespaces created
 288         if os.path.isdir('/tmp/namespaces'):
 289             namespace_list = os.listdir('/tmp/namespaces')
 290             if len(namespace_list):
 291                 self._logger.info('Cleaning up namespaces')
 292             for name in namespace_list:
 293                 namespace.delete_namespace(name)
 294             os.rmdir('/tmp/namespaces')
 295         # cleanup any veth ports created
 296         if os.path.isdir('/tmp/veth'):
 297             veth_list = os.listdir('/tmp/veth')
 298             if len(veth_list):
 299                 self._logger.info('Cleaning up veth ports')
 300             for eth in veth_list:
 301                 port1, port2 = eth.split('-')
 302                 veth.del_veth_port(port1, port2)
 303             os.rmdir('/tmp/veth')
 304
 305     def run_report(self):
 306         """ Report test results
 307         """
 308         self._logger.debug("self._collector Results:")
 309         self._collector.print_results()
 310
 311         results = self._traffic_ctl.get_results()
 312         if results:
 313             self._logger.debug("Traffic Results:")
 314             self._traffic_ctl.print_results()
 315
 316         if self._tc_results is None:
 317             self._tc_results = self._append_results(results)
 318         else:
 319             # integration step driven tests have their status and possible
 320             # failure details stored inside self._tc_results
 321             results = self._append_results(results)
 322             if len(self._tc_results) < len(results):
 323                 if len(self._tc_results) > 1:
 324                     raise RuntimeError('Testcase results do not match:'
 325                                        'results: {}\n'
 326                                        'trafficgen results: {}\n',
 327                                        self._tc_results,
 328                                        results)
 329                 else:
 330                     tmp_results = copy.deepcopy(self._tc_results[0])
 331                     self._tc_results = []
 332                     for res in results:
 333                         tmp_res = copy.deepcopy(tmp_results)
 334                         tmp_res.update(res)
 335                         self._tc_results.append(tmp_res)
 336             else:
 337                 for i, result in enumerate(results):
 338                     self._tc_results[i].update(result)
 339
 340         TestCase.write_result_to_file(self._tc_results, self._output_file)
 341
 342     def run(self):
 343         """Run the test
 344
 345         All setup and teardown through controllers is included.
 346         """
 347         # prepare test execution environment
 348         self.run_initialize()
 349
 350         try:
 351             with self._vswitch_ctl:
 352                 with self._vnf_ctl, self._collector, self._loadgen:
 353                     if not self._vswitch_none:
 354                         self._add_flows()
 355
 356                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 357
 358                     with self._traffic_ctl:
 359                         # execute test based on TestSteps definition if needed...
 360                         if self.step_run():
 361                             # ...and continue with traffic generation, but keep
 362                             # in mind, that clean deployment does not configure
 363                             # OVS nor executes the traffic
 364                             if self.deployment != 'clean' and not self._step_send_traffic:
 365                                 self._traffic_ctl.send_traffic(self._traffic)
 366
 367                         # dump vswitch flows before they are affected by VNF termination
 368                         if not self._vswitch_none:
 369                             self._vswitch_ctl.dump_vswitch_flows()
 370
 371                     # garbage collection for case that TestSteps modify existing deployment
 372                     self.step_stop_vnfs()
 373
 374         finally:
 375             # tear down test execution environment and log results
 376             self.run_finalize()
 377
 378         self._testcase_stop_time = time.time()
 379         self._testcase_run_time = time.strftime("%H:%M:%S",
 380                                                 time.gmtime(self._testcase_stop_time -
 381                                                             self._testcase_start_time))
 382         logging.info("Testcase execution time: " + self._testcase_run_time)
 383         # report test results
 384         self.run_report()
 385
 386     def _append_results(self, results):
 387         """
 388         Method appends mandatory Test Case results to list of dictionaries.
 389
 390         :param results: list of dictionaries which contains results from
 391                 traffic generator.
 392
 393         :returns: modified list of dictionaries.
 394         """
 395         for item in results:
 396             item[ResultsConstants.ID] = self.name
 397             item[ResultsConstants.DEPLOYMENT] = self.deployment
 398             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 399             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 400             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 401             # convert timestamps to human readable format
 402             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 403                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 404             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 405                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 406             if self._traffic['multistream']:
 407                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 408                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 409                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 410             if self._vnf_ctl.get_vnfs_number():
 411                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 412             if self._tunnel_type:
 413                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 414         return results
 415
 416     def _copy_fwd_tools_for_all_guests(self, vm_count):
 417         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 418         """
 419         # consider only VNFs involved in the test
 420         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 421             self._copy_fwd_tools_for_guest(guest_dir)
 422
 423     def _copy_fwd_tools_for_guest(self, guest_dir):
 424         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 425
 426         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 427         """
 428         # remove shared dir if it exists to avoid issues with file consistency
 429         if os.path.exists(guest_dir):
 430             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 431                            'Removing content of shared directory...', True)
 432
 433         # directory to share files between host and guest
 434         os.makedirs(guest_dir)
 435
 436         # copy sources into shared dir only if neccessary
 437         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 438         if 'testpmd' in guest_loopback:
 439             try:
 440                 # exclude whole .git/ subdirectory and all o-files;
 441                 # It is assumed, that the same RTE_TARGET is used in both host
 442                 # and VMs; This simplification significantly speeds up testpmd
 443                 # build. If we will need a different RTE_TARGET in VM,
 444                 # then we have to build whole DPDK from the scratch in VM.
 445                 # In that case we can copy just DPDK sources (e.g. by excluding
 446                 # all items obtained by git status -unormal --porcelain).
 447                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 448                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 449                 exclude = []
 450                 exclude.append(r'--exclude=.git/')
 451                 exclude.append(r'--exclude=*.o')
 452                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 453                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 454                                 os.path.join(guest_dir, 'DPDK')],
 455                                self._logger,
 456                                'Copying DPDK to shared directory...',
 457                                True)
 458             except subprocess.CalledProcessError:
 459                 self._logger.error('Unable to copy DPDK to shared directory')
 460                 raise
 461         if 'l2fwd' in guest_loopback:
 462             try:
 463                 tasks.run_task(['rsync', '-a', '-r', '-l',
 464                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 465                                 os.path.join(guest_dir, 'l2fwd')],
 466                                self._logger,
 467                                'Copying l2fwd to shared directory...',
 468                                True)
 469             except subprocess.CalledProcessError:
 470                 self._logger.error('Unable to copy l2fwd to shared directory')
 471                 raise
 472
 473     def _mount_hugepages(self):
 474         """Mount hugepages if usage of DPDK or Qemu is detected
 475         """
 476         # pylint: disable=too-many-boolean-expressions
 477         # hugepages are needed by DPDK and Qemu
 478         if not self._hugepages_mounted and \
 479             (self.deployment.count('v') or \
 480              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 481              self._vswitch_none or \
 482              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 483             hugepages.mount_hugepages()
 484             self._hugepages_mounted = True
 485
 486     def _umount_hugepages(self):
 487         """Umount hugepages if they were mounted before
 488         """
 489         if self._hugepages_mounted:
 490             hugepages.umount_hugepages()
 491             self._hugepages_mounted = False
 492
 493     def _check_for_enough_hugepages(self):
 494         """Check to make sure enough hugepages are free to satisfy the
 495         test environment.
 496         """
 497         hugepages_needed = 0
 498         hugepage_size = hugepages.get_hugepage_size()
 499         # get hugepage amounts per guest involved in the test
 500         for guest in range(self._vnf_ctl.get_vnfs_number()):
 501             hugepages_needed += math.ceil((int(S.getValue(
 502                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 503
 504         # get hugepage amounts for each socket on dpdk
 505         sock0_mem, sock1_mem = 0, 0
 506
 507         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 508             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 509             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 510                                     int(sock_mem[1]) * 1024 / hugepage_size)
 511
 512         # If hugepages needed, verify the amounts are free
 513         if any([hugepages_needed, sock0_mem, sock1_mem]):
 514             free_hugepages = hugepages.get_free_hugepages()
 515             if hugepages_needed:
 516                 logging.info('Need %s hugepages free for guests',
 517                              hugepages_needed)
 518                 result1 = free_hugepages >= hugepages_needed
 519                 free_hugepages -= hugepages_needed
 520             else:
 521                 result1 = True
 522
 523             if sock0_mem:
 524                 logging.info('Need %s hugepages free for dpdk socket 0',
 525                              sock0_mem)
 526                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 527                 free_hugepages -= sock0_mem
 528             else:
 529                 result2 = True
 530
 531             if sock1_mem:
 532                 logging.info('Need %s hugepages free for dpdk socket 1',
 533                              sock1_mem)
 534                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 535                 free_hugepages -= sock1_mem
 536             else:
 537                 result3 = True
 538
 539             logging.info('Need a total of %s total hugepages',
 540                          hugepages_needed + sock1_mem + sock0_mem)
 541
 542             # The only drawback here is sometimes dpdk doesn't release
 543             # its hugepages on a test failure. This could cause a test
 544             # to fail when dpdk would be OK to start because it will just
 545             # use the previously allocated hugepages.
 546             result4 = True if free_hugepages >= 0 else False
 547
 548             return all([result1, result2, result3, result4])
 549         else:
 550             return True
 551
 552     @staticmethod
 553     def write_result_to_file(results, output):
 554         """Write list of dictionaries to a CSV file.
 555
 556         Each element on list will create separate row in output file.
 557         If output file already exists, data will be appended at the end,
 558         otherwise it will be created.
 559
 560         :param results: list of dictionaries.
 561         :param output: path to output file.
 562         """
 563         with open(output, 'a') as csvfile:
 564
 565             logging.info("Write results to file: " + output)
 566             fieldnames = TestCase._get_unique_keys(results)
 567
 568             writer = csv.DictWriter(csvfile, fieldnames)
 569
 570             if not csvfile.tell():  # file is now empty
 571                 writer.writeheader()
 572
 573             for result in results:
 574                 writer.writerow(result)
 575
 576     @staticmethod
 577     def _get_unique_keys(list_of_dicts):
 578         """Gets unique key values as ordered list of strings in given dicts
 579
 580         :param list_of_dicts: list of dictionaries.
 581
 582         :returns: list of unique keys(strings).
 583         """
 584         result = OrderedDict()
 585         for item in list_of_dicts:
 586             for key in item.keys():
 587                 result[key] = ''
 588
 589         return list(result.keys())
 590
 591     def _add_flows(self):
 592         """Add flows to the vswitch
 593         """
 594         vswitch = self._vswitch_ctl.get_vswitch()
 595         # NOTE BOM 15-08-07 the frame mod code assumes that the
 596         # physical ports are ports 1 & 2. The actual numbers
 597         # need to be retrived from the vSwitch and the metadata value
 598         # updated accordingly.
 599         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 600         if self._frame_mod == "vlan":
 601             # 0x8100 => VLAN ethertype
 602             self._logger.debug(" ****   VLAN   ***** ")
 603             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 604                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 605             vswitch.add_flow(bridge, flow)
 606             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 607                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 608             vswitch.add_flow(bridge, flow)
 609         elif self._frame_mod == "mpls":
 610             # 0x8847 => MPLS unicast ethertype
 611             self._logger.debug(" ****   MPLS  ***** ")
 612             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 613                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 614             vswitch.add_flow(bridge, flow)
 615             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 616                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 617             vswitch.add_flow(bridge, flow)
 618         elif self._frame_mod == "mac":
 619             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 620                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 621                                 'goto_table:3']}
 622             vswitch.add_flow(bridge, flow)
 623             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 624                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 625                                 'goto_table:3']}
 626             vswitch.add_flow(bridge, flow)
 627         elif self._frame_mod == "dscp":
 628             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 629             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 630                     'dl_type':'0x0800',
 631                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 632             vswitch.add_flow(bridge, flow)
 633             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 634                     'dl_type':'0x0800',
 635                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 636             vswitch.add_flow(bridge, flow)
 637         elif self._frame_mod == "ttl":
 638             # 251 and 241 are the highest prime numbers < 255
 639             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 640                     'dl_type':'0x0800',
 641                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 642             vswitch.add_flow(bridge, flow)
 643             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 644                     'dl_type':'0x0800',
 645                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 646             vswitch.add_flow(bridge, flow)
 647         elif self._frame_mod == "ip_addr":
 648             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 649                     'dl_type':'0x0800',
 650                     'actions': ['mod_nw_src:10.10.10.10',
 651                                 'mod_nw_dst:20.20.20.20',
 652                                 'goto_table:3']}
 653             vswitch.add_flow(bridge, flow)
 654             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 655                     'dl_type':'0x0800',
 656                     'actions': ['mod_nw_src:20.20.20.20',
 657                                 'mod_nw_dst:10.10.10.10',
 658                                 'goto_table:3']}
 659             vswitch.add_flow(bridge, flow)
 660         elif self._frame_mod == "ip_port":
 661             # NOTE BOM 15-08-27 The traffic generated is assumed
 662             # to be UDP (nw_proto 17d) which is the default case but
 663             # we will need to pick up the actual traffic params in use.
 664             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 665                     'dl_type':'0x0800', 'nw_proto':'17',
 666                     'actions': ['mod_tp_src:44444',
 667                                 'mod_tp_dst:44444', 'goto_table:3']}
 668             vswitch.add_flow(bridge, flow)
 669             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 670                     'dl_type':'0x0800', 'nw_proto':'17',
 671                     'actions': ['mod_tp_src:44444',
 672                                 'mod_tp_dst:44444', 'goto_table:3']}
 673             vswitch.add_flow(bridge, flow)
 674         else:
 675             pass
 676
 677
 678     #
 679     # TestSteps realted methods
 680     #
 681     def step_report_status(self, label, status):
 682         """ Log status of test step
 683         """
 684         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 685
 686     def step_stop_vnfs(self):
 687         """ Stop all VNFs started by TestSteps
 688         """
 689         for vnf in self._step_vnf_list:
 690             if self._step_vnf_list[vnf]:
 691                 self._step_vnf_list[vnf].stop()
 692
 693     def step_eval_param(self, param, step_result):
 694         """ Helper function for #STEP macro evaluation
 695         """
 696         if isinstance(param, str):
 697             # evaluate every #STEP reference inside parameter itself
 698             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 699
 700             if macros:
 701                 for macro in macros:
 702                     if macro[1] in self._step_result_mapping:
 703                         key = self._step_result_mapping[macro[1]]
 704                     else:
 705                         key = macro[1]
 706                     # pylint: disable=eval-used
 707                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 708                     param = param.replace(macro[0], tmp_val)
 709
 710             # evaluate references to vsperf configuration options
 711             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 712             if macros:
 713                 for macro in macros:
 714                     # pylint: disable=eval-used
 715                     try:
 716                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 717                         param = param.replace('${}'.format(macro[0]), tmp_val)
 718                     # ignore that required option can't be evaluated
 719                     except (IndexError, KeyError, AttributeError):
 720                         self._logger.debug("Skipping %s as it isn't a configuration "
 721                                            "parameter.", '${}'.format(macro[0]))
 722             return param
 723         elif isinstance(param, list) or isinstance(param, tuple):
 724             tmp_list = []
 725             for item in param:
 726                 tmp_list.append(self.step_eval_param(item, step_result))
 727             return tmp_list
 728         elif isinstance(param, dict):
 729             tmp_dict = {}
 730             for (key, value) in param.items():
 731                 tmp_dict[key] = self.step_eval_param(value, step_result)
 732             return tmp_dict
 733         else:
 734             return param
 735
 736     def step_eval_params(self, params, step_result):
 737         """ Evaluates referrences to results from previous steps
 738         """
 739         eval_params = []
 740         # evaluate all parameters if needed
 741         for param in params:
 742             eval_params.append(self.step_eval_param(param, step_result))
 743         return eval_params
 744
 745     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 746     def step_run(self):
 747         """ Execute actions specified by TestSteps list
 748
 749         :return: False if any error was detected
 750                  True otherwise
 751         """
 752         # anything to do?
 753         if not self.test:
 754             return True
 755
 756         # required for VNFs initialization
 757         loader = Loader()
 758         # initialize list with results
 759         self._step_result = [None] * len(self.test)
 760
 761         # We have to suppress pylint report, because test_object has to be set according
 762         # to the test step definition
 763         # pylint: disable=redefined-variable-type
 764         # run test step by step...
 765         for i, step in enumerate(self.test):
 766             step_ok = not self._step_check
 767             step_check = self._step_check
 768             regex = None
 769             # configure step result mapping if step alias/label is detected
 770             if step[0].startswith('#'):
 771                 key = step[0][1:]
 772                 if key.isdigit():
 773                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 774                 if key in self._step_result_mapping:
 775                     raise RuntimeError('Step alias {} has been used already for step '
 776                                        '{}'.format(key, self._step_result_mapping[key]))
 777                 self._step_result_mapping[step[0][1:]] = i
 778                 step = step[1:]
 779
 780             # store regex filter if it is specified
 781             if isinstance(step[-1], str) and step[-1].startswith('|'):
 782                 # evalute macros and variables used in regex
 783                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 784                 step = step[:-1]
 785
 786             # check if step verification should be suppressed
 787             if step[0].startswith('!'):
 788                 step_check = False
 789                 step_ok = True
 790                 step[0] = step[0][1:]
 791             if step[0] == 'vswitch':
 792                 test_object = self._vswitch_ctl.get_vswitch()
 793             elif step[0] == 'namespace':
 794                 test_object = namespace
 795             elif step[0] == 'veth':
 796                 test_object = veth
 797             elif step[0] == 'settings':
 798                 test_object = S
 799             elif step[0] == 'tools':
 800                 test_object = TestStepsTools()
 801                 step[1] = step[1].title()
 802             elif step[0] == 'trafficgen':
 803                 test_object = self._traffic_ctl
 804                 # in case of send_traffic or send_traffic_async methods, ensure
 805                 # that specified traffic values are merged with existing self._traffic
 806                 if step[1].startswith('send_traffic'):
 807                     tmp_traffic = copy.deepcopy(self._traffic)
 808                     tmp_traffic.update(step[2])
 809                     step[2] = tmp_traffic
 810                     # store indication that traffic has been sent
 811                     # so it is not sent again after the execution of teststeps
 812                     self._step_send_traffic = True
 813             elif step[0].startswith('vnf'):
 814                 # use vnf started within TestSteps
 815                 if not self._step_vnf_list[step[0]]:
 816                     # initialize new VM
 817                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 818                 test_object = self._step_vnf_list[step[0]]
 819             elif step[0].startswith('VNF'):
 820                 if step[1] in ('start', 'stop'):
 821                     raise RuntimeError("Cannot execute start() or stop() method of "
 822                                        "VNF deployed automatically by scenario.")
 823                 # use vnf started by scenario deployment (e.g. pvp)
 824                 vnf_index = int(step[0][3:])
 825                 try:
 826                     test_object = self._vnf_list[vnf_index]
 827                 except IndexError:
 828                     raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
 829             elif step[0] == 'wait':
 830                 input(os.linesep + "Step {}: Press Enter to continue with "
 831                       "the next step...".format(i) + os.linesep + os.linesep)
 832                 continue
 833             elif step[0] == 'sleep':
 834                 self._logger.debug("Sleep %s seconds", step[1])
 835                 time.sleep(int(step[1]))
 836                 continue
 837             elif step[0] == 'log':
 838                 test_object = self._logger
 839                 # there isn't a need for validation of log entry
 840                 step_check = False
 841                 step_ok = True
 842             elif step[0] == 'pdb':
 843                 import pdb
 844                 pdb.set_trace()
 845                 continue
 846             else:
 847                 self._logger.error("Unsupported test object %s", step[0])
 848                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 849                 self.step_report_status("Step '{}'".format(' '.join(step)),
 850                                         self._step_status['status'])
 851                 return False
 852
 853             test_method = getattr(test_object, step[1])
 854             if step_check:
 855                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 856             else:
 857                 test_method_check = None
 858
 859             step_params = []
 860             try:
 861                 # eval parameters, but use only valid step_results
 862                 # to support negative indexes
 863                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 864                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 865                 step_log += ' filter "{}"'.format(regex) if regex else ''
 866                 self._logger.debug("Step %s '%s' start", i, step_log)
 867                 self._step_result[i] = test_method(*step_params)
 868                 if regex:
 869                     # apply regex to step output
 870                     self._step_result[i] = functions.filter_output(
 871                         self._step_result[i], regex)
 872
 873                 self._logger.debug("Step %s '%s' results '%s'", i,
 874                                    step_log, self._step_result[i])
 875                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 876                 if step_check:
 877                     step_ok = test_method_check(self._step_result[i], *step_params)
 878             except (AssertionError, AttributeError, IndexError) as ex:
 879                 step_ok = False
 880                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 881
 882             if step_check:
 883                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 884
 885             if not step_ok:
 886                 self._step_status = {'status' : False, 'details' : step_log}
 887                 # Stop all VNFs started by TestSteps
 888                 self.step_stop_vnfs()
 889                 return False
 890
 891         # all steps processed without any issue
 892         return True
 893
 894     #
 895     # get methods for TestCase members, which needs to be publicly available
 896     #
 897     def get_output_file(self):
 898         """Return content of self._output_file member
 899         """
 900         return self._output_file
 901
 902     def get_desc(self):
 903         """Return content of self.desc member
 904         """
 905         return self.desc
 906
 907     def get_versions(self):
 908         """Return content of self.versions member
 909         """
 910         return self._versions
 911
 912     def get_traffic(self):
 913         """Return content of self._traffic member
 914         """
 915         return self._traffic
 916
 917     def get_tc_results(self):
 918         """Return content of self._tc_results member
 919         """
 920         return self._tc_results
 921
 922     def get_collector(self):
 923         """Return content of self._collector member
 924         """
 925         return self._collector