testcases/testcase.py

   1 # Copyright 2015-2018 Intel Corporation, Tieto and others.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39 from tools.llc_management import rmd
  40
  41 # Validation methods required for integration TCs will have following prefix before the name
  42 # of original method.
  43 CHECK_PREFIX = 'validate_'
  44
  45 # Several parameters can be defined by both TC definition keywords and configuration parameters.
  46 # Following mapping table is used to correctly evaluate priority of testcase configuration, where
  47 # TC definition keywords (i.e. mapping table keys) have higher priority than appropriate TC
  48 # parameters (i.e. mapping table values). TC parameters can be defined within "Parameters"
  49 # section, via CLI parameters or within configuration files.
  50 MAPPING_TC_CFG2CONF = {'vSwitch':'VSWITCH', 'VNF':'VNF', 'Trafficgen':'TRAFFICGEN', 'Tunnel Type':'TUNNEL_TYPE'}
  51
  52 # pylint: disable=too-many-instance-attributes
  53 class TestCase(object):
  54     """TestCase base class
  55
  56     In this basic form runs RFC2544 throughput test
  57     """
  58     # pylint: disable=too-many-statements
  59     def __init__(self, test_cfg):
  60         """Pull out fields from test config
  61
  62         :param test_cfg: A dictionary of string-value pairs describing the test
  63             configuration. Both the key and values strings use well-known
  64             values.
  65         :param results_dir: Where the csv formatted results are written.
  66         """
  67         # make a local copy of test configuration to avoid modification of
  68         # original content used in vsperf main script
  69         cfg = copy.deepcopy(test_cfg)
  70
  71         self._testcase_start_time = time.time()
  72         self._testcase_stop_time = self._testcase_start_time
  73         self._hugepages_mounted = False
  74         self._traffic_ctl = None
  75         self._vnf_ctl = None
  76         self._pod_ctl = None
  77         self._pod_list = None
  78         self._vswitch_ctl = None
  79         self._collector = None
  80         self._loadgen = None
  81         self._output_file = None
  82         self._tc_results = None
  83         self._settings_paths_modified = False
  84         self._testcast_run_time = None
  85         self._versions = []
  86         self._k8s = False
  87         # initialization of step driven specific members
  88         self._step_check = False    # by default don't check result for step driven testcases
  89         self._step_vnf_list = {}
  90         self._step_result = []
  91         self._step_result_mapping = {}
  92         self._step_status = None
  93         self._step_send_traffic = False # indication if send_traffic was called within test steps
  94         self._vnf_list = []
  95         self._testcase_run_time = None
  96
  97         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  98         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  99         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
 100         S.setValue('TUNNEL_TYPE', cfg.get('Tunnel Type', S.getValue('TUNNEL_TYPE')))
 101         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
 102         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
 103         test_params = merge_spec(test_params, tc_test_params)
 104
 105         # ensure that parameters from TC definition have the highest priority, see MAPPING_TC_CFG2CONF
 106         for (cfg_param, param) in MAPPING_TC_CFG2CONF.items():
 107             if cfg_param in cfg and param in test_params:
 108                 del test_params[param]
 109
 110         S.setValue('TEST_PARAMS', test_params)
 111         S.check_test_params()
 112
 113         # override all redefined GUEST_ values to have them expanded correctly
 114         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
 115         for key in tmp_test_params:
 116             if key.startswith('GUEST_'):
 117                 S.setValue(key, S.getValue(key))
 118                 S.getValue('TEST_PARAMS').pop(key)
 119
 120         # update global settings
 121         functions.settings_update_paths()
 122
 123         # set test parameters; CLI options take precedence to testcase settings
 124         self._logger = logging.getLogger(__name__)
 125         self.name = cfg['Name']
 126         self.desc = cfg.get('Description', 'No description given.')
 127         self.test = cfg.get('TestSteps', None)
 128
 129         # log testcase name and details
 130         tmp_desc = functions.format_description(self.desc, 50)
 131         self._logger.info('############################################################')
 132         self._logger.info('# Test:    %s', self.name)
 133         self._logger.info('# Details: %s', tmp_desc[0])
 134         for i in range(1, len(tmp_desc)):
 135             self._logger.info('#          %s', tmp_desc[i])
 136         self._logger.info('############################################################')
 137
 138         bidirectional = S.getValue('TRAFFIC')['bidir']
 139         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 140             raise TypeError(
 141                 'Bi-dir value must be of type string')
 142         bidirectional = bidirectional.title()  # Keep things consistent
 143
 144         self.deployment = cfg['Deployment']
 145         self._frame_mod = cfg.get('Frame Modification', None)
 146
 147         self._tunnel_operation = cfg.get('Tunnel Operation', None)
 148
 149         # check if test requires background load and which generator it uses
 150         self._load_cfg = cfg.get('Load', None)
 151
 152         if self._frame_mod:
 153             self._frame_mod = self._frame_mod.lower()
 154         self._results_dir = S.getValue('RESULTS_PATH')
 155
 156         # set traffic details, so they can be passed to vswitch and traffic ctls
 157         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 158         self._traffic.update({'bidir': bidirectional})
 159
 160         # Packet Forwarding mode
 161         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 162
 163         # trafficgen configuration required for tests of tunneling protocols
 164         if self._tunnel_operation:
 165             self._traffic.update({'tunnel_type': S.getValue('TUNNEL_TYPE')})
 166             self._traffic['l2'].update({'srcmac':
 167                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 168                                         'dstmac':
 169                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 170
 171             self._traffic['l3'].update({'srcip':
 172                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 173                                         'dstip':
 174                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 175
 176             if self._tunnel_operation == "decapsulation":
 177                 self._traffic['l2'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L2'))
 178                 self._traffic['l3'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L3'))
 179                 self._traffic['l4'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L4'))
 180                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 181         elif len(S.getValue('NICS')) >= 2 and \
 182              (S.getValue('NICS')[0]['type'] == 'vf' or
 183               S.getValue('NICS')[1]['type'] == 'vf'):
 184             mac1 = S.getValue('NICS')[0]['mac']
 185             mac2 = S.getValue('NICS')[1]['mac']
 186             if mac1 and mac2:
 187                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 188             else:
 189                 self._logger.debug("MAC addresses can not be read")
 190
 191         self._traffic = functions.check_traffic(self._traffic)
 192
 193         # count how many VNFs are involved in TestSteps
 194         if self.test:
 195             for step in self.test:
 196                 if step[0].startswith('vnf'):
 197                     self._step_vnf_list[step[0]] = None
 198
 199         # if llc allocation is required, initialize it.
 200         if S.getValue('LLC_ALLOCATION'):
 201             self._rmd = rmd.CacheAllocator()
 202
 203     def run_initialize(self):
 204         """ Prepare test execution environment
 205         """
 206         # mount hugepages if needed
 207         self._mount_hugepages()
 208
 209         self._logger.debug("Controllers:")
 210         loader = Loader()
 211         self._traffic_ctl = component_factory.create_traffic(
 212             self._traffic['traffic_type'],
 213             loader.get_trafficgen_class())
 214
 215         self._vnf_ctl = component_factory.create_vnf(
 216             self.deployment,
 217             loader.get_vnf_class(),
 218             len(self._step_vnf_list))
 219
 220         self._vnf_list = self._vnf_ctl.get_vnfs()
 221
 222         self._pod_ctl = component_factory.create_pod(
 223             self.deployment,
 224             loader.get_pod_class())
 225
 226         self._pod_list = self._pod_ctl.get_pods()
 227
 228         # verify enough hugepages are free to run the testcase
 229         if not self._check_for_enough_hugepages():
 230             raise RuntimeError('Not enough hugepages free to run test.')
 231
 232         # perform guest related handling
 233         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 234         if tmp_vm_count:
 235             # copy sources of l2 forwarding tools into VM shared dir if needed
 236             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 237
 238             # in case of multi VM in parallel, set the number of streams to the number of VMs
 239             if self.deployment.startswith('pvpv'):
 240                 # for each VM NIC pair we need an unique stream
 241                 streams = 0
 242                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 243                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 244                 self._logger.debug("VMs with parallel connection were detected. "
 245                                    "Thus Number of streams was set to %s", streams)
 246                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 247                 # user can define a proper stream count manually
 248                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 249                     self._traffic.update({'multistream': streams})
 250
 251             # OVS Vanilla requires guest VM MAC address and IPs to work
 252             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 253                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 254                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 255                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 256                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 257
 258         if self._vswitch_none:
 259             self._vswitch_ctl = component_factory.create_pktfwd(
 260                 self.deployment,
 261                 loader.get_pktfwd_class())
 262         else:
 263             self._vswitch_ctl = component_factory.create_vswitch(
 264                 self.deployment,
 265                 loader.get_vswitch_class(),
 266                 self._traffic,
 267                 self._tunnel_operation)
 268
 269         self._collector = component_factory.create_collector(
 270             loader.get_collector_class(),
 271             self._results_dir, self.name)
 272         self._loadgen = component_factory.create_loadgen(
 273             loader.get_loadgen_class(),
 274             self._load_cfg)
 275
 276         self._output_file = os.path.join(self._results_dir, "result_{}_{}_{}.csv".format(
 277             str(S.getValue('_TEST_INDEX')), self.name, self.deployment))
 278
 279         self._step_status = {'status' : True, 'details' : ''}
 280
 281         # Perform LLC-allocations
 282         if S.getValue('LLC_ALLOCATION'):
 283             self._rmd.setup_llc_allocation()
 284
 285         self._logger.debug("Setup:")
 286
 287     def run_finalize(self):
 288         """ Tear down test execution environment and record test results
 289         """
 290         # Stop all VNFs started by TestSteps in case that something went wrong
 291         self.step_stop_vnfs()
 292
 293         if self._k8s:
 294             self._pod_ctl.stop()
 295
 296
 297         # Cleanup any LLC-allocations
 298         if S.getValue('LLC_ALLOCATION'):
 299             self._rmd.cleanup_llc_allocation()
 300
 301         # Stop all processes executed by testcase
 302         tasks.terminate_all_tasks(self._logger)
 303
 304         # umount hugepages if mounted
 305         self._umount_hugepages()
 306
 307         # cleanup any namespaces created
 308         if os.path.isdir('/tmp/namespaces'):
 309             namespace_list = os.listdir('/tmp/namespaces')
 310             if namespace_list:
 311                 self._logger.info('Cleaning up namespaces')
 312             for name in namespace_list:
 313                 namespace.delete_namespace(name)
 314             os.rmdir('/tmp/namespaces')
 315         # cleanup any veth ports created
 316         if os.path.isdir('/tmp/veth'):
 317             veth_list = os.listdir('/tmp/veth')
 318             if veth_list:
 319                 self._logger.info('Cleaning up veth ports')
 320             for eth in veth_list:
 321                 port1, port2 = eth.split('-')
 322                 veth.del_veth_port(port1, port2)
 323             os.rmdir('/tmp/veth')
 324
 325     def run_report(self):
 326         """ Report test results
 327         """
 328         self._logger.debug("self._collector Results:")
 329         self._collector.print_results()
 330
 331         results = self._traffic_ctl.get_results()
 332         if results:
 333             self._logger.debug("Traffic Results:")
 334             self._traffic_ctl.print_results()
 335
 336         if self._tc_results is None:
 337             self._tc_results = self._append_results(results)
 338         else:
 339             # integration step driven tests have their status and possible
 340             # failure details stored inside self._tc_results
 341             results = self._append_results(results)
 342             if len(self._tc_results) < len(results):
 343                 if len(self._tc_results) > 1:
 344                     raise RuntimeError('Testcase results do not match:'
 345                                        'results: %s\n'
 346                                        'trafficgen results: %s\n' %
 347                                        self._tc_results,
 348                                        results)
 349                 else:
 350                     tmp_results = copy.deepcopy(self._tc_results[0])
 351                     self._tc_results = []
 352                     for res in results:
 353                         tmp_res = copy.deepcopy(tmp_results)
 354                         tmp_res.update(res)
 355                         self._tc_results.append(tmp_res)
 356             else:
 357                 for i, result in enumerate(results):
 358                     self._tc_results[i].update(result)
 359
 360         TestCase.write_result_to_file(self._tc_results, self._output_file)
 361
 362     def run(self):
 363         """Run the test
 364
 365         All setup and teardown through controllers is included.
 366
 367         """
 368         # prepare test execution environment
 369         self.run_initialize()
 370
 371         try:
 372             with self._vswitch_ctl:
 373                 with self._vnf_ctl, self._pod_ctl, self._collector, self._loadgen:
 374                     if not self._vswitch_none and not self._k8s:
 375                         self._add_flows()
 376                     if self._k8s:
 377                         self._add_connections()
 378
 379                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 380
 381                     with self._traffic_ctl:
 382                         # execute test based on TestSteps definition if needed...
 383                         if self.step_run():
 384                             # ...and continue with traffic generation, but keep
 385                             # in mind, that clean deployment does not configure
 386                             # OVS nor executes the traffic
 387                             if self.deployment != 'clean' and not self._step_send_traffic:
 388                                 self._traffic_ctl.send_traffic(self._traffic)
 389
 390                         # dump vswitch flows before they are affected by VNF termination
 391                         if not self._vswitch_none:
 392                             self._vswitch_ctl.dump_vswitch_connections()
 393
 394                     # garbage collection for case that TestSteps modify existing deployment
 395                     self.step_stop_vnfs()
 396
 397         finally:
 398             # tear down test execution environment and log results
 399             self.run_finalize()
 400
 401         self._testcase_stop_time = time.time()
 402         self._testcase_run_time = time.strftime("%H:%M:%S",
 403                                                 time.gmtime(self._testcase_stop_time -
 404                                                             self._testcase_start_time))
 405         logging.info("Testcase execution time: %s", self._testcase_run_time)
 406         # report test results
 407         self.run_report()
 408
 409     def _append_results(self, results):
 410         """
 411         Method appends mandatory Test Case results to list of dictionaries.
 412
 413         :param results: list of dictionaries which contains results from
 414                 traffic generator.
 415
 416         :returns: modified list of dictionaries.
 417         """
 418         for item in results:
 419             item[ResultsConstants.ID] = self.name
 420             item[ResultsConstants.DEPLOYMENT] = self.deployment
 421             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 422             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 423             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 424             # convert timestamps to human readable format
 425             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 426                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 427             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 428                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 429             if self._traffic['multistream']:
 430                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 431                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 432                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 433             if self._vnf_ctl.get_vnfs_number():
 434                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 435             if self._tunnel_operation:
 436                 item[ResultsConstants.TUNNEL_TYPE] = S.getValue('TUNNEL_TYPE')
 437         return results
 438
 439     def _copy_fwd_tools_for_all_guests(self, vm_count):
 440         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 441         """
 442         # consider only VNFs involved in the test
 443         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 444             self._copy_fwd_tools_for_guest(guest_dir)
 445
 446     def _copy_fwd_tools_for_guest(self, guest_dir):
 447         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 448
 449         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 450         """
 451         # remove shared dir if it exists to avoid issues with file consistency
 452         if os.path.exists(guest_dir):
 453             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 454                            'Removing content of shared directory...', True)
 455
 456         # directory to share files between host and guest
 457         os.makedirs(guest_dir)
 458
 459         # copy sources into shared dir only if neccessary
 460         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 461         if 'testpmd' in guest_loopback:
 462             try:
 463                 # exclude whole .git/ subdirectory and all o-files;
 464                 # It is assumed, that the same RTE_TARGET is used in both host
 465                 # and VMs; This simplification significantly speeds up testpmd
 466                 # build. If we will need a different RTE_TARGET in VM,
 467                 # then we have to build whole DPDK from the scratch in VM.
 468                 # In that case we can copy just DPDK sources (e.g. by excluding
 469                 # all items obtained by git status -unormal --porcelain).
 470                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 471                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 472                 exclude = []
 473                 exclude.append(r'--exclude=.git/')
 474                 exclude.append(r'--exclude=*.o')
 475                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 476                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 477                                 os.path.join(guest_dir, 'DPDK')],
 478                                self._logger,
 479                                'Copying DPDK to shared directory...',
 480                                True)
 481             except subprocess.CalledProcessError:
 482                 self._logger.error('Unable to copy DPDK to shared directory')
 483                 raise
 484         if 'l2fwd' in guest_loopback:
 485             try:
 486                 tasks.run_task(['rsync', '-a', '-r', '-l',
 487                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 488                                 os.path.join(guest_dir, 'l2fwd')],
 489                                self._logger,
 490                                'Copying l2fwd to shared directory...',
 491                                True)
 492             except subprocess.CalledProcessError:
 493                 self._logger.error('Unable to copy l2fwd to shared directory')
 494                 raise
 495
 496     def _mount_hugepages(self):
 497         """Mount hugepages if usage of DPDK or Qemu is detected
 498         """
 499         # pylint: disable=too-many-boolean-expressions
 500         # hugepages are needed by DPDK and Qemu
 501         if not self._hugepages_mounted and \
 502             (self.deployment.count('v') or \
 503              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 504              self._vswitch_none or \
 505              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 506             hugepages.mount_hugepages()
 507             self._hugepages_mounted = True
 508
 509     def _umount_hugepages(self):
 510         """Umount hugepages if they were mounted before
 511         """
 512         if self._hugepages_mounted:
 513             hugepages.umount_hugepages()
 514             self._hugepages_mounted = False
 515
 516     def _check_for_enough_hugepages(self):
 517         """Check to make sure enough hugepages are free to satisfy the
 518         test environment.
 519         """
 520         hugepages_needed = 0
 521         hugepage_size = hugepages.get_hugepage_size()
 522         # get hugepage amounts per guest involved in the test
 523         for guest in range(self._vnf_ctl.get_vnfs_number()):
 524             hugepages_needed += math.ceil((int(S.getValue(
 525                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 526
 527         # get hugepage amounts for each socket on dpdk
 528         sock0_mem, sock1_mem = 0, 0
 529
 530         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 531             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 532             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 533                                     int(sock_mem[1]) * 1024 / hugepage_size)
 534
 535         # If hugepages needed, verify the amounts are free
 536         if any([hugepages_needed, sock0_mem, sock1_mem]):
 537             free_hugepages = hugepages.get_free_hugepages()
 538             if hugepages_needed:
 539                 logging.info('Need %s hugepages free for guests',
 540                              hugepages_needed)
 541                 result1 = free_hugepages >= hugepages_needed
 542                 free_hugepages -= hugepages_needed
 543             else:
 544                 result1 = True
 545
 546             if sock0_mem:
 547                 logging.info('Need %s hugepages free for dpdk socket 0',
 548                              sock0_mem)
 549                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 550                 free_hugepages -= sock0_mem
 551             else:
 552                 result2 = True
 553
 554             if sock1_mem:
 555                 logging.info('Need %s hugepages free for dpdk socket 1',
 556                              sock1_mem)
 557                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 558                 free_hugepages -= sock1_mem
 559             else:
 560                 result3 = True
 561
 562             logging.info('Need a total of %s total hugepages',
 563                          hugepages_needed + sock1_mem + sock0_mem)
 564
 565             # The only drawback here is sometimes dpdk doesn't release
 566             # its hugepages on a test failure. This could cause a test
 567             # to fail when dpdk would be OK to start because it will just
 568             # use the previously allocated hugepages.
 569             result4 = True if free_hugepages >= 0 else False
 570
 571             return all([result1, result2, result3, result4])
 572         else:
 573             return True
 574
 575     @staticmethod
 576     def write_result_to_file(results, output):
 577         """Write list of dictionaries to a CSV file.
 578
 579         Each element on list will create separate row in output file.
 580         If output file already exists, data will be appended at the end,
 581         otherwise it will be created.
 582
 583         :param results: list of dictionaries.
 584         :param output: path to output file.
 585         """
 586         with open(output, 'a') as csvfile:
 587
 588             logging.info("Write results to file: %s", output)
 589             fieldnames = TestCase._get_unique_keys(results)
 590
 591             writer = csv.DictWriter(csvfile, fieldnames)
 592
 593             if not csvfile.tell():  # file is now empty
 594                 writer.writeheader()
 595
 596             for result in results:
 597                 writer.writerow(result)
 598
 599     @staticmethod
 600     def _get_unique_keys(list_of_dicts):
 601         """Gets unique key values as ordered list of strings in given dicts
 602
 603         :param list_of_dicts: list of dictionaries.
 604
 605         :returns: list of unique keys(strings).
 606         """
 607         result = OrderedDict()
 608         for item in list_of_dicts:
 609             for key in item.keys():
 610                 result[key] = ''
 611
 612         return list(result.keys())
 613
 614     def _add_connections(self):
 615         """
 616         Add connections for Kubernetes Usecases
 617         """
 618         logging.info("Kubernetes: Adding Connections")
 619         vswitch = self._vswitch_ctl.get_vswitch()
 620         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 621         if S.getValue('K8S') and 'sriov' not in S.getValue('PLUGIN'):
 622             if 'Ovs' in S.getValue('VSWITCH'):
 623                 # Add OVS Flows
 624                 logging.info("Kubernetes: Adding OVS Connections")
 625                 flow = {'table':'0', 'in_port':'1',
 626                         'idle_timeout':'0', 'actions': ['output:3']}
 627                 vswitch.add_flow(bridge, flow)
 628                 flow = {'table':'0', 'in_port':'3',
 629                         'idle_timeout':'0', 'actions': ['output:1']}
 630                 vswitch.add_flow(bridge, flow)
 631                 flow = {'table':'0', 'in_port':'2',
 632                         'idle_timeout':'0', 'actions': ['output:4']}
 633                 vswitch.add_flow(bridge, flow)
 634                 flow = {'table':'0', 'in_port':'4',
 635                         'idle_timeout':'0', 'actions': ['output:2']}
 636                 vswitch.add_flow(bridge, flow)
 637             elif 'vpp' in S.getValue('VSWITCH'):
 638                 phy_ports = vswitch.get_ports()
 639                 virt_port0 = 'memif1/0'
 640                 virt_port1 = 'memif2/0'
 641                 vswitch.add_connection(bridge, phy_ports[0],
 642                                        virt_port0, None)
 643                 vswitch.add_connection(bridge, virt_port0,
 644                                        phy_ports[0], None)
 645                 vswitch.add_connection(bridge, phy_ports[1],
 646                                        virt_port1, None)
 647                 vswitch.add_connection(bridge, virt_port1,
 648                                        phy_ports[1], None)
 649
 650
 651     def _add_flows(self):
 652         """Add flows to the vswitch
 653         """
 654         vswitch = self._vswitch_ctl.get_vswitch()
 655         # NOTE BOM 15-08-07 the frame mod code assumes that the
 656         # physical ports are ports 1 & 2. The actual numbers
 657         # need to be retrived from the vSwitch and the metadata value
 658         # updated accordingly.
 659         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 660         if self._frame_mod == "vlan":
 661             # 0x8100 => VLAN ethertype
 662             self._logger.debug(" ****   VLAN   ***** ")
 663             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 664                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 665             vswitch.add_flow(bridge, flow)
 666             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 667                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 668             vswitch.add_flow(bridge, flow)
 669         elif self._frame_mod == "mpls":
 670             # 0x8847 => MPLS unicast ethertype
 671             self._logger.debug(" ****   MPLS  ***** ")
 672             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 673                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 674             vswitch.add_flow(bridge, flow)
 675             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 676                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 677             vswitch.add_flow(bridge, flow)
 678         elif self._frame_mod == "mac":
 679             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 680                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 681                                 'goto_table:3']}
 682             vswitch.add_flow(bridge, flow)
 683             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 684                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 685                                 'goto_table:3']}
 686             vswitch.add_flow(bridge, flow)
 687         elif self._frame_mod == "dscp":
 688             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 689             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 690                     'dl_type':'0x0800',
 691                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 692             vswitch.add_flow(bridge, flow)
 693             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 694                     'dl_type':'0x0800',
 695                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 696             vswitch.add_flow(bridge, flow)
 697         elif self._frame_mod == "ttl":
 698             # 251 and 241 are the highest prime numbers < 255
 699             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 700                     'dl_type':'0x0800',
 701                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 702             vswitch.add_flow(bridge, flow)
 703             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 704                     'dl_type':'0x0800',
 705                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 706             vswitch.add_flow(bridge, flow)
 707         elif self._frame_mod == "ip_addr":
 708             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 709                     'dl_type':'0x0800',
 710                     'actions': ['mod_nw_src:10.10.10.10',
 711                                 'mod_nw_dst:20.20.20.20',
 712                                 'goto_table:3']}
 713             vswitch.add_flow(bridge, flow)
 714             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 715                     'dl_type':'0x0800',
 716                     'actions': ['mod_nw_src:20.20.20.20',
 717                                 'mod_nw_dst:10.10.10.10',
 718                                 'goto_table:3']}
 719             vswitch.add_flow(bridge, flow)
 720         elif self._frame_mod == "ip_port":
 721             # NOTE BOM 15-08-27 The traffic generated is assumed
 722             # to be UDP (nw_proto 17d) which is the default case but
 723             # we will need to pick up the actual traffic params in use.
 724             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 725                     'dl_type':'0x0800', 'nw_proto':'17',
 726                     'actions': ['mod_tp_src:44444',
 727                                 'mod_tp_dst:44444', 'goto_table:3']}
 728             vswitch.add_flow(bridge, flow)
 729             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 730                     'dl_type':'0x0800', 'nw_proto':'17',
 731                     'actions': ['mod_tp_src:44444',
 732                                 'mod_tp_dst:44444', 'goto_table:3']}
 733             vswitch.add_flow(bridge, flow)
 734         else:
 735             pass
 736
 737
 738     #
 739     # TestSteps realted methods
 740     #
 741     def step_report_status(self, label, status):
 742         """ Log status of test step
 743         """
 744         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 745
 746     def step_stop_vnfs(self):
 747         """ Stop all VNFs started by TestSteps
 748         """
 749         for vnf in self._step_vnf_list:
 750             if self._step_vnf_list[vnf]:
 751                 self._step_vnf_list[vnf].stop()
 752
 753     def step_eval_param(self, param, step_result):
 754         """ Helper function for #STEP macro evaluation
 755         """
 756         if isinstance(param, str):
 757             # evaluate every #STEP reference inside parameter itself
 758             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 759
 760             if macros:
 761                 for macro in macros:
 762                     if macro[1] in self._step_result_mapping:
 763                         key = self._step_result_mapping[macro[1]]
 764                     else:
 765                         key = macro[1]
 766                     # pylint: disable=eval-used
 767                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 768                     param = param.replace(macro[0], tmp_val)
 769
 770             # evaluate references to vsperf configuration options
 771             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 772             if macros:
 773                 for macro in macros:
 774                     # pylint: disable=eval-used
 775                     try:
 776                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 777                         param = param.replace('${}'.format(macro[0]), tmp_val)
 778                     # ignore that required option can't be evaluated
 779                     except (IndexError, KeyError, AttributeError):
 780                         self._logger.debug("Skipping %s as it isn't a configuration "
 781                                            "parameter.", '${}'.format(macro[0]))
 782             return param
 783         elif isinstance(param, (list, tuple)):
 784             tmp_list = []
 785             for item in param:
 786                 tmp_list.append(self.step_eval_param(item, step_result))
 787             return tmp_list
 788         elif isinstance(param, dict):
 789             tmp_dict = {}
 790             for (key, value) in param.items():
 791                 tmp_dict[key] = self.step_eval_param(value, step_result)
 792             return tmp_dict
 793         else:
 794             return param
 795
 796     def step_eval_params(self, params, step_result):
 797         """ Evaluates referrences to results from previous steps
 798         """
 799         eval_params = []
 800         # evaluate all parameters if needed
 801         for param in params:
 802             eval_params.append(self.step_eval_param(param, step_result))
 803         return eval_params
 804
 805     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 806     def step_run(self):
 807         """ Execute actions specified by TestSteps list
 808
 809         :return: False if any error was detected
 810                  True otherwise
 811         """
 812         # anything to do?
 813         if not self.test:
 814             return True
 815
 816         # required for VNFs initialization
 817         loader = Loader()
 818         # initialize list with results
 819         self._step_result = [None] * len(self.test)
 820
 821         # run test step by step...
 822         for i, step in enumerate(self.test):
 823             step_ok = not self._step_check
 824             step_check = self._step_check
 825             regex = None
 826             # configure step result mapping if step alias/label is detected
 827             if step[0].startswith('#'):
 828                 key = step[0][1:]
 829                 if key.isdigit():
 830                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 831                 if key in self._step_result_mapping:
 832                     raise RuntimeError('Step alias {} has been used already for step '
 833                                        '{}'.format(key, self._step_result_mapping[key]))
 834                 self._step_result_mapping[step[0][1:]] = i
 835                 step = step[1:]
 836
 837             # store regex filter if it is specified
 838             if isinstance(step[-1], str) and step[-1].startswith('|'):
 839                 # evalute macros and variables used in regex
 840                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 841                 step = step[:-1]
 842
 843             # check if step verification should be suppressed
 844             if step[0].startswith('!'):
 845                 step_check = False
 846                 step_ok = True
 847                 step[0] = step[0][1:]
 848             if step[0] == 'vswitch':
 849                 test_object = self._vswitch_ctl.get_vswitch()
 850             elif step[0] == 'namespace':
 851                 test_object = namespace
 852             elif step[0] == 'veth':
 853                 test_object = veth
 854             elif step[0] == 'settings':
 855                 test_object = S
 856             elif step[0] == 'tools':
 857                 test_object = TestStepsTools()
 858                 step[1] = step[1].title()
 859             elif step[0] == 'trafficgen':
 860                 test_object = self._traffic_ctl
 861                 # in case of send_traffic or send_traffic_async methods, ensure
 862                 # that specified traffic values are merged with existing self._traffic
 863                 if step[1].startswith('send_traffic'):
 864                     tmp_traffic = copy.deepcopy(self._traffic)
 865                     tmp_traffic.update(step[2])
 866                     step[2] = tmp_traffic
 867                     # store indication that traffic has been sent
 868                     # so it is not sent again after the execution of teststeps
 869                     self._step_send_traffic = True
 870             elif step[0].startswith('vnf'):
 871                 # use vnf started within TestSteps
 872                 if not self._step_vnf_list[step[0]]:
 873                     # initialize new VM
 874                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 875                 test_object = self._step_vnf_list[step[0]]
 876             elif step[0].startswith('VNF'):
 877                 if step[1] in ('start', 'stop'):
 878                     raise RuntimeError("Cannot execute start() or stop() method of "
 879                                        "VNF deployed automatically by scenario.")
 880                 # use vnf started by scenario deployment (e.g. pvp)
 881                 vnf_index = int(step[0][3:])
 882                 try:
 883                     test_object = self._vnf_list[vnf_index]
 884                 except IndexError:
 885                     raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
 886             elif step[0] == 'wait':
 887                 input(os.linesep + "Step {}: Press Enter to continue with "
 888                       "the next step...".format(i) + os.linesep + os.linesep)
 889                 continue
 890             elif step[0] == 'sleep':
 891                 self._logger.debug("Sleep %s seconds", step[1])
 892                 time.sleep(int(step[1]))
 893                 continue
 894             elif step[0] == 'log':
 895                 test_object = self._logger
 896                 # there isn't a need for validation of log entry
 897                 step_check = False
 898                 step_ok = True
 899             elif step[0] == 'pdb':
 900                 import pdb
 901                 pdb.set_trace()
 902                 continue
 903             else:
 904                 self._logger.error("Unsupported test object %s", step[0])
 905                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 906                 self.step_report_status("Step '{}'".format(' '.join(step)),
 907                                         self._step_status['status'])
 908                 return False
 909
 910             test_method = getattr(test_object, step[1])
 911             if step_check:
 912                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 913             else:
 914                 test_method_check = None
 915
 916             step_params = []
 917             try:
 918                 # eval parameters, but use only valid step_results
 919                 # to support negative indexes
 920                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 921                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 922                 step_log += ' filter "{}"'.format(regex) if regex else ''
 923                 self._logger.debug("Step %s '%s' start", i, step_log)
 924                 self._step_result[i] = test_method(*step_params)
 925                 if regex:
 926                     # apply regex to step output
 927                     self._step_result[i] = functions.filter_output(
 928                         self._step_result[i], regex)
 929
 930                 self._logger.debug("Step %s '%s' results '%s'", i,
 931                                    step_log, self._step_result[i])
 932                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 933                 if step_check:
 934                     step_ok = test_method_check(self._step_result[i], *step_params)
 935             except (AssertionError, AttributeError, IndexError) as ex:
 936                 step_ok = False
 937                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 938
 939             if step_check:
 940                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 941
 942             if not step_ok:
 943                 self._step_status = {'status' : False, 'details' : step_log}
 944                 # Stop all VNFs started by TestSteps
 945                 self.step_stop_vnfs()
 946                 return False
 947
 948         # all steps processed without any issue
 949         return True
 950
 951     #
 952     # get methods for TestCase members, which needs to be publicly available
 953     #
 954     def get_output_file(self):
 955         """Return content of self._output_file member
 956         """
 957         return self._output_file
 958
 959     def get_desc(self):
 960         """Return content of self.desc member
 961         """
 962         return self.desc
 963
 964     def get_versions(self):
 965         """Return content of self.versions member
 966         """
 967         return self._versions
 968
 969     def get_traffic(self):
 970         """Return content of self._traffic member
 971         """
 972         return self._traffic
 973
 974     def get_tc_results(self):
 975         """Return content of self._tc_results member
 976         """
 977         return self._tc_results
 978
 979     def get_collector(self):
 980         """Return content of self._collector member
 981         """
 982         return self._collector