testcases/testcase.py

   1 # Copyright 2015-2018 Intel Corporation, Tieto and others.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 from collections import OrderedDict
  18 import copy
  19 import csv
  20 import logging
  21 import math
  22 import os
  23 import re
  24 import time
  25 import subprocess
  26
  27 from datetime import datetime as dt
  28 from conf import settings as S
  29 from conf import merge_spec
  30 import core.component_factory as component_factory
  31 from core.loader import Loader
  32 from core.results.results_constants import ResultsConstants
  33 from tools import tasks
  34 from tools import hugepages
  35 from tools import functions
  36 from tools import namespace
  37 from tools import veth
  38 from tools.teststepstools import TestStepsTools
  39 from tools.llc_management import rmd
  40
  41 # Validation methods required for integration TCs will have following prefix before the name
  42 # of original method.
  43 CHECK_PREFIX = 'validate_'
  44
  45 # Several parameters can be defined by both TC definition keywords and configuration parameters.
  46 # Following mapping table is used to correctly evaluate priority of testcase configuration, where
  47 # TC definition keywords (i.e. mapping table keys) have higher priority than appropriate TC
  48 # parameters (i.e. mapping table values). TC parameters can be defined within "Parameters"
  49 # section, via CLI parameters or within configuration files.
  50 MAPPING_TC_CFG2CONF = {'vSwitch':'VSWITCH', 'VNF':'VNF', 'Trafficgen':'TRAFFICGEN', 'Tunnel Type':'TUNNEL_TYPE'}
  51
  52 # pylint: disable=too-many-instance-attributes
  53 class TestCase(object):
  54     """TestCase base class
  55
  56     In this basic form runs RFC2544 throughput test
  57     """
  58     # pylint: disable=too-many-statements
  59     def __init__(self, test_cfg):
  60         """Pull out fields from test config
  61
  62         :param test_cfg: A dictionary of string-value pairs describing the test
  63             configuration. Both the key and values strings use well-known
  64             values.
  65         :param results_dir: Where the csv formatted results are written.
  66         """
  67         # make a local copy of test configuration to avoid modification of
  68         # original content used in vsperf main script
  69         cfg = copy.deepcopy(test_cfg)
  70
  71         self._testcase_start_time = time.time()
  72         self._testcase_stop_time = self._testcase_start_time
  73         self._hugepages_mounted = False
  74         self._traffic_ctl = None
  75         self._vnf_ctl = None
  76         self._vswitch_ctl = None
  77         self._collector = None
  78         self._loadgen = None
  79         self._output_file = None
  80         self._tc_results = None
  81         self._settings_paths_modified = False
  82         self._testcast_run_time = None
  83         self._versions = []
  84         # initialization of step driven specific members
  85         self._step_check = False    # by default don't check result for step driven testcases
  86         self._step_vnf_list = {}
  87         self._step_result = []
  88         self._step_result_mapping = {}
  89         self._step_status = None
  90         self._step_send_traffic = False # indication if send_traffic was called within test steps
  91         self._vnf_list = []
  92         self._testcase_run_time = None
  93
  94         S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  95         S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
  96         S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  97         S.setValue('TUNNEL_TYPE', cfg.get('Tunnel Type', S.getValue('TUNNEL_TYPE')))
  98         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
  99         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
 100         test_params = merge_spec(test_params, tc_test_params)
 101
 102         # ensure that parameters from TC definition have the highest priority, see MAPPING_TC_CFG2CONF
 103         for (cfg_param, param) in MAPPING_TC_CFG2CONF.items():
 104             if cfg_param in cfg and param in test_params:
 105                 del test_params[param]
 106
 107         S.setValue('TEST_PARAMS', test_params)
 108         S.check_test_params()
 109
 110         # override all redefined GUEST_ values to have them expanded correctly
 111         tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
 112         for key in tmp_test_params:
 113             if key.startswith('GUEST_'):
 114                 S.setValue(key, S.getValue(key))
 115                 S.getValue('TEST_PARAMS').pop(key)
 116
 117         # update global settings
 118         functions.settings_update_paths()
 119
 120         # set test parameters; CLI options take precedence to testcase settings
 121         self._logger = logging.getLogger(__name__)
 122         self.name = cfg['Name']
 123         self.desc = cfg.get('Description', 'No description given.')
 124         self.test = cfg.get('TestSteps', None)
 125
 126         # log testcase name and details
 127         tmp_desc = functions.format_description(self.desc, 50)
 128         self._logger.info('############################################################')
 129         self._logger.info('# Test:    %s', self.name)
 130         self._logger.info('# Details: %s', tmp_desc[0])
 131         for i in range(1, len(tmp_desc)):
 132             self._logger.info('#          %s', tmp_desc[i])
 133         self._logger.info('############################################################')
 134
 135         bidirectional = S.getValue('TRAFFIC')['bidir']
 136         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
 137             raise TypeError(
 138                 'Bi-dir value must be of type string')
 139         bidirectional = bidirectional.title()  # Keep things consistent
 140
 141         self.deployment = cfg['Deployment']
 142         self._frame_mod = cfg.get('Frame Modification', None)
 143
 144         self._tunnel_operation = cfg.get('Tunnel Operation', None)
 145
 146         # check if test requires background load and which generator it uses
 147         self._load_cfg = cfg.get('Load', None)
 148
 149         if self._frame_mod:
 150             self._frame_mod = self._frame_mod.lower()
 151         self._results_dir = S.getValue('RESULTS_PATH')
 152
 153         # set traffic details, so they can be passed to vswitch and traffic ctls
 154         self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
 155         self._traffic.update({'bidir': bidirectional})
 156
 157         # Packet Forwarding mode
 158         self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 159
 160         # trafficgen configuration required for tests of tunneling protocols
 161         if self._tunnel_operation:
 162             self._traffic.update({'tunnel_type': S.getValue('TUNNEL_TYPE')})
 163             self._traffic['l2'].update({'srcmac':
 164                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 165                                         'dstmac':
 166                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 167
 168             self._traffic['l3'].update({'srcip':
 169                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 170                                         'dstip':
 171                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 172
 173             if self._tunnel_operation == "decapsulation":
 174                 self._traffic['l2'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L2'))
 175                 self._traffic['l3'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L3'))
 176                 self._traffic['l4'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L4'))
 177                 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
 178         elif len(S.getValue('NICS')) >= 2 and \
 179              (S.getValue('NICS')[0]['type'] == 'vf' or
 180               S.getValue('NICS')[1]['type'] == 'vf'):
 181             mac1 = S.getValue('NICS')[0]['mac']
 182             mac2 = S.getValue('NICS')[1]['mac']
 183             if mac1 and mac2:
 184                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 185             else:
 186                 self._logger.debug("MAC addresses can not be read")
 187
 188         self._traffic = functions.check_traffic(self._traffic)
 189
 190         # count how many VNFs are involved in TestSteps
 191         if self.test:
 192             for step in self.test:
 193                 if step[0].startswith('vnf'):
 194                     self._step_vnf_list[step[0]] = None
 195
 196         # if llc allocation is required, initialize it.
 197         if S.getValue('LLC_ALLOCATION'):
 198             self._rmd = rmd.CacheAllocator()
 199
 200     def run_initialize(self):
 201         """ Prepare test execution environment
 202         """
 203         # mount hugepages if needed
 204         self._mount_hugepages()
 205
 206         self._logger.debug("Controllers:")
 207         loader = Loader()
 208         self._traffic_ctl = component_factory.create_traffic(
 209             self._traffic['traffic_type'],
 210             loader.get_trafficgen_class())
 211
 212         self._vnf_ctl = component_factory.create_vnf(
 213             self.deployment,
 214             loader.get_vnf_class(),
 215             len(self._step_vnf_list))
 216
 217         self._vnf_list = self._vnf_ctl.get_vnfs()
 218
 219         # verify enough hugepages are free to run the testcase
 220         if not self._check_for_enough_hugepages():
 221             raise RuntimeError('Not enough hugepages free to run test.')
 222
 223         # perform guest related handling
 224         tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
 225         if tmp_vm_count:
 226             # copy sources of l2 forwarding tools into VM shared dir if needed
 227             self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 228
 229             # in case of multi VM in parallel, set the number of streams to the number of VMs
 230             if self.deployment.startswith('pvpv'):
 231                 # for each VM NIC pair we need an unique stream
 232                 streams = 0
 233                 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
 234                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
 235                 self._logger.debug("VMs with parallel connection were detected. "
 236                                    "Thus Number of streams was set to %s", streams)
 237                 # update streams if needed; In case of additional VNFs deployed by TestSteps
 238                 # user can define a proper stream count manually
 239                 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
 240                     self._traffic.update({'multistream': streams})
 241
 242             # OVS Vanilla requires guest VM MAC address and IPs to work
 243             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
 244                 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 245                                             'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 246                 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 247                                             'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 248
 249         if self._vswitch_none:
 250             self._vswitch_ctl = component_factory.create_pktfwd(
 251                 self.deployment,
 252                 loader.get_pktfwd_class())
 253         else:
 254             self._vswitch_ctl = component_factory.create_vswitch(
 255                 self.deployment,
 256                 loader.get_vswitch_class(),
 257                 self._traffic,
 258                 self._tunnel_operation)
 259
 260         self._collector = component_factory.create_collector(
 261             loader.get_collector_class(),
 262             self._results_dir, self.name)
 263         self._loadgen = component_factory.create_loadgen(
 264             loader.get_loadgen_class(),
 265             self._load_cfg)
 266
 267         self._output_file = os.path.join(self._results_dir, "result_{}_{}_{}.csv".format(
 268             str(S.getValue('_TEST_INDEX')), self.name, self.deployment))
 269
 270         self._step_status = {'status' : True, 'details' : ''}
 271
 272         # Perform LLC-allocations
 273         if S.getValue('LLC_ALLOCATION'):
 274             self._rmd.setup_llc_allocation()
 275
 276         self._logger.debug("Setup:")
 277
 278     def run_finalize(self):
 279         """ Tear down test execution environment and record test results
 280         """
 281         # Stop all VNFs started by TestSteps in case that something went wrong
 282         self.step_stop_vnfs()
 283
 284         # Cleanup any LLC-allocations
 285         if S.getValue('LLC_ALLOCATION'):
 286             self._rmd.cleanup_llc_allocation()
 287
 288         # Stop all processes executed by testcase
 289         tasks.terminate_all_tasks(self._logger)
 290
 291         # umount hugepages if mounted
 292         self._umount_hugepages()
 293
 294         # cleanup any namespaces created
 295         if os.path.isdir('/tmp/namespaces'):
 296             namespace_list = os.listdir('/tmp/namespaces')
 297             if namespace_list:
 298                 self._logger.info('Cleaning up namespaces')
 299             for name in namespace_list:
 300                 namespace.delete_namespace(name)
 301             os.rmdir('/tmp/namespaces')
 302         # cleanup any veth ports created
 303         if os.path.isdir('/tmp/veth'):
 304             veth_list = os.listdir('/tmp/veth')
 305             if veth_list:
 306                 self._logger.info('Cleaning up veth ports')
 307             for eth in veth_list:
 308                 port1, port2 = eth.split('-')
 309                 veth.del_veth_port(port1, port2)
 310             os.rmdir('/tmp/veth')
 311
 312     def run_report(self):
 313         """ Report test results
 314         """
 315         self._logger.debug("self._collector Results:")
 316         self._collector.print_results()
 317
 318         results = self._traffic_ctl.get_results()
 319         if results:
 320             self._logger.debug("Traffic Results:")
 321             self._traffic_ctl.print_results()
 322
 323         if self._tc_results is None:
 324             self._tc_results = self._append_results(results)
 325         else:
 326             # integration step driven tests have their status and possible
 327             # failure details stored inside self._tc_results
 328             results = self._append_results(results)
 329             if len(self._tc_results) < len(results):
 330                 if len(self._tc_results) > 1:
 331                     raise RuntimeError('Testcase results do not match:'
 332                                        'results: %s\n'
 333                                        'trafficgen results: %s\n' %
 334                                        self._tc_results,
 335                                        results)
 336                 else:
 337                     tmp_results = copy.deepcopy(self._tc_results[0])
 338                     self._tc_results = []
 339                     for res in results:
 340                         tmp_res = copy.deepcopy(tmp_results)
 341                         tmp_res.update(res)
 342                         self._tc_results.append(tmp_res)
 343             else:
 344                 for i, result in enumerate(results):
 345                     self._tc_results[i].update(result)
 346
 347         TestCase.write_result_to_file(self._tc_results, self._output_file)
 348
 349     def run(self):
 350         """Run the test
 351
 352         All setup and teardown through controllers is included.
 353         """
 354         # prepare test execution environment
 355         self.run_initialize()
 356
 357         try:
 358             with self._vswitch_ctl:
 359                 with self._vnf_ctl, self._collector, self._loadgen:
 360                     if not self._vswitch_none:
 361                         self._add_flows()
 362
 363                     self._versions += self._vswitch_ctl.get_vswitch().get_version()
 364
 365                     with self._traffic_ctl:
 366                         # execute test based on TestSteps definition if needed...
 367                         if self.step_run():
 368                             # ...and continue with traffic generation, but keep
 369                             # in mind, that clean deployment does not configure
 370                             # OVS nor executes the traffic
 371                             if self.deployment != 'clean' and not self._step_send_traffic:
 372                                 self._traffic_ctl.send_traffic(self._traffic)
 373
 374                         # dump vswitch flows before they are affected by VNF termination
 375                         if not self._vswitch_none:
 376                             self._vswitch_ctl.dump_vswitch_connections()
 377
 378                     # garbage collection for case that TestSteps modify existing deployment
 379                     self.step_stop_vnfs()
 380
 381         finally:
 382             # tear down test execution environment and log results
 383             self.run_finalize()
 384
 385         self._testcase_stop_time = time.time()
 386         self._testcase_run_time = time.strftime("%H:%M:%S",
 387                                                 time.gmtime(self._testcase_stop_time -
 388                                                             self._testcase_start_time))
 389         logging.info("Testcase execution time: %s", self._testcase_run_time)
 390         # report test results
 391         self.run_report()
 392
 393     def _append_results(self, results):
 394         """
 395         Method appends mandatory Test Case results to list of dictionaries.
 396
 397         :param results: list of dictionaries which contains results from
 398                 traffic generator.
 399
 400         :returns: modified list of dictionaries.
 401         """
 402         for item in results:
 403             item[ResultsConstants.ID] = self.name
 404             item[ResultsConstants.DEPLOYMENT] = self.deployment
 405             item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
 406             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 407             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 408             # convert timestamps to human readable format
 409             item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
 410                 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
 411             item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
 412                 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
 413             if self._traffic['multistream']:
 414                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 415                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 416                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 417             if self._vnf_ctl.get_vnfs_number():
 418                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
 419             if self._tunnel_operation:
 420                 item[ResultsConstants.TUNNEL_TYPE] = S.getValue('TUNNEL_TYPE')
 421         return results
 422
 423     def _copy_fwd_tools_for_all_guests(self, vm_count):
 424         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 425         """
 426         # consider only VNFs involved in the test
 427         for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
 428             self._copy_fwd_tools_for_guest(guest_dir)
 429
 430     def _copy_fwd_tools_for_guest(self, guest_dir):
 431         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 432
 433         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 434         """
 435         # remove shared dir if it exists to avoid issues with file consistency
 436         if os.path.exists(guest_dir):
 437             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 438                            'Removing content of shared directory...', True)
 439
 440         # directory to share files between host and guest
 441         os.makedirs(guest_dir)
 442
 443         # copy sources into shared dir only if neccessary
 444         guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
 445         if 'testpmd' in guest_loopback:
 446             try:
 447                 # exclude whole .git/ subdirectory and all o-files;
 448                 # It is assumed, that the same RTE_TARGET is used in both host
 449                 # and VMs; This simplification significantly speeds up testpmd
 450                 # build. If we will need a different RTE_TARGET in VM,
 451                 # then we have to build whole DPDK from the scratch in VM.
 452                 # In that case we can copy just DPDK sources (e.g. by excluding
 453                 # all items obtained by git status -unormal --porcelain).
 454                 # NOTE: Excluding RTE_TARGET directory won't help on systems,
 455                 # where DPDK is built for multiple targets (e.g. for gcc & icc)
 456                 exclude = []
 457                 exclude.append(r'--exclude=.git/')
 458                 exclude.append(r'--exclude=*.o')
 459                 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
 460                                [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
 461                                 os.path.join(guest_dir, 'DPDK')],
 462                                self._logger,
 463                                'Copying DPDK to shared directory...',
 464                                True)
 465             except subprocess.CalledProcessError:
 466                 self._logger.error('Unable to copy DPDK to shared directory')
 467                 raise
 468         if 'l2fwd' in guest_loopback:
 469             try:
 470                 tasks.run_task(['rsync', '-a', '-r', '-l',
 471                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 472                                 os.path.join(guest_dir, 'l2fwd')],
 473                                self._logger,
 474                                'Copying l2fwd to shared directory...',
 475                                True)
 476             except subprocess.CalledProcessError:
 477                 self._logger.error('Unable to copy l2fwd to shared directory')
 478                 raise
 479
 480     def _mount_hugepages(self):
 481         """Mount hugepages if usage of DPDK or Qemu is detected
 482         """
 483         # pylint: disable=too-many-boolean-expressions
 484         # hugepages are needed by DPDK and Qemu
 485         if not self._hugepages_mounted and \
 486             (self.deployment.count('v') or \
 487              str(S.getValue('VSWITCH')).lower().count('dpdk') or \
 488              self._vswitch_none or \
 489              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 490             hugepages.mount_hugepages()
 491             self._hugepages_mounted = True
 492
 493     def _umount_hugepages(self):
 494         """Umount hugepages if they were mounted before
 495         """
 496         if self._hugepages_mounted:
 497             hugepages.umount_hugepages()
 498             self._hugepages_mounted = False
 499
 500     def _check_for_enough_hugepages(self):
 501         """Check to make sure enough hugepages are free to satisfy the
 502         test environment.
 503         """
 504         hugepages_needed = 0
 505         hugepage_size = hugepages.get_hugepage_size()
 506         # get hugepage amounts per guest involved in the test
 507         for guest in range(self._vnf_ctl.get_vnfs_number()):
 508             hugepages_needed += math.ceil((int(S.getValue(
 509                 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
 510
 511         # get hugepage amounts for each socket on dpdk
 512         sock0_mem, sock1_mem = 0, 0
 513
 514         if str(S.getValue('VSWITCH')).lower().count('dpdk'):
 515             sock_mem = S.getValue('DPDK_SOCKET_MEM')
 516             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
 517                                     int(sock_mem[1]) * 1024 / hugepage_size)
 518
 519         # If hugepages needed, verify the amounts are free
 520         if any([hugepages_needed, sock0_mem, sock1_mem]):
 521             free_hugepages = hugepages.get_free_hugepages()
 522             if hugepages_needed:
 523                 logging.info('Need %s hugepages free for guests',
 524                              hugepages_needed)
 525                 result1 = free_hugepages >= hugepages_needed
 526                 free_hugepages -= hugepages_needed
 527             else:
 528                 result1 = True
 529
 530             if sock0_mem:
 531                 logging.info('Need %s hugepages free for dpdk socket 0',
 532                              sock0_mem)
 533                 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
 534                 free_hugepages -= sock0_mem
 535             else:
 536                 result2 = True
 537
 538             if sock1_mem:
 539                 logging.info('Need %s hugepages free for dpdk socket 1',
 540                              sock1_mem)
 541                 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
 542                 free_hugepages -= sock1_mem
 543             else:
 544                 result3 = True
 545
 546             logging.info('Need a total of %s total hugepages',
 547                          hugepages_needed + sock1_mem + sock0_mem)
 548
 549             # The only drawback here is sometimes dpdk doesn't release
 550             # its hugepages on a test failure. This could cause a test
 551             # to fail when dpdk would be OK to start because it will just
 552             # use the previously allocated hugepages.
 553             result4 = True if free_hugepages >= 0 else False
 554
 555             return all([result1, result2, result3, result4])
 556         else:
 557             return True
 558
 559     @staticmethod
 560     def write_result_to_file(results, output):
 561         """Write list of dictionaries to a CSV file.
 562
 563         Each element on list will create separate row in output file.
 564         If output file already exists, data will be appended at the end,
 565         otherwise it will be created.
 566
 567         :param results: list of dictionaries.
 568         :param output: path to output file.
 569         """
 570         with open(output, 'a') as csvfile:
 571
 572             logging.info("Write results to file: %s", output)
 573             fieldnames = TestCase._get_unique_keys(results)
 574
 575             writer = csv.DictWriter(csvfile, fieldnames)
 576
 577             if not csvfile.tell():  # file is now empty
 578                 writer.writeheader()
 579
 580             for result in results:
 581                 writer.writerow(result)
 582
 583     @staticmethod
 584     def _get_unique_keys(list_of_dicts):
 585         """Gets unique key values as ordered list of strings in given dicts
 586
 587         :param list_of_dicts: list of dictionaries.
 588
 589         :returns: list of unique keys(strings).
 590         """
 591         result = OrderedDict()
 592         for item in list_of_dicts:
 593             for key in item.keys():
 594                 result[key] = ''
 595
 596         return list(result.keys())
 597
 598     def _add_flows(self):
 599         """Add flows to the vswitch
 600         """
 601         vswitch = self._vswitch_ctl.get_vswitch()
 602         # NOTE BOM 15-08-07 the frame mod code assumes that the
 603         # physical ports are ports 1 & 2. The actual numbers
 604         # need to be retrived from the vSwitch and the metadata value
 605         # updated accordingly.
 606         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 607         if self._frame_mod == "vlan":
 608             # 0x8100 => VLAN ethertype
 609             self._logger.debug(" ****   VLAN   ***** ")
 610             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 611                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 612             vswitch.add_flow(bridge, flow)
 613             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 614                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 615             vswitch.add_flow(bridge, flow)
 616         elif self._frame_mod == "mpls":
 617             # 0x8847 => MPLS unicast ethertype
 618             self._logger.debug(" ****   MPLS  ***** ")
 619             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 620                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 621             vswitch.add_flow(bridge, flow)
 622             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 623                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 624             vswitch.add_flow(bridge, flow)
 625         elif self._frame_mod == "mac":
 626             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 627                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 628                                 'goto_table:3']}
 629             vswitch.add_flow(bridge, flow)
 630             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 631                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 632                                 'goto_table:3']}
 633             vswitch.add_flow(bridge, flow)
 634         elif self._frame_mod == "dscp":
 635             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 636             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 637                     'dl_type':'0x0800',
 638                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 639             vswitch.add_flow(bridge, flow)
 640             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 641                     'dl_type':'0x0800',
 642                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 643             vswitch.add_flow(bridge, flow)
 644         elif self._frame_mod == "ttl":
 645             # 251 and 241 are the highest prime numbers < 255
 646             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 647                     'dl_type':'0x0800',
 648                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 649             vswitch.add_flow(bridge, flow)
 650             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 651                     'dl_type':'0x0800',
 652                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 653             vswitch.add_flow(bridge, flow)
 654         elif self._frame_mod == "ip_addr":
 655             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 656                     'dl_type':'0x0800',
 657                     'actions': ['mod_nw_src:10.10.10.10',
 658                                 'mod_nw_dst:20.20.20.20',
 659                                 'goto_table:3']}
 660             vswitch.add_flow(bridge, flow)
 661             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 662                     'dl_type':'0x0800',
 663                     'actions': ['mod_nw_src:20.20.20.20',
 664                                 'mod_nw_dst:10.10.10.10',
 665                                 'goto_table:3']}
 666             vswitch.add_flow(bridge, flow)
 667         elif self._frame_mod == "ip_port":
 668             # NOTE BOM 15-08-27 The traffic generated is assumed
 669             # to be UDP (nw_proto 17d) which is the default case but
 670             # we will need to pick up the actual traffic params in use.
 671             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 672                     'dl_type':'0x0800', 'nw_proto':'17',
 673                     'actions': ['mod_tp_src:44444',
 674                                 'mod_tp_dst:44444', 'goto_table:3']}
 675             vswitch.add_flow(bridge, flow)
 676             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 677                     'dl_type':'0x0800', 'nw_proto':'17',
 678                     'actions': ['mod_tp_src:44444',
 679                                 'mod_tp_dst:44444', 'goto_table:3']}
 680             vswitch.add_flow(bridge, flow)
 681         else:
 682             pass
 683
 684
 685     #
 686     # TestSteps realted methods
 687     #
 688     def step_report_status(self, label, status):
 689         """ Log status of test step
 690         """
 691         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 692
 693     def step_stop_vnfs(self):
 694         """ Stop all VNFs started by TestSteps
 695         """
 696         for vnf in self._step_vnf_list:
 697             if self._step_vnf_list[vnf]:
 698                 self._step_vnf_list[vnf].stop()
 699
 700     def step_eval_param(self, param, step_result):
 701         """ Helper function for #STEP macro evaluation
 702         """
 703         if isinstance(param, str):
 704             # evaluate every #STEP reference inside parameter itself
 705             macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
 706
 707             if macros:
 708                 for macro in macros:
 709                     if macro[1] in self._step_result_mapping:
 710                         key = self._step_result_mapping[macro[1]]
 711                     else:
 712                         key = macro[1]
 713                     # pylint: disable=eval-used
 714                     tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
 715                     param = param.replace(macro[0], tmp_val)
 716
 717             # evaluate references to vsperf configuration options
 718             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
 719             if macros:
 720                 for macro in macros:
 721                     # pylint: disable=eval-used
 722                     try:
 723                         tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
 724                         param = param.replace('${}'.format(macro[0]), tmp_val)
 725                     # ignore that required option can't be evaluated
 726                     except (IndexError, KeyError, AttributeError):
 727                         self._logger.debug("Skipping %s as it isn't a configuration "
 728                                            "parameter.", '${}'.format(macro[0]))
 729             return param
 730         elif isinstance(param, (list, tuple)):
 731             tmp_list = []
 732             for item in param:
 733                 tmp_list.append(self.step_eval_param(item, step_result))
 734             return tmp_list
 735         elif isinstance(param, dict):
 736             tmp_dict = {}
 737             for (key, value) in param.items():
 738                 tmp_dict[key] = self.step_eval_param(value, step_result)
 739             return tmp_dict
 740         else:
 741             return param
 742
 743     def step_eval_params(self, params, step_result):
 744         """ Evaluates referrences to results from previous steps
 745         """
 746         eval_params = []
 747         # evaluate all parameters if needed
 748         for param in params:
 749             eval_params.append(self.step_eval_param(param, step_result))
 750         return eval_params
 751
 752     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 753     def step_run(self):
 754         """ Execute actions specified by TestSteps list
 755
 756         :return: False if any error was detected
 757                  True otherwise
 758         """
 759         # anything to do?
 760         if not self.test:
 761             return True
 762
 763         # required for VNFs initialization
 764         loader = Loader()
 765         # initialize list with results
 766         self._step_result = [None] * len(self.test)
 767
 768         # run test step by step...
 769         for i, step in enumerate(self.test):
 770             step_ok = not self._step_check
 771             step_check = self._step_check
 772             regex = None
 773             # configure step result mapping if step alias/label is detected
 774             if step[0].startswith('#'):
 775                 key = step[0][1:]
 776                 if key.isdigit():
 777                     raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
 778                 if key in self._step_result_mapping:
 779                     raise RuntimeError('Step alias {} has been used already for step '
 780                                        '{}'.format(key, self._step_result_mapping[key]))
 781                 self._step_result_mapping[step[0][1:]] = i
 782                 step = step[1:]
 783
 784             # store regex filter if it is specified
 785             if isinstance(step[-1], str) and step[-1].startswith('|'):
 786                 # evalute macros and variables used in regex
 787                 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
 788                 step = step[:-1]
 789
 790             # check if step verification should be suppressed
 791             if step[0].startswith('!'):
 792                 step_check = False
 793                 step_ok = True
 794                 step[0] = step[0][1:]
 795             if step[0] == 'vswitch':
 796                 test_object = self._vswitch_ctl.get_vswitch()
 797             elif step[0] == 'namespace':
 798                 test_object = namespace
 799             elif step[0] == 'veth':
 800                 test_object = veth
 801             elif step[0] == 'settings':
 802                 test_object = S
 803             elif step[0] == 'tools':
 804                 test_object = TestStepsTools()
 805                 step[1] = step[1].title()
 806             elif step[0] == 'trafficgen':
 807                 test_object = self._traffic_ctl
 808                 # in case of send_traffic or send_traffic_async methods, ensure
 809                 # that specified traffic values are merged with existing self._traffic
 810                 if step[1].startswith('send_traffic'):
 811                     tmp_traffic = copy.deepcopy(self._traffic)
 812                     tmp_traffic.update(step[2])
 813                     step[2] = tmp_traffic
 814                     # store indication that traffic has been sent
 815                     # so it is not sent again after the execution of teststeps
 816                     self._step_send_traffic = True
 817             elif step[0].startswith('vnf'):
 818                 # use vnf started within TestSteps
 819                 if not self._step_vnf_list[step[0]]:
 820                     # initialize new VM
 821                     self._step_vnf_list[step[0]] = loader.get_vnf_class()()
 822                 test_object = self._step_vnf_list[step[0]]
 823             elif step[0].startswith('VNF'):
 824                 if step[1] in ('start', 'stop'):
 825                     raise RuntimeError("Cannot execute start() or stop() method of "
 826                                        "VNF deployed automatically by scenario.")
 827                 # use vnf started by scenario deployment (e.g. pvp)
 828                 vnf_index = int(step[0][3:])
 829                 try:
 830                     test_object = self._vnf_list[vnf_index]
 831                 except IndexError:
 832                     raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
 833             elif step[0] == 'wait':
 834                 input(os.linesep + "Step {}: Press Enter to continue with "
 835                       "the next step...".format(i) + os.linesep + os.linesep)
 836                 continue
 837             elif step[0] == 'sleep':
 838                 self._logger.debug("Sleep %s seconds", step[1])
 839                 time.sleep(int(step[1]))
 840                 continue
 841             elif step[0] == 'log':
 842                 test_object = self._logger
 843                 # there isn't a need for validation of log entry
 844                 step_check = False
 845                 step_ok = True
 846             elif step[0] == 'pdb':
 847                 import pdb
 848                 pdb.set_trace()
 849                 continue
 850             else:
 851                 self._logger.error("Unsupported test object %s", step[0])
 852                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
 853                 self.step_report_status("Step '{}'".format(' '.join(step)),
 854                                         self._step_status['status'])
 855                 return False
 856
 857             test_method = getattr(test_object, step[1])
 858             if step_check:
 859                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 860             else:
 861                 test_method_check = None
 862
 863             step_params = []
 864             try:
 865                 # eval parameters, but use only valid step_results
 866                 # to support negative indexes
 867                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
 868                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 869                 step_log += ' filter "{}"'.format(regex) if regex else ''
 870                 self._logger.debug("Step %s '%s' start", i, step_log)
 871                 self._step_result[i] = test_method(*step_params)
 872                 if regex:
 873                     # apply regex to step output
 874                     self._step_result[i] = functions.filter_output(
 875                         self._step_result[i], regex)
 876
 877                 self._logger.debug("Step %s '%s' results '%s'", i,
 878                                    step_log, self._step_result[i])
 879                 time.sleep(S.getValue('TEST_STEP_DELAY'))
 880                 if step_check:
 881                     step_ok = test_method_check(self._step_result[i], *step_params)
 882             except (AssertionError, AttributeError, IndexError) as ex:
 883                 step_ok = False
 884                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 885
 886             if step_check:
 887                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 888
 889             if not step_ok:
 890                 self._step_status = {'status' : False, 'details' : step_log}
 891                 # Stop all VNFs started by TestSteps
 892                 self.step_stop_vnfs()
 893                 return False
 894
 895         # all steps processed without any issue
 896         return True
 897
 898     #
 899     # get methods for TestCase members, which needs to be publicly available
 900     #
 901     def get_output_file(self):
 902         """Return content of self._output_file member
 903         """
 904         return self._output_file
 905
 906     def get_desc(self):
 907         """Return content of self.desc member
 908         """
 909         return self.desc
 910
 911     def get_versions(self):
 912         """Return content of self.versions member
 913         """
 914         return self._versions
 915
 916     def get_traffic(self):
 917         """Return content of self._traffic member
 918         """
 919         return self._traffic
 920
 921     def get_tc_results(self):
 922         """Return content of self._tc_results member
 923         """
 924         return self._tc_results
 925
 926     def get_collector(self):
 927         """Return content of self._collector member
 928         """
 929         return self._collector