testcases/testcase.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """TestCase base class
  15 """
  16
  17 import csv
  18 import os
  19 import time
  20 import logging
  21 import subprocess
  22 import copy
  23 from collections import OrderedDict
  24
  25 import core.component_factory as component_factory
  26 from core.loader import Loader
  27 from core.results.results_constants import ResultsConstants
  28 from tools import tasks
  29 from tools import hugepages
  30 from tools import functions
  31 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
  32 from conf import settings as S
  33 from conf import get_test_param
  34
  35 class TestCase(object):
  36     """TestCase base class
  37
  38     In this basic form runs RFC2544 throughput test
  39     """
  40     def __init__(self, cfg):
  41         """Pull out fields from test config
  42
  43         :param cfg: A dictionary of string-value pairs describing the test
  44             configuration. Both the key and values strings use well-known
  45             values.
  46         :param results_dir: Where the csv formatted results are written.
  47         """
  48         self._testcase_start_time = time.time()
  49         self._hugepages_mounted = False
  50         self._traffic_ctl = None
  51         self._vnf_ctl = None
  52         self._vswitch_ctl = None
  53         self._collector = None
  54         self._loadgen = None
  55         self._output_file = None
  56         self._tc_results = None
  57         self.guest_loopback = []
  58         self._settings_original = {}
  59         self._settings_paths_modified = False
  60         self._testcast_run_time = None
  61
  62         self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
  63         self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
  64         self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
  65         self._update_settings('TEST_PARAMS', cfg.get('Parameters', S.getValue('TEST_PARAMS')))
  66
  67         # update global settings
  68         guest_loopback = get_test_param('guest_loopback', None)
  69         if guest_loopback:
  70             self._update_settings('GUEST_LOOPBACK', [guest_loopback for dummy in S.getValue('GUEST_LOOPBACK')])
  71
  72         if 'VSWITCH' in self._settings_original or 'VNF' in self._settings_original:
  73             self._settings_original.update({
  74                 'RTE_SDK' : S.getValue('RTE_SDK'),
  75                 'OVS_DIR' : S.getValue('OVS_DIR'),
  76             })
  77             functions.settings_update_paths()
  78
  79         # set test parameters; CLI options take precedence to testcase settings
  80         self._logger = logging.getLogger(__name__)
  81         self.name = cfg['Name']
  82         self.desc = cfg.get('Description', 'No description given.')
  83         self.test = cfg.get('TestSteps', None)
  84
  85         bidirectional = cfg.get('biDirectional', TRAFFIC_DEFAULTS['bidir'])
  86         bidirectional = get_test_param('bidirectional', bidirectional)
  87         if not isinstance(bidirectional, str):
  88             raise TypeError(
  89                 'Bi-dir value must be of type string in testcase configuration')
  90         bidirectional = bidirectional.title()  # Keep things consistent
  91
  92         traffic_type = cfg.get('Traffic Type', TRAFFIC_DEFAULTS['traffic_type'])
  93         traffic_type = get_test_param('traffic_type', traffic_type)
  94
  95         framerate = cfg.get('iLoad', TRAFFIC_DEFAULTS['frame_rate'])
  96         framerate = get_test_param('iload', framerate)
  97
  98         self.deployment = cfg['Deployment']
  99         self._frame_mod = cfg.get('Frame Modification', None)
 100
 101         self._tunnel_type = None
 102         self._tunnel_operation = None
 103
 104         if self.deployment == 'op2p':
 105             self._tunnel_operation = cfg['Tunnel Operation']
 106
 107             if 'Tunnel Type' in cfg:
 108                 self._tunnel_type = cfg['Tunnel Type']
 109                 self._tunnel_type = get_test_param('tunnel_type',
 110                                                    self._tunnel_type)
 111
 112         # identify guest loopback method, so it can be added into reports
 113         if self.deployment == 'pvp':
 114             self.guest_loopback.append(S.getValue('GUEST_LOOPBACK')[0])
 115         else:
 116             self.guest_loopback = S.getValue('GUEST_LOOPBACK').copy()
 117
 118         # read configuration of streams; CLI parameter takes precedence to
 119         # testcase definition
 120         multistream = cfg.get('MultiStream', TRAFFIC_DEFAULTS['multistream'])
 121         multistream = get_test_param('multistream', multistream)
 122         stream_type = cfg.get('Stream Type', TRAFFIC_DEFAULTS['stream_type'])
 123         stream_type = get_test_param('stream_type', stream_type)
 124         pre_installed_flows = cfg.get('Pre-installed Flows', TRAFFIC_DEFAULTS['pre_installed_flows'])
 125         pre_installed_flows = get_test_param('pre-installed_flows', pre_installed_flows)
 126
 127         # check if test requires background load and which generator it uses
 128         self._load_cfg = cfg.get('Load', None)
 129         if self._load_cfg and 'tool' in self._load_cfg:
 130             self._loadgen = self._load_cfg['tool']
 131         else:
 132             # background load is not requested, so use dummy implementation
 133             self._loadgen = "Dummy"
 134
 135         if self._frame_mod:
 136             self._frame_mod = self._frame_mod.lower()
 137         self._results_dir = S.getValue('RESULTS_PATH')
 138
 139         # set traffic details, so they can be passed to vswitch and traffic ctls
 140         self._traffic = copy.deepcopy(TRAFFIC_DEFAULTS)
 141         self._traffic.update({'traffic_type': traffic_type,
 142                               'flow_type': cfg.get('Flow Type', TRAFFIC_DEFAULTS['flow_type']),
 143                               'bidir': bidirectional,
 144                               'tunnel_type': self._tunnel_type,
 145                               'multistream': int(multistream),
 146                               'stream_type': stream_type,
 147                               'pre_installed_flows' : pre_installed_flows,
 148                               'frame_rate': int(framerate)})
 149
 150         # Packet Forwarding mode
 151         self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower()
 152
 153         # OVS Vanilla requires guest VM MAC address and IPs to work
 154         if 'linux_bridge' in self.guest_loopback:
 155             self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
 156                                         'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
 157             self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
 158                                         'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 159
 160         # trafficgen configuration required for tests of tunneling protocols
 161         if self.deployment == "op2p":
 162             self._traffic['l2'].update({'srcmac':
 163                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
 164                                         'dstmac':
 165                                         S.getValue('TRAFFICGEN_PORT2_MAC')})
 166
 167             self._traffic['l3'].update({'srcip':
 168                                         S.getValue('TRAFFICGEN_PORT1_IP'),
 169                                         'dstip':
 170                                         S.getValue('TRAFFICGEN_PORT2_IP')})
 171
 172             if self._tunnel_operation == "decapsulation":
 173                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
 174                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
 175                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
 176         elif S.getValue('NICS')[0]['type'] == 'vf' or S.getValue('NICS')[1]['type'] == 'vf':
 177             mac1 = S.getValue('NICS')[0]['mac']
 178             mac2 = S.getValue('NICS')[1]['mac']
 179             if mac1 and mac2:
 180                 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
 181             else:
 182                 self._logger.debug("MAC addresses can not be read")
 183
 184     def run_initialize(self):
 185         """ Prepare test execution environment
 186         """
 187         self._logger.debug(self.name)
 188
 189         # mount hugepages if needed
 190         self._mount_hugepages()
 191
 192         # copy sources of l2 forwarding tools into VM shared dir if needed
 193         self._copy_fwd_tools_for_all_guests()
 194
 195         self._logger.debug("Controllers:")
 196         loader = Loader()
 197         self._traffic_ctl = component_factory.create_traffic(
 198             self._traffic['traffic_type'],
 199             loader.get_trafficgen_class())
 200
 201         self._vnf_ctl = component_factory.create_vnf(
 202             self.deployment,
 203             loader.get_vnf_class())
 204
 205         if self._vswitch_none:
 206             self._vswitch_ctl = component_factory.create_pktfwd(
 207                 self.deployment,
 208                 loader.get_pktfwd_class())
 209         else:
 210             self._vswitch_ctl = component_factory.create_vswitch(
 211                 self.deployment,
 212                 loader.get_vswitch_class(),
 213                 self._traffic,
 214                 self._tunnel_operation)
 215
 216         self._collector = component_factory.create_collector(
 217             loader.get_collector_class(),
 218             self._results_dir, self.name)
 219         self._loadgen = component_factory.create_loadgen(
 220             self._loadgen,
 221             self._load_cfg)
 222
 223         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
 224                                          "_" + self.deployment + ".csv")
 225
 226         self._logger.debug("Setup:")
 227
 228     def run_finalize(self):
 229         """ Tear down test execution environment and record test results
 230         """
 231         # umount hugepages if mounted
 232         self._umount_hugepages()
 233
 234         # restore original settings
 235         S.load_from_dict(self._settings_original)
 236
 237     def run_report(self):
 238         """ Report test results
 239         """
 240         self._logger.debug("self._collector Results:")
 241         self._collector.print_results()
 242
 243         if S.getValue('mode') != 'trafficgen-off':
 244             self._logger.debug("Traffic Results:")
 245             self._traffic_ctl.print_results()
 246
 247             self._tc_results = self._append_results(self._traffic_ctl.get_results())
 248             TestCase.write_result_to_file(self._tc_results, self._output_file)
 249
 250     def run(self):
 251         """Run the test
 252
 253         All setup and teardown through controllers is included.
 254         """
 255         # prepare test execution environment
 256         self.run_initialize()
 257
 258         with self._vswitch_ctl, self._loadgen:
 259             with self._vnf_ctl, self._collector:
 260                 if not self._vswitch_none:
 261                     self._add_flows()
 262
 263                 # run traffic generator if requested, otherwise wait for manual termination
 264                 if S.getValue('mode') == 'trafficgen-off':
 265                     time.sleep(2)
 266                     self._logger.debug("All is set. Please run traffic generator manually.")
 267                     input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
 268                 else:
 269                     if S.getValue('mode') == 'trafficgen-pause':
 270                         time.sleep(2)
 271                         true_vals = ('yes', 'y', 'ye', None)
 272                         while True:
 273                             choice = input(os.linesep + 'Transmission paused, should'
 274                                            ' transmission be resumed? ' + os.linesep).lower()
 275                             if not choice or choice not in true_vals:
 276                                 print('Please respond with \'yes\' or \'y\' ', end='')
 277                             else:
 278                                 break
 279                     with self._traffic_ctl:
 280                         self._traffic_ctl.send_traffic(self._traffic)
 281
 282                     # dump vswitch flows before they are affected by VNF termination
 283                     if not self._vswitch_none:
 284                         self._vswitch_ctl.dump_vswitch_flows()
 285
 286         # tear down test execution environment and log results
 287         self.run_finalize()
 288
 289         self._testcase_run_time = time.strftime("%H:%M:%S",
 290                                   time.gmtime(time.time() - self._testcase_start_time))
 291         logging.info("Testcase execution time: " + self._testcase_run_time)
 292         # report test results
 293         self.run_report()
 294
 295     def _update_settings(self, param, value):
 296         """ Check value of given configuration parameter
 297         In case that new value is different, then testcase
 298         specific settings is updated and original value stored
 299
 300         :param param: Name of parameter inside settings
 301         :param value: Disired parameter value
 302         """
 303         orig_value = S.getValue(param)
 304         if orig_value != value:
 305             self._settings_original[param] = orig_value
 306             S.setValue(param, value)
 307
 308     def _append_results(self, results):
 309         """
 310         Method appends mandatory Test Case results to list of dictionaries.
 311
 312         :param results: list of dictionaries which contains results from
 313                 traffic generator.
 314
 315         :returns: modified list of dictionaries.
 316         """
 317         for item in results:
 318             item[ResultsConstants.ID] = self.name
 319             item[ResultsConstants.DEPLOYMENT] = self.deployment
 320             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
 321             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
 322             if self._traffic['multistream']:
 323                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
 324                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
 325                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
 326             if self.deployment in ['pvp', 'pvvp'] and len(self.guest_loopback):
 327                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(self.guest_loopback)
 328             if self._tunnel_type:
 329                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
 330         return results
 331
 332     def _copy_fwd_tools_for_all_guests(self):
 333         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
 334         """
 335         # data are copied only for pvp and pvvp, so let's count number of 'v'
 336         counter = 1
 337         while counter <= self.deployment.count('v'):
 338             self._copy_fwd_tools_for_guest(counter)
 339             counter += 1
 340
 341     def _copy_fwd_tools_for_guest(self, index):
 342         """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
 343
 344         :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
 345         """
 346         guest_dir = S.getValue('GUEST_SHARE_DIR')[index-1]
 347
 348         # remove shared dir if it exists to avoid issues with file consistency
 349         if os.path.exists(guest_dir):
 350             tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
 351                            'Removing content of shared directory...', True)
 352
 353         # directory to share files between host and guest
 354         os.makedirs(guest_dir)
 355
 356         # copy sources into shared dir only if neccessary
 357         if 'testpmd' in self.guest_loopback or 'l2fwd' in self.guest_loopback:
 358             try:
 359                 tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"',
 360                                 os.path.join(S.getValue('RTE_SDK_USER'), ''),
 361                                 os.path.join(guest_dir, 'DPDK')],
 362                                self._logger,
 363                                'Copying DPDK to shared directory...',
 364                                True)
 365                 tasks.run_task(['rsync', '-a', '-r', '-l',
 366                                 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
 367                                 os.path.join(guest_dir, 'l2fwd')],
 368                                self._logger,
 369                                'Copying l2fwd to shared directory...',
 370                                True)
 371             except subprocess.CalledProcessError:
 372                 self._logger.error('Unable to copy DPDK and l2fwd to shared directory')
 373
 374
 375     def _mount_hugepages(self):
 376         """Mount hugepages if usage of DPDK or Qemu is detected
 377         """
 378         # hugepages are needed by DPDK and Qemu
 379         if not self._hugepages_mounted and \
 380             (self.deployment.count('v') or \
 381              S.getValue('VSWITCH').lower().count('dpdk') or \
 382              self._vswitch_none or \
 383              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
 384             hugepages.mount_hugepages()
 385             self._hugepages_mounted = True
 386
 387     def _umount_hugepages(self):
 388         """Umount hugepages if they were mounted before
 389         """
 390         if self._hugepages_mounted:
 391             hugepages.umount_hugepages()
 392             self._hugepages_mounted = False
 393
 394     @staticmethod
 395     def write_result_to_file(results, output):
 396         """Write list of dictionaries to a CSV file.
 397
 398         Each element on list will create separate row in output file.
 399         If output file already exists, data will be appended at the end,
 400         otherwise it will be created.
 401
 402         :param results: list of dictionaries.
 403         :param output: path to output file.
 404         """
 405         with open(output, 'a') as csvfile:
 406
 407             logging.info("Write results to file: " + output)
 408             fieldnames = TestCase._get_unique_keys(results)
 409
 410             writer = csv.DictWriter(csvfile, fieldnames)
 411
 412             if not csvfile.tell():  # file is now empty
 413                 writer.writeheader()
 414
 415             for result in results:
 416                 writer.writerow(result)
 417
 418     @staticmethod
 419     def _get_unique_keys(list_of_dicts):
 420         """Gets unique key values as ordered list of strings in given dicts
 421
 422         :param list_of_dicts: list of dictionaries.
 423
 424         :returns: list of unique keys(strings).
 425         """
 426         result = OrderedDict()
 427         for item in list_of_dicts:
 428             for key in item.keys():
 429                 result[key] = ''
 430
 431         return list(result.keys())
 432
 433     def _add_flows(self):
 434         """Add flows to the vswitch
 435         """
 436         vswitch = self._vswitch_ctl.get_vswitch()
 437         # TODO BOM 15-08-07 the frame mod code assumes that the
 438         # physical ports are ports 1 & 2. The actual numbers
 439         # need to be retrived from the vSwitch and the metadata value
 440         # updated accordingly.
 441         bridge = S.getValue('VSWITCH_BRIDGE_NAME')
 442         if self._frame_mod == "vlan":
 443             # 0x8100 => VLAN ethertype
 444             self._logger.debug(" ****   VLAN   ***** ")
 445             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 446                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 447             vswitch.add_flow(bridge, flow)
 448             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 449                     'actions': ['push_vlan:0x8100', 'goto_table:3']}
 450             vswitch.add_flow(bridge, flow)
 451         elif self._frame_mod == "mpls":
 452             # 0x8847 => MPLS unicast ethertype
 453             self._logger.debug(" ****   MPLS  ***** ")
 454             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 455                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 456             vswitch.add_flow(bridge, flow)
 457             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 458                     'actions': ['push_mpls:0x8847', 'goto_table:3']}
 459             vswitch.add_flow(bridge, flow)
 460         elif self._frame_mod == "mac":
 461             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 462                     'actions': ['mod_dl_src:22:22:22:22:22:22',
 463                                 'goto_table:3']}
 464             vswitch.add_flow(bridge, flow)
 465             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 466                     'actions': ['mod_dl_src:11:11:11:11:11:11',
 467                                 'goto_table:3']}
 468             vswitch.add_flow(bridge, flow)
 469         elif self._frame_mod == "dscp":
 470             # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
 471             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 472                     'dl_type':'0x0800',
 473                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 474             vswitch.add_flow(bridge, flow)
 475             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 476                     'dl_type':'0x0800',
 477                     'actions': ['mod_nw_tos:184', 'goto_table:3']}
 478             vswitch.add_flow(bridge, flow)
 479         elif self._frame_mod == "ttl":
 480             # 251 and 241 are the highest prime numbers < 255
 481             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 482                     'dl_type':'0x0800',
 483                     'actions': ['mod_nw_ttl:251', 'goto_table:3']}
 484             vswitch.add_flow(bridge, flow)
 485             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 486                     'dl_type':'0x0800',
 487                     'actions': ['mod_nw_ttl:241', 'goto_table:3']}
 488             vswitch.add_flow(bridge, flow)
 489         elif self._frame_mod == "ip_addr":
 490             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 491                     'dl_type':'0x0800',
 492                     'actions': ['mod_nw_src:10.10.10.10',
 493                                 'mod_nw_dst:20.20.20.20',
 494                                 'goto_table:3']}
 495             vswitch.add_flow(bridge, flow)
 496             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 497                     'dl_type':'0x0800',
 498                     'actions': ['mod_nw_src:20.20.20.20',
 499                                 'mod_nw_dst:10.10.10.10',
 500                                 'goto_table:3']}
 501             vswitch.add_flow(bridge, flow)
 502         elif self._frame_mod == "ip_port":
 503             # TODO BOM 15-08-27 The traffic generated is assumed
 504             # to be UDP (nw_proto 17d) which is the default case but
 505             # we will need to pick up the actual traffic params in use.
 506             flow = {'table':'2', 'priority':'1000', 'metadata':'2',
 507                     'dl_type':'0x0800', 'nw_proto':'17',
 508                     'actions': ['mod_tp_src:44444',
 509                                 'mod_tp_dst:44444', 'goto_table:3']}
 510             vswitch.add_flow(bridge, flow)
 511             flow = {'table':'2', 'priority':'1000', 'metadata':'1',
 512                     'dl_type':'0x0800', 'nw_proto':'17',
 513                     'actions': ['mod_tp_src:44444',
 514                                 'mod_tp_dst:44444', 'goto_table:3']}
 515             vswitch.add_flow(bridge, flow)
 516         else:
 517             pass