nfvbench/traffic_client.py

   1 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   2 #
   3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 #    not use this file except in compliance with the License. You may obtain
   5 #    a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #    Unless required by applicable law or agreed to in writing, software
  10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 #    License for the specific language governing permissions and limitations
  13 #    under the License.
  14
  15 from datetime import datetime
  16 import socket
  17 import struct
  18 import time
  19
  20 from attrdict import AttrDict
  21 import bitmath
  22 from netaddr import IPNetwork
  23 # pylint: disable=import-error
  24 from trex_stl_lib.api import STLError
  25 # pylint: enable=import-error
  26
  27 from log import LOG
  28 from network import Interface
  29 from specs import ChainType
  30 from stats_collector import IntervalCollector
  31 from stats_collector import IterationCollector
  32 import traffic_gen.traffic_utils as utils
  33 from utils import cast_integer
  34
  35
  36 class TrafficClientException(Exception):
  37     pass
  38
  39
  40 class TrafficRunner(object):
  41     def __init__(self, client, duration_sec, interval_sec=0):
  42         self.client = client
  43         self.start_time = None
  44         self.duration_sec = duration_sec
  45         self.interval_sec = interval_sec
  46
  47     def run(self):
  48         LOG.info('Running traffic generator')
  49         self.client.gen.clear_stats()
  50         self.client.gen.start_traffic()
  51         self.start_time = time.time()
  52         return self.poll_stats()
  53
  54     def stop(self):
  55         if self.is_running():
  56             self.start_time = None
  57             self.client.gen.stop_traffic()
  58
  59     def is_running(self):
  60         return self.start_time is not None
  61
  62     def time_elapsed(self):
  63         if self.is_running():
  64             return time.time() - self.start_time
  65         return self.duration_sec
  66
  67     def poll_stats(self):
  68         if not self.is_running():
  69             return None
  70         time_elapsed = self.time_elapsed()
  71         if time_elapsed > self.duration_sec:
  72             self.stop()
  73             return None
  74         time_left = self.duration_sec - time_elapsed
  75         if self.interval_sec > 0.0:
  76             if time_left <= self.interval_sec:
  77                 time.sleep(time_left)
  78                 self.stop()
  79             else:
  80                 time.sleep(self.interval_sec)
  81         else:
  82             time.sleep(self.duration_sec)
  83             self.stop()
  84         return self.client.get_stats()
  85
  86
  87 class IpBlock(object):
  88     def __init__(self, base_ip, step_ip, count_ip):
  89         self.base_ip_int = Device.ip_to_int(base_ip)
  90         self.step = Device.ip_to_int(step_ip)
  91         self.max_available = count_ip
  92         self.next_free = 0
  93
  94     def get_ip(self, index=0):
  95         '''Return the IP address at given index
  96         '''
  97         if index < 0 or index >= self.max_available:
  98             raise IndexError('Index out of bounds')
  99         return Device.int_to_ip(self.base_ip_int + index * self.step)
 100
 101     def reserve_ip_range(self, count):
 102         '''Reserve a range of count consecutive IP addresses spaced by step
 103         '''
 104         if self.next_free + count > self.max_available:
 105             raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d',
 106                              self.next_free,
 107                              self.max_available,
 108                              count)
 109         first_ip = self.get_ip(self.next_free)
 110         last_ip = self.get_ip(self.next_free + count - 1)
 111         self.next_free += count
 112         return (first_ip, last_ip)
 113
 114     def reset_reservation(self):
 115         self.next_free = 0
 116
 117
 118 class Device(object):
 119     def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
 120                  gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
 121                  gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,
 122                  chain_count=1, flow_count=1, vlan_tagging=False):
 123         self.chain_count = chain_count
 124         self.flow_count = flow_count
 125         self.dst = None
 126         self.port = port
 127         self.switch_port = switch_port
 128         self.vtep_vlan = vtep_vlan
 129         self.vlan_tag = None
 130         self.vlan_tagging = vlan_tagging
 131         self.pci = pci
 132         self.mac = None
 133         self.vm_mac_list = None
 134         subnet = IPNetwork(ip)
 135         self.ip = subnet.ip.format()
 136         self.ip_prefixlen = subnet.prefixlen
 137         self.ip_addrs_step = ip_addrs_step
 138         self.tg_gateway_ip_addrs_step = tg_gateway_ip_addrs_step
 139         self.gateway_ip_addrs_step = gateway_ip_addrs_step
 140         self.gateway_ip = gateway_ip
 141         self.tg_gateway_ip = tg_gateway_ip
 142         self.ip_block = IpBlock(self.ip, ip_addrs_step, flow_count)
 143         self.gw_ip_block = IpBlock(gateway_ip,
 144                                    gateway_ip_addrs_step,
 145                                    chain_count)
 146         self.tg_gw_ip_block = IpBlock(tg_gateway_ip,
 147                                       tg_gateway_ip_addrs_step,
 148                                       chain_count)
 149         self.udp_src_port = udp_src_port
 150         self.udp_dst_port = udp_dst_port
 151
 152     def set_mac(self, mac):
 153         if mac is None:
 154             raise TrafficClientException('Trying to set traffic generator MAC address as None')
 155         self.mac = mac
 156
 157     def set_destination(self, dst):
 158         self.dst = dst
 159
 160     def set_vm_mac_list(self, vm_mac_list):
 161         self.vm_mac_list = map(str, vm_mac_list)
 162
 163     def set_vlan_tag(self, vlan_tag):
 164         if self.vlan_tagging and vlan_tag is None:
 165             raise TrafficClientException('Trying to set VLAN tag as None')
 166         self.vlan_tag = vlan_tag
 167
 168     def get_gw_ip(self, chain_index):
 169         '''Retrieve the IP address assigned for the gateway of a given chain
 170         '''
 171         return self.gw_ip_block.get_ip(chain_index)
 172
 173     def get_stream_configs(self, service_chain):
 174         configs = []
 175         # exact flow count for each chain is calculated as follows:
 176         # - all chains except the first will have the same flow count
 177         #   calculated as (total_flows + chain_count - 1) / chain_count
 178         # - the first chain will have the remainder
 179         # example 11 flows and 3 chains => 3, 4, 4
 180         flows_per_chain = (self.flow_count + self.chain_count - 1) / self.chain_count
 181         cur_chain_flow_count = self.flow_count - flows_per_chain * (self.chain_count - 1)
 182
 183         self.ip_block.reset_reservation()
 184         self.dst.ip_block.reset_reservation()
 185
 186         for chain_idx in xrange(self.chain_count):
 187             src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count)
 188             dst_ip_first, dst_ip_last = self.dst.ip_block.reserve_ip_range(cur_chain_flow_count)
 189             configs.append({
 190                 'count': cur_chain_flow_count,
 191                 'mac_src': self.mac,
 192                 'mac_dst': self.dst.mac if service_chain == ChainType.EXT else self.vm_mac_list[
 193                     chain_idx],
 194                 'ip_src_addr': src_ip_first,
 195                 'ip_src_addr_max': src_ip_last,
 196                 'ip_src_count': cur_chain_flow_count,
 197                 'ip_dst_addr': dst_ip_first,
 198                 'ip_dst_addr_max': dst_ip_last,
 199                 'ip_dst_count': cur_chain_flow_count,
 200                 'ip_addrs_step': self.ip_addrs_step,
 201                 'udp_src_port': self.udp_src_port,
 202                 'udp_dst_port': self.udp_dst_port,
 203                 'mac_discovery_gw': self.get_gw_ip(chain_idx),
 204                 'ip_src_tg_gw': self.tg_gw_ip_block.get_ip(chain_idx),
 205                 'ip_dst_tg_gw': self.dst.tg_gw_ip_block.get_ip(chain_idx),
 206                 'vlan_tag': self.vlan_tag if self.vlan_tagging else None
 207             })
 208             # after first chain, fall back to the flow count for all other chains
 209             cur_chain_flow_count = flows_per_chain
 210
 211         return configs
 212
 213     def ip_range_overlaps(self):
 214         '''Check if this device ip range is overlapping with the dst device ip range
 215         '''
 216         src_base_ip = Device.ip_to_int(self.ip)
 217         dst_base_ip = Device.ip_to_int(self.dst.ip)
 218         src_last_ip = src_base_ip + self.flow_count - 1
 219         dst_last_ip = dst_base_ip + self.flow_count - 1
 220         return dst_last_ip >= src_base_ip and src_last_ip >= dst_base_ip
 221
 222     @staticmethod
 223     def mac_to_int(mac):
 224         return int(mac.translate(None, ":.- "), 16)
 225
 226     @staticmethod
 227     def int_to_mac(i):
 228         mac = format(i, 'x').zfill(12)
 229         blocks = [mac[x:x + 2] for x in xrange(0, len(mac), 2)]
 230         return ':'.join(blocks)
 231
 232     @staticmethod
 233     def ip_to_int(addr):
 234         return struct.unpack("!I", socket.inet_aton(addr))[0]
 235
 236     @staticmethod
 237     def int_to_ip(nvalue):
 238         return socket.inet_ntoa(struct.pack("!I", nvalue))
 239
 240
 241 class RunningTrafficProfile(object):
 242     """Represents traffic configuration for currently running traffic profile."""
 243
 244     DEFAULT_IP_STEP = '0.0.0.1'
 245     DEFAULT_SRC_DST_IP_STEP = '0.0.0.1'
 246
 247     def __init__(self, config, generator_profile):
 248         generator_config = self.__match_generator_profile(config.traffic_generator,
 249                                                           generator_profile)
 250         self.generator_config = generator_config
 251         self.service_chain = config.service_chain
 252         self.service_chain_count = config.service_chain_count
 253         self.flow_count = config.flow_count
 254         self.host_name = generator_config.host_name
 255         self.name = generator_config.name
 256         self.tool = generator_config.tool
 257         self.cores = generator_config.get('cores', 1)
 258         self.ip_addrs_step = generator_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
 259         self.tg_gateway_ip_addrs_step = \
 260             generator_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 261         self.gateway_ip_addrs_step = generator_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 262         self.gateway_ips = generator_config.gateway_ip_addrs
 263         self.ip = generator_config.ip
 264         self.intf_speed = bitmath.parse_string(generator_config.intf_speed.replace('ps', '')).bits
 265         self.vlan_tagging = config.vlan_tagging
 266         self.no_arp = config.no_arp
 267         self.src_device = None
 268         self.dst_device = None
 269         self.vm_mac_list = None
 270         self.__prep_interfaces(generator_config)
 271
 272     def to_json(self):
 273         return dict(self.generator_config)
 274
 275     def set_vm_mac_list(self, vm_mac_list):
 276         self.src_device.set_vm_mac_list(vm_mac_list[0])
 277         self.dst_device.set_vm_mac_list(vm_mac_list[1])
 278
 279     @staticmethod
 280     def __match_generator_profile(traffic_generator, generator_profile):
 281         generator_config = AttrDict(traffic_generator)
 282         generator_config.pop('default_profile')
 283         generator_config.pop('generator_profile')
 284         matching_profile = [profile for profile in traffic_generator.generator_profile if
 285                             profile.name == generator_profile]
 286         if len(matching_profile) != 1:
 287             raise Exception('Traffic generator profile not found: ' + generator_profile)
 288
 289         generator_config.update(matching_profile[0])
 290
 291         return generator_config
 292
 293     def __prep_interfaces(self, generator_config):
 294         src_config = {
 295             'chain_count': self.service_chain_count,
 296             'flow_count': self.flow_count / 2,
 297             'ip': generator_config.ip_addrs[0],
 298             'ip_addrs_step': self.ip_addrs_step,
 299             'gateway_ip': self.gateway_ips[0],
 300             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 301             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[0],
 302             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 303             'udp_src_port': generator_config.udp_src_port,
 304             'udp_dst_port': generator_config.udp_dst_port,
 305             'vlan_tagging': self.vlan_tagging
 306         }
 307         dst_config = {
 308             'chain_count': self.service_chain_count,
 309             'flow_count': self.flow_count / 2,
 310             'ip': generator_config.ip_addrs[1],
 311             'ip_addrs_step': self.ip_addrs_step,
 312             'gateway_ip': self.gateway_ips[1],
 313             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 314             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[1],
 315             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 316             'udp_src_port': generator_config.udp_src_port,
 317             'udp_dst_port': generator_config.udp_dst_port,
 318             'vlan_tagging': self.vlan_tagging
 319         }
 320
 321         self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))
 322         self.dst_device = Device(**dict(dst_config, **generator_config.interfaces[1]))
 323         self.src_device.set_destination(self.dst_device)
 324         self.dst_device.set_destination(self.src_device)
 325
 326         if self.service_chain == ChainType.EXT and not self.no_arp \
 327                 and self.src_device.ip_range_overlaps():
 328             raise Exception('Overlapping IP address ranges src=%s dst=%d flows=%d' %
 329                             self.src_device.ip,
 330                             self.dst_device.ip,
 331                             self.flow_count)
 332
 333     @property
 334     def devices(self):
 335         return [self.src_device, self.dst_device]
 336
 337     @property
 338     def vlans(self):
 339         return [self.src_device.vtep_vlan, self.dst_device.vtep_vlan]
 340
 341     @property
 342     def ports(self):
 343         return [self.src_device.port, self.dst_device.port]
 344
 345     @property
 346     def switch_ports(self):
 347         return [self.src_device.switch_port, self.dst_device.switch_port]
 348
 349     @property
 350     def pcis(self):
 351         return [self.src_device.pci, self.dst_device.pci]
 352
 353
 354 class TrafficGeneratorFactory(object):
 355     def __init__(self, config):
 356         self.config = config
 357
 358     def get_tool(self):
 359         return self.config.generator_config.tool
 360
 361     def get_generator_client(self):
 362         tool = self.get_tool().lower()
 363         if tool == 'trex':
 364             from traffic_gen import trex
 365             return trex.TRex(self.config)
 366         elif tool == 'dummy':
 367             from traffic_gen import dummy
 368             return dummy.DummyTG(self.config)
 369         return None
 370
 371     def list_generator_profile(self):
 372         return [profile.name for profile in self.config.traffic_generator.generator_profile]
 373
 374     def get_generator_config(self, generator_profile):
 375         return RunningTrafficProfile(self.config, generator_profile)
 376
 377     def get_matching_profile(self, traffic_profile_name):
 378         matching_profile = [profile for profile in self.config.traffic_profile if
 379                             profile.name == traffic_profile_name]
 380
 381         if len(matching_profile) > 1:
 382             raise Exception('Multiple traffic profiles with the same name found.')
 383         elif not matching_profile:
 384             raise Exception('No traffic profile found.')
 385
 386         return matching_profile[0]
 387
 388     def get_frame_sizes(self, traffic_profile):
 389         matching_profile = self.get_matching_profile(traffic_profile)
 390         return matching_profile.l2frame_size
 391
 392
 393 class TrafficClient(object):
 394     PORTS = [0, 1]
 395
 396     def __init__(self, config, notifier=None):
 397         generator_factory = TrafficGeneratorFactory(config)
 398         self.gen = generator_factory.get_generator_client()
 399         self.tool = generator_factory.get_tool()
 400         self.config = config
 401         self.notifier = notifier
 402         self.interval_collector = None
 403         self.iteration_collector = None
 404         self.runner = TrafficRunner(self, self.config.duration_sec, self.config.interval_sec)
 405         if self.gen is None:
 406             raise TrafficClientException('%s is not a supported traffic generator' % self.tool)
 407
 408         self.run_config = {
 409             'l2frame_size': None,
 410             'duration_sec': self.config.duration_sec,
 411             'bidirectional': True,
 412             'rates': []  # to avoid unsbuscriptable-obj warning
 413         }
 414         self.current_total_rate = {'rate_percent': '10'}
 415         if self.config.single_run:
 416             self.current_total_rate = utils.parse_rate_str(self.config.rate)
 417
 418     def set_macs(self):
 419         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
 420             device.set_mac(mac)
 421
 422     def start_traffic_generator(self):
 423         self.gen.init()
 424         self.gen.connect()
 425
 426     def setup(self):
 427         self.gen.set_mode()
 428         self.gen.config_interface()
 429         self.gen.clear_stats()
 430
 431     def get_version(self):
 432         return self.gen.get_version()
 433
 434     def ensure_end_to_end(self):
 435         """
 436         Ensure traffic generator receives packets it has transmitted.
 437         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 438
 439         At this point all VMs are in active state, but forwarding does not have to work.
 440         Small amount of traffic is sent to every chain. Then total of sent and received packets
 441         is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
 442         N being number of chains, traffic flows through every chain and real measurements can be
 443         performed.
 444
 445         Example:
 446             PVP chain (1 VM per chain)
 447             N = 10 (number of chains)
 448             threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
 449             if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
 450             all 10 VMs are in operational state.
 451         """
 452         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
 453         rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
 454         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 455
 456         # ensures enough traffic is coming back
 457         threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)
 458         retry_count = (self.config.check_traffic_time_sec +
 459                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
 460         for it in xrange(retry_count):
 461             self.gen.clear_stats()
 462             self.gen.start_traffic()
 463             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)
 464             time.sleep(self.config.generic_poll_sec)
 465             self.gen.stop_traffic()
 466             stats = self.gen.get_stats()
 467
 468             # compute total sent and received traffic on both ports
 469             total_rx = 0
 470             total_tx = 0
 471             for port in self.PORTS:
 472                 total_rx += float(stats[port]['rx'].get('total_pkts', 0))
 473                 total_tx += float(stats[port]['tx'].get('total_pkts', 0))
 474
 475             # how much of traffic came back
 476             ratio = total_rx / total_tx if total_tx else 0
 477
 478             if ratio > threshold:
 479                 self.gen.clear_stats()
 480                 self.gen.clear_streamblock()
 481                 LOG.info('End-to-end connectivity ensured')
 482                 return
 483
 484             time.sleep(self.config.generic_poll_sec)
 485
 486         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 487
 488     def ensure_arp_successful(self):
 489         if not self.gen.resolve_arp():
 490             raise TrafficClientException('ARP cannot be resolved')
 491
 492     def set_traffic(self, frame_size, bidirectional):
 493         self.run_config['bidirectional'] = bidirectional
 494         self.run_config['l2frame_size'] = frame_size
 495         self.run_config['rates'] = [self.get_per_direction_rate()]
 496         if bidirectional:
 497             self.run_config['rates'].append(self.get_per_direction_rate())
 498         else:
 499             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
 500             if unidir_reverse_pps > 0:
 501                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})
 502
 503         self.gen.clear_streamblock()
 504         self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)
 505
 506     def modify_load(self, load):
 507         self.current_total_rate = {'rate_percent': str(load)}
 508         rate_per_direction = self.get_per_direction_rate()
 509
 510         self.gen.modify_rate(rate_per_direction, False)
 511         self.run_config['rates'][0] = rate_per_direction
 512         if self.run_config['bidirectional']:
 513             self.gen.modify_rate(rate_per_direction, True)
 514             self.run_config['rates'][1] = rate_per_direction
 515
 516     def get_ndr_and_pdr(self):
 517         dst = 'Bidirectional' if self.run_config['bidirectional'] else 'Unidirectional'
 518         targets = {}
 519         if self.config.ndr_run:
 520             LOG.info('*** Searching NDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 521             targets['ndr'] = self.config.measurement.NDR
 522         if self.config.pdr_run:
 523             LOG.info('*** Searching PDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 524             targets['pdr'] = self.config.measurement.PDR
 525
 526         self.run_config['start_time'] = time.time()
 527         self.interval_collector = IntervalCollector(self.run_config['start_time'])
 528         self.interval_collector.attach_notifier(self.notifier)
 529         self.iteration_collector = IterationCollector(self.run_config['start_time'])
 530         results = {}
 531         self.__range_search(0.0, 200.0, targets, results)
 532
 533         results['iteration_stats'] = {
 534             'ndr_pdr': self.iteration_collector.get()
 535         }
 536
 537         if self.config.ndr_run:
 538             LOG.info('NDR load: %s', results['ndr']['rate_percent'])
 539             results['ndr']['time_taken_sec'] = \
 540                 results['ndr']['timestamp_sec'] - self.run_config['start_time']
 541             if self.config.pdr_run:
 542                 LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 543                 results['pdr']['time_taken_sec'] = \
 544                     results['pdr']['timestamp_sec'] - results['ndr']['timestamp_sec']
 545         else:
 546             LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 547             results['pdr']['time_taken_sec'] = \
 548                 results['pdr']['timestamp_sec'] - self.run_config['start_time']
 549         return results
 550
 551     def __get_dropped_rate(self, result):
 552         dropped_pkts = result['rx']['dropped_pkts']
 553         total_pkts = result['tx']['total_pkts']
 554         if not total_pkts:
 555             return float('inf')
 556         return float(dropped_pkts) / total_pkts * 100
 557
 558     def get_stats(self):
 559         stats = self.gen.get_stats()
 560         retDict = {'total_tx_rate': stats['total_tx_rate']}
 561         for port in self.PORTS:
 562             retDict[port] = {'tx': {}, 'rx': {}}
 563
 564         tx_keys = ['total_pkts', 'total_pkt_bytes', 'pkt_rate', 'pkt_bit_rate']
 565         rx_keys = tx_keys + ['dropped_pkts']
 566
 567         for port in self.PORTS:
 568             for key in tx_keys:
 569                 retDict[port]['tx'][key] = int(stats[port]['tx'][key])
 570             for key in rx_keys:
 571                 try:
 572                     retDict[port]['rx'][key] = int(stats[port]['rx'][key])
 573                 except ValueError:
 574                     retDict[port]['rx'][key] = 0
 575             retDict[port]['rx']['avg_delay_usec'] = cast_integer(
 576                 stats[port]['rx']['avg_delay_usec'])
 577             retDict[port]['rx']['min_delay_usec'] = cast_integer(
 578                 stats[port]['rx']['min_delay_usec'])
 579             retDict[port]['rx']['max_delay_usec'] = cast_integer(
 580                 stats[port]['rx']['max_delay_usec'])
 581             retDict[port]['drop_rate_percent'] = self.__get_dropped_rate(retDict[port])
 582
 583         ports = sorted(retDict.keys())
 584         if self.run_config['bidirectional']:
 585             retDict['overall'] = {'tx': {}, 'rx': {}}
 586             for key in tx_keys:
 587                 retDict['overall']['tx'][key] = \
 588                     retDict[ports[0]]['tx'][key] + retDict[ports[1]]['tx'][key]
 589             for key in rx_keys:
 590                 retDict['overall']['rx'][key] = \
 591                     retDict[ports[0]]['rx'][key] + retDict[ports[1]]['rx'][key]
 592             total_pkts = [retDict[ports[0]]['rx']['total_pkts'],
 593                           retDict[ports[1]]['rx']['total_pkts']]
 594             avg_delays = [retDict[ports[0]]['rx']['avg_delay_usec'],
 595                           retDict[ports[1]]['rx']['avg_delay_usec']]
 596             max_delays = [retDict[ports[0]]['rx']['max_delay_usec'],
 597                           retDict[ports[1]]['rx']['max_delay_usec']]
 598             min_delays = [retDict[ports[0]]['rx']['min_delay_usec'],
 599                           retDict[ports[1]]['rx']['min_delay_usec']]
 600             retDict['overall']['rx']['avg_delay_usec'] = utils.weighted_avg(total_pkts, avg_delays)
 601             retDict['overall']['rx']['min_delay_usec'] = min(min_delays)
 602             retDict['overall']['rx']['max_delay_usec'] = max(max_delays)
 603             for key in ['pkt_bit_rate', 'pkt_rate']:
 604                 for dirc in ['tx', 'rx']:
 605                     retDict['overall'][dirc][key] /= 2.0
 606         else:
 607             retDict['overall'] = retDict[ports[0]]
 608         retDict['overall']['drop_rate_percent'] = self.__get_dropped_rate(retDict['overall'])
 609         return retDict
 610
 611     def __convert_rates(self, rate):
 612         return utils.convert_rates(self.run_config['l2frame_size'],
 613                                    rate,
 614                                    self.config.generator_config.intf_speed)
 615
 616     def __ndr_pdr_found(self, tag, load):
 617         rates = self.__convert_rates({'rate_percent': load})
 618         self.iteration_collector.add_ndr_pdr(tag, rates['rate_pps'])
 619         last_stats = self.iteration_collector.peek()
 620         self.interval_collector.add_ndr_pdr(tag, last_stats)
 621
 622     def __format_output_stats(self, stats):
 623         for key in self.PORTS + ['overall']:
 624             interface = stats[key]
 625             stats[key] = {
 626                 'tx_pkts': interface['tx']['total_pkts'],
 627                 'rx_pkts': interface['rx']['total_pkts'],
 628                 'drop_percentage': interface['drop_rate_percent'],
 629                 'drop_pct': interface['rx']['dropped_pkts'],
 630                 'avg_delay_usec': interface['rx']['avg_delay_usec'],
 631                 'max_delay_usec': interface['rx']['max_delay_usec'],
 632                 'min_delay_usec': interface['rx']['min_delay_usec'],
 633             }
 634
 635         return stats
 636
 637     def __targets_found(self, rate, targets, results):
 638         for tag, target in targets.iteritems():
 639             LOG.info('Found %s (%s) load: %s', tag, target, rate)
 640             self.__ndr_pdr_found(tag, rate)
 641             results[tag]['timestamp_sec'] = time.time()
 642
 643     def __range_search(self, left, right, targets, results):
 644         '''Perform a binary search for a list of targets inside a [left..right] range or rate
 645
 646         left    the left side of the range to search as a % the line rate (100 = 100% line rate)
 647                 indicating the rate to send on each interface
 648         right   the right side of the range to search as a % of line rate
 649                 indicating the rate to send on each interface
 650         targets a dict of drop rates to search (0.1 = 0.1%), indexed by the DR name or "tag"
 651                 ('ndr', 'pdr')
 652         results a dict to store results
 653         '''
 654         if not targets:
 655             return
 656         LOG.info('Range search [%s .. %s] targets: %s', left, right, targets)
 657
 658         # Terminate search when gap is less than load epsilon
 659         if right - left < self.config.measurement.load_epsilon:
 660             self.__targets_found(left, targets, results)
 661             return
 662
 663         # Obtain the average drop rate in for middle load
 664         middle = (left + right) / 2.0
 665         try:
 666             stats, rates = self.__run_search_iteration(middle)
 667         except STLError:
 668             LOG.exception("Got exception from traffic generator during binary search")
 669             self.__targets_found(left, targets, results)
 670             return
 671         # Split target dicts based on the avg drop rate
 672         left_targets = {}
 673         right_targets = {}
 674         for tag, target in targets.iteritems():
 675             if stats['overall']['drop_rate_percent'] <= target:
 676                 # record the best possible rate found for this target
 677                 results[tag] = rates
 678                 results[tag].update({
 679                     'load_percent_per_direction': middle,
 680                     'stats': self.__format_output_stats(dict(stats)),
 681                     'timestamp_sec': None
 682                 })
 683                 right_targets[tag] = target
 684             else:
 685                 # initialize to 0 all fields of result for
 686                 # the worst case scenario of the binary search (if ndr/pdr is not found)
 687                 if tag not in results:
 688                     results[tag] = dict.fromkeys(rates, 0)
 689                     empty_stats = self.__format_output_stats(dict(stats))
 690                     for key in empty_stats:
 691                         if isinstance(empty_stats[key], dict):
 692                             empty_stats[key] = dict.fromkeys(empty_stats[key], 0)
 693                         else:
 694                             empty_stats[key] = 0
 695                     results[tag].update({
 696                         'load_percent_per_direction': 0,
 697                         'stats': empty_stats,
 698                         'timestamp_sec': None
 699                     })
 700                 left_targets[tag] = target
 701
 702         # search lower half
 703         self.__range_search(left, middle, left_targets, results)
 704
 705         # search upper half only if the upper rate does not exceed
 706         # 100%, this only happens when the first search at 100%
 707         # yields a DR that is < target DR
 708         if middle >= 100:
 709             self.__targets_found(100, right_targets, results)
 710         else:
 711             self.__range_search(middle, right, right_targets, results)
 712
 713     def __run_search_iteration(self, rate):
 714         # set load
 715         self.modify_load(rate)
 716
 717         # poll interval stats and collect them
 718         for stats in self.run_traffic():
 719             self.interval_collector.add(stats)
 720             time_elapsed_ratio = self.runner.time_elapsed() / self.run_config['duration_sec']
 721             if time_elapsed_ratio >= 1:
 722                 self.cancel_traffic()
 723         self.interval_collector.reset()
 724
 725         # get stats from the run
 726         stats = self.runner.client.get_stats()
 727         current_traffic_config = self.get_traffic_config()
 728         warning = self.compare_tx_rates(current_traffic_config['direction-total']['rate_pps'],
 729                                         stats['total_tx_rate'])
 730         if warning is not None:
 731             stats['warning'] = warning
 732
 733         # save reliable stats from whole iteration
 734         self.iteration_collector.add(stats, current_traffic_config['direction-total']['rate_pps'])
 735         LOG.info('Average drop rate: %f', stats['overall']['drop_rate_percent'])
 736
 737         return stats, current_traffic_config['direction-total']
 738
 739     @staticmethod
 740     def log_stats(stats):
 741         report = {
 742             'datetime': str(datetime.now()),
 743             'tx_packets': stats['overall']['tx']['total_pkts'],
 744             'rx_packets': stats['overall']['rx']['total_pkts'],
 745             'drop_packets': stats['overall']['rx']['dropped_pkts'],
 746             'drop_rate_percent': stats['overall']['drop_rate_percent']
 747         }
 748         LOG.info('TX: %(tx_packets)d; '
 749                  'RX: %(rx_packets)d; '
 750                  'Dropped: %(drop_packets)d; '
 751                  'Drop rate: %(drop_rate_percent).4f%%',
 752                  report)
 753
 754     def run_traffic(self):
 755         stats = self.runner.run()
 756         while self.runner.is_running:
 757             self.log_stats(stats)
 758             yield stats
 759             stats = self.runner.poll_stats()
 760             if stats is None:
 761                 return
 762         self.log_stats(stats)
 763         LOG.info('Drop rate: %f', stats['overall']['drop_rate_percent'])
 764         yield stats
 765
 766     def cancel_traffic(self):
 767         self.runner.stop()
 768
 769     def get_interface(self, port_index):
 770         port = self.gen.port_handle[port_index]
 771         tx, rx = 0, 0
 772         if not self.config.no_traffic:
 773             stats = self.get_stats()
 774             if port in stats:
 775                 tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])
 776         return Interface('traffic-generator', self.tool.lower(), tx, rx)
 777
 778     def get_traffic_config(self):
 779         config = {}
 780         load_total = 0.0
 781         bps_total = 0.0
 782         pps_total = 0.0
 783         for idx, rate in enumerate(self.run_config['rates']):
 784             key = 'direction-forward' if idx == 0 else 'direction-reverse'
 785             config[key] = {
 786                 'l2frame_size': self.run_config['l2frame_size'],
 787                 'duration_sec': self.run_config['duration_sec']
 788             }
 789             config[key].update(rate)
 790             config[key].update(self.__convert_rates(rate))
 791             load_total += float(config[key]['rate_percent'])
 792             bps_total += float(config[key]['rate_bps'])
 793             pps_total += float(config[key]['rate_pps'])
 794         config['direction-total'] = dict(config['direction-forward'])
 795         config['direction-total'].update({
 796             'rate_percent': load_total,
 797             'rate_pps': cast_integer(pps_total),
 798             'rate_bps': bps_total
 799         })
 800
 801         return config
 802
 803     def get_run_config(self, results):
 804         """Returns configuration which was used for the last run."""
 805         r = {}
 806         for idx, key in enumerate(["direction-forward", "direction-reverse"]):
 807             tx_rate = results["stats"][idx]["tx"]["total_pkts"] / self.config.duration_sec
 808             rx_rate = results["stats"][idx]["rx"]["total_pkts"] / self.config.duration_sec
 809             r[key] = {
 810                 "orig": self.__convert_rates(self.run_config['rates'][idx]),
 811                 "tx": self.__convert_rates({'rate_pps': tx_rate}),
 812                 "rx": self.__convert_rates({'rate_pps': rx_rate})
 813             }
 814
 815         total = {}
 816         for direction in ['orig', 'tx', 'rx']:
 817             total[direction] = {}
 818             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
 819
 820                 total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()])
 821
 822         r['direction-total'] = total
 823         return r
 824
 825     @staticmethod
 826     def compare_tx_rates(required, actual):
 827         threshold = 0.9
 828         are_different = False
 829         try:
 830             if float(actual) / required < threshold:
 831                 are_different = True
 832         except ZeroDivisionError:
 833             are_different = True
 834
 835         if are_different:
 836             msg = "WARNING: There is a significant difference between requested TX rate ({r}) " \
 837                   "and actual TX rate ({a}). The traffic generator may not have sufficient CPU " \
 838                   "to achieve the requested TX rate.".format(r=required, a=actual)
 839             LOG.info(msg)
 840             return msg
 841
 842         return None
 843
 844     def get_per_direction_rate(self):
 845         divisor = 2 if self.run_config['bidirectional'] else 1
 846         if 'rate_percent' in self.current_total_rate:
 847             # don't split rate if it's percentage
 848             divisor = 1
 849
 850         return utils.divide_rate(self.current_total_rate, divisor)
 851
 852     def close(self):
 853         try:
 854             self.gen.stop_traffic()
 855         except Exception:
 856             pass
 857         self.gen.clear_stats()
 858         self.gen.cleanup()