nfvbench/traffic_client.py

   1 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   2 #
   3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 #    not use this file except in compliance with the License. You may obtain
   5 #    a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #    Unless required by applicable law or agreed to in writing, software
  10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 #    License for the specific language governing permissions and limitations
  13 #    under the License.
  14
  15 from attrdict import AttrDict
  16 import bitmath
  17 from datetime import datetime
  18 from log import LOG
  19 from netaddr import IPNetwork
  20 from network import Interface
  21 import socket
  22 from specs import ChainType
  23 from stats_collector import IntervalCollector
  24 from stats_collector import IterationCollector
  25 import struct
  26 import time
  27 import traffic_gen.traffic_utils as utils
  28 from trex_stl_lib.api import STLError
  29 from utils import cast_integer
  30
  31
  32 class TrafficClientException(Exception):
  33     pass
  34
  35
  36 class TrafficRunner(object):
  37
  38     def __init__(self, client, duration_sec, interval_sec=0):
  39         self.client = client
  40         self.start_time = None
  41         self.duration_sec = duration_sec
  42         self.interval_sec = interval_sec
  43
  44     def run(self):
  45         LOG.info('Running traffic generator')
  46         self.client.gen.clear_stats()
  47         self.client.gen.start_traffic()
  48         self.start_time = time.time()
  49         return self.poll_stats()
  50
  51     def stop(self):
  52         if self.is_running():
  53             self.start_time = None
  54             self.client.gen.stop_traffic()
  55
  56     def is_running(self):
  57         return self.start_time is not None
  58
  59     def time_elapsed(self):
  60         if self.is_running():
  61             return time.time() - self.start_time
  62         else:
  63             return self.duration_sec
  64
  65     def poll_stats(self):
  66         if not self.is_running():
  67             return None
  68         time_elapsed = self.time_elapsed()
  69         if time_elapsed > self.duration_sec:
  70             self.stop()
  71             return None
  72         time_left = self.duration_sec - time_elapsed
  73         if self.interval_sec > 0.0:
  74             if time_left <= self.interval_sec:
  75                 time.sleep(time_left)
  76                 self.stop()
  77             else:
  78                 time.sleep(self.interval_sec)
  79         else:
  80             time.sleep(self.duration_sec)
  81             self.stop()
  82         return self.client.get_stats()
  83
  84
  85 class Device(object):
  86
  87     def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
  88                  gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
  89                  gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,
  90                  chain_count=1, flow_count=1, vlan_tagging=False):
  91         self.chain_count = chain_count
  92         self.flow_count = flow_count
  93         self.dst = None
  94         self.port = port
  95         self.switch_port = switch_port
  96         self.vtep_vlan = vtep_vlan
  97         self.vlan_tag = None
  98         self.vlan_tagging = vlan_tagging
  99         self.pci = pci
 100         self.mac = None
 101         self.vm_mac_list = None
 102         subnet = IPNetwork(ip)
 103         self.ip = subnet.ip.format()
 104         self.ip_prefixlen = subnet.prefixlen
 105         self.ip_addrs_step = ip_addrs_step
 106         self.tg_gateway_ip_addrs_step = tg_gateway_ip_addrs_step
 107         self.gateway_ip_addrs_step = gateway_ip_addrs_step
 108         self.ip_list = self.expand_ip(self.ip, self.ip_addrs_step, self.flow_count)
 109         self.gateway_ip = gateway_ip
 110         self.gateway_ip_list = self.expand_ip(self.gateway_ip,
 111                                               self.gateway_ip_addrs_step,
 112                                               self.chain_count)
 113         self.tg_gateway_ip = tg_gateway_ip
 114         self.tg_gateway_ip_list = self.expand_ip(self.tg_gateway_ip,
 115                                                  self.tg_gateway_ip_addrs_step,
 116                                                  self.chain_count)
 117         self.udp_src_port = udp_src_port
 118         self.udp_dst_port = udp_dst_port
 119
 120     def set_mac(self, mac):
 121         if mac is None:
 122             raise TrafficClientException('Trying to set traffic generator MAC address as None')
 123         self.mac = mac
 124
 125     def set_destination(self, dst):
 126         self.dst = dst
 127
 128     def set_vm_mac_list(self, vm_mac_list):
 129         self.vm_mac_list = map(str, vm_mac_list)
 130
 131     def set_vlan_tag(self, vlan_tag):
 132         if self.vlan_tagging and vlan_tag is None:
 133             raise TrafficClientException('Trying to set VLAN tag as None')
 134         self.vlan_tag = vlan_tag
 135
 136     def get_stream_configs(self, service_chain):
 137         configs = []
 138         flow_idx = 0
 139         for chain_idx in xrange(self.chain_count):
 140             current_flow_count = (self.flow_count - flow_idx) / (self.chain_count - chain_idx)
 141             max_idx = flow_idx + current_flow_count - 1
 142             ip_src_count = self.ip_to_int(self.ip_list[max_idx]) - \
 143                 self.ip_to_int(self.ip_list[flow_idx]) + 1
 144             ip_dst_count = self.ip_to_int(self.dst.ip_list[max_idx]) - \
 145                 self.ip_to_int(self.dst.ip_list[flow_idx]) + 1
 146
 147             configs.append({
 148                 'count': current_flow_count,
 149                 'mac_src': self.mac,
 150                 'mac_dst': self.dst.mac if service_chain == ChainType.EXT
 151                 else self.vm_mac_list[chain_idx],
 152                 'ip_src_addr': self.ip_list[flow_idx],
 153                 'ip_src_addr_max': self.ip_list[max_idx],
 154                 'ip_src_count': ip_src_count,
 155                 'ip_dst_addr': self.dst.ip_list[flow_idx],
 156                 'ip_dst_addr_max': self.dst.ip_list[max_idx],
 157                 'ip_dst_count': ip_dst_count,
 158                 'ip_addrs_step': self.ip_addrs_step,
 159                 'udp_src_port': self.udp_src_port,
 160                 'udp_dst_port': self.udp_dst_port,
 161                 'mac_discovery_gw': self.gateway_ip_list[chain_idx],
 162                 'ip_src_tg_gw': self.tg_gateway_ip_list[chain_idx],
 163                 'ip_dst_tg_gw': self.dst.tg_gateway_ip_list[chain_idx],
 164                 'vlan_tag': self.vlan_tag if self.vlan_tagging else None
 165             })
 166
 167             flow_idx += current_flow_count
 168         return configs
 169
 170     @classmethod
 171     def expand_ip(cls, ip, step_ip, count):
 172         if step_ip == 'random':
 173             # Repeatable Random will used in the stream src/dst IP pairs, but we still need
 174             # to expand the IP based on the number of chains and flows configured. So we use
 175             # "0.0.0.1" as the step to have the exact IP flow ranges for every chain.
 176             step_ip = '0.0.0.1'
 177
 178         step_ip_in_int = cls.ip_to_int(step_ip)
 179         subnet = IPNetwork(ip)
 180         ip_list = []
 181         for _ in xrange(count):
 182             ip_list.append(subnet.ip.format())
 183             subnet = subnet.next(step_ip_in_int)
 184         return ip_list
 185
 186     @staticmethod
 187     def mac_to_int(mac):
 188         return int(mac.translate(None, ":.- "), 16)
 189
 190     @staticmethod
 191     def int_to_mac(i):
 192         mac = format(i, 'x').zfill(12)
 193         blocks = [mac[x:x + 2] for x in xrange(0, len(mac), 2)]
 194         return ':'.join(blocks)
 195
 196     @staticmethod
 197     def ip_to_int(addr):
 198         return struct.unpack("!I", socket.inet_aton(addr))[0]
 199
 200
 201 class RunningTrafficProfile(object):
 202     """Represents traffic configuration for currently running traffic profile."""
 203
 204     DEFAULT_IP_STEP = '0.0.0.1'
 205     DEFAULT_SRC_DST_IP_STEP = '0.0.0.1'
 206
 207     def __init__(self, config, generator_profile):
 208         generator_config = self.__match_generator_profile(config.traffic_generator,
 209                                                           generator_profile)
 210         self.generator_config = generator_config
 211         self.service_chain = config.service_chain
 212         self.service_chain_count = config.service_chain_count
 213         self.flow_count = config.flow_count
 214         self.host_name = generator_config.host_name
 215         self.name = generator_config.name
 216         self.tool = generator_config.tool
 217         self.cores = generator_config.get('cores', 1)
 218         self.ip_addrs_step = generator_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
 219         self.tg_gateway_ip_addrs_step = \
 220             generator_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 221         self.gateway_ip_addrs_step = generator_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 222         self.gateway_ips = generator_config.gateway_ip_addrs
 223         self.ip = generator_config.ip
 224         self.intf_speed = bitmath.parse_string(generator_config.intf_speed.replace('ps', '')).bits
 225         self.vlan_tagging = config.vlan_tagging
 226         self.no_arp = config.no_arp
 227         self.src_device = None
 228         self.dst_device = None
 229         self.vm_mac_list = None
 230         self.__prep_interfaces(generator_config)
 231
 232     def to_json(self):
 233         return dict(self.generator_config)
 234
 235     def set_vm_mac_list(self, vm_mac_list):
 236         self.src_device.set_vm_mac_list(vm_mac_list[0])
 237         self.dst_device.set_vm_mac_list(vm_mac_list[1])
 238
 239     @staticmethod
 240     def __match_generator_profile(traffic_generator, generator_profile):
 241         generator_config = AttrDict(traffic_generator)
 242         generator_config.pop('default_profile')
 243         generator_config.pop('generator_profile')
 244         matching_profile = filter(lambda profile: profile.name == generator_profile,
 245                                   traffic_generator.generator_profile)
 246         if len(matching_profile) != 1:
 247             raise Exception('Traffic generator profile not found: ' + generator_profile)
 248
 249         generator_config.update(matching_profile[0])
 250
 251         return generator_config
 252
 253     def __prep_interfaces(self, generator_config):
 254         src_config = {
 255             'chain_count': self.service_chain_count,
 256             'flow_count': self.flow_count / 2,
 257             'ip': generator_config.ip_addrs[0],
 258             'ip_addrs_step': self.ip_addrs_step,
 259             'gateway_ip': self.gateway_ips[0],
 260             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 261             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[0],
 262             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 263             'udp_src_port': generator_config.udp_src_port,
 264             'udp_dst_port': generator_config.udp_dst_port,
 265             'vlan_tagging': self.vlan_tagging
 266         }
 267         dst_config = {
 268             'chain_count': self.service_chain_count,
 269             'flow_count': self.flow_count / 2,
 270             'ip': generator_config.ip_addrs[1],
 271             'ip_addrs_step': self.ip_addrs_step,
 272             'gateway_ip': self.gateway_ips[1],
 273             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 274             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[1],
 275             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 276             'udp_src_port': generator_config.udp_src_port,
 277             'udp_dst_port': generator_config.udp_dst_port,
 278             'vlan_tagging': self.vlan_tagging
 279         }
 280
 281         self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))
 282         self.dst_device = Device(**dict(dst_config, **generator_config.interfaces[1]))
 283         self.src_device.set_destination(self.dst_device)
 284         self.dst_device.set_destination(self.src_device)
 285
 286         if self.service_chain == ChainType.EXT and not self.no_arp \
 287                 and not self.__are_unique(self.src_device.ip_list, self.dst_device.ip_list):
 288             raise Exception('Computed IP addresses are not unique, choose different base. '
 289                             'Start IPs: {start}. End IPs: {end}'
 290                             .format(start=self.src_device.ip_list,
 291                                     end=self.dst_device.ip_list))
 292
 293     def __are_unique(self, list1, list2):
 294         return set(list1).isdisjoint(set(list2))
 295
 296     @property
 297     def devices(self):
 298         return [self.src_device, self.dst_device]
 299
 300     @property
 301     def vlans(self):
 302         return [self.src_device.vtep_vlan, self.dst_device.vtep_vlan]
 303
 304     @property
 305     def ports(self):
 306         return [self.src_device.port, self.dst_device.port]
 307
 308     @property
 309     def switch_ports(self):
 310         return [self.src_device.switch_port, self.dst_device.switch_port]
 311
 312     @property
 313     def pcis(self):
 314         return [self.src_device.pci, self.dst_device.pci]
 315
 316
 317 class TrafficGeneratorFactory(object):
 318
 319     def __init__(self, config):
 320         self.config = config
 321
 322     def get_tool(self):
 323         return self.config.generator_config.tool
 324
 325     def get_generator_client(self):
 326         tool = self.get_tool().lower()
 327         if tool == 'trex':
 328             from traffic_gen import trex
 329             return trex.TRex(self.config)
 330         elif tool == 'dummy':
 331             from traffic_gen import dummy
 332             return dummy.DummyTG(self.config)
 333         else:
 334             return None
 335
 336     def list_generator_profile(self):
 337         return [profile.name for profile in self.config.traffic_generator.generator_profile]
 338
 339     def get_generator_config(self, generator_profile):
 340         return RunningTrafficProfile(self.config, generator_profile)
 341
 342     def get_matching_profile(self, traffic_profile_name):
 343         matching_profile = filter(lambda profile: profile.name == traffic_profile_name,
 344                                   self.config.traffic_profile)
 345
 346         if len(matching_profile) > 1:
 347             raise Exception('Multiple traffic profiles with the same name found.')
 348         elif len(matching_profile) == 0:
 349             raise Exception('No traffic profile found.')
 350
 351         return matching_profile[0]
 352
 353     def get_frame_sizes(self, traffic_profile):
 354         matching_profile = self.get_matching_profile(traffic_profile)
 355         return matching_profile.l2frame_size
 356
 357
 358 class TrafficClient(object):
 359
 360     PORTS = [0, 1]
 361
 362     def __init__(self, config, notifier=None):
 363         generator_factory = TrafficGeneratorFactory(config)
 364         self.gen = generator_factory.get_generator_client()
 365         self.tool = generator_factory.get_tool()
 366         self.config = config
 367         self.notifier = notifier
 368         self.interval_collector = None
 369         self.iteration_collector = None
 370         self.runner = TrafficRunner(self, self.config.duration_sec, self.config.interval_sec)
 371         if self.gen is None:
 372             raise TrafficClientException('%s is not a supported traffic generator' % self.tool)
 373
 374         self.run_config = {
 375             'l2frame_size': None,
 376             'duration_sec': self.config.duration_sec,
 377             'bidirectional': True,
 378             'rates': None
 379         }
 380         self.current_total_rate = {'rate_percent': '10'}
 381         if self.config.single_run:
 382             self.current_total_rate = utils.parse_rate_str(self.config.rate)
 383
 384     def set_macs(self):
 385         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
 386             device.set_mac(mac)
 387
 388     def start_traffic_generator(self):
 389         self.gen.init()
 390         self.gen.connect()
 391
 392     def setup(self):
 393         self.gen.set_mode()
 394         self.gen.config_interface()
 395         self.gen.clear_stats()
 396
 397     def get_version(self):
 398         return self.gen.get_version()
 399
 400     def ensure_end_to_end(self):
 401         """
 402         Ensure traffic generator receives packets it has transmitted.
 403         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 404
 405         At this point all VMs are in active state, but forwarding does not have to work.
 406         Small amount of traffic is sent to every chain. Then total of sent and received packets
 407         is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
 408         N being number of chains, traffic flows through every chain and real measurements can be
 409         performed.
 410
 411         Example:
 412             PVP chain (1 VM per chain)
 413             N = 10 (number of chains)
 414             threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
 415             if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
 416             all 10 VMs are in operational state.
 417         """
 418         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
 419         rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
 420         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 421
 422         # ensures enough traffic is coming back
 423         threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)
 424         retry_count = (self.config.check_traffic_time_sec +
 425                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
 426         for it in xrange(retry_count):
 427             self.gen.clear_stats()
 428             self.gen.start_traffic()
 429             LOG.info('Waiting for packets to be received back... ({} / {})'.format(it + 1,
 430                      retry_count))
 431             time.sleep(self.config.generic_poll_sec)
 432             self.gen.stop_traffic()
 433             stats = self.gen.get_stats()
 434
 435             # compute total sent and received traffic on both ports
 436             total_rx = 0
 437             total_tx = 0
 438             for port in self.PORTS:
 439                 total_rx += float(stats[port]['rx'].get('total_pkts', 0))
 440                 total_tx += float(stats[port]['tx'].get('total_pkts', 0))
 441
 442             # how much of traffic came back
 443             ratio = total_rx / total_tx if total_tx else 0
 444
 445             if ratio > threshold:
 446                 self.gen.clear_stats()
 447                 self.gen.clear_streamblock()
 448                 LOG.info('End-to-end connectivity ensured')
 449                 return
 450
 451             time.sleep(self.config.generic_poll_sec)
 452
 453         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 454
 455     def ensure_arp_successful(self):
 456         if not self.gen.resolve_arp():
 457             raise TrafficClientException('ARP cannot be resolved')
 458
 459     def set_traffic(self, frame_size, bidirectional):
 460         self.run_config['bidirectional'] = bidirectional
 461         self.run_config['l2frame_size'] = frame_size
 462         self.run_config['rates'] = [self.get_per_direction_rate()]
 463         if bidirectional:
 464             self.run_config['rates'].append(self.get_per_direction_rate())
 465         else:
 466             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
 467             if unidir_reverse_pps > 0:
 468                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})
 469
 470         self.gen.clear_streamblock()
 471         self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)
 472
 473     def modify_load(self, load):
 474         self.current_total_rate = {'rate_percent': str(load)}
 475         rate_per_direction = self.get_per_direction_rate()
 476
 477         self.gen.modify_rate(rate_per_direction, False)
 478         self.run_config['rates'][0] = rate_per_direction
 479         if self.run_config['bidirectional']:
 480             self.gen.modify_rate(rate_per_direction, True)
 481             self.run_config['rates'][1] = rate_per_direction
 482
 483     def get_ndr_and_pdr(self):
 484         dst = 'Bidirectional' if self.run_config['bidirectional'] else 'Unidirectional'
 485         targets = {}
 486         if self.config.ndr_run:
 487             LOG.info('*** Searching NDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 488             targets['ndr'] = self.config.measurement.NDR
 489         if self.config.pdr_run:
 490             LOG.info('*** Searching PDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 491             targets['pdr'] = self.config.measurement.PDR
 492
 493         self.run_config['start_time'] = time.time()
 494         self.interval_collector = IntervalCollector(self.run_config['start_time'])
 495         self.interval_collector.attach_notifier(self.notifier)
 496         self.iteration_collector = IterationCollector(self.run_config['start_time'])
 497         results = {}
 498         self.__range_search(0.0, 200.0, targets, results)
 499
 500         results['iteration_stats'] = {
 501             'ndr_pdr': self.iteration_collector.get()
 502         }
 503
 504         if self.config.ndr_run:
 505             LOG.info('NDR load: %s', results['ndr']['rate_percent'])
 506             results['ndr']['time_taken_sec'] = \
 507                 results['ndr']['timestamp_sec'] - self.run_config['start_time']
 508             if self.config.pdr_run:
 509                 LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 510                 results['pdr']['time_taken_sec'] = \
 511                     results['pdr']['timestamp_sec'] - results['ndr']['timestamp_sec']
 512         else:
 513             LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 514             results['pdr']['time_taken_sec'] = \
 515                 results['pdr']['timestamp_sec'] - self.run_config['start_time']
 516         return results
 517
 518     def __get_dropped_rate(self, result):
 519         dropped_pkts = result['rx']['dropped_pkts']
 520         total_pkts = result['tx']['total_pkts']
 521         if not total_pkts:
 522             return float('inf')
 523         else:
 524             return float(dropped_pkts) / total_pkts * 100
 525
 526     def get_stats(self):
 527         stats = self.gen.get_stats()
 528         retDict = {'total_tx_rate': stats['total_tx_rate']}
 529         for port in self.PORTS:
 530             retDict[port] = {'tx': {}, 'rx': {}}
 531
 532         tx_keys = ['total_pkts', 'total_pkt_bytes', 'pkt_rate', 'pkt_bit_rate']
 533         rx_keys = tx_keys + ['dropped_pkts']
 534
 535         for port in self.PORTS:
 536             for key in tx_keys:
 537                 retDict[port]['tx'][key] = int(stats[port]['tx'][key])
 538             for key in rx_keys:
 539                 try:
 540                     retDict[port]['rx'][key] = int(stats[port]['rx'][key])
 541                 except ValueError:
 542                     retDict[port]['rx'][key] = 0
 543             retDict[port]['rx']['avg_delay_usec'] = cast_integer(
 544                 stats[port]['rx']['avg_delay_usec'])
 545             retDict[port]['rx']['min_delay_usec'] = cast_integer(
 546                 stats[port]['rx']['min_delay_usec'])
 547             retDict[port]['rx']['max_delay_usec'] = cast_integer(
 548                 stats[port]['rx']['max_delay_usec'])
 549             retDict[port]['drop_rate_percent'] = self.__get_dropped_rate(retDict[port])
 550
 551         ports = sorted(retDict.keys())
 552         if self.run_config['bidirectional']:
 553             retDict['overall'] = {'tx': {}, 'rx': {}}
 554             for key in tx_keys:
 555                 retDict['overall']['tx'][key] = \
 556                     retDict[ports[0]]['tx'][key] + retDict[ports[1]]['tx'][key]
 557             for key in rx_keys:
 558                 retDict['overall']['rx'][key] = \
 559                     retDict[ports[0]]['rx'][key] + retDict[ports[1]]['rx'][key]
 560             total_pkts = [retDict[ports[0]]['rx']['total_pkts'],
 561                           retDict[ports[1]]['rx']['total_pkts']]
 562             avg_delays = [retDict[ports[0]]['rx']['avg_delay_usec'],
 563                           retDict[ports[1]]['rx']['avg_delay_usec']]
 564             max_delays = [retDict[ports[0]]['rx']['max_delay_usec'],
 565                           retDict[ports[1]]['rx']['max_delay_usec']]
 566             min_delays = [retDict[ports[0]]['rx']['min_delay_usec'],
 567                           retDict[ports[1]]['rx']['min_delay_usec']]
 568             retDict['overall']['rx']['avg_delay_usec'] = utils.weighted_avg(total_pkts, avg_delays)
 569             retDict['overall']['rx']['min_delay_usec'] = min(min_delays)
 570             retDict['overall']['rx']['max_delay_usec'] = max(max_delays)
 571             for key in ['pkt_bit_rate', 'pkt_rate']:
 572                 for dirc in ['tx', 'rx']:
 573                     retDict['overall'][dirc][key] /= 2.0
 574         else:
 575             retDict['overall'] = retDict[ports[0]]
 576         retDict['overall']['drop_rate_percent'] = self.__get_dropped_rate(retDict['overall'])
 577         return retDict
 578
 579     def __convert_rates(self, rate):
 580         return utils.convert_rates(self.run_config['l2frame_size'],
 581                                    rate,
 582                                    self.config.generator_config.intf_speed)
 583
 584     def __ndr_pdr_found(self, tag, load):
 585         rates = self.__convert_rates({'rate_percent': load})
 586         self.iteration_collector.add_ndr_pdr(tag, rates['rate_pps'])
 587         last_stats = self.iteration_collector.peek()
 588         self.interval_collector.add_ndr_pdr(tag, last_stats)
 589
 590     def __format_output_stats(self, stats):
 591         for key in (self.PORTS + ['overall']):
 592             interface = stats[key]
 593             stats[key] = {
 594                 'tx_pkts': interface['tx']['total_pkts'],
 595                 'rx_pkts': interface['rx']['total_pkts'],
 596                 'drop_percentage': interface['drop_rate_percent'],
 597                 'drop_pct': interface['rx']['dropped_pkts'],
 598                 'avg_delay_usec': interface['rx']['avg_delay_usec'],
 599                 'max_delay_usec': interface['rx']['max_delay_usec'],
 600                 'min_delay_usec': interface['rx']['min_delay_usec'],
 601             }
 602
 603         return stats
 604
 605     def __targets_found(self, rate, targets, results):
 606         for tag, target in targets.iteritems():
 607             LOG.info('Found {} ({}) load: {}'.format(tag, target, rate))
 608             self.__ndr_pdr_found(tag, rate)
 609             results[tag]['timestamp_sec'] = time.time()
 610
 611     def __range_search(self, left, right, targets, results):
 612         '''Perform a binary search for a list of targets inside a [left..right] range or rate
 613
 614         left    the left side of the range to search as a % the line rate (100 = 100% line rate)
 615                 indicating the rate to send on each interface
 616         right   the right side of the range to search as a % of line rate
 617                 indicating the rate to send on each interface
 618         targets a dict of drop rates to search (0.1 = 0.1%), indexed by the DR name or "tag"
 619                 ('ndr', 'pdr')
 620         results a dict to store results
 621         '''
 622         if len(targets) == 0:
 623             return
 624         LOG.info('Range search [{} .. {}] targets: {}'.format(left, right, targets))
 625
 626         # Terminate search when gap is less than load epsilon
 627         if right - left < self.config.measurement.load_epsilon:
 628             self.__targets_found(left, targets, results)
 629             return
 630
 631         # Obtain the average drop rate in for middle load
 632         middle = (left + right) / 2.0
 633         try:
 634             stats, rates = self.__run_search_iteration(middle)
 635         except STLError:
 636             LOG.exception("Got exception from traffic generator during binary search")
 637             self.__targets_found(left, targets, results)
 638             return
 639         # Split target dicts based on the avg drop rate
 640         left_targets = {}
 641         right_targets = {}
 642         for tag, target in targets.iteritems():
 643             if stats['overall']['drop_rate_percent'] <= target:
 644                 # record the best possible rate found for this target
 645                 results[tag] = rates
 646                 results[tag].update({
 647                     'load_percent_per_direction': middle,
 648                     'stats': self.__format_output_stats(dict(stats)),
 649                     'timestamp_sec': None
 650                 })
 651                 right_targets[tag] = target
 652             else:
 653                 # initialize to 0 all fields of result for
 654                 # the worst case scenario of the binary search (if ndr/pdr is not found)
 655                 if tag not in results:
 656                     results[tag] = dict.fromkeys(rates, 0)
 657                     empty_stats = self.__format_output_stats(dict(stats))
 658                     for key in empty_stats:
 659                         if isinstance(empty_stats[key], dict):
 660                             empty_stats[key] = dict.fromkeys(empty_stats[key], 0)
 661                         else:
 662                             empty_stats[key] = 0
 663                     results[tag].update({
 664                         'load_percent_per_direction': 0,
 665                         'stats': empty_stats,
 666                         'timestamp_sec': None
 667                     })
 668                 left_targets[tag] = target
 669
 670         # search lower half
 671         self.__range_search(left, middle, left_targets, results)
 672
 673         # search upper half only if the upper rate does not exceed
 674         # 100%, this only happens when the first search at 100%
 675         # yields a DR that is < target DR
 676         if middle >= 100:
 677             self.__targets_found(100, right_targets, results)
 678         else:
 679             self.__range_search(middle, right, right_targets, results)
 680
 681     def __run_search_iteration(self, rate):
 682         # set load
 683         self.modify_load(rate)
 684
 685         # poll interval stats and collect them
 686         for stats in self.run_traffic():
 687             self.interval_collector.add(stats)
 688             time_elapsed_ratio = self.runner.time_elapsed() / self.run_config['duration_sec']
 689             if time_elapsed_ratio >= 1:
 690                 self.cancel_traffic()
 691         self.interval_collector.reset()
 692
 693         # get stats from the run
 694         stats = self.runner.client.get_stats()
 695         current_traffic_config = self.get_traffic_config()
 696         warning = self.compare_tx_rates(current_traffic_config['direction-total']['rate_pps'],
 697                                         stats['total_tx_rate'])
 698         if warning is not None:
 699             stats['warning'] = warning
 700
 701         # save reliable stats from whole iteration
 702         self.iteration_collector.add(stats, current_traffic_config['direction-total']['rate_pps'])
 703         LOG.info('Average drop rate: {}'.format(stats['overall']['drop_rate_percent']))
 704
 705         return stats, current_traffic_config['direction-total']
 706
 707     @staticmethod
 708     def log_stats(stats):
 709         report = {
 710             'datetime': str(datetime.now()),
 711             'tx_packets': stats['overall']['tx']['total_pkts'],
 712             'rx_packets': stats['overall']['rx']['total_pkts'],
 713             'drop_packets': stats['overall']['rx']['dropped_pkts'],
 714             'drop_rate_percent': stats['overall']['drop_rate_percent']
 715         }
 716         LOG.info('TX: %(tx_packets)d; '
 717                  'RX: %(rx_packets)d; '
 718                  'Dropped: %(drop_packets)d; '
 719                  'Drop rate: %(drop_rate_percent).4f%%',
 720                  report)
 721
 722     def run_traffic(self):
 723         stats = self.runner.run()
 724         while self.runner.is_running:
 725             self.log_stats(stats)
 726             yield stats
 727             stats = self.runner.poll_stats()
 728             if stats is None:
 729                 return
 730         self.log_stats(stats)
 731         LOG.info('Drop rate: {}'.format(stats['overall']['drop_rate_percent']))
 732         yield stats
 733
 734     def cancel_traffic(self):
 735         self.runner.stop()
 736
 737     def get_interface(self, port_index):
 738         port = self.gen.port_handle[port_index]
 739         tx, rx = 0, 0
 740         if not self.config.no_traffic:
 741             stats = self.get_stats()
 742             if port in stats:
 743                 tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])
 744         return Interface('traffic-generator', self.tool.lower(), tx, rx)
 745
 746     def get_traffic_config(self):
 747         config = {}
 748         load_total = 0.0
 749         bps_total = 0.0
 750         pps_total = 0.0
 751         for idx, rate in enumerate(self.run_config['rates']):
 752             key = 'direction-forward' if idx == 0 else 'direction-reverse'
 753             config[key] = {
 754                 'l2frame_size': self.run_config['l2frame_size'],
 755                 'duration_sec': self.run_config['duration_sec']
 756             }
 757             config[key].update(rate)
 758             config[key].update(self.__convert_rates(rate))
 759             load_total += float(config[key]['rate_percent'])
 760             bps_total += float(config[key]['rate_bps'])
 761             pps_total += float(config[key]['rate_pps'])
 762         config['direction-total'] = dict(config['direction-forward'])
 763         config['direction-total'].update({
 764             'rate_percent': load_total,
 765             'rate_pps': cast_integer(pps_total),
 766             'rate_bps': bps_total
 767         })
 768
 769         return config
 770
 771     def get_run_config(self, results):
 772         """Returns configuration which was used for the last run."""
 773         r = {}
 774         for idx, key in enumerate(["direction-forward", "direction-reverse"]):
 775             tx_rate = results["stats"][idx]["tx"]["total_pkts"] / self.config.duration_sec
 776             rx_rate = results["stats"][idx]["rx"]["total_pkts"] / self.config.duration_sec
 777             r[key] = {
 778                 "orig": self.__convert_rates(self.run_config['rates'][idx]),
 779                 "tx": self.__convert_rates({'rate_pps': tx_rate}),
 780                 "rx": self.__convert_rates({'rate_pps': rx_rate})
 781             }
 782
 783         total = {}
 784         for direction in ['orig', 'tx', 'rx']:
 785             total[direction] = {}
 786             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
 787                 total[direction][unit] = sum(map(lambda x: float(x[direction][unit]), r.values()))
 788
 789         r['direction-total'] = total
 790         return r
 791
 792     @staticmethod
 793     def compare_tx_rates(required, actual):
 794         threshold = 0.9
 795         are_different = False
 796         try:
 797             if float(actual) / required < threshold:
 798                 are_different = True
 799         except ZeroDivisionError:
 800             are_different = True
 801
 802         if are_different:
 803             msg = "WARNING: There is a significant difference between requested TX rate ({r}) " \
 804                   "and actual TX rate ({a}). The traffic generator may not have sufficient CPU " \
 805                   "to achieve the requested TX rate.".format(r=required, a=actual)
 806             LOG.info(msg)
 807             return msg
 808
 809         return None
 810
 811     def get_per_direction_rate(self):
 812         divisor = 2 if self.run_config['bidirectional'] else 1
 813         if 'rate_percent' in self.current_total_rate:
 814             # don't split rate if it's percentage
 815             divisor = 1
 816
 817         return utils.divide_rate(self.current_total_rate, divisor)
 818
 819     def close(self):
 820         try:
 821             self.gen.stop_traffic()
 822         except Exception:
 823             pass
 824         self.gen.clear_stats()
 825         self.gen.cleanup()