nfvbench/traffic_client.py

   1 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   2 #
   3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 #    not use this file except in compliance with the License. You may obtain
   5 #    a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #    Unless required by applicable law or agreed to in writing, software
  10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 #    License for the specific language governing permissions and limitations
  13 #    under the License.
  14
  15 from attrdict import AttrDict
  16 import bitmath
  17 from datetime import datetime
  18 from log import LOG
  19 from netaddr import IPNetwork
  20 from network import Interface
  21 import socket
  22 from specs import ChainType
  23 from stats_collector import IntervalCollector
  24 from stats_collector import IterationCollector
  25 import struct
  26 import time
  27 import traffic_gen.traffic_utils as utils
  28 from trex_stl_lib.api import STLError
  29 from utils import cast_integer
  30
  31
  32 class TrafficClientException(Exception):
  33     pass
  34
  35
  36 class TrafficRunner(object):
  37
  38     def __init__(self, client, duration_sec, interval_sec=0):
  39         self.client = client
  40         self.start_time = None
  41         self.duration_sec = duration_sec
  42         self.interval_sec = interval_sec
  43
  44     def run(self):
  45         LOG.info('Running traffic generator')
  46         self.client.gen.clear_stats()
  47         self.client.gen.start_traffic()
  48         self.start_time = time.time()
  49         return self.poll_stats()
  50
  51     def stop(self):
  52         if self.is_running():
  53             self.start_time = None
  54             self.client.gen.stop_traffic()
  55
  56     def is_running(self):
  57         return self.start_time is not None
  58
  59     def time_elapsed(self):
  60         if self.is_running():
  61             return time.time() - self.start_time
  62         else:
  63             return self.duration_sec
  64
  65     def poll_stats(self):
  66         if not self.is_running():
  67             return None
  68         time_elapsed = self.time_elapsed()
  69         if time_elapsed > self.duration_sec:
  70             self.stop()
  71             return None
  72         time_left = self.duration_sec - time_elapsed
  73         if self.interval_sec > 0.0:
  74             if time_left <= self.interval_sec:
  75                 time.sleep(time_left)
  76                 self.stop()
  77             else:
  78                 time.sleep(self.interval_sec)
  79         else:
  80             time.sleep(self.duration_sec)
  81             self.stop()
  82         return self.client.get_stats()
  83
  84 class IpBlock(object):
  85     def __init__(self, base_ip, step_ip, count_ip):
  86         self.base_ip_int = Device.ip_to_int(base_ip)
  87         self.step = Device.ip_to_int(step_ip)
  88         self.max_available = count_ip
  89         self.next_free = 0
  90
  91     def get_ip(self, index=0):
  92         '''Return the IP address at given index
  93         '''
  94         if index < 0 or index >= self.max_available:
  95             raise IndexError('Index out of bounds')
  96         return Device.int_to_ip(self.base_ip_int + index * self.step)
  97
  98     def reserve_ip_range(self, count):
  99         '''Reserve a range of count consecutive IP addresses spaced by step
 100         '''
 101         if self.next_free + count > self.max_available:
 102             raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d',
 103                               self.next_free,
 104                               self.max_available,
 105                               count)
 106         first_ip = self.get_ip(self.next_free)
 107         last_ip = self.get_ip(self.next_free + count - 1)
 108         self.next_free += count
 109         return (first_ip, last_ip)
 110
 111     def reset_reservation(self):
 112         self.next_free = 0
 113
 114 class Device(object):
 115
 116     def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
 117                  gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
 118                  gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,
 119                  chain_count=1, flow_count=1, vlan_tagging=False):
 120         self.chain_count = chain_count
 121         self.flow_count = flow_count
 122         self.dst = None
 123         self.port = port
 124         self.switch_port = switch_port
 125         self.vtep_vlan = vtep_vlan
 126         self.vlan_tag = None
 127         self.vlan_tagging = vlan_tagging
 128         self.pci = pci
 129         self.mac = None
 130         self.vm_mac_list = None
 131         subnet = IPNetwork(ip)
 132         self.ip = subnet.ip.format()
 133         self.ip_prefixlen = subnet.prefixlen
 134         self.ip_addrs_step = ip_addrs_step
 135         self.tg_gateway_ip_addrs_step = tg_gateway_ip_addrs_step
 136         self.gateway_ip_addrs_step = gateway_ip_addrs_step
 137         self.gateway_ip = gateway_ip
 138         self.tg_gateway_ip = tg_gateway_ip
 139         self.ip_block = IpBlock(self.ip, ip_addrs_step, flow_count)
 140         self.gw_ip_block = IpBlock(gateway_ip,
 141                                    gateway_ip_addrs_step,
 142                                    chain_count)
 143         self.tg_gw_ip_block = IpBlock(tg_gateway_ip,
 144                                       tg_gateway_ip_addrs_step,
 145                                       chain_count)
 146         self.udp_src_port = udp_src_port
 147         self.udp_dst_port = udp_dst_port
 148
 149     def set_mac(self, mac):
 150         if mac is None:
 151             raise TrafficClientException('Trying to set traffic generator MAC address as None')
 152         self.mac = mac
 153
 154     def set_destination(self, dst):
 155         self.dst = dst
 156
 157     def set_vm_mac_list(self, vm_mac_list):
 158         self.vm_mac_list = map(str, vm_mac_list)
 159
 160     def set_vlan_tag(self, vlan_tag):
 161         if self.vlan_tagging and vlan_tag is None:
 162             raise TrafficClientException('Trying to set VLAN tag as None')
 163         self.vlan_tag = vlan_tag
 164
 165     def get_gw_ip(self, chain_index):
 166         '''Retrieve the IP address assigned for the gateway of a given chain
 167         '''
 168         return self.gw_ip_block.get_ip(chain_index)
 169
 170     def get_stream_configs(self, service_chain):
 171         configs = []
 172         # exact flow count for each chain is calculated as follows:
 173         # - all chains except the first will have the same flow count
 174         #   calculated as (total_flows + chain_count - 1) / chain_count
 175         # - the first chain will have the remainder
 176         # example 11 flows and 3 chains => 3, 4, 4
 177         flows_per_chain = (self.flow_count + self.chain_count -1) / self.chain_count
 178         cur_chain_flow_count = self.flow_count - flows_per_chain * (self.chain_count - 1)
 179
 180         self.ip_block.reset_reservation()
 181         self.dst.ip_block.reset_reservation()
 182
 183         for chain_idx in xrange(self.chain_count):
 184             src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count)
 185             dst_ip_first, dst_ip_last = self.dst.ip_block.reserve_ip_range(cur_chain_flow_count)
 186             configs.append({
 187                 'count': cur_chain_flow_count,
 188                 'mac_src': self.mac,
 189                 'mac_dst': self.dst.mac if service_chain == ChainType.EXT
 190                 else self.vm_mac_list[chain_idx],
 191                 'ip_src_addr': src_ip_first,
 192                 'ip_src_addr_max': src_ip_last,
 193                 'ip_src_count': cur_chain_flow_count,
 194                 'ip_dst_addr': dst_ip_first,
 195                 'ip_dst_addr_max': dst_ip_last,
 196                 'ip_dst_count': cur_chain_flow_count,
 197                 'ip_addrs_step': self.ip_addrs_step,
 198                 'udp_src_port': self.udp_src_port,
 199                 'udp_dst_port': self.udp_dst_port,
 200                 'mac_discovery_gw': self.get_gw_ip(chain_idx),
 201                 'ip_src_tg_gw': self.tg_gw_ip_block.get_ip(chain_idx),
 202                 'ip_dst_tg_gw': self.dst.tg_gw_ip_block.get_ip(chain_idx),
 203                 'vlan_tag': self.vlan_tag if self.vlan_tagging else None
 204             })
 205             # after first chain, fall back to the flow count for all other chains
 206             cur_chain_flow_count = flows_per_chain
 207
 208         return configs
 209
 210     def ip_range_overlaps(self):
 211         '''Check if this device ip range is overlapping with the dst device ip range
 212         '''
 213         src_base_ip = Device.ip_to_int(self.ip)
 214         dst_base_ip = Device.ip_to_int(self.dst.ip)
 215         src_last_ip = src_base_ip + self.flow_count - 1
 216         dst_last_ip = dst_base_ip + self.flow_count - 1
 217         return dst_last_ip >= src_base_ip and src_last_ip >= dst_base_ip
 218
 219     @staticmethod
 220     def mac_to_int(mac):
 221         return int(mac.translate(None, ":.- "), 16)
 222
 223     @staticmethod
 224     def int_to_mac(i):
 225         mac = format(i, 'x').zfill(12)
 226         blocks = [mac[x:x + 2] for x in xrange(0, len(mac), 2)]
 227         return ':'.join(blocks)
 228
 229     @staticmethod
 230     def ip_to_int(addr):
 231         return struct.unpack("!I", socket.inet_aton(addr))[0]
 232
 233     @staticmethod
 234     def int_to_ip(nvalue):
 235         return socket.inet_ntoa(struct.pack("!I", nvalue))
 236
 237 class RunningTrafficProfile(object):
 238     """Represents traffic configuration for currently running traffic profile."""
 239
 240     DEFAULT_IP_STEP = '0.0.0.1'
 241     DEFAULT_SRC_DST_IP_STEP = '0.0.0.1'
 242
 243     def __init__(self, config, generator_profile):
 244         generator_config = self.__match_generator_profile(config.traffic_generator,
 245                                                           generator_profile)
 246         self.generator_config = generator_config
 247         self.service_chain = config.service_chain
 248         self.service_chain_count = config.service_chain_count
 249         self.flow_count = config.flow_count
 250         self.host_name = generator_config.host_name
 251         self.name = generator_config.name
 252         self.tool = generator_config.tool
 253         self.cores = generator_config.get('cores', 1)
 254         self.ip_addrs_step = generator_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
 255         self.tg_gateway_ip_addrs_step = \
 256             generator_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 257         self.gateway_ip_addrs_step = generator_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
 258         self.gateway_ips = generator_config.gateway_ip_addrs
 259         self.ip = generator_config.ip
 260         self.intf_speed = bitmath.parse_string(generator_config.intf_speed.replace('ps', '')).bits
 261         self.vlan_tagging = config.vlan_tagging
 262         self.no_arp = config.no_arp
 263         self.src_device = None
 264         self.dst_device = None
 265         self.vm_mac_list = None
 266         self.__prep_interfaces(generator_config)
 267
 268     def to_json(self):
 269         return dict(self.generator_config)
 270
 271     def set_vm_mac_list(self, vm_mac_list):
 272         self.src_device.set_vm_mac_list(vm_mac_list[0])
 273         self.dst_device.set_vm_mac_list(vm_mac_list[1])
 274
 275     @staticmethod
 276     def __match_generator_profile(traffic_generator, generator_profile):
 277         generator_config = AttrDict(traffic_generator)
 278         generator_config.pop('default_profile')
 279         generator_config.pop('generator_profile')
 280         matching_profile = filter(lambda profile: profile.name == generator_profile,
 281                                   traffic_generator.generator_profile)
 282         if len(matching_profile) != 1:
 283             raise Exception('Traffic generator profile not found: ' + generator_profile)
 284
 285         generator_config.update(matching_profile[0])
 286
 287         return generator_config
 288
 289     def __prep_interfaces(self, generator_config):
 290         src_config = {
 291             'chain_count': self.service_chain_count,
 292             'flow_count': self.flow_count / 2,
 293             'ip': generator_config.ip_addrs[0],
 294             'ip_addrs_step': self.ip_addrs_step,
 295             'gateway_ip': self.gateway_ips[0],
 296             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 297             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[0],
 298             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 299             'udp_src_port': generator_config.udp_src_port,
 300             'udp_dst_port': generator_config.udp_dst_port,
 301             'vlan_tagging': self.vlan_tagging
 302         }
 303         dst_config = {
 304             'chain_count': self.service_chain_count,
 305             'flow_count': self.flow_count / 2,
 306             'ip': generator_config.ip_addrs[1],
 307             'ip_addrs_step': self.ip_addrs_step,
 308             'gateway_ip': self.gateway_ips[1],
 309             'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
 310             'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[1],
 311             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
 312             'udp_src_port': generator_config.udp_src_port,
 313             'udp_dst_port': generator_config.udp_dst_port,
 314             'vlan_tagging': self.vlan_tagging
 315         }
 316
 317         self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))
 318         self.dst_device = Device(**dict(dst_config, **generator_config.interfaces[1]))
 319         self.src_device.set_destination(self.dst_device)
 320         self.dst_device.set_destination(self.src_device)
 321
 322         if self.service_chain == ChainType.EXT and not self.no_arp \
 323                 and self.src_device.ip_range_overlaps():
 324             raise Exception('Overlapping IP address ranges src=%s dst=%d flows=%d' %
 325                             self.src_device.ip,
 326                             self.dst_device.ip,
 327                             self.flow_count)
 328
 329     @property
 330     def devices(self):
 331         return [self.src_device, self.dst_device]
 332
 333     @property
 334     def vlans(self):
 335         return [self.src_device.vtep_vlan, self.dst_device.vtep_vlan]
 336
 337     @property
 338     def ports(self):
 339         return [self.src_device.port, self.dst_device.port]
 340
 341     @property
 342     def switch_ports(self):
 343         return [self.src_device.switch_port, self.dst_device.switch_port]
 344
 345     @property
 346     def pcis(self):
 347         return [self.src_device.pci, self.dst_device.pci]
 348
 349
 350 class TrafficGeneratorFactory(object):
 351
 352     def __init__(self, config):
 353         self.config = config
 354
 355     def get_tool(self):
 356         return self.config.generator_config.tool
 357
 358     def get_generator_client(self):
 359         tool = self.get_tool().lower()
 360         if tool == 'trex':
 361             from traffic_gen import trex
 362             return trex.TRex(self.config)
 363         elif tool == 'dummy':
 364             from traffic_gen import dummy
 365             return dummy.DummyTG(self.config)
 366         else:
 367             return None
 368
 369     def list_generator_profile(self):
 370         return [profile.name for profile in self.config.traffic_generator.generator_profile]
 371
 372     def get_generator_config(self, generator_profile):
 373         return RunningTrafficProfile(self.config, generator_profile)
 374
 375     def get_matching_profile(self, traffic_profile_name):
 376         matching_profile = filter(lambda profile: profile.name == traffic_profile_name,
 377                                   self.config.traffic_profile)
 378
 379         if len(matching_profile) > 1:
 380             raise Exception('Multiple traffic profiles with the same name found.')
 381         elif len(matching_profile) == 0:
 382             raise Exception('No traffic profile found.')
 383
 384         return matching_profile[0]
 385
 386     def get_frame_sizes(self, traffic_profile):
 387         matching_profile = self.get_matching_profile(traffic_profile)
 388         return matching_profile.l2frame_size
 389
 390
 391 class TrafficClient(object):
 392
 393     PORTS = [0, 1]
 394
 395     def __init__(self, config, notifier=None):
 396         generator_factory = TrafficGeneratorFactory(config)
 397         self.gen = generator_factory.get_generator_client()
 398         self.tool = generator_factory.get_tool()
 399         self.config = config
 400         self.notifier = notifier
 401         self.interval_collector = None
 402         self.iteration_collector = None
 403         self.runner = TrafficRunner(self, self.config.duration_sec, self.config.interval_sec)
 404         if self.gen is None:
 405             raise TrafficClientException('%s is not a supported traffic generator' % self.tool)
 406
 407         self.run_config = {
 408             'l2frame_size': None,
 409             'duration_sec': self.config.duration_sec,
 410             'bidirectional': True,
 411             'rates': None
 412         }
 413         self.current_total_rate = {'rate_percent': '10'}
 414         if self.config.single_run:
 415             self.current_total_rate = utils.parse_rate_str(self.config.rate)
 416
 417     def set_macs(self):
 418         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
 419             device.set_mac(mac)
 420
 421     def start_traffic_generator(self):
 422         self.gen.init()
 423         self.gen.connect()
 424
 425     def setup(self):
 426         self.gen.set_mode()
 427         self.gen.config_interface()
 428         self.gen.clear_stats()
 429
 430     def get_version(self):
 431         return self.gen.get_version()
 432
 433     def ensure_end_to_end(self):
 434         """
 435         Ensure traffic generator receives packets it has transmitted.
 436         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 437
 438         At this point all VMs are in active state, but forwarding does not have to work.
 439         Small amount of traffic is sent to every chain. Then total of sent and received packets
 440         is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
 441         N being number of chains, traffic flows through every chain and real measurements can be
 442         performed.
 443
 444         Example:
 445             PVP chain (1 VM per chain)
 446             N = 10 (number of chains)
 447             threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
 448             if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
 449             all 10 VMs are in operational state.
 450         """
 451         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
 452         rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
 453         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 454
 455         # ensures enough traffic is coming back
 456         threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)
 457         retry_count = (self.config.check_traffic_time_sec +
 458                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
 459         for it in xrange(retry_count):
 460             self.gen.clear_stats()
 461             self.gen.start_traffic()
 462             LOG.info('Waiting for packets to be received back... ({} / {})'.format(it + 1,
 463                      retry_count))
 464             time.sleep(self.config.generic_poll_sec)
 465             self.gen.stop_traffic()
 466             stats = self.gen.get_stats()
 467
 468             # compute total sent and received traffic on both ports
 469             total_rx = 0
 470             total_tx = 0
 471             for port in self.PORTS:
 472                 total_rx += float(stats[port]['rx'].get('total_pkts', 0))
 473                 total_tx += float(stats[port]['tx'].get('total_pkts', 0))
 474
 475             # how much of traffic came back
 476             ratio = total_rx / total_tx if total_tx else 0
 477
 478             if ratio > threshold:
 479                 self.gen.clear_stats()
 480                 self.gen.clear_streamblock()
 481                 LOG.info('End-to-end connectivity ensured')
 482                 return
 483
 484             time.sleep(self.config.generic_poll_sec)
 485
 486         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 487
 488     def ensure_arp_successful(self):
 489         if not self.gen.resolve_arp():
 490             raise TrafficClientException('ARP cannot be resolved')
 491
 492     def set_traffic(self, frame_size, bidirectional):
 493         self.run_config['bidirectional'] = bidirectional
 494         self.run_config['l2frame_size'] = frame_size
 495         self.run_config['rates'] = [self.get_per_direction_rate()]
 496         if bidirectional:
 497             self.run_config['rates'].append(self.get_per_direction_rate())
 498         else:
 499             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
 500             if unidir_reverse_pps > 0:
 501                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})
 502
 503         self.gen.clear_streamblock()
 504         self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)
 505
 506     def modify_load(self, load):
 507         self.current_total_rate = {'rate_percent': str(load)}
 508         rate_per_direction = self.get_per_direction_rate()
 509
 510         self.gen.modify_rate(rate_per_direction, False)
 511         self.run_config['rates'][0] = rate_per_direction
 512         if self.run_config['bidirectional']:
 513             self.gen.modify_rate(rate_per_direction, True)
 514             self.run_config['rates'][1] = rate_per_direction
 515
 516     def get_ndr_and_pdr(self):
 517         dst = 'Bidirectional' if self.run_config['bidirectional'] else 'Unidirectional'
 518         targets = {}
 519         if self.config.ndr_run:
 520             LOG.info('*** Searching NDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 521             targets['ndr'] = self.config.measurement.NDR
 522         if self.config.pdr_run:
 523             LOG.info('*** Searching PDR for %s (%s)...', self.run_config['l2frame_size'], dst)
 524             targets['pdr'] = self.config.measurement.PDR
 525
 526         self.run_config['start_time'] = time.time()
 527         self.interval_collector = IntervalCollector(self.run_config['start_time'])
 528         self.interval_collector.attach_notifier(self.notifier)
 529         self.iteration_collector = IterationCollector(self.run_config['start_time'])
 530         results = {}
 531         self.__range_search(0.0, 200.0, targets, results)
 532
 533         results['iteration_stats'] = {
 534             'ndr_pdr': self.iteration_collector.get()
 535         }
 536
 537         if self.config.ndr_run:
 538             LOG.info('NDR load: %s', results['ndr']['rate_percent'])
 539             results['ndr']['time_taken_sec'] = \
 540                 results['ndr']['timestamp_sec'] - self.run_config['start_time']
 541             if self.config.pdr_run:
 542                 LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 543                 results['pdr']['time_taken_sec'] = \
 544                     results['pdr']['timestamp_sec'] - results['ndr']['timestamp_sec']
 545         else:
 546             LOG.info('PDR load: %s', results['pdr']['rate_percent'])
 547             results['pdr']['time_taken_sec'] = \
 548                 results['pdr']['timestamp_sec'] - self.run_config['start_time']
 549         return results
 550
 551     def __get_dropped_rate(self, result):
 552         dropped_pkts = result['rx']['dropped_pkts']
 553         total_pkts = result['tx']['total_pkts']
 554         if not total_pkts:
 555             return float('inf')
 556         else:
 557             return float(dropped_pkts) / total_pkts * 100
 558
 559     def get_stats(self):
 560         stats = self.gen.get_stats()
 561         retDict = {'total_tx_rate': stats['total_tx_rate']}
 562         for port in self.PORTS:
 563             retDict[port] = {'tx': {}, 'rx': {}}
 564
 565         tx_keys = ['total_pkts', 'total_pkt_bytes', 'pkt_rate', 'pkt_bit_rate']
 566         rx_keys = tx_keys + ['dropped_pkts']
 567
 568         for port in self.PORTS:
 569             for key in tx_keys:
 570                 retDict[port]['tx'][key] = int(stats[port]['tx'][key])
 571             for key in rx_keys:
 572                 try:
 573                     retDict[port]['rx'][key] = int(stats[port]['rx'][key])
 574                 except ValueError:
 575                     retDict[port]['rx'][key] = 0
 576             retDict[port]['rx']['avg_delay_usec'] = cast_integer(
 577                 stats[port]['rx']['avg_delay_usec'])
 578             retDict[port]['rx']['min_delay_usec'] = cast_integer(
 579                 stats[port]['rx']['min_delay_usec'])
 580             retDict[port]['rx']['max_delay_usec'] = cast_integer(
 581                 stats[port]['rx']['max_delay_usec'])
 582             retDict[port]['drop_rate_percent'] = self.__get_dropped_rate(retDict[port])
 583
 584         ports = sorted(retDict.keys())
 585         if self.run_config['bidirectional']:
 586             retDict['overall'] = {'tx': {}, 'rx': {}}
 587             for key in tx_keys:
 588                 retDict['overall']['tx'][key] = \
 589                     retDict[ports[0]]['tx'][key] + retDict[ports[1]]['tx'][key]
 590             for key in rx_keys:
 591                 retDict['overall']['rx'][key] = \
 592                     retDict[ports[0]]['rx'][key] + retDict[ports[1]]['rx'][key]
 593             total_pkts = [retDict[ports[0]]['rx']['total_pkts'],
 594                           retDict[ports[1]]['rx']['total_pkts']]
 595             avg_delays = [retDict[ports[0]]['rx']['avg_delay_usec'],
 596                           retDict[ports[1]]['rx']['avg_delay_usec']]
 597             max_delays = [retDict[ports[0]]['rx']['max_delay_usec'],
 598                           retDict[ports[1]]['rx']['max_delay_usec']]
 599             min_delays = [retDict[ports[0]]['rx']['min_delay_usec'],
 600                           retDict[ports[1]]['rx']['min_delay_usec']]
 601             retDict['overall']['rx']['avg_delay_usec'] = utils.weighted_avg(total_pkts, avg_delays)
 602             retDict['overall']['rx']['min_delay_usec'] = min(min_delays)
 603             retDict['overall']['rx']['max_delay_usec'] = max(max_delays)
 604             for key in ['pkt_bit_rate', 'pkt_rate']:
 605                 for dirc in ['tx', 'rx']:
 606                     retDict['overall'][dirc][key] /= 2.0
 607         else:
 608             retDict['overall'] = retDict[ports[0]]
 609         retDict['overall']['drop_rate_percent'] = self.__get_dropped_rate(retDict['overall'])
 610         return retDict
 611
 612     def __convert_rates(self, rate):
 613         return utils.convert_rates(self.run_config['l2frame_size'],
 614                                    rate,
 615                                    self.config.generator_config.intf_speed)
 616
 617     def __ndr_pdr_found(self, tag, load):
 618         rates = self.__convert_rates({'rate_percent': load})
 619         self.iteration_collector.add_ndr_pdr(tag, rates['rate_pps'])
 620         last_stats = self.iteration_collector.peek()
 621         self.interval_collector.add_ndr_pdr(tag, last_stats)
 622
 623     def __format_output_stats(self, stats):
 624         for key in (self.PORTS + ['overall']):
 625             interface = stats[key]
 626             stats[key] = {
 627                 'tx_pkts': interface['tx']['total_pkts'],
 628                 'rx_pkts': interface['rx']['total_pkts'],
 629                 'drop_percentage': interface['drop_rate_percent'],
 630                 'drop_pct': interface['rx']['dropped_pkts'],
 631                 'avg_delay_usec': interface['rx']['avg_delay_usec'],
 632                 'max_delay_usec': interface['rx']['max_delay_usec'],
 633                 'min_delay_usec': interface['rx']['min_delay_usec'],
 634             }
 635
 636         return stats
 637
 638     def __targets_found(self, rate, targets, results):
 639         for tag, target in targets.iteritems():
 640             LOG.info('Found {} ({}) load: {}'.format(tag, target, rate))
 641             self.__ndr_pdr_found(tag, rate)
 642             results[tag]['timestamp_sec'] = time.time()
 643
 644     def __range_search(self, left, right, targets, results):
 645         '''Perform a binary search for a list of targets inside a [left..right] range or rate
 646
 647         left    the left side of the range to search as a % the line rate (100 = 100% line rate)
 648                 indicating the rate to send on each interface
 649         right   the right side of the range to search as a % of line rate
 650                 indicating the rate to send on each interface
 651         targets a dict of drop rates to search (0.1 = 0.1%), indexed by the DR name or "tag"
 652                 ('ndr', 'pdr')
 653         results a dict to store results
 654         '''
 655         if len(targets) == 0:
 656             return
 657         LOG.info('Range search [{} .. {}] targets: {}'.format(left, right, targets))
 658
 659         # Terminate search when gap is less than load epsilon
 660         if right - left < self.config.measurement.load_epsilon:
 661             self.__targets_found(left, targets, results)
 662             return
 663
 664         # Obtain the average drop rate in for middle load
 665         middle = (left + right) / 2.0
 666         try:
 667             stats, rates = self.__run_search_iteration(middle)
 668         except STLError:
 669             LOG.exception("Got exception from traffic generator during binary search")
 670             self.__targets_found(left, targets, results)
 671             return
 672         # Split target dicts based on the avg drop rate
 673         left_targets = {}
 674         right_targets = {}
 675         for tag, target in targets.iteritems():
 676             if stats['overall']['drop_rate_percent'] <= target:
 677                 # record the best possible rate found for this target
 678                 results[tag] = rates
 679                 results[tag].update({
 680                     'load_percent_per_direction': middle,
 681                     'stats': self.__format_output_stats(dict(stats)),
 682                     'timestamp_sec': None
 683                 })
 684                 right_targets[tag] = target
 685             else:
 686                 # initialize to 0 all fields of result for
 687                 # the worst case scenario of the binary search (if ndr/pdr is not found)
 688                 if tag not in results:
 689                     results[tag] = dict.fromkeys(rates, 0)
 690                     empty_stats = self.__format_output_stats(dict(stats))
 691                     for key in empty_stats:
 692                         if isinstance(empty_stats[key], dict):
 693                             empty_stats[key] = dict.fromkeys(empty_stats[key], 0)
 694                         else:
 695                             empty_stats[key] = 0
 696                     results[tag].update({
 697                         'load_percent_per_direction': 0,
 698                         'stats': empty_stats,
 699                         'timestamp_sec': None
 700                     })
 701                 left_targets[tag] = target
 702
 703         # search lower half
 704         self.__range_search(left, middle, left_targets, results)
 705
 706         # search upper half only if the upper rate does not exceed
 707         # 100%, this only happens when the first search at 100%
 708         # yields a DR that is < target DR
 709         if middle >= 100:
 710             self.__targets_found(100, right_targets, results)
 711         else:
 712             self.__range_search(middle, right, right_targets, results)
 713
 714     def __run_search_iteration(self, rate):
 715         # set load
 716         self.modify_load(rate)
 717
 718         # poll interval stats and collect them
 719         for stats in self.run_traffic():
 720             self.interval_collector.add(stats)
 721             time_elapsed_ratio = self.runner.time_elapsed() / self.run_config['duration_sec']
 722             if time_elapsed_ratio >= 1:
 723                 self.cancel_traffic()
 724         self.interval_collector.reset()
 725
 726         # get stats from the run
 727         stats = self.runner.client.get_stats()
 728         current_traffic_config = self.get_traffic_config()
 729         warning = self.compare_tx_rates(current_traffic_config['direction-total']['rate_pps'],
 730                                         stats['total_tx_rate'])
 731         if warning is not None:
 732             stats['warning'] = warning
 733
 734         # save reliable stats from whole iteration
 735         self.iteration_collector.add(stats, current_traffic_config['direction-total']['rate_pps'])
 736         LOG.info('Average drop rate: {}'.format(stats['overall']['drop_rate_percent']))
 737
 738         return stats, current_traffic_config['direction-total']
 739
 740     @staticmethod
 741     def log_stats(stats):
 742         report = {
 743             'datetime': str(datetime.now()),
 744             'tx_packets': stats['overall']['tx']['total_pkts'],
 745             'rx_packets': stats['overall']['rx']['total_pkts'],
 746             'drop_packets': stats['overall']['rx']['dropped_pkts'],
 747             'drop_rate_percent': stats['overall']['drop_rate_percent']
 748         }
 749         LOG.info('TX: %(tx_packets)d; '
 750                  'RX: %(rx_packets)d; '
 751                  'Dropped: %(drop_packets)d; '
 752                  'Drop rate: %(drop_rate_percent).4f%%',
 753                  report)
 754
 755     def run_traffic(self):
 756         stats = self.runner.run()
 757         while self.runner.is_running:
 758             self.log_stats(stats)
 759             yield stats
 760             stats = self.runner.poll_stats()
 761             if stats is None:
 762                 return
 763         self.log_stats(stats)
 764         LOG.info('Drop rate: {}'.format(stats['overall']['drop_rate_percent']))
 765         yield stats
 766
 767     def cancel_traffic(self):
 768         self.runner.stop()
 769
 770     def get_interface(self, port_index):
 771         port = self.gen.port_handle[port_index]
 772         tx, rx = 0, 0
 773         if not self.config.no_traffic:
 774             stats = self.get_stats()
 775             if port in stats:
 776                 tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])
 777         return Interface('traffic-generator', self.tool.lower(), tx, rx)
 778
 779     def get_traffic_config(self):
 780         config = {}
 781         load_total = 0.0
 782         bps_total = 0.0
 783         pps_total = 0.0
 784         for idx, rate in enumerate(self.run_config['rates']):
 785             key = 'direction-forward' if idx == 0 else 'direction-reverse'
 786             config[key] = {
 787                 'l2frame_size': self.run_config['l2frame_size'],
 788                 'duration_sec': self.run_config['duration_sec']
 789             }
 790             config[key].update(rate)
 791             config[key].update(self.__convert_rates(rate))
 792             load_total += float(config[key]['rate_percent'])
 793             bps_total += float(config[key]['rate_bps'])
 794             pps_total += float(config[key]['rate_pps'])
 795         config['direction-total'] = dict(config['direction-forward'])
 796         config['direction-total'].update({
 797             'rate_percent': load_total,
 798             'rate_pps': cast_integer(pps_total),
 799             'rate_bps': bps_total
 800         })
 801
 802         return config
 803
 804     def get_run_config(self, results):
 805         """Returns configuration which was used for the last run."""
 806         r = {}
 807         for idx, key in enumerate(["direction-forward", "direction-reverse"]):
 808             tx_rate = results["stats"][idx]["tx"]["total_pkts"] / self.config.duration_sec
 809             rx_rate = results["stats"][idx]["rx"]["total_pkts"] / self.config.duration_sec
 810             r[key] = {
 811                 "orig": self.__convert_rates(self.run_config['rates'][idx]),
 812                 "tx": self.__convert_rates({'rate_pps': tx_rate}),
 813                 "rx": self.__convert_rates({'rate_pps': rx_rate})
 814             }
 815
 816         total = {}
 817         for direction in ['orig', 'tx', 'rx']:
 818             total[direction] = {}
 819             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
 820                 total[direction][unit] = sum(map(lambda x: float(x[direction][unit]), r.values()))
 821
 822         r['direction-total'] = total
 823         return r
 824
 825     @staticmethod
 826     def compare_tx_rates(required, actual):
 827         threshold = 0.9
 828         are_different = False
 829         try:
 830             if float(actual) / required < threshold:
 831                 are_different = True
 832         except ZeroDivisionError:
 833             are_different = True
 834
 835         if are_different:
 836             msg = "WARNING: There is a significant difference between requested TX rate ({r}) " \
 837                   "and actual TX rate ({a}). The traffic generator may not have sufficient CPU " \
 838                   "to achieve the requested TX rate.".format(r=required, a=actual)
 839             LOG.info(msg)
 840             return msg
 841
 842         return None
 843
 844     def get_per_direction_rate(self):
 845         divisor = 2 if self.run_config['bidirectional'] else 1
 846         if 'rate_percent' in self.current_total_rate:
 847             # don't split rate if it's percentage
 848             divisor = 1
 849
 850         return utils.divide_rate(self.current_total_rate, divisor)
 851
 852     def close(self):
 853         try:
 854             self.gen.stop_traffic()
 855         except Exception:
 856             pass
 857         self.gen.clear_stats()
 858         self.gen.cleanup()