NFVBENCH-155 Add options to disable extra stats, latency stats and latency streams
[nfvbench.git] / nfvbench / traffic_client.py
old mode 100644 (file)
new mode 100755 (executable)
index 7542d0b..6d870f6
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+"""Interface to the traffic generator clients including NDR/PDR binary search."""
+
+from datetime import datetime
+import socket
+import struct
+import time
+
 from attrdict import AttrDict
 import bitmath
-from datetime import datetime
-from log import LOG
 from netaddr import IPNetwork
-from network import Interface
-import socket
-from specs import ChainType
+# pylint: disable=import-error
+from trex.stl.api import Ether
+from trex.stl.api import STLError
+from trex.stl.api import UDP
+# pylint: enable=import-error
+
+from log import LOG
+from packet_stats import InterfaceStats
+from packet_stats import PacketPathStats
 from stats_collector import IntervalCollector
 from stats_collector import IterationCollector
-import struct
-import time
 import traffic_gen.traffic_utils as utils
+from utils import cast_integer
 
 
 class TrafficClientException(Exception):
+    """Generic traffic client exception."""
+
     pass
 
 
 class TrafficRunner(object):
+    """Serialize various steps required to run traffic."""
 
-    def __init__(self, client, duration_sec, interval_sec=0):
+    def __init__(self, client, duration_sec, interval_sec=0, service_mode=False):
+        """Create a traffic runner."""
         self.client = client
         self.start_time = None
         self.duration_sec = duration_sec
         self.interval_sec = interval_sec
+        self.service_mode = service_mode
 
     def run(self):
+        """Clear stats and instruct the traffic generator to start generating traffic."""
+        if self.is_running():
+            return None
         LOG.info('Running traffic generator')
         self.client.gen.clear_stats()
+        # Debug use only : new '--service-mode' option available for the NFVBench command line.
+        # A read-only mode TRex console would be able to capture the generated traffic.
+        self.client.gen.set_service_mode(enabled=self.service_mode)
+        LOG.info('Service mode is %sabled', 'en' if self.service_mode else 'dis')
         self.client.gen.start_traffic()
         self.start_time = time.time()
         return self.poll_stats()
 
     def stop(self):
+        """Stop the current run and instruct the traffic generator to stop traffic."""
         if self.is_running():
             self.start_time = None
             self.client.gen.stop_traffic()
 
     def is_running(self):
+        """Check if a run is still pending."""
         return self.start_time is not None
 
     def time_elapsed(self):
+        """Return time elapsed since start of run."""
         if self.is_running():
             return time.time() - self.start_time
-        else:
-            return self.duration_sec
+        return self.duration_sec
 
     def poll_stats(self):
+        """Poll latest stats from the traffic generator at fixed interval - sleeps if necessary.
+
+        return: latest stats or None if traffic is stopped
+        """
         if not self.is_running():
             return None
+        if self.client.skip_sleep():
+            self.stop()
+            return self.client.get_stats()
         time_elapsed = self.time_elapsed()
         if time_elapsed > self.duration_sec:
             self.stop()
@@ -80,381 +111,559 @@ class TrafficRunner(object):
         return self.client.get_stats()
 
 
+class IpBlock(object):
+    """Manage a block of IP addresses."""
+
+    def __init__(self, base_ip, step_ip, count_ip):
+        """Create an IP block."""
+        self.base_ip_int = Device.ip_to_int(base_ip)
+        self.step = Device.ip_to_int(step_ip)
+        self.max_available = count_ip
+        self.next_free = 0
+
+    def get_ip(self, index=0):
+        """Return the IP address at given index."""
+        if index < 0 or index >= self.max_available:
+            raise IndexError('Index out of bounds: %d (max=%d)' % (index, self.max_available))
+        return Device.int_to_ip(self.base_ip_int + index * self.step)
+
+    def reserve_ip_range(self, count):
+        """Reserve a range of count consecutive IP addresses spaced by step."""
+        if self.next_free + count > self.max_available:
+            raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d' %
+                             (self.next_free,
+                              self.max_available,
+                              count))
+        first_ip = self.get_ip(self.next_free)
+        last_ip = self.get_ip(self.next_free + count - 1)
+        self.next_free += count
+        return (first_ip, last_ip)
+
+    def reset_reservation(self):
+        """Reset all reservations and restart with a completely unused IP block."""
+        self.next_free = 0
+
+
 class Device(object):
+    """Represent a port device and all information associated to it.
 
-    def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
-                 gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
-                 gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,
-                 chain_count=1, flow_count=1, vlan_tagging=False):
-        self.chain_count = chain_count
-        self.flow_count = flow_count
-        self.dst = None
+    In the curent version we only support 2 port devices for the traffic generator
+    identified as port 0 or port 1.
+    """
+
+    def __init__(self, port, generator_config):
+        """Create a new device for a given port."""
+        self.generator_config = generator_config
+        self.chain_count = generator_config.service_chain_count
+        self.flow_count = generator_config.flow_count / 2
         self.port = port
-        self.switch_port = switch_port
-        self.vtep_vlan = vtep_vlan
-        self.vlan_tag = None
-        self.vlan_tagging = vlan_tagging
-        self.pci = pci
+        self.switch_port = generator_config.interfaces[port].get('switch_port', None)
+        self.vtep_vlan = None
+        self.vtep_src_mac = None
+        self.vxlan = False
+        self.pci = generator_config.interfaces[port].pci
         self.mac = None
-        self.vm_mac_list = None
-        subnet = IPNetwork(ip)
+        self.dest_macs = None
+        self.vtep_dst_mac = None
+        self.vtep_dst_ip = None
+        if generator_config.vteps is None:
+            self.vtep_src_ip = None
+        else:
+            self.vtep_src_ip = generator_config.vteps[port]
+        self.vnis = None
+        self.vlans = None
+        self.ip_addrs = generator_config.ip_addrs[port]
+        subnet = IPNetwork(self.ip_addrs)
         self.ip = subnet.ip.format()
-        self.ip_prefixlen = subnet.prefixlen
-        self.ip_addrs_step = ip_addrs_step
-        self.tg_gateway_ip_addrs_step = tg_gateway_ip_addrs_step
-        self.gateway_ip_addrs_step = gateway_ip_addrs_step
-        self.ip_list = self.expand_ip(self.ip, self.ip_addrs_step, self.flow_count)
-        self.gateway_ip = gateway_ip
-        self.gateway_ip_list = self.expand_ip(self.gateway_ip,
-                                              self.gateway_ip_addrs_step,
-                                              self.chain_count)
-        self.tg_gateway_ip = tg_gateway_ip
-        self.tg_gateway_ip_list = self.expand_ip(self.tg_gateway_ip,
-                                                 self.tg_gateway_ip_addrs_step,
-                                                 self.chain_count)
-        self.udp_src_port = udp_src_port
-        self.udp_dst_port = udp_dst_port
+        self.ip_addrs_step = generator_config.ip_addrs_step
+        self.ip_block = IpBlock(self.ip, self.ip_addrs_step, self.flow_count)
+        self.gw_ip_block = IpBlock(generator_config.gateway_ips[port],
+                                   generator_config.gateway_ip_addrs_step,
+                                   self.chain_count)
+        self.tg_gateway_ip_addrs = generator_config.tg_gateway_ip_addrs[port]
+        self.tg_gw_ip_block = IpBlock(self.tg_gateway_ip_addrs,
+                                      generator_config.tg_gateway_ip_addrs_step,
+                                      self.chain_count)
+        self.udp_src_port = generator_config.udp_src_port
+        self.udp_dst_port = generator_config.udp_dst_port
 
     def set_mac(self, mac):
+        """Set the local MAC for this port device."""
         if mac is None:
             raise TrafficClientException('Trying to set traffic generator MAC address as None')
         self.mac = mac
 
-    def set_destination(self, dst):
-        self.dst = dst
+    def get_peer_device(self):
+        """Get the peer device (device 0 -> device 1, or device 1 -> device 0)."""
+        return self.generator_config.devices[1 - self.port]
+
+    def set_vtep_dst_mac(self, dest_macs):
+        """Set the list of dest MACs indexed by the chain id.
+
+        This is only called in 2 cases:
+        - VM macs discovered using openstack API
+        - dest MACs provisioned in config file
+        """
+        self.vtep_dst_mac = map(str, dest_macs)
+
+    def set_dest_macs(self, dest_macs):
+        """Set the list of dest MACs indexed by the chain id.
 
-    def set_vm_mac_list(self, vm_mac_list):
-        self.vm_mac_list = map(str, vm_mac_list)
+        This is only called in 2 cases:
+        - VM macs discovered using openstack API
+        - dest MACs provisioned in config file
+        """
+        self.dest_macs = map(str, dest_macs)
 
-    def set_vlan_tag(self, vlan_tag):
-        if self.vlan_tagging and vlan_tag is None:
-            raise TrafficClientException('Trying to set VLAN tag as None')
-        self.vlan_tag = vlan_tag
+    def get_dest_macs(self):
+        """Get the list of dest macs for this device.
 
-    def get_stream_configs(self, service_chain):
+        If set_dest_macs was never called, assumes l2-loopback and return
+        a list of peer mac (as many as chains but normally only 1 chain)
+        """
+        if self.dest_macs:
+            return self.dest_macs
+        # assume this is l2-loopback
+        return [self.get_peer_device().mac] * self.chain_count
+
+    def set_vlans(self, vlans):
+        """Set the list of vlans to use indexed by the chain id."""
+        self.vlans = vlans
+        LOG.info("Port %d: VLANs %s", self.port, self.vlans)
+
+    def set_vtep_vlan(self, vlan):
+        """Set the vtep vlan to use indexed by specific port."""
+        self.vtep_vlan = vlan
+        self.vxlan = True
+        self.vlan_tagging = None
+        LOG.info("Port %d: VTEP VLANs %s", self.port, self.vtep_vlan)
+
+    def set_vxlan_endpoints(self, src_ip, dst_ip):
+        self.vtep_dst_ip = dst_ip
+        self.vtep_src_ip = src_ip
+        LOG.info("Port %d: src_vtep %s, dst_vtep %s", self.port,
+                 self.vtep_src_ip, self.vtep_dst_ip)
+
+    def set_vxlans(self, vnis):
+        self.vnis = vnis
+        LOG.info("Port %d: VNIs %s", self.port, self.vnis)
+
+    def set_gw_ip(self, gateway_ip):
+        self.gw_ip_block = IpBlock(gateway_ip,
+                                   self.generator_config.gateway_ip_addrs_step,
+                                   self.chain_count)
+
+    def get_gw_ip(self, chain_index):
+        """Retrieve the IP address assigned for the gateway of a given chain."""
+        return self.gw_ip_block.get_ip(chain_index)
+
+    def get_stream_configs(self):
+        """Get the stream config for a given chain on this device.
+
+        Called by the traffic generator driver to program the traffic generator properly
+        before generating traffic
+        """
         configs = []
-        flow_idx = 0
+        # exact flow count for each chain is calculated as follows:
+        # - all chains except the first will have the same flow count
+        #   calculated as (total_flows + chain_count - 1) / chain_count
+        # - the first chain will have the remainder
+        # example 11 flows and 3 chains => 3, 4, 4
+        flows_per_chain = (self.flow_count + self.chain_count - 1) / self.chain_count
+        cur_chain_flow_count = self.flow_count - flows_per_chain * (self.chain_count - 1)
+        peer = self.get_peer_device()
+        self.ip_block.reset_reservation()
+        peer.ip_block.reset_reservation()
+        dest_macs = self.get_dest_macs()
+
         for chain_idx in xrange(self.chain_count):
-            current_flow_count = (self.flow_count - flow_idx) / (self.chain_count - chain_idx)
-            max_idx = flow_idx + current_flow_count - 1
-            ip_src_count = self.ip_to_int(self.ip_list[max_idx]) - \
-                self.ip_to_int(self.ip_list[flow_idx]) + 1
-            ip_dst_count = self.ip_to_int(self.dst.ip_list[max_idx]) - \
-                self.ip_to_int(self.dst.ip_list[flow_idx]) + 1
+            src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count)
+            dst_ip_first, dst_ip_last = peer.ip_block.reserve_ip_range(cur_chain_flow_count)
 
             configs.append({
-                'count': current_flow_count,
+                'count': cur_chain_flow_count,
                 'mac_src': self.mac,
-                'mac_dst': self.dst.mac if service_chain == ChainType.EXT
-                else self.vm_mac_list[chain_idx],
-                'ip_src_addr': self.ip_list[flow_idx],
-                'ip_src_addr_max': self.ip_list[max_idx],
-                'ip_src_count': ip_src_count,
-                'ip_dst_addr': self.dst.ip_list[flow_idx],
-                'ip_dst_addr_max': self.dst.ip_list[max_idx],
-                'ip_dst_count': ip_dst_count,
+                'mac_dst': dest_macs[chain_idx],
+                'ip_src_addr': src_ip_first,
+                'ip_src_addr_max': src_ip_last,
+                'ip_src_count': cur_chain_flow_count,
+                'ip_dst_addr': dst_ip_first,
+                'ip_dst_addr_max': dst_ip_last,
+                'ip_dst_count': cur_chain_flow_count,
                 'ip_addrs_step': self.ip_addrs_step,
                 'udp_src_port': self.udp_src_port,
                 'udp_dst_port': self.udp_dst_port,
-                'mac_discovery_gw': self.gateway_ip_list[chain_idx],
-                'ip_src_tg_gw': self.tg_gateway_ip_list[chain_idx],
-                'ip_dst_tg_gw': self.dst.tg_gateway_ip_list[chain_idx],
-                'vlan_tag': self.vlan_tag if self.vlan_tagging else None
+                'mac_discovery_gw': self.get_gw_ip(chain_idx),
+                'ip_src_tg_gw': self.tg_gw_ip_block.get_ip(chain_idx),
+                'ip_dst_tg_gw': peer.tg_gw_ip_block.get_ip(chain_idx),
+                'vlan_tag': self.vlans[chain_idx] if self.vlans else None,
+                'vxlan': self.vxlan,
+                'vtep_vlan': self.vtep_vlan if self.vtep_vlan else None,
+                'vtep_src_mac': self.mac if self.vxlan is True else None,
+                'vtep_dst_mac': self.vtep_dst_mac if self.vxlan is True else None,
+                'vtep_dst_ip': self.vtep_dst_ip if self.vxlan is True else None,
+                'vtep_src_ip': self.vtep_src_ip if self.vxlan is True else None,
+                'net_vni': self.vnis[chain_idx] if self.vxlan is True else None
             })
-
-            flow_idx += current_flow_count
+            # after first chain, fall back to the flow count for all other chains
+            cur_chain_flow_count = flows_per_chain
         return configs
 
-    @classmethod
-    def expand_ip(cls, ip, step_ip, count):
-        if step_ip == 'random':
-            # Repeatable Random will used in the stream src/dst IP pairs, but we still need
-            # to expand the IP based on the number of chains and flows configured. So we use
-            # "0.0.0.1" as the step to have the exact IP flow ranges for every chain.
-            step_ip = '0.0.0.1'
-
-        step_ip_in_int = cls.ip_to_int(step_ip)
-        subnet = IPNetwork(ip)
-        ip_list = []
-        for _ in xrange(count):
-            ip_list.append(subnet.ip.format())
-            subnet = subnet.next(step_ip_in_int)
-        return ip_list
-
-    @staticmethod
-    def mac_to_int(mac):
-        return int(mac.translate(None, ":.- "), 16)
-
-    @staticmethod
-    def int_to_mac(i):
-        mac = format(i, 'x').zfill(12)
-        blocks = [mac[x:x + 2] for x in xrange(0, len(mac), 2)]
-        return ':'.join(blocks)
-
     @staticmethod
     def ip_to_int(addr):
+        """Convert an IP address from string to numeric."""
         return struct.unpack("!I", socket.inet_aton(addr))[0]
 
+    @staticmethod
+    def int_to_ip(nvalue):
+        """Convert an IP address from numeric to string."""
+        return socket.inet_ntoa(struct.pack("!I", nvalue))
+
 
-class RunningTrafficProfile(object):
+class GeneratorConfig(object):
     """Represents traffic configuration for currently running traffic profile."""
 
     DEFAULT_IP_STEP = '0.0.0.1'
     DEFAULT_SRC_DST_IP_STEP = '0.0.0.1'
 
-    def __init__(self, config, generator_profile):
-        generator_config = self.__match_generator_profile(config.traffic_generator,
-                                                          generator_profile)
-        self.generator_config = generator_config
+    def __init__(self, config):
+        """Create a generator config."""
+        self.config = config
+        # name of the generator profile (normally trex or dummy)
+        # pick the default one if not specified explicitly from cli options
+        if not config.generator_profile:
+            config.generator_profile = config.traffic_generator.default_profile
+        # pick up the profile dict based on the name
+        gen_config = self.__match_generator_profile(config.traffic_generator,
+                                                    config.generator_profile)
+        self.gen_config = gen_config
+        # copy over fields from the dict
+        self.tool = gen_config.tool
+        self.ip = gen_config.ip
+        # overrides on config.cores and config.mbuf_factor
+        if config.cores:
+            self.cores = config.cores
+        else:
+            self.cores = gen_config.get('cores', 1)
+        self.mbuf_factor = config.mbuf_factor
+        self.mbuf_64 = config.mbuf_64
+        self.hdrh = not config.disable_hdrh
+        if gen_config.intf_speed:
+            # interface speed is overriden from config
+            self.intf_speed = bitmath.parse_string(gen_config.intf_speed.replace('ps', '')).bits
+        else:
+            # interface speed is discovered/provided by the traffic generator
+            self.intf_speed = 0
+        self.name = gen_config.name
+        self.zmq_pub_port = gen_config.get('zmq_pub_port', 4500)
+        self.zmq_rpc_port = gen_config.get('zmq_rpc_port', 4501)
+        self.limit_memory = gen_config.get('limit_memory', 1024)
+        self.software_mode = gen_config.get('software_mode', False)
+        self.interfaces = gen_config.interfaces
+        if self.interfaces[0].port != 0 or self.interfaces[1].port != 1:
+            raise TrafficClientException('Invalid port order/id in generator_profile.interfaces')
         self.service_chain = config.service_chain
         self.service_chain_count = config.service_chain_count
         self.flow_count = config.flow_count
-        self.host_name = generator_config.host_name
-        self.name = generator_config.name
-        self.tool = generator_config.tool
-        self.cores = generator_config.get('cores', 1)
-        self.ip_addrs_step = generator_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
+        self.host_name = gen_config.host_name
+
+        self.tg_gateway_ip_addrs = gen_config.tg_gateway_ip_addrs
+        self.ip_addrs = gen_config.ip_addrs
+        self.ip_addrs_step = gen_config.ip_addrs_step or self.DEFAULT_SRC_DST_IP_STEP
         self.tg_gateway_ip_addrs_step = \
-            generator_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
-        self.gateway_ip_addrs_step = generator_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
-        self.gateway_ips = generator_config.gateway_ip_addrs
-        self.ip = generator_config.ip
-        self.intf_speed = bitmath.parse_string(generator_config.intf_speed.replace('ps', '')).bits
+            gen_config.tg_gateway_ip_addrs_step or self.DEFAULT_IP_STEP
+        self.gateway_ip_addrs_step = gen_config.gateway_ip_addrs_step or self.DEFAULT_IP_STEP
+        self.gateway_ips = gen_config.gateway_ip_addrs
+        self.udp_src_port = gen_config.udp_src_port
+        self.udp_dst_port = gen_config.udp_dst_port
+        self.vteps = gen_config.get('vteps')
+        self.devices = [Device(port, self) for port in [0, 1]]
+        # This should normally always be [0, 1]
+        self.ports = [device.port for device in self.devices]
+
+        # check that pci is not empty
+        if not gen_config.interfaces[0].get('pci', None) or \
+           not gen_config.interfaces[1].get('pci', None):
+            raise TrafficClientException("configuration interfaces pci fields cannot be empty")
+
+        self.pcis = [tgif['pci'] for tgif in gen_config.interfaces]
         self.vlan_tagging = config.vlan_tagging
-        self.no_arp = config.no_arp
-        self.src_device = None
-        self.dst_device = None
-        self.vm_mac_list = None
-        self.__prep_interfaces(generator_config)
-
-    def to_json(self):
-        return dict(self.generator_config)
-
-    def set_vm_mac_list(self, vm_mac_list):
-        self.src_device.set_vm_mac_list(vm_mac_list[0])
-        self.dst_device.set_vm_mac_list(vm_mac_list[1])
-
-    @staticmethod
-    def __match_generator_profile(traffic_generator, generator_profile):
-        generator_config = AttrDict(traffic_generator)
-        generator_config.pop('default_profile')
-        generator_config.pop('generator_profile')
-        matching_profile = filter(lambda profile: profile.name == generator_profile,
-                                  traffic_generator.generator_profile)
-        if len(matching_profile) != 1:
-            raise Exception('Traffic generator profile not found: ' + generator_profile)
-
-        generator_config.update(matching_profile[0])
-
-        return generator_config
-
-    def __prep_interfaces(self, generator_config):
-        src_config = {
-            'chain_count': self.service_chain_count,
-            'flow_count': self.flow_count / 2,
-            'ip': generator_config.ip_addrs[0],
-            'ip_addrs_step': self.ip_addrs_step,
-            'gateway_ip': self.gateway_ips[0],
-            'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
-            'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[0],
-            'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
-            'udp_src_port': generator_config.udp_src_port,
-            'udp_dst_port': generator_config.udp_dst_port,
-            'vlan_tagging': self.vlan_tagging
-        }
-        dst_config = {
-            'chain_count': self.service_chain_count,
-            'flow_count': self.flow_count / 2,
-            'ip': generator_config.ip_addrs[1],
-            'ip_addrs_step': self.ip_addrs_step,
-            'gateway_ip': self.gateway_ips[1],
-            'gateway_ip_addrs_step': self.gateway_ip_addrs_step,
-            'tg_gateway_ip': generator_config.tg_gateway_ip_addrs[1],
-            'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
-            'udp_src_port': generator_config.udp_src_port,
-            'udp_dst_port': generator_config.udp_dst_port,
-            'vlan_tagging': self.vlan_tagging
-        }
-
-        self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))
-        self.dst_device = Device(**dict(dst_config, **generator_config.interfaces[1]))
-        self.src_device.set_destination(self.dst_device)
-        self.dst_device.set_destination(self.src_device)
-
-        if self.service_chain == ChainType.EXT and not self.no_arp \
-                and not self.__are_unique(self.src_device.ip_list, self.dst_device.ip_list):
-            raise Exception('Computed IP addresses are not unique, choose different base. '
-                            'Start IPs: {start}. End IPs: {end}'
-                            .format(start=self.src_device.ip_list,
-                                    end=self.dst_device.ip_list))
 
-    def __are_unique(self, list1, list2):
-        return set(list1).isdisjoint(set(list2))
-
-    @property
-    def devices(self):
-        return [self.src_device, self.dst_device]
-
-    @property
-    def vlans(self):
-        return [self.src_device.vtep_vlan, self.dst_device.vtep_vlan]
-
-    @property
-    def ports(self):
-        return [self.src_device.port, self.dst_device.port]
-
-    @property
-    def switch_ports(self):
-        return [self.src_device.switch_port, self.dst_device.switch_port]
-
-    @property
-    def pcis(self):
-        return [self.src_device.pci, self.dst_device.pci]
+        # needed for result/summarizer
+        config['tg-name'] = gen_config.name
+        config['tg-tool'] = self.tool
 
+    def to_json(self):
+        """Get json form to display the content into the overall result dict."""
+        return dict(self.gen_config)
 
-class TrafficGeneratorFactory(object):
+    def set_dest_macs(self, port_index, dest_macs):
+        """Set the list of dest MACs indexed by the chain id on given port.
 
-    def __init__(self, config):
-        self.config = config
+        port_index: the port for which dest macs must be set
+        dest_macs: a list of dest MACs indexed by chain id
+        """
+        if len(dest_macs) < self.config.service_chain_count:
+            raise TrafficClientException('Dest MAC list %s must have %d entries' %
+                                         (dest_macs, self.config.service_chain_count))
+        # only pass the first scc dest MACs
+        self.devices[port_index].set_dest_macs(dest_macs[:self.config.service_chain_count])
+        LOG.info('Port %d: dst MAC %s', port_index, [str(mac) for mac in dest_macs])
+
+    def set_vtep_dest_macs(self, port_index, dest_macs):
+        """Set the list of dest MACs indexed by the chain id on given port.
+
+        port_index: the port for which dest macs must be set
+        dest_macs: a list of dest MACs indexed by chain id
+        """
+        if len(dest_macs) != self.config.service_chain_count:
+            raise TrafficClientException('Dest MAC list %s must have %d entries' %
+                                         (dest_macs, self.config.service_chain_count))
+        self.devices[port_index].set_vtep_dst_mac(dest_macs)
+        LOG.info('Port %d: vtep dst MAC %s', port_index, set([str(mac) for mac in dest_macs]))
 
-    def get_tool(self):
-        return self.config.generator_config.tool
+    def get_dest_macs(self):
+        """Return the list of dest macs indexed by port."""
+        return [dev.get_dest_macs() for dev in self.devices]
 
-    def get_generator_client(self):
-        tool = self.get_tool().lower()
-        if tool == 'trex':
-            from traffic_gen import trex
-            return trex.TRex(self.config)
-        elif tool == 'dummy':
-            from traffic_gen import dummy
-            return dummy.DummyTG(self.config)
-        else:
-            return None
+    def set_vlans(self, port_index, vlans):
+        """Set the list of vlans to use indexed by the chain id on given port.
 
-    def list_generator_profile(self):
-        return [profile.name for profile in self.config.traffic_generator.generator_profile]
+        port_index: the port for which VLANs must be set
+        vlans: a  list of vlan lists indexed by chain id
+        """
+        if len(vlans) != self.config.service_chain_count:
+            raise TrafficClientException('VLAN list %s must have %d entries' %
+                                         (vlans, self.config.service_chain_count))
+        self.devices[port_index].set_vlans(vlans)
 
-    def get_generator_config(self, generator_profile):
-        return RunningTrafficProfile(self.config, generator_profile)
+    def set_vxlans(self, port_index, vxlans):
+        """Set the list of vxlans (VNIs) to use indexed by the chain id on given port.
 
-    def get_matching_profile(self, traffic_profile_name):
-        matching_profile = filter(lambda profile: profile.name == traffic_profile_name,
-                                  self.config.traffic_profile)
+        port_index: the port for which VXLANs must be set
+        VXLANs: a  list of VNIs lists indexed by chain id
+        """
+        if len(vxlans) != self.config.service_chain_count:
+            raise TrafficClientException('VXLAN list %s must have %d entries' %
+                                         (vxlans, self.config.service_chain_count))
+        self.devices[port_index].set_vxlans(vxlans)
+
+    def set_vtep_vlan(self, port_index, vlan):
+        """Set the vtep vlan to use indexed by the chain id on given port.
+        port_index: the port for which VLAN must be set
+        """
+        self.devices[port_index].set_vtep_vlan(vlan)
 
-        if len(matching_profile) > 1:
-            raise Exception('Multiple traffic profiles with the same name found.')
-        elif len(matching_profile) == 0:
-            raise Exception('No traffic profile found.')
+    def set_vxlan_endpoints(self, port_index, src_ip, dst_ip):
+        self.devices[port_index].set_vxlan_endpoints(src_ip, dst_ip)
 
-        return matching_profile[0]
+    @staticmethod
+    def __match_generator_profile(traffic_generator, generator_profile):
+        gen_config = AttrDict(traffic_generator)
+        gen_config.pop('default_profile')
+        gen_config.pop('generator_profile')
+        matching_profile = [profile for profile in traffic_generator.generator_profile if
+                            profile.name == generator_profile]
+        if len(matching_profile) != 1:
+            raise Exception('Traffic generator profile not found: ' + generator_profile)
 
-    def get_frame_sizes(self, traffic_profile):
-        matching_profile = self.get_matching_profile(traffic_profile)
-        return matching_profile.l2frame_size
+        gen_config.update(matching_profile[0])
+        return gen_config
 
 
 class TrafficClient(object):
+    """Traffic generator client with NDR/PDR binary seearch."""
 
     PORTS = [0, 1]
 
     def __init__(self, config, notifier=None):
-        generator_factory = TrafficGeneratorFactory(config)
-        self.gen = generator_factory.get_generator_client()
-        self.tool = generator_factory.get_tool()
+        """Create a new TrafficClient instance.
+
+        config: nfvbench config
+        notifier: notifier (optional)
+
+        A new instance is created everytime the nfvbench config may have changed.
+        """
         self.config = config
+        self.generator_config = GeneratorConfig(config)
+        self.tool = self.generator_config.tool
+        self.gen = self._get_generator()
         self.notifier = notifier
         self.interval_collector = None
         self.iteration_collector = None
-        self.runner = TrafficRunner(self, self.config.duration_sec, self.config.interval_sec)
-        if self.gen is None:
-            raise TrafficClientException('%s is not a supported traffic generator' % self.tool)
-
+        self.runner = TrafficRunner(self, self.config.duration_sec, self.config.interval_sec,
+                                    self.config.service_mode)
+        self.config.frame_sizes = self._get_frame_sizes()
         self.run_config = {
             'l2frame_size': None,
             'duration_sec': self.config.duration_sec,
             'bidirectional': True,
-            'rates': None
+            'rates': []  # to avoid unsbuscriptable-obj warning
         }
         self.current_total_rate = {'rate_percent': '10'}
         if self.config.single_run:
             self.current_total_rate = utils.parse_rate_str(self.config.rate)
+        self.ifstats = None
+        # Speed is either discovered when connecting to TG or set from config
+        # This variable is 0 if not yet discovered from TG or must be the speed of
+        # each interface in bits per second
+        self.intf_speed = self.generator_config.intf_speed
+
+    def _get_generator(self):
+        tool = self.tool.lower()
+        if tool == 'trex':
+            from traffic_gen import trex_gen
+            return trex_gen.TRex(self)
+        if tool == 'dummy':
+            from traffic_gen import dummy
+            return dummy.DummyTG(self)
+        raise TrafficClientException('Unsupported generator tool name:' + self.tool)
 
-    def set_macs(self):
-        for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
-            device.set_mac(mac)
+    def skip_sleep(self):
+        """Skip all sleeps when doing unit testing with dummy TG.
+
+        Must be overriden using mock.patch
+        """
+        return False
+
+    def _get_frame_sizes(self):
+        traffic_profile_name = self.config.traffic.profile
+        matching_profiles = [profile for profile in self.config.traffic_profile if
+                             profile.name == traffic_profile_name]
+        if len(matching_profiles) > 1:
+            raise TrafficClientException('Multiple traffic profiles with name: ' +
+                                         traffic_profile_name)
+        elif not matching_profiles:
+            raise TrafficClientException('Cannot find traffic profile: ' + traffic_profile_name)
+        return matching_profiles[0].l2frame_size
 
     def start_traffic_generator(self):
-        self.gen.init()
+        """Start the traffic generator process (traffic not started yet)."""
         self.gen.connect()
+        # pick up the interface speed if it is not set from config
+        intf_speeds = self.gen.get_port_speed_gbps()
+        # convert Gbps unit into bps
+        tg_if_speed = bitmath.parse_string(str(intf_speeds[0]) + 'Gb').bits
+        if self.intf_speed:
+            # interface speed is overriden from config
+            if self.intf_speed != tg_if_speed:
+                # Warn the user if the speed in the config is different
+                LOG.warning('Interface speed provided is different from actual speed (%d Gbps)',
+                            intf_speeds[0])
+        else:
+            # interface speed not provisioned by config
+            self.intf_speed = tg_if_speed
+            # also update the speed in the tg config
+            self.generator_config.intf_speed = tg_if_speed
+
+        # Save the traffic generator local MAC
+        for mac, device in zip(self.gen.get_macs(), self.generator_config.devices):
+            device.set_mac(mac)
 
     def setup(self):
-        self.gen.set_mode()
-        self.gen.config_interface()
+        """Set up the traffic client."""
         self.gen.clear_stats()
 
     def get_version(self):
+        """Get the traffic generator version."""
         return self.gen.get_version()
 
     def ensure_end_to_end(self):
-        """
-        Ensure traffic generator receives packets it has transmitted.
+        """Ensure traffic generator receives packets it has transmitted.
+
         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 
-        At this point all VMs are in active state, but forwarding does not have to work.
-        Small amount of traffic is sent to every chain. Then total of sent and received packets
-        is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
-        N being number of chains, traffic flows through every chain and real measurements can be
-        performed.
+        VMs that are started and in active state may not pass traffic yet. It is imperative to make
+        sure that all VMs are passing traffic in both directions before starting any benchmarking.
+        To verify this, we need to send at a low frequency bi-directional packets and make sure
+        that we receive all packets back from all VMs. The number of flows is equal to 2 times
+        the number of chains (1 per direction) and we need to make sure we receive packets coming
+        from exactly 2 x chain count different source MAC addresses.
 
         Example:
             PVP chain (1 VM per chain)
             N = 10 (number of chains)
-            threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
-            if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
-            all 10 VMs are in operational state.
+            Flow count = 20 (number of flows)
+            If the number of unique source MAC addresses from received packets is 20 then
+            all 10 VMs 10 VMs are in operational state.
         """
         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
-        rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
-        self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
-
+        # send 2pps on each chain and each direction
+        rate_pps = {'rate_pps': str(self.config.service_chain_count * 2)}
+        self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False,
+                                e2e=True)
         # ensures enough traffic is coming back
-        threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)
         retry_count = (self.config.check_traffic_time_sec +
                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
+
+        # we expect to see packets coming from 2 unique MAC per chain
+        # because there can be flooding in the case of shared net
+        # we must verify that packets from the right VMs are received
+        # and not just count unique src MAC
+        # create a dict of (port, chain) tuples indexed by dest mac
+        mac_map = {}
+        for port, dest_macs in enumerate(self.generator_config.get_dest_macs()):
+            for chain, mac in enumerate(dest_macs):
+                mac_map[mac] = (port, chain)
+        unique_src_mac_count = len(mac_map)
+        if self.config.vxlan and self.config.traffic_generator.vtep_vlan:
+            get_mac_id = lambda packet: packet['binary'][60:66]
+        elif self.config.vxlan:
+            get_mac_id = lambda packet: packet['binary'][56:62]
+        else:
+            get_mac_id = lambda packet: packet['binary'][6:12]
         for it in xrange(retry_count):
             self.gen.clear_stats()
             self.gen.start_traffic()
-            LOG.info('Waiting for packets to be received back... ({} / {})'.format(it + 1,
-                     retry_count))
-            time.sleep(self.config.generic_poll_sec)
+            self.gen.start_capture()
+            LOG.info('Captured unique src mac %d/%d, capturing return packets (retry %d/%d)...',
+                     unique_src_mac_count - len(mac_map), unique_src_mac_count,
+                     it + 1, retry_count)
+            if not self.skip_sleep():
+                time.sleep(self.config.generic_poll_sec)
             self.gen.stop_traffic()
-            stats = self.gen.get_stats()
-
-            # compute total sent and received traffic on both ports
-            total_rx = 0
-            total_tx = 0
-            for port in self.PORTS:
-                total_rx += float(stats[port]['rx'].get('total_pkts', 0))
-                total_tx += float(stats[port]['tx'].get('total_pkts', 0))
-
-            # how much of traffic came back
-            ratio = total_rx / total_tx if total_tx else 0
-
-            if ratio > threshold:
-                self.gen.clear_stats()
-                self.gen.clear_streamblock()
-                LOG.info('End-to-end connectivity ensured')
-                return
-
-            time.sleep(self.config.generic_poll_sec)
-
+            self.gen.fetch_capture_packets()
+            self.gen.stop_capture()
+            for packet in self.gen.packet_list:
+                mac_id = get_mac_id(packet)
+                src_mac = ':'.join(["%02x" % ord(x) for x in mac_id])
+                if src_mac in mac_map and self.is_udp(packet):
+                    port, chain = mac_map[src_mac]
+                    LOG.info('Received packet from mac: %s (chain=%d, port=%d)',
+                             src_mac, chain, port)
+                    mac_map.pop(src_mac, None)
+
+                if not mac_map:
+                    LOG.info('End-to-end connectivity established')
+                    return
+            if self.config.l3_router and not self.config.no_arp:
+                # In case of L3 traffic mode, routers are not able to route traffic
+                # until VM interfaces are up and ARP requests are done
+                LOG.info('Waiting for loopback service completely started...')
+                LOG.info('Sending ARP request to assure end-to-end connectivity established')
+                self.ensure_arp_successful()
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 
+    def is_udp(self, packet):
+        pkt = Ether(packet['binary'])
+        return UDP in pkt
+
     def ensure_arp_successful(self):
-        if not self.gen.resolve_arp():
+        """Resolve all IP using ARP and throw an exception in case of failure."""
+        dest_macs = self.gen.resolve_arp()
+        if dest_macs:
+            # all dest macs are discovered, saved them into the generator config
+            if self.config.vxlan:
+                self.generator_config.set_vtep_dest_macs(0, dest_macs[0])
+                self.generator_config.set_vtep_dest_macs(1, dest_macs[1])
+            else:
+                self.generator_config.set_dest_macs(0, dest_macs[0])
+                self.generator_config.set_dest_macs(1, dest_macs[1])
+        else:
             raise TrafficClientException('ARP cannot be resolved')
 
     def set_traffic(self, frame_size, bidirectional):
+        """Reconfigure the traffic generator for a new frame size."""
         self.run_config['bidirectional'] = bidirectional
         self.run_config['l2frame_size'] = frame_size
         self.run_config['rates'] = [self.get_per_direction_rate()]
@@ -464,11 +673,18 @@ class TrafficClient(object):
             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
             if unidir_reverse_pps > 0:
                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})
+        # Fix for [NFVBENCH-67], convert the rate string to PPS
+        for idx, rate in enumerate(self.run_config['rates']):
+            if 'rate_pps' not in rate:
+                self.run_config['rates'][idx] = {'rate_pps': self.__convert_rates(rate)['rate_pps']}
 
         self.gen.clear_streamblock()
-        self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)
+        if self.config.no_latency_streams:
+            LOG.info("Latency streams are disabled")
+        self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional,
+                                latency=not self.config.no_latency_streams)
 
-    def modify_load(self, load):
+    def _modify_load(self, load):
         self.current_total_rate = {'rate_percent': str(load)}
         rate_per_direction = self.get_per_direction_rate()
 
@@ -479,6 +695,7 @@ class TrafficClient(object):
             self.run_config['rates'][1] = rate_per_direction
 
     def get_ndr_and_pdr(self):
+        """Start the NDR/PDR iteration and return the results."""
         dst = 'Bidirectional' if self.run_config['bidirectional'] else 'Unidirectional'
         targets = {}
         if self.config.ndr_run:
@@ -518,10 +735,10 @@ class TrafficClient(object):
         total_pkts = result['tx']['total_pkts']
         if not total_pkts:
             return float('inf')
-        else:
-            return float(dropped_pkts) / total_pkts * 100
+        return float(dropped_pkts) / total_pkts * 100
 
     def get_stats(self):
+        """Collect final stats for previous run."""
         stats = self.gen.get_stats()
         retDict = {'total_tx_rate': stats['total_tx_rate']}
         for port in self.PORTS:
@@ -538,9 +755,12 @@ class TrafficClient(object):
                     retDict[port]['rx'][key] = int(stats[port]['rx'][key])
                 except ValueError:
                     retDict[port]['rx'][key] = 0
-            retDict[port]['rx']['avg_delay_usec'] = int(stats[port]['rx']['avg_delay_usec'])
-            retDict[port]['rx']['min_delay_usec'] = int(stats[port]['rx']['min_delay_usec'])
-            retDict[port]['rx']['max_delay_usec'] = int(stats[port]['rx']['max_delay_usec'])
+            retDict[port]['rx']['avg_delay_usec'] = cast_integer(
+                stats[port]['rx']['avg_delay_usec'])
+            retDict[port]['rx']['min_delay_usec'] = cast_integer(
+                stats[port]['rx']['min_delay_usec'])
+            retDict[port]['rx']['max_delay_usec'] = cast_integer(
+                stats[port]['rx']['max_delay_usec'])
             retDict[port]['drop_rate_percent'] = self.__get_dropped_rate(retDict[port])
 
         ports = sorted(retDict.keys())
@@ -574,7 +794,7 @@ class TrafficClient(object):
     def __convert_rates(self, rate):
         return utils.convert_rates(self.run_config['l2frame_size'],
                                    rate,
-                                   self.config.generator_config.intf_speed)
+                                   self.intf_speed)
 
     def __ndr_pdr_found(self, tag, load):
         rates = self.__convert_rates({'rate_percent': load})
@@ -583,7 +803,7 @@ class TrafficClient(object):
         self.interval_collector.add_ndr_pdr(tag, last_stats)
 
     def __format_output_stats(self, stats):
-        for key in (self.PORTS + ['overall']):
+        for key in self.PORTS + ['overall']:
             interface = stats[key]
             stats[key] = {
                 'tx_pkts': interface['tx']['total_pkts'],
@@ -599,12 +819,12 @@ class TrafficClient(object):
 
     def __targets_found(self, rate, targets, results):
         for tag, target in targets.iteritems():
-            LOG.info('Found {} ({}) load: {}'.format(tag, target, rate))
+            LOG.info('Found %s (%s) load: %s', tag, target, rate)
             self.__ndr_pdr_found(tag, rate)
             results[tag]['timestamp_sec'] = time.time()
 
     def __range_search(self, left, right, targets, results):
-        '''Perform a binary search for a list of targets inside a [left..right] range or rate
+        """Perform a binary search for a list of targets inside a [left..right] range or rate.
 
         left    the left side of the range to search as a % the line rate (100 = 100% line rate)
                 indicating the rate to send on each interface
@@ -613,10 +833,10 @@ class TrafficClient(object):
         targets a dict of drop rates to search (0.1 = 0.1%), indexed by the DR name or "tag"
                 ('ndr', 'pdr')
         results a dict to store results
-        '''
-        if len(targets) == 0:
+        """
+        if not targets:
             return
-        LOG.info('Range search [{} .. {}] targets: {}'.format(left, right, targets))
+        LOG.info('Range search [%s .. %s] targets: %s', left, right, targets)
 
         # Terminate search when gap is less than load epsilon
         if right - left < self.config.measurement.load_epsilon:
@@ -625,8 +845,12 @@ class TrafficClient(object):
 
         # Obtain the average drop rate in for middle load
         middle = (left + right) / 2.0
-        stats, rates = self.__run_search_iteration(middle)
-
+        try:
+            stats, rates = self.__run_search_iteration(middle)
+        except STLError:
+            LOG.exception("Got exception from traffic generator during binary search")
+            self.__targets_found(left, targets, results)
+            return
         # Split target dicts based on the avg drop rate
         left_targets = {}
         right_targets = {}
@@ -670,8 +894,11 @@ class TrafficClient(object):
             self.__range_search(middle, right, right_targets, results)
 
     def __run_search_iteration(self, rate):
-        # set load
-        self.modify_load(rate)
+        """Run one iteration at the given rate level.
+
+        rate: the rate to send on each port in percent (0 to 100)
+        """
+        self._modify_load(rate)
 
         # poll interval stats and collect them
         for stats in self.run_traffic():
@@ -679,11 +906,13 @@ class TrafficClient(object):
             time_elapsed_ratio = self.runner.time_elapsed() / self.run_config['duration_sec']
             if time_elapsed_ratio >= 1:
                 self.cancel_traffic()
+                if not self.skip_sleep():
+                    time.sleep(self.config.pause_sec)
         self.interval_collector.reset()
 
         # get stats from the run
         stats = self.runner.client.get_stats()
-        current_traffic_config = self.get_traffic_config()
+        current_traffic_config = self._get_traffic_config()
         warning = self.compare_tx_rates(current_traffic_config['direction-total']['rate_pps'],
                                         stats['total_tx_rate'])
         if warning is not None:
@@ -691,12 +920,12 @@ class TrafficClient(object):
 
         # save reliable stats from whole iteration
         self.iteration_collector.add(stats, current_traffic_config['direction-total']['rate_pps'])
-        LOG.info('Average drop rate: {}'.format(stats['overall']['drop_rate_percent']))
-
+        LOG.info('Average drop rate: %f', stats['overall']['drop_rate_percent'])
         return stats, current_traffic_config['direction-total']
 
     @staticmethod
     def log_stats(stats):
+        """Log estimated stats during run."""
         report = {
             'datetime': str(datetime.now()),
             'tx_packets': stats['overall']['tx']['total_pkts'],
@@ -706,11 +935,12 @@ class TrafficClient(object):
         }
         LOG.info('TX: %(tx_packets)d; '
                  'RX: %(rx_packets)d; '
-                 'Dropped: %(drop_packets)d; '
-                 'Drop rate: %(drop_rate_percent).4f%%',
+                 'Est. Dropped: %(drop_packets)d; '
+                 'Est. Drop rate: %(drop_rate_percent).4f%%',
                  report)
 
     def run_traffic(self):
+        """Start traffic and return intermediate stats for each interval."""
         stats = self.runner.run()
         while self.runner.is_running:
             self.log_stats(stats)
@@ -719,22 +949,14 @@ class TrafficClient(object):
             if stats is None:
                 return
         self.log_stats(stats)
-        LOG.info('Drop rate: {}'.format(stats['overall']['drop_rate_percent']))
+        LOG.info('Drop rate: %f', stats['overall']['drop_rate_percent'])
         yield stats
 
     def cancel_traffic(self):
+        """Stop traffic."""
         self.runner.stop()
 
-    def get_interface(self, port_index):
-        port = self.gen.port_handle[port_index]
-        tx, rx = 0, 0
-        if not self.config.no_traffic:
-            stats = self.get_stats()
-            if port in stats:
-                tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])
-        return Interface('traffic-generator', self.tool.lower(), tx, rx)
-
-    def get_traffic_config(self):
+    def _get_traffic_config(self):
         config = {}
         load_total = 0.0
         bps_total = 0.0
@@ -753,18 +975,20 @@ class TrafficClient(object):
         config['direction-total'] = dict(config['direction-forward'])
         config['direction-total'].update({
             'rate_percent': load_total,
-            'rate_pps': pps_total,
+            'rate_pps': cast_integer(pps_total),
             'rate_bps': bps_total
         })
 
         return config
 
     def get_run_config(self, results):
-        """Returns configuration which was used for the last run."""
+        """Return configuration which was used for the last run."""
         r = {}
+        # because we want each direction to have the far end RX rates,
+        # use the far end index (1-idx) to retrieve the RX rates
         for idx, key in enumerate(["direction-forward", "direction-reverse"]):
             tx_rate = results["stats"][idx]["tx"]["total_pkts"] / self.config.duration_sec
-            rx_rate = results["stats"][idx]["rx"]["total_pkts"] / self.config.duration_sec
+            rx_rate = results["stats"][1 - idx]["rx"]["total_pkts"] / self.config.duration_sec
             r[key] = {
                 "orig": self.__convert_rates(self.run_config['rates'][idx]),
                 "tx": self.__convert_rates({'rate_pps': tx_rate}),
@@ -775,13 +999,90 @@ class TrafficClient(object):
         for direction in ['orig', 'tx', 'rx']:
             total[direction] = {}
             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
-                total[direction][unit] = sum(map(lambda x: float(x[direction][unit]), r.values()))
+                total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()])
 
         r['direction-total'] = total
         return r
 
+    def insert_interface_stats(self, pps_list):
+        """Insert interface stats to a list of packet path stats.
+
+        pps_list: a list of packet path stats instances indexed by chain index
+
+        This function will insert the packet path stats for the traffic gen ports 0 and 1
+        with itemized per chain tx/rx counters.
+        There will be as many packet path stats as chains.
+        Each packet path stats will have exactly 2 InterfaceStats for port 0 and port 1
+        self.pps_list:
+        [
+        PacketPathStats(InterfaceStats(chain 0, port 0), InterfaceStats(chain 0, port 1)),
+        PacketPathStats(InterfaceStats(chain 1, port 0), InterfaceStats(chain 1, port 1)),
+        ...
+        ]
+        """
+        def get_if_stats(chain_idx):
+            return [InterfaceStats('p' + str(port), self.tool)
+                    for port in range(2)]
+        # keep the list of list of interface stats indexed by the chain id
+        self.ifstats = [get_if_stats(chain_idx)
+                        for chain_idx in range(self.config.service_chain_count)]
+        # note that we need to make a copy of the ifs list so that any modification in the
+        # list from pps will not change the list saved in self.ifstats
+        self.pps_list = [PacketPathStats(list(ifs)) for ifs in self.ifstats]
+        # insert the corresponding pps in the passed list
+        pps_list.extend(self.pps_list)
+
+    def update_interface_stats(self, diff=False):
+        """Update all interface stats.
+
+        diff: if False, simply refresh the interface stats values with latest values
+              if True, diff the interface stats with the latest values
+        Make sure that the interface stats inserted in insert_interface_stats() are updated
+        with proper values.
+        self.ifstats:
+        [
+        [InterfaceStats(chain 0, port 0), InterfaceStats(chain 0, port 1)],
+        [InterfaceStats(chain 1, port 0), InterfaceStats(chain 1, port 1)],
+        ...
+        ]
+        """
+        if diff:
+            stats = self.gen.get_stats()
+            for chain_idx, ifs in enumerate(self.ifstats):
+                # each ifs has exactly 2 InterfaceStats and 2 Latency instances
+                # corresponding to the
+                # port 0 and port 1 for the given chain_idx
+                # Note that we cannot use self.pps_list[chain_idx].if_stats to pick the
+                # interface stats for the pps because it could have been modified to contain
+                # additional interface stats
+                self.gen.get_stream_stats(stats, ifs, self.pps_list[chain_idx].latencies, chain_idx)
+            # special handling for vxlan
+            # in case of vxlan, flow stats are not available so all rx counters will be
+            # zeros when the total rx port counter is non zero.
+            # in that case,
+            for port in range(2):
+                total_rx = 0
+                for ifs in self.ifstats:
+                    total_rx += ifs[port].rx
+                if total_rx == 0:
+                    # check if the total port rx from Trex is also zero
+                    port_rx = stats[port]['rx']['total_pkts']
+                    if port_rx:
+                        # the total rx for all chains from port level stats is non zero
+                        # which means that the per-chain stats are not available
+                        if len(self.ifstats) == 1:
+                            # only one chain, simply report the port level rx to the chain rx stats
+                            self.ifstats[0][port].rx = port_rx
+                        else:
+                            for ifs in self.ifstats:
+                                # mark this data as unavailable
+                                ifs[port].rx = None
+                            # pitch in the total rx only in the last chain pps
+                            self.ifstats[-1][port].rx_total = port_rx
+
     @staticmethod
     def compare_tx_rates(required, actual):
+        """Compare the actual TX rate to the required TX rate."""
         threshold = 0.9
         are_different = False
         try:
@@ -800,6 +1101,7 @@ class TrafficClient(object):
         return None
 
     def get_per_direction_rate(self):
+        """Get the rate for each direction."""
         divisor = 2 if self.run_config['bidirectional'] else 1
         if 'rate_percent' in self.current_total_rate:
             # don't split rate if it's percentage
@@ -808,6 +1110,7 @@ class TrafficClient(object):
         return utils.divide_rate(self.current_total_rate, divisor)
 
     def close(self):
+        """Close this instance."""
         try:
             self.gen.stop_traffic()
         except Exception: