NFVBENCH-171 Not accurate flow count with some IP and UDP ranges combinations
[nfvbench.git] / nfvbench / traffic_gen / trex_gen.py
index bbb67c1..f5c2afb 100644 (file)
@@ -18,19 +18,19 @@ import os
 import random
 import time
 import traceback
+from functools import reduce
 
 from itertools import count
+# pylint: disable=import-error
+from scapy.contrib.mpls import MPLS  # flake8: noqa
+# pylint: enable=import-error
 from nfvbench.log import LOG
 from nfvbench.traffic_server import TRexTrafficServer
 from nfvbench.utils import cast_integer
 from nfvbench.utils import timeout
 from nfvbench.utils import TimeoutError
-from traffic_base import AbstractTrafficGenerator
-from traffic_base import TrafficGeneratorException
-import traffic_utils as utils
-from traffic_utils import IMIX_AVG_L2_FRAME_SIZE
-from traffic_utils import IMIX_L2_SIZES
-from traffic_utils import IMIX_RATIOS
+
+from hdrh.histogram import HdrHistogram
 
 # pylint: disable=import-error
 from trex.common.services.trex_service_arp import ServiceARP
@@ -50,16 +50,24 @@ from trex.stl.api import STLScVmRaw
 from trex.stl.api import STLStream
 from trex.stl.api import STLTXCont
 from trex.stl.api import STLVmFixChecksumHw
+from trex.stl.api import STLVmFixIpv4
 from trex.stl.api import STLVmFlowVar
 from trex.stl.api import STLVmFlowVarRepeatableRandom
+from trex.stl.api import STLVmTupleGen
 from trex.stl.api import STLVmWrFlowVar
 from trex.stl.api import ThreeBytesField
 from trex.stl.api import UDP
 from trex.stl.api import XByteField
 
-
 # pylint: enable=import-error
 
+from .traffic_base import AbstractTrafficGenerator
+from .traffic_base import TrafficGeneratorException
+from . import traffic_utils as utils
+from .traffic_utils import IMIX_AVG_L2_FRAME_SIZE
+from .traffic_utils import IMIX_L2_SIZES
+from .traffic_utils import IMIX_RATIOS
+
 class VXLAN(Packet):
     """VxLAN class."""
 
@@ -91,6 +99,7 @@ class TRex(AbstractTrafficGenerator):
         self.rates = []
         self.capture_id = None
         self.packet_list = []
+        self.l2_frame_size = 0
 
     def get_version(self):
         """Get the Trex version."""
@@ -111,7 +120,7 @@ class TRex(AbstractTrafficGenerator):
         pg_id = port * TRex.PORT_PG_ID_MASK | chain_id
         return pg_id, pg_id | TRex.LATENCY_PG_ID_MASK
 
-    def extract_stats(self, in_stats):
+    def extract_stats(self, in_stats, ifstats):
         """Extract stats from dict returned by Trex API.
 
         :param in_stats: dict as returned by TRex api
@@ -147,8 +156,39 @@ class TRex(AbstractTrafficGenerator):
 
         total_tx_pkts = result[0]['tx']['total_pkts'] + result[1]['tx']['total_pkts']
         result["total_tx_rate"] = cast_integer(total_tx_pkts / self.config.duration_sec)
+        # actual offered tx rate in bps
+        avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
+        total_tx_bps = utils.pps_to_bps(result["total_tx_rate"], avg_packet_size)
+        result['offered_tx_rate_bps'] = total_tx_bps
+
+        result.update(self.get_theoretical_rates(avg_packet_size))
+
         result["flow_stats"] = in_stats["flow_stats"]
         result["latency"] = in_stats["latency"]
+
+        # Merge HDRHistogram to have an overall value for all chains and ports
+        try:
+            hdrh_list = []
+            if ifstats:
+                for chain_id, _ in enumerate(ifstats):
+                    for ph in self.port_handle:
+                        _, lat_pg_id = self.get_pg_id(ph, chain_id)
+                        hdrh_list.append(
+                            HdrHistogram.decode(in_stats['latency'][lat_pg_id]['latency']['hdrh']))
+            else:
+                for pg_id in in_stats['latency']:
+                    if pg_id != 'global':
+                        hdrh_list.append(
+                            HdrHistogram.decode(in_stats['latency'][pg_id]['latency']['hdrh']))
+
+            def add_hdrh(x, y):
+                x.add(y)
+                return x
+            decoded_hdrh = reduce(add_hdrh, hdrh_list)
+            result["hdrh"] = HdrHistogram.encode(decoded_hdrh).decode('utf-8')
+        except KeyError:
+            pass
+
         return result
 
     def get_stream_stats(self, trex_stats, if_stats, latencies, chain_idx):
@@ -245,11 +285,74 @@ class TRex(AbstractTrafficGenerator):
                 else:
                     latencies[port].min_usec = get_latency(lat['total_min'])
                     latencies[port].avg_usec = get_latency(lat['average'])
+                # pick up the HDR histogram if present (otherwise will raise KeyError)
+                latencies[port].hdrh = lat['hdrh']
             except KeyError:
                 pass
 
     def __combine_latencies(self, in_stats, results, port_handle):
-        """Traverse TRex result dictionary and combines chosen latency stats."""
+        """Traverse TRex result dictionary and combines chosen latency stats.
+
+          example of latency dict returned by trex (2 chains):
+         'latency': {256: {'err_cntrs': {'dropped': 0,
+                                 'dup': 0,
+                                 'out_of_order': 0,
+                                 'seq_too_high': 0,
+                                 'seq_too_low': 0},
+                            'latency': {'average': 26.5,
+                                        'hdrh': u'HISTFAAAAEx4nJNpmSgz...bFRgxi',
+                                        'histogram': {20: 303,
+                                                        30: 320,
+                                                        40: 300,
+                                                        50: 73,
+                                                        60: 4,
+                                                        70: 1},
+                                        'jitter': 14,
+                                        'last_max': 63,
+                                        'total_max': 63,
+                                        'total_min': 20}},
+                    257: {'err_cntrs': {'dropped': 0,
+                                        'dup': 0,
+                                        'out_of_order': 0,
+                                        'seq_too_high': 0,
+                                        'seq_too_low': 0},
+                            'latency': {'average': 29.75,
+                                        'hdrh': u'HISTFAAAAEV4nJN...CALilDG0=',
+                                        'histogram': {20: 261,
+                                                        30: 431,
+                                                        40: 3,
+                                                        50: 80,
+                                                        60: 225},
+                                        'jitter': 23,
+                                        'last_max': 67,
+                                        'total_max': 67,
+                                        'total_min': 20}},
+                    384: {'err_cntrs': {'dropped': 0,
+                                        'dup': 0,
+                                        'out_of_order': 0,
+                                        'seq_too_high': 0,
+                                        'seq_too_low': 0},
+                            'latency': {'average': 18.0,
+                                        'hdrh': u'HISTFAAAADR4nJNpm...MjCwDDxAZG',
+                                        'histogram': {20: 987, 30: 14},
+                                        'jitter': 0,
+                                        'last_max': 34,
+                                        'total_max': 34,
+                                        'total_min': 20}},
+                    385: {'err_cntrs': {'dropped': 0,
+                                    'dup': 0,
+                                    'out_of_order': 0,
+                                    'seq_too_high': 0,
+                                    'seq_too_low': 0},
+                            'latency': {'average': 19.0,
+                                        'hdrh': u'HISTFAAAADR4nJNpm...NkYmJgDdagfK',
+                                        'histogram': {20: 989, 30: 11},
+                                        'jitter': 0,
+                                        'last_max': 38,
+                                        'total_max': 38,
+                                        'total_min': 20}},
+                    'global': {'bad_hdr': 0, 'old_flow': 0}},
+        """
         total_max = 0
         average = 0
         total_min = float("inf")
@@ -273,14 +376,14 @@ class TRex(AbstractTrafficGenerator):
         bind_layers(UDP, VXLAN, dport=4789)
         bind_layers(VXLAN, Ether)
 
-    def _create_pkt(self, stream_cfg, l2frame_size):
+    def _create_pkt(self, stream_cfg, l2frame_size, disable_random_latency_flow=False):
         """Create a packet of given size.
 
         l2frame_size: size of the L2 frame in bytes (including the 32-bit FCS)
         """
         # Trex will add the FCS field, so we need to remove 4 bytes from the l2 frame size
         frame_size = int(l2frame_size) - 4
-
+        vm_param = []
         if stream_cfg['vxlan'] is True:
             self._bind_vxlan()
             encap_level = '1'
@@ -291,6 +394,26 @@ class TRex(AbstractTrafficGenerator):
             pkt_base /= UDP(sport=random.randint(1337, 32767), dport=4789)
             pkt_base /= VXLAN(vni=stream_cfg['net_vni'])
             pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
+            # need to randomize the outer header UDP src port based on flow
+            vxlan_udp_src_fv = STLVmFlowVar(
+                name="vxlan_udp_src",
+                min_value=1337,
+                max_value=32767,
+                size=2,
+                op="random")
+            vm_param = [vxlan_udp_src_fv,
+                        STLVmWrFlowVar(fv_name="vxlan_udp_src", pkt_offset="UDP.sport")]
+        elif stream_cfg['mpls'] is True:
+            encap_level = '0'
+            pkt_base = Ether(src=stream_cfg['vtep_src_mac'], dst=stream_cfg['vtep_dst_mac'])
+            if stream_cfg['vtep_vlan'] is not None:
+                pkt_base /= Dot1Q(vlan=stream_cfg['vtep_vlan'])
+            if stream_cfg['mpls_outer_label'] is not None:
+                pkt_base /= MPLS(label=stream_cfg['mpls_outer_label'], cos=1, s=0, ttl=255)
+            if stream_cfg['mpls_inner_label'] is not None:
+                pkt_base /= MPLS(label=stream_cfg['mpls_inner_label'], cos=1, s=1, ttl=255)
+            #  Flow stats and MPLS labels randomization TBD
+            pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
         else:
             encap_level = '0'
             pkt_base = Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
@@ -301,53 +424,156 @@ class TRex(AbstractTrafficGenerator):
         udp_args = {}
         if stream_cfg['udp_src_port']:
             udp_args['sport'] = int(stream_cfg['udp_src_port'])
+            if stream_cfg['udp_port_step'] == 'random':
+                step = 1
+            else:
+                step = stream_cfg['udp_port_step']
+            udp_args['sport_step'] = int(step)
+            udp_args['sport_max'] = int(stream_cfg['udp_src_port_max'])
         if stream_cfg['udp_dst_port']:
             udp_args['dport'] = int(stream_cfg['udp_dst_port'])
-        pkt_base /= IP() / UDP(**udp_args)
-
-        if stream_cfg['ip_addrs_step'] == 'random':
-            src_fv = STLVmFlowVarRepeatableRandom(
-                name="ip_src",
-                min_value=stream_cfg['ip_src_addr'],
-                max_value=stream_cfg['ip_src_addr_max'],
-                size=4,
-                seed=random.randint(0, 32767),
-                limit=stream_cfg['ip_src_count'])
-            dst_fv = STLVmFlowVarRepeatableRandom(
-                name="ip_dst",
-                min_value=stream_cfg['ip_dst_addr'],
-                max_value=stream_cfg['ip_dst_addr_max'],
-                size=4,
-                seed=random.randint(0, 32767),
-                limit=stream_cfg['ip_dst_count'])
+            if stream_cfg['udp_port_step'] == 'random':
+                step = 1
+            else:
+                step = stream_cfg['udp_port_step']
+            udp_args['dport_step'] = int(step)
+            udp_args['dport_max'] = int(stream_cfg['udp_dst_port_max'])
+
+        pkt_base /= IP(src=stream_cfg['ip_src_addr'], dst=stream_cfg['ip_dst_addr']) / \
+                    UDP(dport=udp_args['dport'], sport=udp_args['sport'])
+
+        # STLVmTupleGen need flow count >= cores used by TRex, if FC < cores we used STLVmFlowVar
+        if stream_cfg['ip_addrs_step'] == '0.0.0.1' and stream_cfg['udp_port_step'] == '1' and \
+                stream_cfg['count'] >= self.generator_config.cores:
+            src_fv = STLVmTupleGen(ip_min=stream_cfg['ip_src_addr'],
+                                   ip_max=stream_cfg['ip_src_addr_max'],
+                                   port_min=udp_args['sport'],
+                                   port_max=udp_args['sport_max'],
+                                   name="tuple_src",
+                                   limit_flows=stream_cfg['count'])
+            dst_fv = STLVmTupleGen(ip_min=stream_cfg['ip_dst_addr'],
+                                   ip_max=stream_cfg['ip_dst_addr_max'],
+                                   port_min=udp_args['dport'],
+                                   port_max=udp_args['dport_max'],
+                                   name="tuple_dst",
+                                   limit_flows=stream_cfg['count'])
+            vm_param = [
+                src_fv,
+                STLVmWrFlowVar(fv_name="tuple_src.ip",
+                               pkt_offset="IP:{}.src".format(encap_level)),
+                STLVmWrFlowVar(fv_name="tuple_src.port",
+                               pkt_offset="UDP:{}.sport".format(encap_level)),
+                dst_fv,
+                STLVmWrFlowVar(fv_name="tuple_dst.ip",
+                               pkt_offset="IP:{}.dst".format(encap_level)),
+                STLVmWrFlowVar(fv_name="tuple_dst.port",
+                               pkt_offset="UDP:{}.dport".format(encap_level)),
+            ]
         else:
-            src_fv = STLVmFlowVar(
-                name="ip_src",
-                min_value=stream_cfg['ip_src_addr'],
-                max_value=stream_cfg['ip_src_addr'],
-                size=4,
-                op="inc",
-                step=stream_cfg['ip_addrs_step'])
-            dst_fv = STLVmFlowVar(
-                name="ip_dst",
-                min_value=stream_cfg['ip_dst_addr'],
-                max_value=stream_cfg['ip_dst_addr_max'],
-                size=4,
-                op="inc",
-                step=stream_cfg['ip_addrs_step'])
-
-        vm_param = [
-            src_fv,
-            STLVmWrFlowVar(fv_name="ip_src", pkt_offset="IP:{}.src".format(encap_level)),
-            dst_fv,
-            STLVmWrFlowVar(fv_name="ip_dst", pkt_offset="IP:{}.dst".format(encap_level)),
-            STLVmFixChecksumHw(l3_offset="IP:{}".format(encap_level),
-                               l4_offset="UDP:{}".format(encap_level),
-                               l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP)
-        ]
+            if disable_random_latency_flow:
+                src_fv_ip = STLVmFlowVar(
+                    name="ip_src",
+                    min_value=stream_cfg['ip_src_addr'],
+                    max_value=stream_cfg['ip_src_addr'],
+                    size=4)
+                dst_fv_ip = STLVmFlowVar(
+                    name="ip_dst",
+                    min_value=stream_cfg['ip_dst_addr'],
+                    max_value=stream_cfg['ip_dst_addr'],
+                    size=4)
+            elif stream_cfg['ip_addrs_step'] == 'random':
+                src_fv_ip = STLVmFlowVarRepeatableRandom(
+                    name="ip_src",
+                    min_value=stream_cfg['ip_src_addr'],
+                    max_value=stream_cfg['ip_src_addr_max'],
+                    size=4,
+                    seed=random.randint(0, 32767),
+                    limit=stream_cfg['ip_src_count'])
+                dst_fv_ip = STLVmFlowVarRepeatableRandom(
+                    name="ip_dst",
+                    min_value=stream_cfg['ip_dst_addr'],
+                    max_value=stream_cfg['ip_dst_addr_max'],
+                    size=4,
+                    seed=random.randint(0, 32767),
+                    limit=stream_cfg['ip_dst_count'])
+            else:
+                src_fv_ip = STLVmFlowVar(
+                    name="ip_src",
+                    min_value=stream_cfg['ip_src_addr'],
+                    max_value=stream_cfg['ip_src_addr_max'],
+                    size=4,
+                    op="inc",
+                    step=stream_cfg['ip_addrs_step'])
+                dst_fv_ip = STLVmFlowVar(
+                    name="ip_dst",
+                    min_value=stream_cfg['ip_dst_addr'],
+                    max_value=stream_cfg['ip_dst_addr_max'],
+                    size=4,
+                    op="inc",
+                    step=stream_cfg['ip_addrs_step'])
+
+            if disable_random_latency_flow:
+                src_fv_port = STLVmFlowVar(
+                    name="p_src",
+                    min_value=udp_args['sport'],
+                    max_value=udp_args['sport'],
+                    size=2)
+                dst_fv_port = STLVmFlowVar(
+                    name="p_dst",
+                    min_value=udp_args['dport'],
+                    max_value=udp_args['dport'],
+                    size=2)
+            elif stream_cfg['udp_port_step'] == 'random':
+                src_fv_port = STLVmFlowVarRepeatableRandom(
+                    name="p_src",
+                    min_value=udp_args['sport'],
+                    max_value=udp_args['sport_max'],
+                    size=2,
+                    seed=random.randint(0, 32767),
+                    limit=stream_cfg['udp_src_count'])
+                dst_fv_port = STLVmFlowVarRepeatableRandom(
+                    name="p_dst",
+                    min_value=udp_args['dport'],
+                    max_value=udp_args['dport_max'],
+                    size=2,
+                    seed=random.randint(0, 32767),
+                    limit=stream_cfg['udp_dst_count'])
+            else:
+                src_fv_port = STLVmFlowVar(
+                    name="p_src",
+                    min_value=udp_args['sport'],
+                    max_value=udp_args['sport_max'],
+                    size=2,
+                    op="inc",
+                    step=udp_args['sport_step'])
+                dst_fv_port = STLVmFlowVar(
+                    name="p_dst",
+                    min_value=udp_args['dport'],
+                    max_value=udp_args['dport_max'],
+                    size=2,
+                    op="inc",
+                    step=udp_args['dport_step'])
+            vm_param = [
+                src_fv_ip,
+                STLVmWrFlowVar(fv_name="ip_src", pkt_offset="IP:{}.src".format(encap_level)),
+                src_fv_port,
+                STLVmWrFlowVar(fv_name="p_src", pkt_offset="UDP:{}.sport".format(encap_level)),
+                dst_fv_ip,
+                STLVmWrFlowVar(fv_name="ip_dst", pkt_offset="IP:{}.dst".format(encap_level)),
+                dst_fv_port,
+                STLVmWrFlowVar(fv_name="p_dst", pkt_offset="UDP:{}.dport".format(encap_level)),
+            ]
+        # Use HW Offload to calculate the outter IP/UDP packet
+        vm_param.append(STLVmFixChecksumHw(l3_offset="IP:0",
+                                           l4_offset="UDP:0",
+                                           l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP))
+        # Use software to fix the inner IP/UDP payload for VxLAN packets
+        if int(encap_level):
+            vm_param.append(STLVmFixIpv4(offset="IP:1"))
         pad = max(0, frame_size - len(pkt_base)) * 'x'
 
-        return STLPktBuilder(pkt=pkt_base / pad, vm=STLScVmRaw(vm_param))
+        return STLPktBuilder(pkt=pkt_base / pad,
+                             vm=STLScVmRaw(vm_param, cache_size=int(self.config.cache_size)))
 
     def generate_streams(self, port, chain_id, stream_cfg, l2frame, latency=True,
                          e2e=False):
@@ -362,55 +588,86 @@ class TRex(AbstractTrafficGenerator):
         """
         streams = []
         pg_id, lat_pg_id = self.get_pg_id(port, chain_id)
+        if self.config.no_flow_stats:
+            LOG.info("Traffic flow statistics are disabled.")
         if l2frame == 'IMIX':
             for ratio, l2_frame_size in zip(IMIX_RATIOS, IMIX_L2_SIZES):
                 pkt = self._create_pkt(stream_cfg, l2_frame_size)
-                if e2e:
+                if e2e or stream_cfg['mpls']:
                     streams.append(STLStream(packet=pkt,
                                              mode=STLTXCont(pps=ratio)))
                 else:
                     if stream_cfg['vxlan'] is True:
                         streams.append(STLStream(packet=pkt,
                                                  flow_stats=STLFlowStats(pg_id=pg_id,
-                                                                         vxlan=True),
+                                                                         vxlan=True)
+                                                 if not self.config.no_flow_stats else None,
                                                  mode=STLTXCont(pps=ratio)))
                     else:
                         streams.append(STLStream(packet=pkt,
-                                                 flow_stats=STLFlowStats(pg_id=pg_id),
+                                                 flow_stats=STLFlowStats(pg_id=pg_id)
+                                                 if not self.config.no_flow_stats else None,
                                                  mode=STLTXCont(pps=ratio)))
 
             if latency:
                 # for IMIX, the latency packets have the average IMIX packet size
-                pkt = self._create_pkt(stream_cfg, IMIX_AVG_L2_FRAME_SIZE)
+                if stream_cfg['ip_addrs_step'] == 'random' or \
+                        stream_cfg['udp_port_step'] == 'random':
+                    # Force latency flow to only one flow to avoid creating flows
+                    # over requested flow count
+                    pkt = self._create_pkt(stream_cfg, IMIX_AVG_L2_FRAME_SIZE, True)
+                else:
+                    pkt = self._create_pkt(stream_cfg, IMIX_AVG_L2_FRAME_SIZE)
 
         else:
             l2frame_size = int(l2frame)
             pkt = self._create_pkt(stream_cfg, l2frame_size)
-            if e2e:
+            if e2e or stream_cfg['mpls']:
                 streams.append(STLStream(packet=pkt,
+                                         # Flow stats is disabled for MPLS now
+                                         # flow_stats=STLFlowStats(pg_id=pg_id),
                                          mode=STLTXCont()))
             else:
                 if stream_cfg['vxlan'] is True:
                     streams.append(STLStream(packet=pkt,
                                              flow_stats=STLFlowStats(pg_id=pg_id,
-                                                                     vxlan=True),
+                                                                     vxlan=True)
+                                             if not self.config.no_flow_stats else None,
                                              mode=STLTXCont()))
                 else:
                     streams.append(STLStream(packet=pkt,
-                                             flow_stats=STLFlowStats(pg_id=pg_id),
+                                             flow_stats=STLFlowStats(pg_id=pg_id)
+                                             if not self.config.no_flow_stats else None,
                                              mode=STLTXCont()))
             # for the latency stream, the minimum payload is 16 bytes even in case of vlan tagging
             # without vlan, the min l2 frame size is 64
             # with vlan it is 68
             # This only applies to the latency stream
-            if latency and stream_cfg['vlan_tag'] and l2frame_size < 68:
-                pkt = self._create_pkt(stream_cfg, 68)
+            if latency:
+                if stream_cfg['vlan_tag'] and l2frame_size < 68:
+                    l2frame_size = 68
+                if stream_cfg['ip_addrs_step'] == 'random' or \
+                        stream_cfg['udp_port_step'] == 'random':
+                        # Force latency flow to only one flow to avoid creating flows
+                        # over requested flow count
+                    pkt = self._create_pkt(stream_cfg, l2frame_size, True)
+                else:
+                    pkt = self._create_pkt(stream_cfg, l2frame_size)
 
         if latency:
-            # TRex limitation: VXLAN skip is not supported for latency stream
-            streams.append(STLStream(packet=pkt,
-                                     flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id),
-                                     mode=STLTXCont(pps=self.LATENCY_PPS)))
+            if self.config.no_latency_stats:
+                LOG.info("Latency flow statistics are disabled.")
+            if stream_cfg['vxlan'] is True:
+                streams.append(STLStream(packet=pkt,
+                                         flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id,
+                                                                        vxlan=True)
+                                         if not self.config.no_latency_stats else None,
+                                         mode=STLTXCont(pps=self.LATENCY_PPS)))
+            else:
+                streams.append(STLStream(packet=pkt,
+                                         flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id)
+                                         if not self.config.no_latency_stats else None,
+                                         mode=STLTXCont(pps=self.LATENCY_PPS)))
         return streams
 
     @timeout(5)
@@ -420,7 +677,7 @@ class TRex(AbstractTrafficGenerator):
     def __connect_after_start(self):
         # after start, Trex may take a bit of time to initialize
         # so we need to retry a few times
-        for it in xrange(self.config.generic_retry_count):
+        for it in range(self.config.generic_retry_count):
             try:
                 time.sleep(1)
                 self.client.connect()
@@ -428,7 +685,7 @@ class TRex(AbstractTrafficGenerator):
             except Exception as ex:
                 if it == (self.config.generic_retry_count - 1):
                     raise
-                LOG.info("Retrying connection to TRex (%s)...", ex.message)
+                LOG.info("Retrying connection to TRex (%s)...", ex.msg)
 
     def connect(self):
         """Connect to the TRex server."""
@@ -440,32 +697,15 @@ class TRex(AbstractTrafficGenerator):
                                 async_port=self.generator_config.zmq_pub_port)
         try:
             self.__connect(self.client)
+            if server_ip == '127.0.0.1':
+                config_updated = self.__check_config()
+                if config_updated or self.config.restart:
+                    self.__restart()
         except (TimeoutError, STLError) as e:
             if server_ip == '127.0.0.1':
-                try:
-                    self.__start_server()
-                    self.__connect_after_start()
-                except (TimeoutError, STLError) as e:
-                    LOG.error('Cannot connect to TRex')
-                    LOG.error(traceback.format_exc())
-                    logpath = '/tmp/trex.log'
-                    if os.path.isfile(logpath):
-                        # Wait for TRex to finish writing error message
-                        last_size = 0
-                        for _ in xrange(self.config.generic_retry_count):
-                            size = os.path.getsize(logpath)
-                            if size == last_size:
-                                # probably not writing anymore
-                                break
-                            last_size = size
-                            time.sleep(1)
-                        with open(logpath, 'r') as f:
-                            message = f.read()
-                    else:
-                        message = e.message
-                    raise TrafficGeneratorException(message)
+                self.__start_local_server()
             else:
-                raise TrafficGeneratorException(e.message)
+                raise TrafficGeneratorException(e.message) from e
 
         ports = list(self.generator_config.ports)
         self.port_handle = ports
@@ -498,10 +738,63 @@ class TRex(AbstractTrafficGenerator):
                                             (self.port_info[0]['speed'],
                                              self.port_info[1]['speed']))
 
+    def __start_local_server(self):
+        try:
+            LOG.info("Starting TRex ...")
+            self.__start_server()
+            self.__connect_after_start()
+        except (TimeoutError, STLError) as e:
+            LOG.error('Cannot connect to TRex')
+            LOG.error(traceback.format_exc())
+            logpath = '/tmp/trex.log'
+            if os.path.isfile(logpath):
+                # Wait for TRex to finish writing error message
+                last_size = 0
+                for _ in range(self.config.generic_retry_count):
+                    size = os.path.getsize(logpath)
+                    if size == last_size:
+                        # probably not writing anymore
+                        break
+                    last_size = size
+                    time.sleep(1)
+                with open(logpath, 'r') as f:
+                    message = f.read()
+            else:
+                message = e.message
+            raise TrafficGeneratorException(message) from e
+
     def __start_server(self):
         server = TRexTrafficServer()
         server.run_server(self.generator_config)
 
+    def __check_config(self):
+        server = TRexTrafficServer()
+        return server.check_config_updated(self.generator_config)
+
+    def __restart(self):
+        LOG.info("Restarting TRex ...")
+        self.__stop_server()
+        # Wait for server stopped
+        for _ in range(self.config.generic_retry_count):
+            time.sleep(1)
+            if not self.client.is_connected():
+                LOG.info("TRex is stopped...")
+                break
+        self.__start_local_server()
+
+    def __stop_server(self):
+        if self.generator_config.ip == '127.0.0.1':
+            ports = self.client.get_acquired_ports()
+            LOG.info('Release ports %s and stopping TRex...', ports)
+            try:
+                if ports:
+                    self.client.release(ports=ports)
+                self.client.server_shutdown()
+            except STLError as e:
+                LOG.warning('Unable to stop TRex. Error: %s', e)
+        else:
+            LOG.info('Using remote TRex. Unable to stop TRex')
+
     def resolve_arp(self):
         """Resolve all configured remote IP addresses.
 
@@ -521,7 +814,7 @@ class TRex(AbstractTrafficGenerator):
             dst_macs = [None] * chain_count
             dst_macs_count = 0
             # the index in the list is the chain id
-            if self.config.vxlan:
+            if self.config.vxlan or self.config.mpls:
                 arps = [
                     ServiceARP(ctx,
                                src_ip=device.vtep_src_ip,
@@ -563,12 +856,12 @@ class TRex(AbstractTrafficGenerator):
                     arp_dest_macs[port] = dst_macs
                     LOG.info('ARP resolved successfully for port %s', port)
                     break
-                else:
-                    retry = attempt + 1
-                    LOG.info('Retrying ARP for: %s (retry %d/%d)',
-                             unresolved, retry, self.config.generic_retry_count)
-                    if retry < self.config.generic_retry_count:
-                        time.sleep(self.config.generic_poll_sec)
+
+                retry = attempt + 1
+                LOG.info('Retrying ARP for: %s (retry %d/%d)',
+                         unresolved, retry, self.config.generic_retry_count)
+                if retry < self.config.generic_retry_count:
+                    time.sleep(self.config.generic_poll_sec)
             else:
                 LOG.error('ARP timed out for port %s (resolved %d out of %d)',
                           port,
@@ -576,7 +869,10 @@ class TRex(AbstractTrafficGenerator):
                           chain_count)
                 break
 
-        self.client.set_service_mode(ports=self.port_handle, enabled=False)
+        # if the capture from the TRex console was started before the arp request step,
+        # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
+        if not self.config.service_mode:
+            self.client.set_service_mode(ports=self.port_handle, enabled=False)
         if len(arp_dest_macs) == len(self.port_handle):
             return arp_dest_macs
         return None
@@ -621,6 +917,7 @@ class TRex(AbstractTrafficGenerator):
                 .format(pps=r['rate_pps'],
                         bps=r['rate_bps'],
                         load=r['rate_percent']))
+        self.l2_frame_size = l2frame_size
         # a dict of list of streams indexed by port#
         # in case of fixed size, has self.chain_count * 2 * 2 streams
         # (1 normal + 1 latency stream per direction per chain)
@@ -630,6 +927,11 @@ class TRex(AbstractTrafficGenerator):
         for port in self.port_handle:
             streamblock[port] = []
         stream_cfgs = [d.get_stream_configs() for d in self.generator_config.devices]
+        if self.generator_config.ip_addrs_step == 'random' \
+                or self.generator_config.gen_config.udp_port_step == 'random':
+            LOG.warning("Using random step, the number of flows can be less than "
+                        "the requested number of flows due to repeatable multivariate random "
+                        "generation which can reproduce the same pattern of values")
         self.rates = [utils.to_rate_str(rate) for rate in rates]
         for chain_id, (fwd_stream_cfg, rev_stream_cfg) in enumerate(zip(*stream_cfgs)):
             streamblock[0].extend(self.generate_streams(self.port_handle[0],
@@ -660,10 +962,10 @@ class TRex(AbstractTrafficGenerator):
         self.client.reset(self.port_handle)
         LOG.info('Cleared all existing streams')
 
-    def get_stats(self):
+    def get_stats(self, if_stats=None):
         """Get stats from Trex."""
         stats = self.client.get_stats()
-        return self.extract_stats(stats)
+        return self.extract_stats(stats, if_stats)
 
     def get_macs(self):
         """Return the Trex local port MAC addresses.
@@ -703,8 +1005,8 @@ class TRex(AbstractTrafficGenerator):
         bpf_filter = "ether dst %s or ether dst %s" % (src_mac_list[0], src_mac_list[1])
         # ports must be set in service in order to enable capture
         self.client.set_service_mode(ports=self.port_handle)
-        self.capture_id = self.client.start_capture(rx_ports=self.port_handle,
-                                                    bpf_filter=bpf_filter)
+        self.capture_id = self.client.start_capture \
+            (rx_ports=self.port_handle, bpf_filter=bpf_filter)
 
     def fetch_capture_packets(self):
         """Fetch capture packets in capture mode."""
@@ -718,7 +1020,10 @@ class TRex(AbstractTrafficGenerator):
         if self.capture_id:
             self.client.stop_capture(capture_id=self.capture_id['id'])
             self.capture_id = None
-            self.client.set_service_mode(ports=self.port_handle, enabled=False)
+            # if the capture from TRex console was started before the connectivity step,
+            # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
+            if not self.config.service_mode:
+                self.client.set_service_mode(ports=self.port_handle, enabled=False)
 
     def cleanup(self):
         """Cleanup Trex driver."""
@@ -729,3 +1034,7 @@ class TRex(AbstractTrafficGenerator):
             except STLError:
                 # TRex does not like a reset while in disconnected state
                 pass
+
+    def set_service_mode(self, enabled=True):
+        """Enable/disable the 'service_mode'."""
+        self.client.set_service_mode(ports=self.port_handle, enabled=enabled)